From c6abf6cd3fbd688b111b339775cbd2d66d509ddc Mon Sep 17 00:00:00 2001 From: Arun Raghavan Date: Tue, 12 Dec 2023 10:42:58 -0500 Subject: [PATCH] Bump to WebRTC M120 release Some API deprecation -- ExperimentalAgc and ExperimentalNs are gone. We're continuing to carry iSAC even though it's gone upstream, but maybe we'll want to drop that soon. --- meson.build | 4 +- webrtc/BUILD.gn | 231 +- webrtc/api/array_view.h | 43 +- webrtc/api/audio/audio_frame.cc | 26 +- webrtc/api/audio/audio_frame.h | 14 +- webrtc/api/audio/channel_layout.cc | 2 +- webrtc/api/audio/echo_canceller3_config.cc | 8 + webrtc/api/audio/echo_canceller3_config.h | 22 + webrtc/api/audio/echo_control.h | 7 + webrtc/api/audio_codecs/audio_decoder.cc | 5 +- webrtc/api/audio_codecs/audio_decoder.h | 42 +- webrtc/api/audio_codecs/audio_encoder.cc | 3 +- webrtc/api/audio_codecs/audio_encoder.h | 23 +- webrtc/api/call/bitrate_allocation.h | 2 +- webrtc/api/location.h | 31 + webrtc/api/make_ref_counted.h | 120 + webrtc/api/ref_counted_base.h | 61 +- webrtc/api/rtp_headers.cc | 1 - webrtc/api/rtp_headers.h | 17 +- webrtc/api/rtp_packet_info.cc | 24 +- webrtc/api/rtp_packet_info.h | 44 +- webrtc/api/rtp_packet_infos.h | 15 +- webrtc/api/scoped_refptr.h | 72 +- webrtc/api/sequence_checker.h | 140 ++ webrtc/api/task_queue/task_queue_base.cc | 2 + webrtc/api/task_queue/task_queue_base.h | 149 +- webrtc/api/units/data_rate.h | 10 +- webrtc/api/units/data_size.h | 10 +- webrtc/api/units/frequency.h | 10 +- webrtc/api/units/time_delta.h | 15 +- webrtc/api/units/timestamp.h | 9 +- webrtc/api/video/color_space.cc | 80 +- webrtc/api/video/color_space.h | 5 +- webrtc/api/video/video_content_type.cc | 68 +- webrtc/api/video/video_content_type.h | 9 +- webrtc/api/video/video_timing.cc | 30 + webrtc/api/video/video_timing.h | 53 +- webrtc/audio/utility/BUILD.gn | 9 +- .../audio/utility/audio_frame_operations.cc | 16 +- webrtc/audio/utility/audio_frame_operations.h | 62 +- webrtc/common_audio/BUILD.gn | 19 +- webrtc/common_audio/audio_converter.h | 13 +- webrtc/common_audio/channel_buffer.h | 26 +- webrtc/common_audio/fir_filter.h | 4 +- webrtc/common_audio/fir_filter_avx2.cc | 2 +- webrtc/common_audio/fir_filter_c.cc | 2 +- webrtc/common_audio/fir_filter_factory.cc | 2 +- webrtc/common_audio/fir_filter_factory.h | 2 +- webrtc/common_audio/fir_filter_neon.cc | 2 +- webrtc/common_audio/fir_filter_sse.cc | 2 +- webrtc/common_audio/include/audio_util.h | 24 +- webrtc/common_audio/real_fourier.h | 2 +- .../common_audio/resampler/push_resampler.cc | 50 +- .../resampler/push_sinc_resampler.cc | 4 +- .../resampler/push_sinc_resampler.h | 14 +- webrtc/common_audio/resampler/resampler.cc | 1 - .../common_audio/resampler/sinc_resampler.cc | 22 +- .../common_audio/resampler/sinc_resampler.h | 44 +- .../resampler/sinc_resampler_avx2.cc | 2 +- .../resampler/sinc_resampler_sse.cc | 2 +- .../sinusoidal_linear_chirp_source.h | 11 +- webrtc/common_audio/ring_buffer.c | 10 +- webrtc/common_audio/ring_buffer.h | 18 +- .../cross_correlation_neon.c | 4 +- .../signal_processing/division_operations.c | 7 +- .../dot_product_with_scale.h | 2 +- .../signal_processing/downsample_fast_neon.c | 33 +- .../signal_processing/include/real_fft.h | 2 +- .../include/signal_processing_library.h | 226 +- .../signal_processing/include/spl_inl.h | 2 + .../signal_processing/include/spl_inl_armv7.h | 2 + .../signal_processing/min_max_operations.c | 36 +- .../min_max_operations_neon.c | 50 + .../signal_processing/splitting_filter.c | 38 +- webrtc/common_audio/smoothing_filter.cc | 12 +- webrtc/common_audio/smoothing_filter.h | 6 +- .../third_party/ooura/fft_size_256/fft4g.cc | 4 +- .../third_party/ooura/fft_size_256/fft4g.h | 2 + .../spl_sqrt_floor/spl_sqrt_floor.h | 4 +- webrtc/common_audio/vad/include/webrtc_vad.h | 4 +- webrtc/common_audio/vad/vad.cc | 2 +- webrtc/common_audio/vad/vad_core.c | 46 +- webrtc/common_audio/vad/vad_core.h | 25 +- webrtc/common_audio/vad/vad_filterbank.c | 98 +- webrtc/common_audio/vad/vad_filterbank.h | 8 +- webrtc/common_audio/vad/vad_gmm.c | 28 +- webrtc/common_audio/vad/vad_gmm.h | 10 +- webrtc/common_audio/vad/vad_sp.c | 14 +- webrtc/common_audio/vad/vad_sp.h | 6 +- webrtc/common_audio/vad/webrtc_vad.c | 2 +- webrtc/common_audio/wav_file.cc | 4 +- webrtc/common_audio/wav_file.h | 4 +- webrtc/common_audio/wav_header.cc | 12 +- webrtc/common_audio/wav_header.h | 1 + webrtc/experiments/registered_field_trials.h | 297 +++ webrtc/modules/BUILD.gn | 27 +- webrtc/modules/audio_coding/BUILD.gn | 1552 +++++------- .../isac/main/source/filter_functions.h | 2 + .../codecs/isac/main/source/pitch_filter.c | 16 +- webrtc/modules/audio_processing/BUILD.gn | 462 ++-- webrtc/modules/audio_processing/aec3/BUILD.gn | 34 +- .../aec3/adaptive_fir_filter.cc | 32 +- .../aec3/adaptive_fir_filter.h | 3 +- .../aec3/adaptive_fir_filter_avx2.cc | 19 +- .../aec3/adaptive_fir_filter_erl_avx2.cc | 4 +- .../audio_processing/aec3/aec3_common.cc | 4 +- .../audio_processing/aec3/aec3_common.h | 6 +- .../modules/audio_processing/aec3/aec3_fft.cc | 8 +- .../modules/audio_processing/aec3/aec3_fft.h | 6 +- .../audio_processing/aec3/aec_state.cc | 60 +- .../modules/audio_processing/aec3/aec_state.h | 28 +- .../audio_processing/aec3/alignment_mixer.cc | 27 +- .../audio_processing/aec3/alignment_mixer.h | 9 +- webrtc/modules/audio_processing/aec3/block.h | 91 + .../audio_processing/aec3/block_buffer.cc | 20 +- .../audio_processing/aec3/block_buffer.h | 8 +- .../aec3/block_delay_buffer.cc | 17 +- .../audio_processing/aec3/block_framer.cc | 27 +- .../audio_processing/aec3/block_framer.h | 5 +- .../audio_processing/aec3/block_processor.cc | 62 +- .../audio_processing/aec3/block_processor.h | 19 +- .../aec3/block_processor_metrics.h | 7 +- .../aec3/comfort_noise_generator.h | 1 - .../audio_processing/aec3/config_selector.cc | 71 + .../audio_processing/aec3/config_selector.h | 41 + .../modules/audio_processing/aec3/decimator.h | 6 +- .../audio_processing/aec3/delay_estimate.h | 2 + .../audio_processing/aec3/echo_audibility.cc | 5 +- .../audio_processing/aec3/echo_audibility.h | 1 - .../audio_processing/aec3/echo_canceller3.cc | 410 ++-- .../audio_processing/aec3/echo_canceller3.h | 80 +- .../aec3/echo_path_delay_estimator.cc | 20 +- .../aec3/echo_path_delay_estimator.h | 9 +- .../aec3/echo_path_variability.h | 6 +- .../audio_processing/aec3/echo_remover.cc | 155 +- .../audio_processing/aec3/echo_remover.h | 11 +- .../aec3/echo_remover_metrics.cc | 95 +- .../aec3/echo_remover_metrics.h | 9 +- .../audio_processing/aec3/erl_estimator.h | 5 +- .../audio_processing/aec3/erle_estimator.cc | 9 +- .../audio_processing/aec3/erle_estimator.h | 23 +- .../audio_processing/aec3/fft_data_avx2.cc | 3 +- .../audio_processing/aec3/filter_analyzer.cc | 35 +- .../audio_processing/aec3/filter_analyzer.h | 7 +- .../audio_processing/aec3/frame_blocker.cc | 34 +- .../audio_processing/aec3/frame_blocker.h | 5 +- .../aec3/fullband_erle_estimator.cc | 43 +- .../aec3/fullband_erle_estimator.h | 6 +- .../audio_processing/aec3/matched_filter.cc | 644 ++++- .../audio_processing/aec3/matched_filter.h | 77 +- .../aec3/matched_filter_avx2.cc | 159 +- .../aec3/matched_filter_lag_aggregator.cc | 191 +- .../aec3/matched_filter_lag_aggregator.h | 57 +- .../aec3/multi_channel_content_detector.cc | 148 ++ .../aec3/multi_channel_content_detector.h | 95 + .../aec3/refined_filter_update_gain.cc | 9 +- .../aec3/refined_filter_update_gain.h | 4 +- .../audio_processing/aec3/render_buffer.cc | 17 +- .../audio_processing/aec3/render_buffer.h | 3 +- .../aec3/render_delay_buffer.cc | 70 +- .../aec3/render_delay_buffer.h | 4 +- .../aec3/render_delay_controller.cc | 38 +- .../aec3/render_delay_controller.h | 3 +- .../aec3/render_delay_controller_metrics.cc | 29 +- .../aec3/render_delay_controller_metrics.h | 16 +- .../aec3/render_signal_analyzer.cc | 16 +- .../aec3/render_signal_analyzer.h | 6 +- .../aec3/residual_echo_estimator.cc | 90 +- .../aec3/residual_echo_estimator.h | 17 +- .../aec3/reverb_decay_estimator.cc | 5 +- .../aec3/reverb_decay_estimator.h | 12 +- .../aec3/reverb_frequency_response.cc | 14 +- .../aec3/reverb_frequency_response.h | 5 +- .../audio_processing/aec3/reverb_model.h | 1 - .../aec3/reverb_model_estimator.cc | 5 +- .../aec3/reverb_model_estimator.h | 9 +- .../aec3/signal_dependent_erle_estimator.cc | 10 + .../aec3/signal_dependent_erle_estimator.h | 10 +- .../aec3/stationarity_estimator.cc | 6 +- .../aec3/stationarity_estimator.h | 3 +- .../aec3/subband_erle_estimator.cc | 83 +- .../aec3/subband_erle_estimator.h | 23 +- .../audio_processing/aec3/subtractor.cc | 53 +- .../audio_processing/aec3/subtractor.h | 15 +- .../aec3/subtractor_output_analyzer.cc | 11 +- .../aec3/subtractor_output_analyzer.h | 1 + .../aec3/suppression_filter.cc | 43 +- .../aec3/suppression_filter.h | 9 +- .../audio_processing/aec3/suppression_gain.cc | 161 +- .../audio_processing/aec3/suppression_gain.h | 29 +- .../audio_processing/aec3/transparent_mode.cc | 28 +- .../audio_processing/aec3/transparent_mode.h | 1 + .../audio_processing/aec3/vector_math_avx2.cc | 3 +- .../audio_processing/aec_dump/BUILD.gn | 12 +- .../aec_dump/aec_dump_factory.h | 12 +- .../aec_dump/null_aec_dump_factory.cc | 3 +- webrtc/modules/audio_processing/aecm/BUILD.gn | 2 +- .../audio_processing/aecm/aecm_core.cc | 15 +- .../modules/audio_processing/aecm/aecm_core.h | 18 +- .../audio_processing/aecm/aecm_core_c.cc | 35 +- .../audio_processing/aecm/aecm_core_mips.cc | 32 +- webrtc/modules/audio_processing/agc/BUILD.gn | 26 +- webrtc/modules/audio_processing/agc/agc.cc | 16 +- webrtc/modules/audio_processing/agc/agc.h | 7 +- .../agc/agc_manager_direct.cc | 464 ++-- .../audio_processing/agc/agc_manager_direct.h | 230 +- .../audio_processing/agc/gain_control.h | 14 +- .../audio_processing/agc/legacy/analog_agc.cc | 2 +- .../audio_processing/agc/legacy/analog_agc.h | 7 +- .../agc/legacy/digital_agc.cc | 20 +- .../agc/legacy/gain_control.h | 3 + .../agc/loudness_histogram.cc | 2 +- .../audio_processing/agc/loudness_histogram.h | 8 +- .../modules/audio_processing/agc/mock_agc.h | 6 +- webrtc/modules/audio_processing/agc2/BUILD.gn | 359 ++- .../audio_processing/agc2/adaptive_agc.cc | 90 - .../audio_processing/agc2/adaptive_agc.h | 50 - .../agc2/adaptive_digital_gain_applier.cc | 179 -- .../agc2/adaptive_digital_gain_applier.h | 69 - .../agc2/adaptive_digital_gain_controller.cc | 216 ++ .../agc2/adaptive_digital_gain_controller.h | 66 + .../agc2/adaptive_mode_level_estimator.cc | 198 -- .../agc2/adaptive_mode_level_estimator_agc.cc | 65 - .../agc2/adaptive_mode_level_estimator_agc.h | 51 - .../audio_processing/agc2/agc2_common.h | 78 +- .../agc2/agc2_testing_common.cc | 73 +- .../agc2/agc2_testing_common.h | 76 +- .../audio_processing/agc2/biquad_filter.cc | 52 +- .../audio_processing/agc2/biquad_filter.h | 52 +- .../agc2/clipping_predictor.cc | 384 +++ .../agc2/clipping_predictor.h | 62 + .../agc2/clipping_predictor_level_buffer.cc | 77 + .../agc2/clipping_predictor_level_buffer.h | 71 + .../agc2/compute_interpolated_gain_curve.cc | 4 +- .../agc2/compute_interpolated_gain_curve.h | 4 +- .../audio_processing/agc2/cpu_features.cc | 62 + .../audio_processing/agc2/cpu_features.h | 39 + .../audio_processing/agc2/down_sampler.cc | 99 - .../audio_processing/agc2/down_sampler.h | 42 - .../agc2/fixed_digital_level_estimator.cc | 29 +- .../agc2/fixed_digital_level_estimator.h | 15 +- .../agc2/fixed_gain_controller.cc | 101 - .../audio_processing/agc2/gain_applier.cc | 11 +- .../audio_processing/agc2/gain_applier.h | 2 +- .../{agc => agc2}/gain_map_internal.h | 16 +- .../agc2/input_volume_controller.cc | 580 +++++ .../agc2/input_volume_controller.h | 282 +++ .../agc2/input_volume_stats_reporter.cc | 171 ++ .../agc2/input_volume_stats_reporter.h | 96 + .../agc2/interpolated_gain_curve.cc | 43 +- .../agc2/interpolated_gain_curve.h | 20 +- .../modules/audio_processing/agc2/limiter.cc | 51 +- .../modules/audio_processing/agc2/limiter.h | 11 +- .../agc2/limiter_db_gain_curve.cc | 2 +- .../agc2/noise_level_estimator.cc | 206 +- .../agc2/noise_level_estimator.h | 29 +- .../agc2/noise_spectrum_estimator.cc | 70 - .../agc2/noise_spectrum_estimator.h | 42 - .../audio_processing/agc2/rnn_vad/BUILD.gn | 140 +- .../agc2/rnn_vad/auto_correlation.cc | 23 +- .../agc2/rnn_vad/auto_correlation.h | 4 +- .../audio_processing/agc2/rnn_vad/common.cc | 34 - .../audio_processing/agc2/rnn_vad/common.h | 57 +- .../agc2/rnn_vad/features_extraction.cc | 26 +- .../agc2/rnn_vad/features_extraction.h | 7 +- .../agc2/rnn_vad/lp_residual.cc | 93 +- .../agc2/rnn_vad/lp_residual.h | 10 +- .../agc2/rnn_vad/pitch_info.h | 29 - .../agc2/rnn_vad/pitch_search.cc | 66 +- .../agc2/rnn_vad/pitch_search.h | 25 +- .../agc2/rnn_vad/pitch_search_internal.cc | 648 ++--- .../agc2/rnn_vad/pitch_search_internal.h | 117 +- .../agc2/rnn_vad/ring_buffer.h | 15 +- .../audio_processing/agc2/rnn_vad/rnn.cc | 430 +--- .../audio_processing/agc2/rnn_vad/rnn.h | 103 +- .../audio_processing/agc2/rnn_vad/rnn_fc.cc | 104 + .../audio_processing/agc2/rnn_vad/rnn_fc.h | 72 + .../audio_processing/agc2/rnn_vad/rnn_gru.cc | 198 ++ .../audio_processing/agc2/rnn_vad/rnn_gru.h | 70 + .../agc2/rnn_vad/sequence_buffer.h | 6 +- .../agc2/rnn_vad/spectral_features.cc | 23 +- .../rnn_vad/spectral_features_internal.cc | 19 +- .../agc2/rnn_vad/spectral_features_internal.h | 16 +- .../agc2/rnn_vad/symmetric_matrix_buffer.h | 21 +- .../agc2/rnn_vad/test_utils.cc | 154 +- .../agc2/rnn_vad/test_utils.h | 171 +- .../agc2/rnn_vad/vector_math.h | 114 + .../agc2/rnn_vad/vector_math_avx2.cc | 54 + .../agc2/saturation_protector.cc | 200 +- .../agc2/saturation_protector.h | 76 +- .../agc2/saturation_protector_buffer.cc | 77 + .../agc2/saturation_protector_buffer.h | 59 + .../agc2/signal_classifier.cc | 177 -- .../audio_processing/agc2/signal_classifier.h | 73 - .../agc2/speech_level_estimator.cc | 174 ++ ...l_estimator.h => speech_level_estimator.h} | 49 +- .../agc2/speech_probability_buffer.cc | 105 + .../agc2/speech_probability_buffer.h | 80 + .../audio_processing/agc2/vad_with_level.cc | 114 - .../audio_processing/agc2/vad_with_level.h | 58 - .../audio_processing/agc2/vad_wrapper.cc | 113 + .../audio_processing/agc2/vad_wrapper.h | 82 + .../modules/audio_processing/audio_buffer.cc | 17 +- .../modules/audio_processing/audio_buffer.h | 24 +- .../audio_processing_builder_impl.cc | 25 +- .../audio_processing/audio_processing_impl.cc | 1561 +++++++++---- .../audio_processing/audio_processing_impl.h | 204 +- .../capture_levels_adjuster/BUILD.gn | 45 + .../audio_samples_scaler.cc | 92 + .../audio_samples_scaler.h | 46 + .../capture_levels_adjuster.cc | 96 + .../capture_levels_adjuster.h | 88 + webrtc/modules/audio_processing/common.h | 34 - webrtc/modules/audio_processing/debug.proto | 3 +- .../echo_control_mobile_impl.cc | 7 +- .../echo_control_mobile_impl.h | 2 +- .../audio_processing/gain_control_impl.cc | 37 +- .../audio_processing/gain_control_impl.h | 1 - .../audio_processing/gain_controller2.cc | 337 ++- .../audio_processing/gain_controller2.h | 88 +- .../audio_processing/high_pass_filter.cc | 4 +- .../audio_processing/include/aec_dump.cc | 2 +- .../audio_processing/include/aec_dump.h | 10 +- .../include/audio_frame_proxies.cc | 12 +- .../include/audio_frame_proxies.h | 12 +- .../include/audio_frame_view.h | 25 +- .../include/audio_processing.cc | 184 +- .../include/audio_processing.h | 475 ++-- .../include/audio_processing_statistics.h | 16 +- .../audio_processing/level_estimator.cc | 29 - .../audio_processing/level_estimator.h | 47 - .../logging/apm_data_dumper.cc | 17 +- .../logging/apm_data_dumper.h | 182 +- webrtc/modules/audio_processing/meson.build | 30 +- webrtc/modules/audio_processing/ns/BUILD.gn | 8 +- .../audio_processing/ns/noise_suppressor.cc | 7 + .../audio_processing/ns/noise_suppressor.h | 9 + .../ns/prior_signal_model_estimator.cc | 1 + .../ns/quantile_noise_estimator.h | 1 + .../ns/speech_probability_estimator.cc | 1 + .../audio_processing/ns/suppression_params.cc | 2 +- .../audio_processing/ns/wiener_filter.cc | 1 + .../optionally_built_submodule_creators.cc | 9 +- .../optionally_built_submodule_creators.h | 10 +- .../residual_echo_detector.cc | 14 +- .../audio_processing/residual_echo_detector.h | 7 +- webrtc/modules/audio_processing/rms_level.cc | 14 +- webrtc/modules/audio_processing/rms_level.h | 6 +- .../three_band_filter_bank.cc | 36 +- .../audio_processing/three_band_filter_bank.h | 8 +- .../audio_processing/transient/BUILD.gn | 97 +- .../transient/click_annotate.cc | 107 - .../transient/dyadic_decimator.h | 6 +- .../audio_processing/transient/file_utils.h | 47 +- .../transient/moving_moments.h | 8 +- .../transient/transient_detector.cc | 6 +- .../transient/transient_detector.h | 6 +- .../transient/transient_suppressor.h | 69 +- .../transient/transient_suppressor_impl.cc | 140 +- .../transient/transient_suppressor_impl.h | 54 +- .../transient/voice_probability_delay_unit.cc | 56 + .../transient/voice_probability_delay_unit.h | 43 + .../audio_processing/transient/wpd_node.h | 3 +- .../audio_processing/transient/wpd_tree.h | 2 +- .../audio_processing/typing_detection.cc | 93 - .../audio_processing/typing_detection.h | 92 - .../modules/audio_processing/utility/BUILD.gn | 2 - .../utility/cascaded_biquad_filter.cc | 29 +- .../utility/delay_estimator.cc | 146 +- .../utility/delay_estimator.h | 6 +- .../utility/delay_estimator_internal.h | 4 +- .../utility/delay_estimator_wrapper.cc | 32 +- .../utility/delay_estimator_wrapper.h | 38 +- .../audio_processing/utility/pffft_wrapper.h | 8 +- webrtc/modules/audio_processing/vad/gmm.h | 10 +- .../audio_processing/vad/pitch_based_vad.h | 2 +- .../audio_processing/vad/pitch_internal.h | 2 +- .../audio_processing/vad/standalone_vad.h | 4 +- .../audio_processing/vad/vad_audio_proc.cc | 4 +- .../audio_processing/vad/vad_audio_proc.h | 32 +- .../vad/vad_audio_proc_internal.h | 2 + .../vad/vad_circular_buffer.h | 6 +- .../vad/voice_activity_detector.cc | 5 +- .../vad/voice_activity_detector.h | 2 + .../audio_processing/voice_detection.cc | 92 - .../audio_processing/voice_detection.h | 59 - webrtc/rtc_base/BUILD.gn | 2075 +++++++++++------ webrtc/rtc_base/atomic_ops.h | 79 - webrtc/rtc_base/buffer.h | 33 +- webrtc/rtc_base/checks.cc | 85 +- webrtc/rtc_base/checks.h | 79 +- webrtc/rtc_base/containers/BUILD.gn | 56 + webrtc/rtc_base/containers/flat_set.h | 178 ++ .../flat_tree.cc} | 18 +- webrtc/rtc_base/containers/flat_tree.h | 1099 +++++++++ webrtc/rtc_base/containers/identity.h | 36 + webrtc/rtc_base/deprecation.h | 45 - webrtc/rtc_base/event.cc | 35 +- webrtc/rtc_base/event.h | 69 +- webrtc/rtc_base/event_tracer.cc | 88 +- webrtc/rtc_base/event_tracer.h | 13 +- .../experiments/field_trial_parser.cc | 105 +- .../rtc_base/experiments/field_trial_parser.h | 49 +- webrtc/rtc_base/logging.cc | 213 +- webrtc/rtc_base/logging.h | 182 +- webrtc/rtc_base/memory/BUILD.gn | 17 +- webrtc/rtc_base/memory/aligned_malloc.cc | 2 +- webrtc/rtc_base/memory/aligned_malloc.h | 8 +- webrtc/rtc_base/meson.build | 22 +- webrtc/rtc_base/numerics/divide_round.h | 60 + webrtc/rtc_base/numerics/safe_conversions.h | 6 +- webrtc/rtc_base/numerics/safe_minmax.h | 6 +- webrtc/rtc_base/platform_thread.cc | 267 ++- webrtc/rtc_base/platform_thread.h | 140 +- webrtc/rtc_base/platform_thread_types.cc | 13 +- webrtc/rtc_base/race_checker.cc | 6 +- webrtc/rtc_base/random.cc | 87 + webrtc/rtc_base/random.h | 96 + webrtc/rtc_base/ref_counted_object.h | 41 +- webrtc/rtc_base/string_encode.cc | 179 +- webrtc/rtc_base/string_encode.h | 79 +- webrtc/rtc_base/string_to_number.cc | 50 +- webrtc/rtc_base/string_to_number.h | 31 +- webrtc/rtc_base/string_utils.cc | 26 +- webrtc/rtc_base/string_utils.h | 67 +- webrtc/rtc_base/strings/string_builder.cc | 33 +- webrtc/rtc_base/strings/string_builder.h | 13 +- webrtc/rtc_base/swap_queue.h | 6 +- webrtc/rtc_base/synchronization/BUILD.gn | 51 +- webrtc/rtc_base/synchronization/mutex.cc | 39 - webrtc/rtc_base/synchronization/mutex.h | 51 +- .../synchronization/mutex_critical_section.h | 4 +- .../rtc_base/synchronization/mutex_pthread.h | 56 +- .../rtc_base/synchronization/rw_lock_posix.cc | 52 - .../rtc_base/synchronization/rw_lock_posix.h | 40 - .../rtc_base/synchronization/rw_lock_win.cc | 41 - webrtc/rtc_base/synchronization/rw_lock_win.h | 38 - .../synchronization/rw_lock_wrapper.h | 66 - .../synchronization/sequence_checker.h | 187 -- ...hecker.cc => sequence_checker_internal.cc} | 60 +- .../sequence_checker_internal.h | 90 + webrtc/rtc_base/system/BUILD.gn | 111 + webrtc/rtc_base/system/arch.h | 10 + webrtc/rtc_base/system/file_wrapper.cc | 60 +- webrtc/rtc_base/system/file_wrapper.h | 22 +- webrtc/rtc_base/system/no_unique_address.h | 37 + webrtc/rtc_base/system/unused.h | 21 +- .../warn_current_thread_is_deadlocked.cc | 28 + .../warn_current_thread_is_deadlocked.h | 4 - webrtc/rtc_base/system_time.cc | 102 + webrtc/rtc_base/system_time.h | 24 + webrtc/rtc_base/thread_annotations.h | 3 + webrtc/rtc_base/thread_checker.h | 27 - webrtc/rtc_base/time_utils.cc | 129 +- webrtc/rtc_base/time_utils.h | 25 +- webrtc/rtc_base/trace_event.h | 920 ++++---- webrtc/rtc_base/units/BUILD.gn | 3 +- webrtc/rtc_base/units/unit_base.h | 71 +- webrtc/rtc_base/win32.h | 62 +- webrtc/system_wrappers/BUILD.gn | 45 +- .../include/cpu_features_wrapper.h | 2 +- .../include/denormal_disabler.h | 56 + webrtc/system_wrappers/include/field_trial.h | 30 +- webrtc/system_wrappers/include/metrics.h | 162 +- webrtc/system_wrappers/meson.build | 1 + webrtc/system_wrappers/source/cpu_features.cc | 14 +- .../source/denormal_disabler.cc | 111 + webrtc/system_wrappers/source/field_trial.cc | 63 +- webrtc/system_wrappers/source/metrics.cc | 81 +- webrtc/third_party/pffft/BUILD.gn | 7 +- webrtc/third_party/pffft/LICENSE | 46 +- webrtc/third_party/pffft/README.md | 69 + webrtc/third_party/pffft/README.txt | 379 +++ .../pffft/patches/01-rmv_printf.diff | 60 + .../pffft/patches/02-decl_validate_simd.diff | 16 + .../third_party/pffft/patches/03-malloca.diff | 82 + .../pffft/patches/04-fix_ptr_cast.diff | 48 + .../pffft/patches/05-fix-arch-detection.diff | 22 + webrtc/third_party/rnnoise/BUILD.gn | 18 +- 479 files changed, 20900 insertions(+), 11996 deletions(-) create mode 100644 webrtc/api/location.h create mode 100644 webrtc/api/make_ref_counted.h create mode 100644 webrtc/api/sequence_checker.h create mode 100644 webrtc/experiments/registered_field_trials.h create mode 100644 webrtc/modules/audio_processing/aec3/block.h create mode 100644 webrtc/modules/audio_processing/aec3/config_selector.cc create mode 100644 webrtc/modules/audio_processing/aec3/config_selector.h create mode 100644 webrtc/modules/audio_processing/aec3/multi_channel_content_detector.cc create mode 100644 webrtc/modules/audio_processing/aec3/multi_channel_content_detector.h delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_agc.cc delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_agc.h delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.h create mode 100644 webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.cc create mode 100644 webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.h delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.cc delete mode 100644 webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.h create mode 100644 webrtc/modules/audio_processing/agc2/clipping_predictor.cc create mode 100644 webrtc/modules/audio_processing/agc2/clipping_predictor.h create mode 100644 webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.cc create mode 100644 webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.h create mode 100644 webrtc/modules/audio_processing/agc2/cpu_features.cc create mode 100644 webrtc/modules/audio_processing/agc2/cpu_features.h delete mode 100644 webrtc/modules/audio_processing/agc2/down_sampler.cc delete mode 100644 webrtc/modules/audio_processing/agc2/down_sampler.h delete mode 100644 webrtc/modules/audio_processing/agc2/fixed_gain_controller.cc rename webrtc/modules/audio_processing/{agc => agc2}/gain_map_internal.h (74%) create mode 100644 webrtc/modules/audio_processing/agc2/input_volume_controller.cc create mode 100644 webrtc/modules/audio_processing/agc2/input_volume_controller.h create mode 100644 webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.cc create mode 100644 webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.h delete mode 100644 webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.cc delete mode 100644 webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.h delete mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/common.cc delete mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/pitch_info.h create mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.cc create mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.h create mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.cc create mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.h create mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/vector_math.h create mode 100644 webrtc/modules/audio_processing/agc2/rnn_vad/vector_math_avx2.cc create mode 100644 webrtc/modules/audio_processing/agc2/saturation_protector_buffer.cc create mode 100644 webrtc/modules/audio_processing/agc2/saturation_protector_buffer.h delete mode 100644 webrtc/modules/audio_processing/agc2/signal_classifier.cc delete mode 100644 webrtc/modules/audio_processing/agc2/signal_classifier.h create mode 100644 webrtc/modules/audio_processing/agc2/speech_level_estimator.cc rename webrtc/modules/audio_processing/agc2/{adaptive_mode_level_estimator.h => speech_level_estimator.h} (55%) create mode 100644 webrtc/modules/audio_processing/agc2/speech_probability_buffer.cc create mode 100644 webrtc/modules/audio_processing/agc2/speech_probability_buffer.h delete mode 100644 webrtc/modules/audio_processing/agc2/vad_with_level.cc delete mode 100644 webrtc/modules/audio_processing/agc2/vad_with_level.h create mode 100644 webrtc/modules/audio_processing/agc2/vad_wrapper.cc create mode 100644 webrtc/modules/audio_processing/agc2/vad_wrapper.h create mode 100644 webrtc/modules/audio_processing/capture_levels_adjuster/BUILD.gn create mode 100644 webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc create mode 100644 webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h create mode 100644 webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc create mode 100644 webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h delete mode 100644 webrtc/modules/audio_processing/common.h delete mode 100644 webrtc/modules/audio_processing/level_estimator.cc delete mode 100644 webrtc/modules/audio_processing/level_estimator.h delete mode 100644 webrtc/modules/audio_processing/transient/click_annotate.cc create mode 100644 webrtc/modules/audio_processing/transient/voice_probability_delay_unit.cc create mode 100644 webrtc/modules/audio_processing/transient/voice_probability_delay_unit.h delete mode 100644 webrtc/modules/audio_processing/typing_detection.cc delete mode 100644 webrtc/modules/audio_processing/typing_detection.h delete mode 100644 webrtc/modules/audio_processing/voice_detection.cc delete mode 100644 webrtc/modules/audio_processing/voice_detection.h delete mode 100644 webrtc/rtc_base/atomic_ops.h create mode 100644 webrtc/rtc_base/containers/BUILD.gn create mode 100644 webrtc/rtc_base/containers/flat_set.h rename webrtc/rtc_base/{synchronization/rw_lock_wrapper.cc => containers/flat_tree.cc} (50%) create mode 100644 webrtc/rtc_base/containers/flat_tree.h create mode 100644 webrtc/rtc_base/containers/identity.h delete mode 100644 webrtc/rtc_base/deprecation.h create mode 100644 webrtc/rtc_base/numerics/divide_round.h create mode 100644 webrtc/rtc_base/random.cc create mode 100644 webrtc/rtc_base/random.h delete mode 100644 webrtc/rtc_base/synchronization/mutex.cc delete mode 100644 webrtc/rtc_base/synchronization/rw_lock_posix.cc delete mode 100644 webrtc/rtc_base/synchronization/rw_lock_posix.h delete mode 100644 webrtc/rtc_base/synchronization/rw_lock_win.cc delete mode 100644 webrtc/rtc_base/synchronization/rw_lock_win.h delete mode 100644 webrtc/rtc_base/synchronization/rw_lock_wrapper.h delete mode 100644 webrtc/rtc_base/synchronization/sequence_checker.h rename webrtc/rtc_base/synchronization/{sequence_checker.cc => sequence_checker_internal.cc} (59%) create mode 100644 webrtc/rtc_base/synchronization/sequence_checker_internal.h create mode 100644 webrtc/rtc_base/system/BUILD.gn create mode 100644 webrtc/rtc_base/system/no_unique_address.h create mode 100644 webrtc/rtc_base/system/warn_current_thread_is_deadlocked.cc create mode 100644 webrtc/rtc_base/system_time.cc create mode 100644 webrtc/rtc_base/system_time.h delete mode 100644 webrtc/rtc_base/thread_checker.h create mode 100644 webrtc/system_wrappers/include/denormal_disabler.h create mode 100644 webrtc/system_wrappers/source/denormal_disabler.cc create mode 100644 webrtc/third_party/pffft/README.md create mode 100644 webrtc/third_party/pffft/README.txt create mode 100644 webrtc/third_party/pffft/patches/01-rmv_printf.diff create mode 100644 webrtc/third_party/pffft/patches/02-decl_validate_simd.diff create mode 100644 webrtc/third_party/pffft/patches/03-malloca.diff create mode 100644 webrtc/third_party/pffft/patches/04-fix_ptr_cast.diff create mode 100644 webrtc/third_party/pffft/patches/05-fix-arch-detection.diff diff --git a/meson.build b/meson.build index de07ded..7aff86c 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,5 @@ project('webrtc-audio-processing', 'c', 'cpp', - version : '1.3', + version : '1.0', meson_version : '>= 0.63', default_options : [ 'warning_level=1', 'buildtype=debugoptimized', @@ -166,6 +166,8 @@ endif common_cflags = [ '-DWEBRTC_LIBRARY_IMPL', '-DWEBRTC_ENABLE_SYMBOL_EXPORT', + # avoid windows.h/winsock2.h conflicts + '-D_WINSOCKAPI_', '-DNDEBUG' ] + platform_cflags + os_cflags + arch_cflags common_cxxflags = common_cflags diff --git a/webrtc/BUILD.gn b/webrtc/BUILD.gn index cec97c5..7e8325e 100644 --- a/webrtc/BUILD.gn +++ b/webrtc/BUILD.gn @@ -12,6 +12,15 @@ # you add a new build file, there must be some path of dependencies from this # file to your new one or GN won't know about it. +# Use of visibility = clauses: +# The default visibility for all rtc_ targets is equivalent to "//*", or +# "all targets in webrtc can depend on this, nothing outside can". +# +# When overriding, the choices are: +# - visibility = [ "*" ] - public. Stuff outside webrtc can use this. +# - visibility = [ ":*" ] - directory private. +# As a general guideline, only targets in api/ should have public visibility. + import("//build/config/linux/pkg_config.gni") import("//build/config/sanitizers/sanitizers.gni") import("webrtc.gni") @@ -21,6 +30,7 @@ if (rtc_enable_protobuf) { if (is_android) { import("//build/config/android/config.gni") import("//build/config/android/rules.gni") + import("//third_party/jni_zero/jni_zero.gni") } if (!build_with_chromium) { @@ -38,7 +48,6 @@ if (!build_with_chromium) { if (rtc_include_tests) { deps += [ ":rtc_unittests", - ":slow_tests", ":video_engine_tests", ":voip_unittests", ":webrtc_nonparallel_tests", @@ -54,9 +63,14 @@ if (!build_with_chromium) { "modules/remote_bitrate_estimator:rtp_to_text", "modules/rtp_rtcp:test_packet_masks_metrics", "modules/video_capture:video_capture_internal_impl", + "modules/video_coding:video_codec_perf_tests", + "net/dcsctp:dcsctp_unittests", "pc:peerconnection_unittests", "pc:rtc_pc_unittests", + "pc:slow_peer_connection_unittests", + "pc:svc_tests", "rtc_tools:rtp_generator", + "rtc_tools:video_encoder", "rtc_tools:video_replay", "stats:rtc_stats_unittests", "system_wrappers:system_wrappers_unittests", @@ -71,6 +85,13 @@ if (!build_with_chromium) { # see bugs.webrtc.org/11027#c5. deps += [ ":webrtc_lib_link_test" ] } + if (is_ios) { + deps += [ + "examples:apprtcmobile_tests", + "sdk:sdk_framework_unittests", + "sdk:sdk_unittests", + ] + } if (is_android) { deps += [ "examples:android_examples_junit_tests", @@ -82,11 +103,16 @@ if (!build_with_chromium) { } if (rtc_enable_protobuf) { deps += [ - "audio:low_bandwidth_audio_test", "logging:rtc_event_log_rtp_dump", "tools_webrtc/perf:webrtc_dashboard_upload", ] } + if ((is_linux || is_chromeos) && rtc_use_pipewire) { + deps += [ "modules/desktop_capture:shared_screencast_stream_test" ] + } + } + if (target_os == "android") { + deps += [ "tools_webrtc:binary_version_check" ] } } } @@ -113,12 +139,23 @@ config("common_inherited_config") { cflags = [] ldflags = [] - if (rtc_enable_symbol_export || is_component_build) { - defines = [ "WEBRTC_ENABLE_SYMBOL_EXPORT" ] + if (rtc_jni_generator_legacy_symbols) { + defines += [ "RTC_JNI_GENERATOR_LEGACY_SYMBOLS" ] } - if (build_with_mozilla) { - defines += [ "WEBRTC_MOZILLA_BUILD" ] + if (rtc_objc_prefix != "") { + defines += [ "RTC_OBJC_TYPE_PREFIX=${rtc_objc_prefix}" ] + } + + if (rtc_dlog_always_on) { + defines += [ "DLOG_ALWAYS_ON" ] + } + + if (rtc_enable_symbol_export || is_component_build) { + defines += [ "WEBRTC_ENABLE_SYMBOL_EXPORT" ] + } + if (rtc_enable_objc_symbol_export) { + defines += [ "WEBRTC_ENABLE_OBJC_SYMBOL_EXPORT" ] } if (!rtc_builtin_ssl_root_certificates) { @@ -133,6 +170,10 @@ config("common_inherited_config") { defines += [ "WEBRTC_ENABLE_AVX2" ] } + if (rtc_enable_win_wgc) { + defines += [ "RTC_ENABLE_WIN_WGC" ] + } + # Some tests need to declare their own trace event handlers. If this define is # not set, the first time TRACE_EVENT_* is called it will store the return # value for the current handler in an static variable, so that subsequent @@ -210,14 +251,6 @@ config("common_inherited_config") { } } -# TODO(bugs.webrtc.org/9693): Remove the possibility to suppress this warning -# as soon as WebRTC compiles without it. -config("no_exit_time_destructors") { - if (is_clang) { - cflags = [ "-Wno-exit-time-destructors" ] - } -} - # TODO(bugs.webrtc.org/9693): Remove the possibility to suppress this warning # as soon as WebRTC compiles without it. config("no_global_constructors") { @@ -249,6 +282,18 @@ config("common_config") { defines += [ "WEBRTC_ENABLE_PROTOBUF=0" ] } + if (rtc_strict_field_trials == "") { + defines += [ "WEBRTC_STRICT_FIELD_TRIALS=0" ] + } else if (rtc_strict_field_trials == "dcheck") { + defines += [ "WEBRTC_STRICT_FIELD_TRIALS=1" ] + } else if (rtc_strict_field_trials == "warn") { + defines += [ "WEBRTC_STRICT_FIELD_TRIALS=2" ] + } else { + assert(false, + "Unsupported value for rtc_strict_field_trials: " + + "$rtc_strict_field_trials") + } + if (rtc_include_internal_audio_device) { defines += [ "WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE" ] } @@ -257,8 +302,16 @@ config("common_config") { defines += [ "RTC_ENABLE_VP9" ] } + if (rtc_use_h265) { + defines += [ "RTC_ENABLE_H265" ] + } + + if (rtc_include_dav1d_in_internal_decoder_factory) { + defines += [ "RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY" ] + } + if (rtc_enable_sctp) { - defines += [ "HAVE_SCTP" ] + defines += [ "WEBRTC_HAVE_SCTP" ] } if (rtc_enable_external_auth) { @@ -273,6 +326,10 @@ config("common_config") { defines += [ "WEBRTC_ABSL_MUTEX" ] } + if (rtc_enable_libevent) { + defines += [ "WEBRTC_ENABLE_LIBEVENT" ] + } + if (rtc_disable_logging) { defines += [ "RTC_DISABLE_LOGGING" ] } @@ -293,7 +350,16 @@ config("common_config") { defines += [ "WEBRTC_EXCLUDE_AUDIO_PROCESSING_MODULE" ] } - cflags = [] + if (is_clang) { + cflags += [ + # TODO(webrtc:13219): Fix -Wshadow instances and enable. + "-Wno-shadow", + + # See https://reviews.llvm.org/D56731 for details about this + # warning. + "-Wctad-maybe-unsupported", + ] + } if (build_with_chromium) { defines += [ @@ -329,20 +395,17 @@ config("common_config") { } if (is_clang) { - cflags += [ - "-Wc++11-narrowing", - "-Wimplicit-fallthrough", - "-Wthread-safety", - "-Winconsistent-missing-override", - "-Wundef", - ] + cflags += [ "-Wc++11-narrowing" ] - # use_xcode_clang only refers to the iOS toolchain, host binaries use - # chromium's clang always. - if (!is_nacl && - (!use_xcode_clang || current_toolchain == host_toolchain)) { - # Flags NaCl (Clang 3.7) and Xcode 7.3 (Clang clang-703.0.31) do not - # recognize. + if (!is_fuchsia) { + # Compiling with the Fuchsia SDK results in Wundef errors + # TODO(bugs.fuchsia.dev/100722): Remove from (!is_fuchsia) branch when + # Fuchsia build errors are fixed. + cflags += [ "-Wundef" ] + } + + if (!is_nacl) { + # Flags NaCl (Clang 3.7) do not recognize. cflags += [ "-Wunused-lambda-capture" ] } } @@ -404,7 +467,7 @@ config("common_config") { ] } - if (use_fuzzing_engine && optimize_for_fuzzing) { + if (use_fuzzing_engine) { # Used in Chromium's overrides to disable logging defines += [ "WEBRTC_UNSAFE_FUZZER_MODE" ] } @@ -419,10 +482,6 @@ config("common_config") { config("common_objc") { frameworks = [ "Foundation.framework" ] - - if (rtc_use_metal_rendering) { - defines = [ "RTC_SUPPORTS_METAL" ] - } } if (!build_with_chromium) { @@ -448,6 +507,17 @@ if (!build_with_chromium) { "api/rtc_event_log:rtc_event_log_factory", "api/task_queue", "api/task_queue:default_task_queue_factory", + "api/test/metrics", + "api/video_codecs:video_decoder_factory_template", + "api/video_codecs:video_decoder_factory_template_dav1d_adapter", + "api/video_codecs:video_decoder_factory_template_libvpx_vp8_adapter", + "api/video_codecs:video_decoder_factory_template_libvpx_vp9_adapter", + "api/video_codecs:video_decoder_factory_template_open_h264_adapter", + "api/video_codecs:video_encoder_factory_template", + "api/video_codecs:video_encoder_factory_template_libaom_av1_adapter", + "api/video_codecs:video_encoder_factory_template_libvpx_vp8_adapter", + "api/video_codecs:video_encoder_factory_template_libvpx_vp9_adapter", + "api/video_codecs:video_encoder_factory_template_open_h264_adapter", "audio", "call", "common_audio", @@ -458,10 +528,7 @@ if (!build_with_chromium) { "modules/video_capture:video_capture_internal_impl", "p2p:rtc_p2p", "pc:libjingle_peerconnection", - "pc:peerconnection", "pc:rtc_pc", - "pc:rtc_pc_base", - "rtc_base", "sdk", "video", ] @@ -473,13 +540,6 @@ if (!build_with_chromium) { ] } - if (rtc_include_builtin_video_codecs) { - deps += [ - "api/video_codecs:builtin_video_decoder_factory", - "api/video_codecs:builtin_video_encoder_factory", - ] - } - if (build_with_mozilla) { deps += [ "api/video:video_frame", @@ -504,6 +564,10 @@ if (!build_with_chromium) { rtc_executable("webrtc_lib_link_test") { testonly = true + # This target is used for checking to link, so do not check dependencies + # on gn check. + check_includes = false # no-presubmit-check TODO(bugs.webrtc.org/12785) + sources = [ "webrtc_lib_link_test.cc" ] deps = [ # NOTE: Don't add deps here. If this test fails to link, it means you @@ -523,7 +587,7 @@ if (use_libfuzzer || use_afl) { } } -if (rtc_include_tests) { +if (rtc_include_tests && !build_with_chromium) { rtc_test("rtc_unittests") { testonly = true @@ -533,13 +597,17 @@ if (rtc_include_tests) { "api/audio/test:audio_api_unittests", "api/audio_codecs/test:audio_codecs_api_unittests", "api/numerics:numerics_unittests", + "api/task_queue:pending_task_safety_flag_unittests", + "api/test/metrics:metrics_unittests", "api/transport:stun_unittest", "api/video/test:rtc_api_video_unittests", "api/video_codecs/test:video_codecs_api_unittests", + "api/voip:compile_all_headers", "call:fake_network_pipe_unittests", "p2p:libstunprober_unittests", "p2p:rtc_p2p_unittests", - "rtc_base:robo_caller_unittests", + "rtc_base:async_dns_resolver_unittests", + "rtc_base:callback_list_unittests", "rtc_base:rtc_base_approved_unittests", "rtc_base:rtc_base_unittests", "rtc_base:rtc_json_unittests", @@ -547,12 +615,13 @@ if (rtc_include_tests) { "rtc_base:rtc_operations_chain_unittests", "rtc_base:rtc_task_queue_unittests", "rtc_base:sigslot_unittest", + "rtc_base:task_queue_stdlib_unittest", "rtc_base:untyped_function_unittest", "rtc_base:weak_ptr_unittests", "rtc_base/experiments:experiments_unittests", - "rtc_base/synchronization:sequence_checker_unittests", - "rtc_base/task_utils:pending_task_safety_flag_unittests", - "rtc_base/task_utils:to_queued_task_unittests", + "rtc_base/system:file_wrapper_unittests", + "rtc_base/task_utils:repeating_task_unittests", + "rtc_base/units:units_unittests", "sdk:sdk_tests", "test:rtp_test_utils", "test:test_main", @@ -574,31 +643,18 @@ if (rtc_include_tests) { ] shard_timeout = 900 } + } - if (is_ios || is_mac) { - deps += [ "sdk:rtc_unittests_objc" ] + if (rtc_enable_google_benchmarks) { + rtc_test("benchmarks") { + testonly = true + deps = [ + "rtc_base/synchronization:mutex_benchmark", + "test:benchmark_main", + ] } } - rtc_test("benchmarks") { - testonly = true - deps = [ - "rtc_base/synchronization:mutex_benchmark", - "test:benchmark_main", - ] - } - - # This runs tests that must run in real time and therefore can take some - # time to execute. They are in a separate executable to avoid making the - # regular unittest suite too slow to run frequently. - rtc_test("slow_tests") { - testonly = true - deps = [ - "rtc_base/task_utils:repeating_task_unittests", - "test:test_main", - ] - } - # TODO(pbos): Rename test suite, this is no longer "just" for video targets. video_engine_tests_resources = [ "resources/foreman_cif_short.yuv", @@ -630,7 +686,12 @@ if (rtc_include_tests) { ] data = video_engine_tests_resources if (is_android) { - deps += [ "//testing/android/native_test:native_test_native_code" ] + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] shard_timeout = 900 } if (is_ios) { @@ -663,7 +724,6 @@ if (rtc_include_tests) { rtc_test("webrtc_perf_tests") { testonly = true deps = [ - "audio:audio_perf_tests", "call:call_perf_tests", "modules/audio_coding:audio_coding_perf_tests", "modules/audio_processing:audio_processing_perf_tests", @@ -675,7 +735,12 @@ if (rtc_include_tests) { data = webrtc_perf_tests_resources if (is_android) { - deps += [ "//testing/android/native_test:native_test_native_code" ] + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] shard_timeout = 4500 } if (is_ios) { @@ -695,6 +760,7 @@ if (rtc_include_tests) { rtc_test("voip_unittests") { testonly = true deps = [ + "api/voip:compile_all_headers", "api/voip:voip_engine_factory_unittests", "audio/voip/test:audio_channel_unittests", "audio/voip/test:audio_egress_unittests", @@ -705,6 +771,23 @@ if (rtc_include_tests) { } } +# Build target for standalone dcsctp +rtc_static_library("dcsctp") { + # Only the root target should depend on this. + visibility = [ "//:default" ] + sources = [] + complete_static_lib = true + suppressed_configs += [ "//build/config/compiler:thin_archive" ] + defines = [] + deps = [ + "net/dcsctp/public:factory", + "net/dcsctp/public:socket", + "net/dcsctp/public:types", + "net/dcsctp/socket:dcsctp_socket", + "net/dcsctp/timer:task_queue_timeout", + ] +} + # ---- Poisons ---- # # Here is one empty dummy target for each poison type (needed because @@ -720,7 +803,7 @@ group("poison_audio_codecs") { group("poison_default_task_queue") { } -group("poison_rtc_json") { +group("poison_default_echo_detector") { } group("poison_software_video_codecs") { diff --git a/webrtc/api/array_view.h b/webrtc/api/array_view.h index a66369a..7e01959 100644 --- a/webrtc/api/array_view.h +++ b/webrtc/api/array_view.h @@ -13,6 +13,7 @@ #include #include +#include #include #include "rtc_base/checks.h" @@ -83,7 +84,7 @@ namespace rtc { // a pointer if fix-sized) and trivially copyable, so it's probably cheaper to // pass it by value than by const reference. -namespace impl { +namespace array_view_internal { // Magic constant for indicating that the size of an ArrayView is variable // instead of fixed. @@ -124,7 +125,7 @@ class ArrayViewBase { // Specialized base class for ArrayViews of variable size. template -class ArrayViewBase { +class ArrayViewBase { public: ArrayViewBase(T* data, size_t size) : data_(size == 0 ? nullptr : data), size_(size) {} @@ -141,18 +142,23 @@ class ArrayViewBase { size_t size_; }; -} // namespace impl +} // namespace array_view_internal -template -class ArrayView final : public impl::ArrayViewBase { +template +class ArrayView final : public array_view_internal::ArrayViewBase { public: using value_type = T; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; using const_iterator = const T*; // Construct an ArrayView from a pointer and a length. template ArrayView(U* data, size_t size) - : impl::ArrayViewBase::ArrayViewBase(data, size) { + : array_view_internal::ArrayViewBase::ArrayViewBase(data, size) { RTC_DCHECK_EQ(size == 0 ? nullptr : data, this->data()); RTC_DCHECK_EQ(size, this->size()); RTC_DCHECK_EQ(!this->data(), @@ -166,7 +172,8 @@ class ArrayView final : public impl::ArrayViewBase { : ArrayView() {} ArrayView(std::nullptr_t, size_t size) : ArrayView(static_cast(nullptr), size) { - static_assert(Size == 0 || Size == impl::kArrayViewVarSize, ""); + static_assert(Size == 0 || Size == array_view_internal::kArrayViewVarSize, + ""); RTC_DCHECK_EQ(0, size); } @@ -174,7 +181,7 @@ class ArrayView final : public impl::ArrayViewBase { template ArrayView(U (&array)[N]) // NOLINT : ArrayView(array, N) { - static_assert(Size == N || Size == impl::kArrayViewVarSize, + static_assert(Size == N || Size == array_view_internal::kArrayViewVarSize, "Array size must match ArrayView size"); } @@ -207,7 +214,7 @@ class ArrayView final : public impl::ArrayViewBase { // N> when M != N. template < typename U, - typename std::enable_if::value>::type* = nullptr> ArrayView(U& u) // NOLINT : ArrayView(u.data(), u.size()) { @@ -215,7 +222,7 @@ class ArrayView final : public impl::ArrayViewBase { } template < typename U, - typename std::enable_if::value>::type* = nullptr> ArrayView(const U& u) // NOLINT(runtime/explicit) : ArrayView(u.data(), u.size()) { @@ -235,13 +242,13 @@ class ArrayView final : public impl::ArrayViewBase { // const rtc::Buffer to ArrayView. template < typename U, - typename std::enable_if::value>::type* = nullptr> ArrayView(U& u) // NOLINT : ArrayView(u.data(), u.size()) {} template < typename U, - typename std::enable_if::value>::type* = nullptr> ArrayView(const U& u) // NOLINT(runtime/explicit) : ArrayView(u.data(), u.size()) {} @@ -258,6 +265,18 @@ class ArrayView final : public impl::ArrayViewBase { T* end() const { return this->data() + this->size(); } const T* cbegin() const { return this->data(); } const T* cend() const { return this->data() + this->size(); } + std::reverse_iterator rbegin() const { + return std::make_reverse_iterator(end()); + } + std::reverse_iterator rend() const { + return std::make_reverse_iterator(begin()); + } + std::reverse_iterator crbegin() const { + return std::make_reverse_iterator(cend()); + } + std::reverse_iterator crend() const { + return std::make_reverse_iterator(cbegin()); + } ArrayView subview(size_t offset, size_t size) const { return offset < this->size() diff --git a/webrtc/api/audio/audio_frame.cc b/webrtc/api/audio/audio_frame.cc index c6e5cf4..3e12006 100644 --- a/webrtc/api/audio/audio_frame.cc +++ b/webrtc/api/audio/audio_frame.cc @@ -11,8 +11,6 @@ #include "api/audio/audio_frame.h" #include -#include -#include #include "rtc_base/checks.h" #include "rtc_base/time_utils.h" @@ -24,35 +22,13 @@ AudioFrame::AudioFrame() { static_assert(sizeof(data_) == kMaxDataSizeBytes, "kMaxDataSizeBytes"); } -void swap(AudioFrame& a, AudioFrame& b) { - using std::swap; - swap(a.timestamp_, b.timestamp_); - swap(a.elapsed_time_ms_, b.elapsed_time_ms_); - swap(a.ntp_time_ms_, b.ntp_time_ms_); - swap(a.samples_per_channel_, b.samples_per_channel_); - swap(a.sample_rate_hz_, b.sample_rate_hz_); - swap(a.num_channels_, b.num_channels_); - swap(a.channel_layout_, b.channel_layout_); - swap(a.speech_type_, b.speech_type_); - swap(a.vad_activity_, b.vad_activity_); - swap(a.profile_timestamp_ms_, b.profile_timestamp_ms_); - swap(a.packet_infos_, b.packet_infos_); - const size_t length_a = a.samples_per_channel_ * a.num_channels_; - const size_t length_b = b.samples_per_channel_ * b.num_channels_; - RTC_DCHECK_LE(length_a, AudioFrame::kMaxDataSizeSamples); - RTC_DCHECK_LE(length_b, AudioFrame::kMaxDataSizeSamples); - std::swap_ranges(a.data_, a.data_ + std::max(length_a, length_b), b.data_); - swap(a.muted_, b.muted_); - swap(a.absolute_capture_timestamp_ms_, b.absolute_capture_timestamp_ms_); -} - void AudioFrame::Reset() { ResetWithoutMuting(); muted_ = true; } void AudioFrame::ResetWithoutMuting() { - // TODO(wu): Zero is a valid value for |timestamp_|. We should initialize + // TODO(wu): Zero is a valid value for `timestamp_`. We should initialize // to an invalid value, or add a new member to indicate invalidity. timestamp_ = 0; elapsed_time_ms_ = -1; diff --git a/webrtc/api/audio/audio_frame.h b/webrtc/api/audio/audio_frame.h index 78539f5..d5dcb5f 100644 --- a/webrtc/api/audio/audio_frame.h +++ b/webrtc/api/audio/audio_frame.h @@ -14,11 +14,8 @@ #include #include -#include - #include "api/audio/channel_layout.h" #include "api/rtp_packet_infos.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -60,7 +57,8 @@ class AudioFrame { AudioFrame(); - friend void swap(AudioFrame& a, AudioFrame& b); + AudioFrame(const AudioFrame&) = delete; + AudioFrame& operator=(const AudioFrame&) = delete; // Resets all members to their default state. void Reset(); @@ -139,7 +137,7 @@ class AudioFrame { int64_t profile_timestamp_ms_ = 0; // Information about packets used to assemble this audio frame. This is needed - // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's + // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's // MediaStreamTrack, in order to implement getContributingSources(). See: // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources // @@ -149,7 +147,7 @@ class AudioFrame { // sync buffer is the small sample-holding buffer located after the audio // decoder and before where samples are assembled into output frames. // - // |RtpPacketInfos| may also be empty if the audio samples did not come from + // `RtpPacketInfos` may also be empty if the audio samples did not come from // RTP packets. E.g. if the audio were locally generated by packet loss // concealment, comfort noise generation, etc. RtpPacketInfos packet_infos_; @@ -165,11 +163,9 @@ class AudioFrame { // Absolute capture timestamp when this audio frame was originally captured. // This is only valid for audio frames captured on this machine. The absolute - // capture timestamp of a received frame is found in |packet_infos_|. + // capture timestamp of a received frame is found in `packet_infos_`. // This timestamp MUST be based on the same clock as rtc::TimeMillis(). absl::optional absolute_capture_timestamp_ms_; - - RTC_DISALLOW_COPY_AND_ASSIGN(AudioFrame); }; } // namespace webrtc diff --git a/webrtc/api/audio/channel_layout.cc b/webrtc/api/audio/channel_layout.cc index 567f4d9..e4ae356 100644 --- a/webrtc/api/audio/channel_layout.cc +++ b/webrtc/api/audio/channel_layout.cc @@ -275,7 +275,7 @@ const char* ChannelLayoutToString(ChannelLayout layout) { case CHANNEL_LAYOUT_BITSTREAM: return "BITSTREAM"; } - RTC_NOTREACHED() << "Invalid channel layout provided: " << layout; + RTC_DCHECK_NOTREACHED() << "Invalid channel layout provided: " << layout; return ""; } diff --git a/webrtc/api/audio/echo_canceller3_config.cc b/webrtc/api/audio/echo_canceller3_config.cc index aeb809e..0224c71 100644 --- a/webrtc/api/audio/echo_canceller3_config.cc +++ b/webrtc/api/audio/echo_canceller3_config.cc @@ -153,6 +153,7 @@ bool EchoCanceller3Config::Validate(EchoCanceller3Config* config) { res = res & Limit(&c->filter.config_change_duration_blocks, 0, 100000); res = res & Limit(&c->filter.initial_state_seconds, 0.f, 100.f); + res = res & Limit(&c->filter.coarse_reset_hangover_blocks, 0, 250000); res = res & Limit(&c->erle.min, 1.f, 100000.f); res = res & Limit(&c->erle.max_l, 1.f, 100000.f); @@ -165,6 +166,7 @@ bool EchoCanceller3Config::Validate(EchoCanceller3Config* config) { res = res & Limit(&c->ep_strength.default_gain, 0.f, 1000000.f); res = res & Limit(&c->ep_strength.default_len, -1.f, 1.f); + res = res & Limit(&c->ep_strength.nearend_len, -1.0f, 1.0f); res = res & Limit(&c->echo_audibility.low_render_limit, 0.f, 32768.f * 32768.f); @@ -228,6 +230,12 @@ bool EchoCanceller3Config::Validate(EchoCanceller3Config* config) { res = res & Limit(&c->suppressor.nearend_tuning.max_dec_factor_lf, 0.f, 100.f); + res = res & Limit(&c->suppressor.last_permanent_lf_smoothing_band, 0, 64); + res = res & Limit(&c->suppressor.last_lf_smoothing_band, 0, 64); + res = res & Limit(&c->suppressor.last_lf_band, 0, 63); + res = res & + Limit(&c->suppressor.first_hf_band, c->suppressor.last_lf_band + 1, 64); + res = res & Limit(&c->suppressor.dominant_nearend_detection.enr_threshold, 0.f, 1000000.f); res = res & Limit(&c->suppressor.dominant_nearend_detection.snr_threshold, diff --git a/webrtc/api/audio/echo_canceller3_config.h b/webrtc/api/audio/echo_canceller3_config.h index af57d04..4b1c7fb 100644 --- a/webrtc/api/audio/echo_canceller3_config.h +++ b/webrtc/api/audio/echo_canceller3_config.h @@ -43,6 +43,7 @@ struct RTC_EXPORT EchoCanceller3Config { size_t hysteresis_limit_blocks = 1; size_t fixed_capture_delay_samples = 0; float delay_estimate_smoothing = 0.7f; + float delay_estimate_smoothing_delay_found = 0.7f; float delay_candidate_detection_threshold = 0.2f; struct DelaySelectionThresholds { int initial; @@ -58,6 +59,7 @@ struct RTC_EXPORT EchoCanceller3Config { }; AlignmentMixing render_alignment_mixing = {false, true, 10000.f, true}; AlignmentMixing capture_alignment_mixing = {false, true, 10000.f, false}; + bool detect_pre_echo = true; } delay; struct Filter { @@ -86,9 +88,11 @@ struct RTC_EXPORT EchoCanceller3Config { size_t config_change_duration_blocks = 250; float initial_state_seconds = 2.5f; + int coarse_reset_hangover_blocks = 25; bool conservative_initial_phase = false; bool enable_coarse_filter_output_usage = true; bool use_linear_filter = true; + bool high_pass_filter_echo_reference = false; bool export_linear_aec_output = false; } filter; @@ -105,8 +109,11 @@ struct RTC_EXPORT EchoCanceller3Config { struct EpStrength { float default_gain = 1.f; float default_len = 0.83f; + float nearend_len = 0.83f; bool echo_can_saturate = true; bool bounded_erl = false; + bool erle_onset_compensation_in_dominant_nearend = false; + bool use_conservative_tail_frequency_response = true; } ep_strength; struct EchoAudibility { @@ -190,6 +197,12 @@ struct RTC_EXPORT EchoCanceller3Config { 2.0f, 0.25f); + bool lf_smoothing_during_initial_phase = true; + int last_permanent_lf_smoothing_band = 0; + int last_lf_smoothing_band = 5; + int last_lf_band = 5; + int first_hf_band = 8; + struct DominantNearendDetection { float enr_threshold = .25f; float enr_exit_threshold = 10.f; @@ -197,6 +210,7 @@ struct RTC_EXPORT EchoCanceller3Config { int hold_duration = 50; int trigger_threshold = 12; bool use_during_initial_phase = true; + bool use_unbounded_echo_spectrum = true; } dominant_nearend_detection; struct SubbandNearendDetection { @@ -221,7 +235,15 @@ struct RTC_EXPORT EchoCanceller3Config { } high_bands_suppression; float floor_first_increase = 0.00001f; + bool conservative_hf_suppression = false; } suppressor; + + struct MultiChannel { + bool detect_stereo_content = true; + float stereo_detection_threshold = 0.0f; + int stereo_detection_timeout_threshold_seconds = 300; + float stereo_detection_hysteresis_seconds = 2.0f; + } multi_channel; }; } // namespace webrtc diff --git a/webrtc/api/audio/echo_control.h b/webrtc/api/audio/echo_control.h index 8d567bf..74fbc27 100644 --- a/webrtc/api/audio/echo_control.h +++ b/webrtc/api/audio/echo_control.h @@ -48,6 +48,13 @@ class EchoControl { // Provides an optional external estimate of the audio buffer delay. virtual void SetAudioBufferDelay(int delay_ms) = 0; + // Specifies whether the capture output will be used. The purpose of this is + // to allow the echo controller to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + // TODO(b/177830919): Make pure virtual. + virtual void SetCaptureOutputUsage(bool capture_output_used) {} + // Returns wheter the signal is altered. virtual bool ActiveProcessing() const = 0; diff --git a/webrtc/api/audio_codecs/audio_decoder.cc b/webrtc/api/audio_codecs/audio_decoder.cc index 97cda27..0a131f1 100644 --- a/webrtc/api/audio_codecs/audio_decoder.cc +++ b/webrtc/api/audio_codecs/audio_decoder.cc @@ -10,8 +10,6 @@ #include "api/audio_codecs/audio_decoder.h" -#include - #include #include @@ -162,9 +160,10 @@ AudioDecoder::SpeechType AudioDecoder::ConvertSpeechType(int16_t type) { case 2: return kComfortNoise; default: - assert(false); + RTC_DCHECK_NOTREACHED(); return kSpeech; } } +constexpr int AudioDecoder::kMaxNumberOfChannels; } // namespace webrtc diff --git a/webrtc/api/audio_codecs/audio_decoder.h b/webrtc/api/audio_codecs/audio_decoder.h index 557ffe2..4113874 100644 --- a/webrtc/api/audio_codecs/audio_decoder.h +++ b/webrtc/api/audio_codecs/audio_decoder.h @@ -20,7 +20,6 @@ #include "absl/types/optional.h" #include "api/array_view.h" #include "rtc_base/buffer.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -37,6 +36,9 @@ class AudioDecoder { AudioDecoder() = default; virtual ~AudioDecoder() = default; + AudioDecoder(const AudioDecoder&) = delete; + AudioDecoder& operator=(const AudioDecoder&) = delete; + class EncodedAudioFrame { public: struct DecodeResult { @@ -53,8 +55,8 @@ class AudioDecoder { // Returns true if this packet contains DTX. virtual bool IsDtxPacket() const; - // Decodes this frame of audio and writes the result in |decoded|. - // |decoded| must be large enough to store as many samples as indicated by a + // Decodes this frame of audio and writes the result in `decoded`. + // `decoded` must be large enough to store as many samples as indicated by a // call to Duration() . On success, returns an absl::optional containing the // total number of samples across all channels, as well as whether the // decoder produced comfort noise or speech. On failure, returns an empty @@ -85,8 +87,8 @@ class AudioDecoder { // Let the decoder parse this payload and prepare zero or more decodable // frames. Each frame must be between 10 ms and 120 ms long. The caller must // ensure that the AudioDecoder object outlives any frame objects returned by - // this call. The decoder is free to swap or move the data from the |payload| - // buffer. |timestamp| is the input timestamp, in samples, corresponding to + // this call. The decoder is free to swap or move the data from the `payload` + // buffer. `timestamp` is the input timestamp, in samples, corresponding to // the start of the payload. virtual std::vector ParsePayload(rtc::Buffer&& payload, uint32_t timestamp); @@ -95,12 +97,12 @@ class AudioDecoder { // obsolete; callers should call ParsePayload instead. For now, subclasses // must still implement DecodeInternal. - // Decodes |encode_len| bytes from |encoded| and writes the result in - // |decoded|. The maximum bytes allowed to be written into |decoded| is - // |max_decoded_bytes|. Returns the total number of samples across all - // channels. If the decoder produced comfort noise, |speech_type| + // Decodes `encode_len` bytes from `encoded` and writes the result in + // `decoded`. The maximum bytes allowed to be written into `decoded` is + // `max_decoded_bytes`. Returns the total number of samples across all + // channels. If the decoder produced comfort noise, `speech_type` // is set to kComfortNoise, otherwise it is kSpeech. The desired output - // sample rate is provided in |sample_rate_hz|, which must be valid for the + // sample rate is provided in `sample_rate_hz`, which must be valid for the // codec at hand. int Decode(const uint8_t* encoded, size_t encoded_len, @@ -123,11 +125,11 @@ class AudioDecoder { // Calls the packet-loss concealment of the decoder to update the state after // one or several lost packets. The caller has to make sure that the - // memory allocated in |decoded| should accommodate |num_frames| frames. + // memory allocated in `decoded` should accommodate `num_frames` frames. virtual size_t DecodePlc(size_t num_frames, int16_t* decoded); // Asks the decoder to generate packet-loss concealment and append it to the - // end of |concealment_audio|. The concealment audio should be in + // end of `concealment_audio`. The concealment audio should be in // channel-interleaved format, with as many channels as the last decoded // packet produced. The implementation must produce at least // requested_samples_per_channel, or nothing at all. This is a signal to the @@ -136,7 +138,7 @@ class AudioDecoder { // with the decoded audio on either side of the concealment. // Note: The default implementation of GeneratePlc will be deleted soon. All // implementations must provide their own, which can be a simple as a no-op. - // TODO(bugs.webrtc.org/9676): Remove default impementation. + // TODO(bugs.webrtc.org/9676): Remove default implementation. virtual void GeneratePlc(size_t requested_samples_per_channel, rtc::BufferT* concealment_audio); @@ -146,19 +148,19 @@ class AudioDecoder { // Returns the last error code from the decoder. virtual int ErrorCode(); - // Returns the duration in samples-per-channel of the payload in |encoded| - // which is |encoded_len| bytes long. Returns kNotImplemented if no duration + // Returns the duration in samples-per-channel of the payload in `encoded` + // which is `encoded_len` bytes long. Returns kNotImplemented if no duration // estimate is available, or -1 in case of an error. virtual int PacketDuration(const uint8_t* encoded, size_t encoded_len) const; // Returns the duration in samples-per-channel of the redandant payload in - // |encoded| which is |encoded_len| bytes long. Returns kNotImplemented if no + // `encoded` which is `encoded_len` bytes long. Returns kNotImplemented if no // duration estimate is available, or -1 in case of an error. virtual int PacketDurationRedundant(const uint8_t* encoded, size_t encoded_len) const; // Detects whether a packet has forward error correction. The packet is - // comprised of the samples in |encoded| which is |encoded_len| bytes long. + // comprised of the samples in `encoded` which is `encoded_len` bytes long. // Returns true if the packet has FEC and false otherwise. virtual bool PacketHasFec(const uint8_t* encoded, size_t encoded_len) const; @@ -170,6 +172,9 @@ class AudioDecoder { // during the lifetime of the decoder. virtual size_t Channels() const = 0; + // The maximum number of audio channels supported by WebRTC decoders. + static constexpr int kMaxNumberOfChannels = 24; + protected: static SpeechType ConvertSpeechType(int16_t type); @@ -184,9 +189,6 @@ class AudioDecoder { int sample_rate_hz, int16_t* decoded, SpeechType* speech_type); - - private: - RTC_DISALLOW_COPY_AND_ASSIGN(AudioDecoder); }; } // namespace webrtc diff --git a/webrtc/api/audio_codecs/audio_encoder.cc b/webrtc/api/audio_codecs/audio_encoder.cc index cd4d200..31bb873 100644 --- a/webrtc/api/audio_codecs/audio_encoder.cc +++ b/webrtc/api/audio_codecs/audio_encoder.cc @@ -83,7 +83,7 @@ void AudioEncoder::OnReceivedUplinkPacketLossFraction( void AudioEncoder::OnReceivedUplinkRecoverablePacketLossFraction( float uplink_recoverable_packet_loss_fraction) { - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) { @@ -110,4 +110,5 @@ ANAStats AudioEncoder::GetANAStats() const { return ANAStats(); } +constexpr int AudioEncoder::kMaxNumberOfChannels; } // namespace webrtc diff --git a/webrtc/api/audio_codecs/audio_encoder.h b/webrtc/api/audio_codecs/audio_encoder.h index fd2d948..7f5a342 100644 --- a/webrtc/api/audio_codecs/audio_encoder.h +++ b/webrtc/api/audio_codecs/audio_encoder.h @@ -16,12 +16,12 @@ #include #include +#include "absl/base/attributes.h" #include "absl/types/optional.h" #include "api/array_view.h" #include "api/call/bitrate_allocation.h" #include "api/units/time_delta.h" #include "rtc_base/buffer.h" -#include "rtc_base/deprecation.h" namespace webrtc { @@ -95,13 +95,13 @@ class AudioEncoder { // This is the main struct for auxiliary encoding information. Each encoded // packet should be accompanied by one EncodedInfo struct, containing the - // total number of |encoded_bytes|, the |encoded_timestamp| and the - // |payload_type|. If the packet contains redundant encodings, the |redundant| + // total number of `encoded_bytes`, the `encoded_timestamp` and the + // `payload_type`. If the packet contains redundant encodings, the `redundant` // vector will be populated with EncodedInfoLeaf structs. Each struct in the // vector represents one encoding; the order of structs in the vector is the // same as the order in which the actual payloads are written to the byte // stream. When EncoderInfoLeaf structs are present in the vector, the main - // struct's |encoded_bytes| will be the sum of all the |encoded_bytes| in the + // struct's `encoded_bytes` will be the sum of all the `encoded_bytes` in the // vector. struct EncodedInfo : public EncodedInfoLeaf { EncodedInfo(); @@ -143,7 +143,7 @@ class AudioEncoder { // Accepts one 10 ms block of input audio (i.e., SampleRateHz() / 100 * // NumChannels() samples). Multi-channel audio must be sample-interleaved. - // The encoder appends zero or more bytes of output to |encoded| and returns + // The encoder appends zero or more bytes of output to `encoded` and returns // additional encoding information. Encode() checks some preconditions, calls // EncodeImpl() which does the actual work, and then checks some // postconditions. @@ -182,12 +182,11 @@ class AudioEncoder { // implementation does nothing. virtual void SetMaxPlaybackRate(int frequency_hz); - // This is to be deprecated. Please use |OnReceivedTargetAudioBitrate| - // instead. // Tells the encoder what average bitrate we'd like it to produce. The // encoder is free to adjust or disregard the given bitrate (the default // implementation does the latter). - RTC_DEPRECATED virtual void SetTargetBitrate(int target_bps); + ABSL_DEPRECATED("Use OnReceivedTargetAudioBitrate instead") + virtual void SetTargetBitrate(int target_bps); // Causes this encoder to let go of any other encoders it contains, and // returns a pointer to an array where they are stored (which is required to @@ -206,11 +205,12 @@ class AudioEncoder { virtual void DisableAudioNetworkAdaptor(); // Provides uplink packet loss fraction to this encoder to allow it to adapt. - // |uplink_packet_loss_fraction| is in the range [0.0, 1.0]. + // `uplink_packet_loss_fraction` is in the range [0.0, 1.0]. virtual void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction); - RTC_DEPRECATED virtual void OnReceivedUplinkRecoverablePacketLossFraction( + ABSL_DEPRECATED("") + virtual void OnReceivedUplinkRecoverablePacketLossFraction( float uplink_recoverable_packet_loss_fraction); // Provides target audio bitrate to this encoder to allow it to adapt. @@ -246,6 +246,9 @@ class AudioEncoder { virtual absl::optional> GetFrameLengthRange() const = 0; + // The maximum number of audio channels supported by WebRTC encoders. + static constexpr int kMaxNumberOfChannels = 24; + protected: // Subclasses implement this to perform the actual encoding. Called by // Encode(). diff --git a/webrtc/api/call/bitrate_allocation.h b/webrtc/api/call/bitrate_allocation.h index 13c7f74..4b4e5e7 100644 --- a/webrtc/api/call/bitrate_allocation.h +++ b/webrtc/api/call/bitrate_allocation.h @@ -32,7 +32,7 @@ struct BitrateAllocationUpdate { double packet_loss_ratio = 0; // Predicted round trip time. TimeDelta round_trip_time = TimeDelta::PlusInfinity(); - // |bwe_period| is deprecated, use |stable_target_bitrate| allocation instead. + // `bwe_period` is deprecated, use `stable_target_bitrate` allocation instead. TimeDelta bwe_period = TimeDelta::PlusInfinity(); // Congestion window pushback bitrate reduction fraction. Used in // VideoStreamEncoder to reduce the bitrate by the given fraction diff --git a/webrtc/api/location.h b/webrtc/api/location.h new file mode 100644 index 0000000..81e9a15 --- /dev/null +++ b/webrtc/api/location.h @@ -0,0 +1,31 @@ +/* + * Copyright 2023 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_LOCATION_H_ +#define API_LOCATION_H_ + +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Location provides basic info where of an object was constructed, or was +// significantly brought to life. This is a stripped down version of +// https://source.chromium.org/chromium/chromium/src/+/main:base/location.h +// that only specifies an interface compatible to how base::Location is +// supposed to be used. +// The declaration is overriden inside the Chromium build. +class RTC_EXPORT Location { + public: + static Location Current() { return Location(); } +}; + +} // namespace webrtc + +#endif // API_LOCATION_H_ diff --git a/webrtc/api/make_ref_counted.h b/webrtc/api/make_ref_counted.h new file mode 100644 index 0000000..e958da9 --- /dev/null +++ b/webrtc/api/make_ref_counted.h @@ -0,0 +1,120 @@ +/* + * Copyright 2022 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_MAKE_REF_COUNTED_H_ +#define API_MAKE_REF_COUNTED_H_ + +#include +#include + +#include "rtc_base/ref_counted_object.h" + +namespace rtc { + +namespace webrtc_make_ref_counted_internal { +// Determines if the given class has AddRef and Release methods. +template +class HasAddRefAndRelease { + private: + template ().AddRef())* = nullptr, + decltype(std::declval().Release())* = nullptr> + static int Test(int); + template + static char Test(...); + + public: + static constexpr bool value = std::is_same_v(0)), int>; +}; +} // namespace webrtc_make_ref_counted_internal + +// General utilities for constructing a reference counted class and the +// appropriate reference count implementation for that class. +// +// These utilities select either the `RefCountedObject` implementation or +// `FinalRefCountedObject` depending on whether the to-be-shared class is +// derived from the RefCountInterface interface or not (respectively). + +// `make_ref_counted`: +// +// Use this when you want to construct a reference counted object of type T and +// get a `scoped_refptr<>` back. Example: +// +// auto p = make_ref_counted("bar", 123); +// +// For a class that inherits from RefCountInterface, this is equivalent to: +// +// auto p = scoped_refptr(new RefCountedObject("bar", 123)); +// +// If the class does not inherit from RefCountInterface, but does have +// AddRef/Release methods (so a T* is convertible to rtc::scoped_refptr), this +// is equivalent to just +// +// auto p = scoped_refptr(new Foo("bar", 123)); +// +// Otherwise, the example is equivalent to: +// +// auto p = scoped_refptr>( +// new FinalRefCountedObject("bar", 123)); +// +// In these cases, `make_ref_counted` reduces the amount of boilerplate code but +// also helps with the most commonly intended usage of RefCountedObject whereby +// methods for reference counting, are virtual and designed to satisfy the need +// of an interface. When such a need does not exist, it is more efficient to use +// the `FinalRefCountedObject` template, which does not add the vtable overhead. +// +// Note that in some cases, using RefCountedObject directly may still be what's +// needed. + +// `make_ref_counted` for abstract classes that are convertible to +// RefCountInterface. The is_abstract requirement rejects classes that inherit +// both RefCountInterface and RefCounted object, which is a a discouraged +// pattern, and would result in double inheritance of RefCountedObject if this +// template was applied. +template < + typename T, + typename... Args, + typename std::enable_if && + std::is_abstract_v, + T>::type* = nullptr> +scoped_refptr make_ref_counted(Args&&... args) { + return scoped_refptr(new RefCountedObject(std::forward(args)...)); +} + +// `make_ref_counted` for complete classes that are not convertible to +// RefCountInterface and already carry a ref count. +template < + typename T, + typename... Args, + typename std::enable_if< + !std::is_convertible_v && + webrtc_make_ref_counted_internal::HasAddRefAndRelease::value, + T>::type* = nullptr> +scoped_refptr make_ref_counted(Args&&... args) { + return scoped_refptr(new T(std::forward(args)...)); +} + +// `make_ref_counted` for complete classes that are not convertible to +// RefCountInterface and have no ref count of their own. +template < + typename T, + typename... Args, + typename std::enable_if< + !std::is_convertible_v && + !webrtc_make_ref_counted_internal::HasAddRefAndRelease::value, + + T>::type* = nullptr> +scoped_refptr> make_ref_counted(Args&&... args) { + return scoped_refptr>( + new FinalRefCountedObject(std::forward(args)...)); +} + +} // namespace rtc + +#endif // API_MAKE_REF_COUNTED_H_ diff --git a/webrtc/api/ref_counted_base.h b/webrtc/api/ref_counted_base.h index a1761db..f20228b 100644 --- a/webrtc/api/ref_counted_base.h +++ b/webrtc/api/ref_counted_base.h @@ -10,8 +10,8 @@ #ifndef API_REF_COUNTED_BASE_H_ #define API_REF_COUNTED_BASE_H_ -#include "rtc_base/constructor_magic.h" -#include "rtc_base/ref_count.h" +#include + #include "rtc_base/ref_counter.h" namespace rtc { @@ -20,6 +20,9 @@ class RefCountedBase { public: RefCountedBase() = default; + RefCountedBase(const RefCountedBase&) = delete; + RefCountedBase& operator=(const RefCountedBase&) = delete; + void AddRef() const { ref_count_.IncRef(); } RefCountReleaseStatus Release() const { const auto status = ref_count_.DecRef(); @@ -30,12 +33,64 @@ class RefCountedBase { } protected: + // Provided for internal webrtc subclasses for corner cases where it's + // necessary to know whether or not a reference is exclusively held. + bool HasOneRef() const { return ref_count_.HasOneRef(); } + virtual ~RefCountedBase() = default; private: mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; - RTC_DISALLOW_COPY_AND_ASSIGN(RefCountedBase); +// Template based version of `RefCountedBase` for simple implementations that do +// not need (or want) destruction via virtual destructor or the overhead of a +// vtable. +// +// To use: +// struct MyInt : public rtc::RefCountedNonVirtual { +// int foo_ = 0; +// }; +// +// rtc::scoped_refptr my_int(new MyInt()); +// +// sizeof(MyInt) on a 32 bit system would then be 8, int + refcount and no +// vtable generated. +template +class RefCountedNonVirtual { + public: + RefCountedNonVirtual() = default; + + RefCountedNonVirtual(const RefCountedNonVirtual&) = delete; + RefCountedNonVirtual& operator=(const RefCountedNonVirtual&) = delete; + + void AddRef() const { ref_count_.IncRef(); } + RefCountReleaseStatus Release() const { + // If you run into this assert, T has virtual methods. There are two + // options: + // 1) The class doesn't actually need virtual methods, the type is complete + // so the virtual attribute(s) can be removed. + // 2) The virtual methods are a part of the design of the class. In this + // case you can consider using `RefCountedBase` instead or alternatively + // use `rtc::RefCountedObject`. + static_assert(!std::is_polymorphic::value, + "T has virtual methods. RefCountedBase is a better fit."); + const auto status = ref_count_.DecRef(); + if (status == RefCountReleaseStatus::kDroppedLastRef) { + delete static_cast(this); + } + return status; + } + + protected: + // Provided for internal webrtc subclasses for corner cases where it's + // necessary to know whether or not a reference is exclusively held. + bool HasOneRef() const { return ref_count_.HasOneRef(); } + + ~RefCountedNonVirtual() = default; + + private: + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; }; } // namespace rtc diff --git a/webrtc/api/rtp_headers.cc b/webrtc/api/rtp_headers.cc index e0ad9eb..0573e54 100644 --- a/webrtc/api/rtp_headers.cc +++ b/webrtc/api/rtp_headers.cc @@ -44,7 +44,6 @@ RTPHeader::RTPHeader() arrOfCSRCs(), paddingLength(0), headerLength(0), - payload_type_frequency(0), extension() {} RTPHeader::RTPHeader(const RTPHeader& other) = default; diff --git a/webrtc/api/rtp_headers.h b/webrtc/api/rtp_headers.h index b9a97c8..5d4d419 100644 --- a/webrtc/api/rtp_headers.h +++ b/webrtc/api/rtp_headers.h @@ -103,15 +103,6 @@ struct RTPHeaderExtension { (1 << kAbsSendTimeFraction)); } - TimeDelta GetAbsoluteSendTimeDelta(uint32_t previous_sendtime) const { - RTC_DCHECK(hasAbsoluteSendTime); - RTC_DCHECK(absoluteSendTime < (1ul << 24)); - RTC_DCHECK(previous_sendtime < (1ul << 24)); - int32_t delta = - static_cast((absoluteSendTime - previous_sendtime) << 8) >> 8; - return TimeDelta::Micros((delta * 1000000ll) / (1 << kAbsSendTimeFraction)); - } - bool hasTransmissionTimeOffset; int32_t transmissionTimeOffset; bool hasAbsoluteSendTime; @@ -144,13 +135,12 @@ struct RTPHeaderExtension { VideoPlayoutDelay playout_delay; // For identification of a stream when ssrc is not signaled. See - // https://tools.ietf.org/html/draft-ietf-avtext-rid-09 - // TODO(danilchap): Update url from draft to release version. + // https://tools.ietf.org/html/rfc8852 std::string stream_id; std::string repaired_stream_id; // For identifying the media section used to interpret this RTP packet. See - // https://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-38 + // https://tools.ietf.org/html/rfc8843 std::string mid; absl::optional color_space; @@ -158,7 +148,7 @@ struct RTPHeaderExtension { enum { kRtpCsrcSize = 15 }; // RFC 3550 page 13 -struct RTPHeader { +struct RTC_EXPORT RTPHeader { RTPHeader(); RTPHeader(const RTPHeader& other); RTPHeader& operator=(const RTPHeader& other); @@ -172,7 +162,6 @@ struct RTPHeader { uint32_t arrOfCSRCs[kRtpCsrcSize]; size_t paddingLength; size_t headerLength; - int payload_type_frequency; RTPHeaderExtension extension; }; diff --git a/webrtc/api/rtp_packet_info.cc b/webrtc/api/rtp_packet_info.cc index a9ebd9d..cba274e 100644 --- a/webrtc/api/rtp_packet_info.cc +++ b/webrtc/api/rtp_packet_info.cc @@ -16,27 +16,22 @@ namespace webrtc { RtpPacketInfo::RtpPacketInfo() - : ssrc_(0), rtp_timestamp_(0), receive_time_ms_(-1) {} + : ssrc_(0), rtp_timestamp_(0), receive_time_(Timestamp::MinusInfinity()) {} -RtpPacketInfo::RtpPacketInfo( - uint32_t ssrc, - std::vector csrcs, - uint32_t rtp_timestamp, - absl::optional audio_level, - absl::optional absolute_capture_time, - int64_t receive_time_ms) +RtpPacketInfo::RtpPacketInfo(uint32_t ssrc, + std::vector csrcs, + uint32_t rtp_timestamp, + Timestamp receive_time) : ssrc_(ssrc), csrcs_(std::move(csrcs)), rtp_timestamp_(rtp_timestamp), - audio_level_(audio_level), - absolute_capture_time_(absolute_capture_time), - receive_time_ms_(receive_time_ms) {} + receive_time_(receive_time) {} RtpPacketInfo::RtpPacketInfo(const RTPHeader& rtp_header, - int64_t receive_time_ms) + Timestamp receive_time) : ssrc_(rtp_header.ssrc), rtp_timestamp_(rtp_header.timestamp), - receive_time_ms_(receive_time_ms) { + receive_time_(receive_time) { const auto& extension = rtp_header.extension; const auto csrcs_count = std::min(rtp_header.numCSRCs, kRtpCsrcSize); @@ -52,9 +47,10 @@ RtpPacketInfo::RtpPacketInfo(const RTPHeader& rtp_header, bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs) { return (lhs.ssrc() == rhs.ssrc()) && (lhs.csrcs() == rhs.csrcs()) && (lhs.rtp_timestamp() == rhs.rtp_timestamp()) && + (lhs.receive_time() == rhs.receive_time()) && (lhs.audio_level() == rhs.audio_level()) && (lhs.absolute_capture_time() == rhs.absolute_capture_time()) && - (lhs.receive_time_ms() == rhs.receive_time_ms()); + (lhs.local_capture_clock_offset() == rhs.local_capture_clock_offset()); } } // namespace webrtc diff --git a/webrtc/api/rtp_packet_info.h b/webrtc/api/rtp_packet_info.h index 639ba32..8df12a3 100644 --- a/webrtc/api/rtp_packet_info.h +++ b/webrtc/api/rtp_packet_info.h @@ -17,14 +17,16 @@ #include "absl/types/optional.h" #include "api/rtp_headers.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { // -// Structure to hold information about a received |RtpPacket|. It is primarily +// Structure to hold information about a received `RtpPacket`. It is primarily // used to carry per-packet information from when a packet is received until -// the information is passed to |SourceTracker|. +// the information is passed to `SourceTracker`. // class RTC_EXPORT RtpPacketInfo { public: @@ -33,11 +35,9 @@ class RTC_EXPORT RtpPacketInfo { RtpPacketInfo(uint32_t ssrc, std::vector csrcs, uint32_t rtp_timestamp, - absl::optional audio_level, - absl::optional absolute_capture_time, - int64_t receive_time_ms); + Timestamp receive_time); - RtpPacketInfo(const RTPHeader& rtp_header, int64_t receive_time_ms); + RtpPacketInfo(const RTPHeader& rtp_header, Timestamp receive_time); RtpPacketInfo(const RtpPacketInfo& other) = default; RtpPacketInfo(RtpPacketInfo&& other) = default; @@ -53,19 +53,32 @@ class RTC_EXPORT RtpPacketInfo { uint32_t rtp_timestamp() const { return rtp_timestamp_; } void set_rtp_timestamp(uint32_t value) { rtp_timestamp_ = value; } + Timestamp receive_time() const { return receive_time_; } + void set_receive_time(Timestamp value) { receive_time_ = value; } + absl::optional audio_level() const { return audio_level_; } - void set_audio_level(absl::optional value) { audio_level_ = value; } + RtpPacketInfo& set_audio_level(absl::optional value) { + audio_level_ = value; + return *this; + } const absl::optional& absolute_capture_time() const { return absolute_capture_time_; } - void set_absolute_capture_time( + RtpPacketInfo& set_absolute_capture_time( const absl::optional& value) { absolute_capture_time_ = value; + return *this; } - int64_t receive_time_ms() const { return receive_time_ms_; } - void set_receive_time_ms(int64_t value) { receive_time_ms_ = value; } + const absl::optional& local_capture_clock_offset() const { + return local_capture_clock_offset_; + } + RtpPacketInfo& set_local_capture_clock_offset( + absl::optional value) { + local_capture_clock_offset_ = value; + return *this; + } private: // Fields from the RTP header: @@ -74,6 +87,9 @@ class RTC_EXPORT RtpPacketInfo { std::vector csrcs_; uint32_t rtp_timestamp_; + // Local `webrtc::Clock`-based timestamp of when the packet was received. + Timestamp receive_time_; + // Fields from the Audio Level header extension: // https://tools.ietf.org/html/rfc6464#section-3 absl::optional audio_level_; @@ -82,8 +98,12 @@ class RTC_EXPORT RtpPacketInfo { // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time absl::optional absolute_capture_time_; - // Local |webrtc::Clock|-based timestamp of when the packet was received. - int64_t receive_time_ms_; + // Clock offset between the local clock and the capturer's clock. + // Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset` + // which instead represents the clock offset between a remote sender and the + // capturer. The following holds: + // Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset + absl::optional local_capture_clock_offset_; }; bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs); diff --git a/webrtc/api/rtp_packet_infos.h b/webrtc/api/rtp_packet_infos.h index d636464..7445729 100644 --- a/webrtc/api/rtp_packet_infos.h +++ b/webrtc/api/rtp_packet_infos.h @@ -15,6 +15,7 @@ #include #include +#include "api/make_ref_counted.h" #include "api/ref_counted_base.h" #include "api/rtp_packet_info.h" #include "api/scoped_refptr.h" @@ -26,8 +27,8 @@ namespace webrtc { // an audio or video frame. Uses internal reference counting to make it very // cheap to copy. // -// We should ideally just use |std::vector| and have it -// |std::move()|-ed as the per-packet information is transferred from one object +// We should ideally just use `std::vector` and have it +// `std::move()`-ed as the per-packet information is transferred from one object // to another. But moving the info, instead of copying it, is not easily done // for the current video code. class RTC_EXPORT RtpPacketInfos { @@ -79,7 +80,7 @@ class RTC_EXPORT RtpPacketInfos { size_type size() const { return entries().size(); } private: - class Data : public rtc::RefCountedBase { + class Data final : public rtc::RefCountedNonVirtual { public: static rtc::scoped_refptr Create(const vector_type& entries) { // Performance optimization for the empty case. @@ -87,7 +88,7 @@ class RTC_EXPORT RtpPacketInfos { return nullptr; } - return new Data(entries); + return rtc::make_ref_counted(entries); } static rtc::scoped_refptr Create(vector_type&& entries) { @@ -96,16 +97,16 @@ class RTC_EXPORT RtpPacketInfos { return nullptr; } - return new Data(std::move(entries)); + return rtc::make_ref_counted(std::move(entries)); } const vector_type& entries() const { return entries_; } - private: explicit Data(const vector_type& entries) : entries_(entries) {} explicit Data(vector_type&& entries) : entries_(std::move(entries)) {} - ~Data() override {} + ~Data() = default; + private: const vector_type entries_; }; diff --git a/webrtc/api/scoped_refptr.h b/webrtc/api/scoped_refptr.h index fa4e83d..e145509 100644 --- a/webrtc/api/scoped_refptr.h +++ b/webrtc/api/scoped_refptr.h @@ -24,13 +24,13 @@ // void some_function() { // scoped_refptr foo = new MyFoo(); // foo->Method(param); -// // |foo| is released when this function returns +// // `foo` is released when this function returns // } // // void some_other_function() { // scoped_refptr foo = new MyFoo(); // ... -// foo = nullptr; // explicitly releases |foo| +// foo = nullptr; // explicitly releases `foo` // ... // if (foo) // foo->Method(param); @@ -45,10 +45,10 @@ // scoped_refptr b; // // b.swap(a); -// // now, |b| references the MyFoo object, and |a| references null. +// // now, `b` references the MyFoo object, and `a` references null. // } // -// To make both |a| and |b| in the above example reference the same MyFoo +// To make both `a` and `b` in the above example reference the same MyFoo // object, simply use the assignment operator: // // { @@ -56,7 +56,7 @@ // scoped_refptr b; // // b = a; -// // now, |a| and |b| each own a reference to the same MyFoo object. +// // now, `a` and `b` each own a reference to the same MyFoo object. // } // @@ -74,8 +74,9 @@ class scoped_refptr { typedef T element_type; scoped_refptr() : ptr_(nullptr) {} + scoped_refptr(std::nullptr_t) : ptr_(nullptr) {} // NOLINT(runtime/explicit) - scoped_refptr(T* p) : ptr_(p) { // NOLINT(runtime/explicit) + explicit scoped_refptr(T* p) : ptr_(p) { if (ptr_) ptr_->AddRef(); } @@ -103,7 +104,8 @@ class scoped_refptr { } T* get() const { return ptr_; } - operator T*() const { return ptr_; } + explicit operator bool() const { return ptr_ != nullptr; } + T& operator*() const { return *ptr_; } T* operator->() const { return ptr_; } // Returns the (possibly null) raw pointer, and makes the scoped_refptr hold a @@ -159,6 +161,62 @@ class scoped_refptr { T* ptr_; }; +template +bool operator==(const rtc::scoped_refptr& a, + const rtc::scoped_refptr& b) { + return a.get() == b.get(); +} +template +bool operator!=(const rtc::scoped_refptr& a, + const rtc::scoped_refptr& b) { + return !(a == b); +} + +template +bool operator==(const rtc::scoped_refptr& a, std::nullptr_t) { + return a.get() == nullptr; +} + +template +bool operator!=(const rtc::scoped_refptr& a, std::nullptr_t) { + return !(a == nullptr); +} + +template +bool operator==(std::nullptr_t, const rtc::scoped_refptr& a) { + return a.get() == nullptr; +} + +template +bool operator!=(std::nullptr_t, const rtc::scoped_refptr& a) { + return !(a == nullptr); +} + +// Comparison with raw pointer. +template +bool operator==(const rtc::scoped_refptr& a, const U* b) { + return a.get() == b; +} +template +bool operator!=(const rtc::scoped_refptr& a, const U* b) { + return !(a == b); +} + +template +bool operator==(const T* a, const rtc::scoped_refptr& b) { + return a == b.get(); +} +template +bool operator!=(const T* a, const rtc::scoped_refptr& b) { + return !(a == b); +} + +// Ordered comparison, needed for use as a std::map key. +template +bool operator<(const rtc::scoped_refptr& a, const rtc::scoped_refptr& b) { + return a.get() < b.get(); +} + } // namespace rtc #endif // API_SCOPED_REFPTR_H_ diff --git a/webrtc/api/sequence_checker.h b/webrtc/api/sequence_checker.h new file mode 100644 index 0000000..5ff5860 --- /dev/null +++ b/webrtc/api/sequence_checker.h @@ -0,0 +1,140 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_SEQUENCE_CHECKER_H_ +#define API_SEQUENCE_CHECKER_H_ + +#include "rtc_base/checks.h" +#include "rtc_base/synchronization/sequence_checker_internal.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// SequenceChecker is a helper class used to help verify that some methods +// of a class are called on the same task queue or thread. A +// SequenceChecker is bound to a a task queue if the object is +// created on a task queue, or a thread otherwise. +// +// +// Example: +// class MyClass { +// public: +// void Foo() { +// RTC_DCHECK_RUN_ON(&sequence_checker_); +// ... (do stuff) ... +// } +// +// private: +// SequenceChecker sequence_checker_; +// } +// +// In Release mode, IsCurrent will always return true. +class RTC_LOCKABLE SequenceChecker +#if RTC_DCHECK_IS_ON + : public webrtc_sequence_checker_internal::SequenceCheckerImpl { + using Impl = webrtc_sequence_checker_internal::SequenceCheckerImpl; +#else + : public webrtc_sequence_checker_internal::SequenceCheckerDoNothing { + using Impl = webrtc_sequence_checker_internal::SequenceCheckerDoNothing; +#endif + public: + enum InitialState : bool { kDetached = false, kAttached = true }; + + // TODO(tommi): We could maybe join these two ctors and have fewer factory + // functions. At the moment they're separate to minimize code changes when + // we added the second ctor as well as avoiding to have unnecessary code at + // the SequenceChecker which much only run for the SequenceCheckerImpl + // implementation. + // In theory we could have something like: + // + // SequenceChecker(InitialState initial_state = kAttached, + // TaskQueueBase* attached_queue = TaskQueueBase::Current()); + // + // But the problem with that is having the call to `Current()` exist for + // `SequenceCheckerDoNothing`. + explicit SequenceChecker(InitialState initial_state = kAttached) + : Impl(initial_state) {} + explicit SequenceChecker(TaskQueueBase* attached_queue) + : Impl(attached_queue) {} + + // Returns true if sequence checker is attached to the current sequence. + bool IsCurrent() const { return Impl::IsCurrent(); } + // Detaches checker from sequence to which it is attached. Next attempt + // to do a check with this checker will result in attaching this checker + // to the sequence on which check was performed. + void Detach() { Impl::Detach(); } +}; + +} // namespace webrtc + +// RTC_RUN_ON/RTC_GUARDED_BY/RTC_DCHECK_RUN_ON macros allows to annotate +// variables are accessed from same thread/task queue. +// Using tools designed to check mutexes, it checks at compile time everywhere +// variable is access, there is a run-time dcheck thread/task queue is correct. +// +// class SequenceCheckerExample { +// public: +// int CalledFromPacer() RTC_RUN_ON(pacer_sequence_checker_) { +// return var2_; +// } +// +// void CallMeFromPacer() { +// RTC_DCHECK_RUN_ON(&pacer_sequence_checker_) +// << "Should be called from pacer"; +// CalledFromPacer(); +// } +// +// private: +// int pacer_var_ RTC_GUARDED_BY(pacer_sequence_checker_); +// SequenceChecker pacer_sequence_checker_; +// }; +// +// class TaskQueueExample { +// public: +// class Encoder { +// public: +// rtc::TaskQueueBase& Queue() { return encoder_queue_; } +// void Encode() { +// RTC_DCHECK_RUN_ON(&encoder_queue_); +// DoSomething(var_); +// } +// +// private: +// rtc::TaskQueueBase& encoder_queue_; +// Frame var_ RTC_GUARDED_BY(encoder_queue_); +// }; +// +// void Encode() { +// // Will fail at runtime when DCHECK is enabled: +// // encoder_->Encode(); +// // Will work: +// rtc::scoped_refptr encoder = encoder_; +// encoder_->Queue().PostTask([encoder] { encoder->Encode(); }); +// } +// +// private: +// rtc::scoped_refptr encoder_; +// } + +// Document if a function expected to be called from same thread/task queue. +#define RTC_RUN_ON(x) \ + RTC_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(x)) + +// Checks current code is running on the desired sequence. +// +// First statement validates it is running on the sequence `x`. +// Second statement annotates for the thread safety analyzer the check was done. +// Such annotation has to be attached to a function, and that function has to be +// called. Thus current implementation creates a noop lambda and calls it. +#define RTC_DCHECK_RUN_ON(x) \ + RTC_DCHECK((x)->IsCurrent()) \ + << webrtc::webrtc_sequence_checker_internal::ExpectationToString(x); \ + []() RTC_ASSERT_EXCLUSIVE_LOCK(x) {}() + +#endif // API_SEQUENCE_CHECKER_H_ diff --git a/webrtc/api/task_queue/task_queue_base.cc b/webrtc/api/task_queue/task_queue_base.cc index 7d3539a..ecdc7f7 100644 --- a/webrtc/api/task_queue/task_queue_base.cc +++ b/webrtc/api/task_queue/task_queue_base.cc @@ -11,6 +11,8 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/functional/any_invocable.h" +#include "api/units/time_delta.h" #include "rtc_base/checks.h" #if defined(ABSL_HAVE_THREAD_LOCAL) diff --git a/webrtc/api/task_queue/task_queue_base.h b/webrtc/api/task_queue/task_queue_base.h index 90b1efd..89e9e9e 100644 --- a/webrtc/api/task_queue/task_queue_base.h +++ b/webrtc/api/task_queue/task_queue_base.h @@ -11,8 +11,11 @@ #define API_TASK_QUEUE_TASK_QUEUE_BASE_H_ #include +#include -#include "api/task_queue/queued_task.h" +#include "absl/functional/any_invocable.h" +#include "api/location.h" +#include "api/units/time_delta.h" #include "rtc_base/system/rtc_export.h" #include "rtc_base/thread_annotations.h" @@ -24,41 +27,139 @@ namespace webrtc { // known task queue, use IsCurrent(). class RTC_LOCKABLE RTC_EXPORT TaskQueueBase { public: + enum class DelayPrecision { + // This may include up to a 17 ms leeway in addition to OS timer precision. + // See PostDelayedTask() for more information. + kLow, + // This does not have the additional delay that kLow has, but it is still + // limited by OS timer precision. See PostDelayedHighPrecisionTask() for + // more information. + kHigh, + }; + // Starts destruction of the task queue. // On return ensures no task are running and no new tasks are able to start // on the task queue. - // Responsible for deallocation. Deallocation may happen syncrhoniously during + // Responsible for deallocation. Deallocation may happen synchronously during // Delete or asynchronously after Delete returns. // Code not running on the TaskQueue should not make any assumption when // TaskQueue is deallocated and thus should not call any methods after Delete. // Code running on the TaskQueue should not call Delete, but can assume // TaskQueue still exists and may call other methods, e.g. PostTask. + // Should be called on the same task queue or thread that this task queue + // was created on. virtual void Delete() = 0; - // Schedules a task to execute. Tasks are executed in FIFO order. - // If |task->Run()| returns true, task is deleted on the task queue - // before next QueuedTask starts executing. + // Schedules a `task` to execute. Tasks are executed in FIFO order. // When a TaskQueue is deleted, pending tasks will not be executed but they - // will be deleted. The deletion of tasks may happen synchronously on the - // TaskQueue or it may happen asynchronously after TaskQueue is deleted. - // This may vary from one implementation to the next so assumptions about - // lifetimes of pending tasks should not be made. - virtual void PostTask(std::unique_ptr task) = 0; + // will be deleted. + // + // As long as tasks are not posted from task destruction, posted tasks are + // guaranteed to be destroyed with Current() pointing to the task queue they + // were posted to, whether they're executed or not. That means SequenceChecker + // works during task destruction, a fact that can be used to guarantee + // thread-compatible object deletion happening on a particular task queue + // which can simplify class design. + // Note that this guarantee does not apply to delayed tasks. + // + // May be called on any thread or task queue, including this task queue. + void PostTask(absl::AnyInvocable task, + const Location& location = Location::Current()) { + PostTaskImpl(std::move(task), PostTaskTraits{}, location); + } - // Schedules a task to execute a specified number of milliseconds from when - // the call is made. The precision should be considered as "best effort" - // and in some cases, such as on Windows when all high precision timers have - // been used up, can be off by as much as 15 millseconds. - virtual void PostDelayedTask(std::unique_ptr task, - uint32_t milliseconds) = 0; + // Prefer PostDelayedTask() over PostDelayedHighPrecisionTask() whenever + // possible. + // + // Schedules a `task` to execute a specified `delay` from when the call is + // made, using "low" precision. All scheduling is affected by OS-specific + // leeway and current workloads which means that in terms of precision there + // are no hard guarantees, but in addition to the OS induced leeway, "low" + // precision adds up to a 17 ms additional leeway. The purpose of this leeway + // is to achieve more efficient CPU scheduling and reduce Idle Wake Up + // frequency. + // + // The task may execute with [-1, 17 + OS induced leeway) ms additional delay. + // + // Avoid making assumptions about the precision of the OS scheduler. On macOS, + // the OS induced leeway may be 10% of sleep interval. On Windows, 1 ms + // precision timers may be used but there are cases, such as when running on + // battery, when the timer precision can be as poor as 15 ms. + // + // "Low" precision is not implemented everywhere yet. Where not yet + // implemented, PostDelayedTask() has "high" precision. See + // https://crbug.com/webrtc/13583 for more information. + // + // May be called on any thread or task queue, including this task queue. + void PostDelayedTask(absl::AnyInvocable task, + TimeDelta delay, + const Location& location = Location::Current()) { + PostDelayedTaskImpl(std::move(task), delay, PostDelayedTaskTraits{}, + location); + } + + // Prefer PostDelayedTask() over PostDelayedHighPrecisionTask() whenever + // possible. + // + // Schedules a `task` to execute a specified `delay` from when the call is + // made, using "high" precision. All scheduling is affected by OS-specific + // leeway and current workloads which means that in terms of precision there + // are no hard guarantees. + // + // The task may execute with [-1, OS induced leeway] ms additional delay. + // + // Avoid making assumptions about the precision of the OS scheduler. On macOS, + // the OS induced leeway may be 10% of sleep interval. On Windows, 1 ms + // precision timers may be used but there are cases, such as when running on + // battery, when the timer precision can be as poor as 15 ms. + // + // May be called on any thread or task queue, including this task queue. + void PostDelayedHighPrecisionTask( + absl::AnyInvocable task, + TimeDelta delay, + const Location& location = Location::Current()) { + PostDelayedTaskTraits traits; + traits.high_precision = true; + PostDelayedTaskImpl(std::move(task), delay, traits, location); + } + + // As specified by `precision`, calls either PostDelayedTask() or + // PostDelayedHighPrecisionTask(). + void PostDelayedTaskWithPrecision( + DelayPrecision precision, + absl::AnyInvocable task, + TimeDelta delay, + const Location& location = Location::Current()) { + switch (precision) { + case DelayPrecision::kLow: + PostDelayedTask(std::move(task), delay, location); + break; + case DelayPrecision::kHigh: + PostDelayedHighPrecisionTask(std::move(task), delay, location); + break; + } + } // Returns the task queue that is running the current thread. // Returns nullptr if this thread is not associated with any task queue. + // May be called on any thread or task queue, including this task queue. static TaskQueueBase* Current(); bool IsCurrent() const { return Current() == this; } protected: - class CurrentTaskQueueSetter { + // This is currently only present here to simplify introduction of future + // planned task queue changes. + struct PostTaskTraits {}; + + struct PostDelayedTaskTraits { + // If `high_precision` is false, tasks may execute within up to a 17 ms + // leeway in addition to OS timer precision. Otherwise the task should be + // limited to OS timer precision. See PostDelayedTask() and + // PostDelayedHighPrecisionTask() for more information. + bool high_precision = false; + }; + + class RTC_EXPORT CurrentTaskQueueSetter { public: explicit CurrentTaskQueueSetter(TaskQueueBase* task_queue); CurrentTaskQueueSetter(const CurrentTaskQueueSetter&) = delete; @@ -69,6 +170,20 @@ class RTC_LOCKABLE RTC_EXPORT TaskQueueBase { TaskQueueBase* const previous_; }; + // Subclasses should implement this method to support the behavior defined in + // the PostTask and PostTaskTraits docs above. + virtual void PostTaskImpl(absl::AnyInvocable task, + const PostTaskTraits& traits, + const Location& location) = 0; + + // Subclasses should implement this method to support the behavior defined in + // the PostDelayedTask/PostHighPrecisionDelayedTask and PostDelayedTaskTraits + // docs above. + virtual void PostDelayedTaskImpl(absl::AnyInvocable task, + TimeDelta delay, + const PostDelayedTaskTraits& traits, + const Location& location) = 0; + // Users of the TaskQueue should call Delete instead of directly deleting // this object. virtual ~TaskQueueBase() = default; diff --git a/webrtc/api/units/data_rate.h b/webrtc/api/units/data_rate.h index 5c8a61f..d813c61 100644 --- a/webrtc/api/units/data_rate.h +++ b/webrtc/api/units/data_rate.h @@ -11,9 +11,9 @@ #ifndef API_UNITS_DATA_RATE_H_ #define API_UNITS_DATA_RATE_H_ -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST #include // no-presubmit-check TODO(webrtc:8982) -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST #include #include @@ -23,7 +23,7 @@ #include "api/units/frequency.h" #include "api/units/time_delta.h" #include "rtc_base/checks.h" -#include "rtc_base/units/unit_base.h" +#include "rtc_base/units/unit_base.h" // IWYU pragma: export namespace webrtc { // DataRate is a class that represents a given data rate. This can be used to @@ -142,13 +142,13 @@ inline std::string ToLogString(DataRate value) { return ToString(value); } -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982) std::ostream& stream, // no-presubmit-check TODO(webrtc:8982) DataRate value) { return stream << ToString(value); } -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST } // namespace webrtc diff --git a/webrtc/api/units/data_size.h b/webrtc/api/units/data_size.h index 27a2a4e..9df6434 100644 --- a/webrtc/api/units/data_size.h +++ b/webrtc/api/units/data_size.h @@ -11,14 +11,14 @@ #ifndef API_UNITS_DATA_SIZE_H_ #define API_UNITS_DATA_SIZE_H_ -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST #include // no-presubmit-check TODO(webrtc:8982) -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST #include #include -#include "rtc_base/units/unit_base.h" +#include "rtc_base/units/unit_base.h" // IWYU pragma: export namespace webrtc { // DataSize is a class represeting a count of bytes. @@ -53,13 +53,13 @@ inline std::string ToLogString(DataSize value) { return ToString(value); } -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982) std::ostream& stream, // no-presubmit-check TODO(webrtc:8982) DataSize value) { return stream << ToString(value); } -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST } // namespace webrtc diff --git a/webrtc/api/units/frequency.h b/webrtc/api/units/frequency.h index 88912c6..06081e4 100644 --- a/webrtc/api/units/frequency.h +++ b/webrtc/api/units/frequency.h @@ -10,9 +10,9 @@ #ifndef API_UNITS_FREQUENCY_H_ #define API_UNITS_FREQUENCY_H_ -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST #include // no-presubmit-check TODO(webrtc:8982) -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST #include #include @@ -20,7 +20,7 @@ #include #include "api/units/time_delta.h" -#include "rtc_base/units/unit_base.h" +#include "rtc_base/units/unit_base.h" // IWYU pragma: export namespace webrtc { @@ -89,13 +89,13 @@ inline std::string ToLogString(Frequency value) { return ToString(value); } -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982) std::ostream& stream, // no-presubmit-check TODO(webrtc:8982) Frequency value) { return stream << ToString(value); } -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST } // namespace webrtc #endif // API_UNITS_FREQUENCY_H_ diff --git a/webrtc/api/units/time_delta.h b/webrtc/api/units/time_delta.h index 173affc..5981e32 100644 --- a/webrtc/api/units/time_delta.h +++ b/webrtc/api/units/time_delta.h @@ -11,15 +11,15 @@ #ifndef API_UNITS_TIME_DELTA_H_ #define API_UNITS_TIME_DELTA_H_ -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST #include // no-presubmit-check TODO(webrtc:8982) -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST #include #include #include -#include "rtc_base/units/unit_base.h" +#include "rtc_base/units/unit_base.h" // IWYU pragma: export namespace webrtc { @@ -32,6 +32,11 @@ namespace webrtc { // microseconds (us). class TimeDelta final : public rtc_units_impl::RelativeUnit { public: + template + static constexpr TimeDelta Minutes(T value) { + static_assert(std::is_arithmetic::value, ""); + return Seconds(value * 60); + } template static constexpr TimeDelta Seconds(T value) { static_assert(std::is_arithmetic::value, ""); @@ -92,13 +97,13 @@ inline std::string ToLogString(TimeDelta value) { return ToString(value); } -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982) std::ostream& stream, // no-presubmit-check TODO(webrtc:8982) TimeDelta value) { return stream << ToString(value); } -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST } // namespace webrtc diff --git a/webrtc/api/units/timestamp.h b/webrtc/api/units/timestamp.h index f83477e..8aabe05 100644 --- a/webrtc/api/units/timestamp.h +++ b/webrtc/api/units/timestamp.h @@ -11,15 +11,16 @@ #ifndef API_UNITS_TIMESTAMP_H_ #define API_UNITS_TIMESTAMP_H_ -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST #include // no-presubmit-check TODO(webrtc:8982) -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST #include #include #include "api/units/time_delta.h" #include "rtc_base/checks.h" +#include "rtc_base/units/unit_base.h" // IWYU pragma: export namespace webrtc { // Timestamp represents the time that has passed since some unspecified epoch. @@ -125,13 +126,13 @@ inline std::string ToLogString(Timestamp value) { return ToString(value); } -#ifdef UNIT_TEST +#ifdef WEBRTC_UNIT_TEST inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982) std::ostream& stream, // no-presubmit-check TODO(webrtc:8982) Timestamp value) { return stream << ToString(value); } -#endif // UNIT_TEST +#endif // WEBRTC_UNIT_TEST } // namespace webrtc diff --git a/webrtc/api/video/color_space.cc b/webrtc/api/video/color_space.cc index 710bb43..dcb9c67 100644 --- a/webrtc/api/video/color_space.cc +++ b/webrtc/api/video/color_space.cc @@ -10,9 +10,11 @@ #include "api/video/color_space.h" +#include "rtc_base/strings/string_builder.h" + namespace webrtc { namespace { -// Try to convert |enum_value| into the enum class T. |enum_bitmask| is created +// Try to convert `enum_value` into the enum class T. `enum_bitmask` is created // by the funciton below. Returns true if conversion was successful, false // otherwise. template @@ -43,7 +45,7 @@ constexpr int MakeMask(const int index, const int length, T (&values)[N]) { } // Create a bitmask where each bit corresponds to one potential enum value. -// |values| should be an array listing all possible enum values. The bit is set +// `values` should be an array listing all possible enum values. The bit is set // to one if the corresponding enum exists. Only works for enums with values // less than 64. template @@ -124,6 +126,80 @@ const HdrMetadata* ColorSpace::hdr_metadata() const { return hdr_metadata_ ? &*hdr_metadata_ : nullptr; } +#define PRINT_ENUM_CASE(TYPE, NAME) \ + case TYPE::NAME: \ + ss << #NAME; \ + break; + +std::string ColorSpace::AsString() const { + char buf[1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{primaries:"; + switch (primaries_) { + PRINT_ENUM_CASE(PrimaryID, kBT709) + PRINT_ENUM_CASE(PrimaryID, kUnspecified) + PRINT_ENUM_CASE(PrimaryID, kBT470M) + PRINT_ENUM_CASE(PrimaryID, kBT470BG) + PRINT_ENUM_CASE(PrimaryID, kSMPTE170M) + PRINT_ENUM_CASE(PrimaryID, kSMPTE240M) + PRINT_ENUM_CASE(PrimaryID, kFILM) + PRINT_ENUM_CASE(PrimaryID, kBT2020) + PRINT_ENUM_CASE(PrimaryID, kSMPTEST428) + PRINT_ENUM_CASE(PrimaryID, kSMPTEST431) + PRINT_ENUM_CASE(PrimaryID, kSMPTEST432) + PRINT_ENUM_CASE(PrimaryID, kJEDECP22) + } + ss << ", transfer:"; + switch (transfer_) { + PRINT_ENUM_CASE(TransferID, kBT709) + PRINT_ENUM_CASE(TransferID, kUnspecified) + PRINT_ENUM_CASE(TransferID, kGAMMA22) + PRINT_ENUM_CASE(TransferID, kGAMMA28) + PRINT_ENUM_CASE(TransferID, kSMPTE170M) + PRINT_ENUM_CASE(TransferID, kSMPTE240M) + PRINT_ENUM_CASE(TransferID, kLINEAR) + PRINT_ENUM_CASE(TransferID, kLOG) + PRINT_ENUM_CASE(TransferID, kLOG_SQRT) + PRINT_ENUM_CASE(TransferID, kIEC61966_2_4) + PRINT_ENUM_CASE(TransferID, kBT1361_ECG) + PRINT_ENUM_CASE(TransferID, kIEC61966_2_1) + PRINT_ENUM_CASE(TransferID, kBT2020_10) + PRINT_ENUM_CASE(TransferID, kBT2020_12) + PRINT_ENUM_CASE(TransferID, kSMPTEST2084) + PRINT_ENUM_CASE(TransferID, kSMPTEST428) + PRINT_ENUM_CASE(TransferID, kARIB_STD_B67) + } + ss << ", matrix:"; + switch (matrix_) { + PRINT_ENUM_CASE(MatrixID, kRGB) + PRINT_ENUM_CASE(MatrixID, kBT709) + PRINT_ENUM_CASE(MatrixID, kUnspecified) + PRINT_ENUM_CASE(MatrixID, kFCC) + PRINT_ENUM_CASE(MatrixID, kBT470BG) + PRINT_ENUM_CASE(MatrixID, kSMPTE170M) + PRINT_ENUM_CASE(MatrixID, kSMPTE240M) + PRINT_ENUM_CASE(MatrixID, kYCOCG) + PRINT_ENUM_CASE(MatrixID, kBT2020_NCL) + PRINT_ENUM_CASE(MatrixID, kBT2020_CL) + PRINT_ENUM_CASE(MatrixID, kSMPTE2085) + PRINT_ENUM_CASE(MatrixID, kCDNCLS) + PRINT_ENUM_CASE(MatrixID, kCDCLS) + PRINT_ENUM_CASE(MatrixID, kBT2100_ICTCP) + } + + ss << ", range:"; + switch (range_) { + PRINT_ENUM_CASE(RangeID, kInvalid) + PRINT_ENUM_CASE(RangeID, kLimited) + PRINT_ENUM_CASE(RangeID, kFull) + PRINT_ENUM_CASE(RangeID, kDerived) + } + ss << "}"; + return ss.str(); +} + +#undef PRINT_ENUM_CASE + bool ColorSpace::set_primaries_from_uint8(uint8_t enum_value) { constexpr PrimaryID kPrimaryIds[] = { PrimaryID::kBT709, PrimaryID::kUnspecified, PrimaryID::kBT470M, diff --git a/webrtc/api/video/color_space.h b/webrtc/api/video/color_space.h index a7ad86b..31963a1 100644 --- a/webrtc/api/video/color_space.h +++ b/webrtc/api/video/color_space.h @@ -13,6 +13,8 @@ #include +#include + #include "absl/types/optional.h" #include "api/video/hdr_metadata.h" #include "rtc_base/system/rtc_export.h" @@ -101,7 +103,7 @@ class RTC_EXPORT ColorSpace { kInvalid = 0, // Limited Rec. 709 color range with RGB values ranging from 16 to 235. kLimited = 1, - // Full RGB color range with RGB valees from 0 to 255. + // Full RGB color range with RGB values from 0 to 255. kFull = 2, // Range is defined by MatrixCoefficients/TransferCharacteristics. kDerived = 3, @@ -155,6 +157,7 @@ class RTC_EXPORT ColorSpace { ChromaSiting chroma_siting_horizontal() const; ChromaSiting chroma_siting_vertical() const; const HdrMetadata* hdr_metadata() const; + std::string AsString() const; bool set_primaries_from_uint8(uint8_t enum_value); bool set_transfer_from_uint8(uint8_t enum_value); diff --git a/webrtc/api/video/video_content_type.cc b/webrtc/api/video/video_content_type.cc index 9ba3ece..79da9ff 100644 --- a/webrtc/api/video/video_content_type.cc +++ b/webrtc/api/video/video_content_type.cc @@ -10,21 +10,7 @@ #include "api/video/video_content_type.h" -// VideoContentType stored as a single byte, which is sent over the network. -// Structure: -// -// 0 1 2 3 4 5 6 7 -// +---------------+ -// |r r e e e s s c| -// -// where: -// r - reserved bits. -// e - 3-bit number of an experiment group counted from 1. 0 means there's no -// experiment ongoing. -// s - 2-bit simulcast stream id or spatial layer, counted from 1. 0 means that -// no simulcast information is set. -// c - content type. 0 means real-time video, 1 means screenshare. -// +#include "rtc_base/checks.h" namespace webrtc { namespace videocontenttypehelpers { @@ -33,57 +19,21 @@ namespace { static constexpr uint8_t kScreenshareBitsSize = 1; static constexpr uint8_t kScreenshareBitsMask = (1u << kScreenshareBitsSize) - 1; - -static constexpr uint8_t kSimulcastShift = 1; -static constexpr uint8_t kSimulcastBitsSize = 2; -static constexpr uint8_t kSimulcastBitsMask = ((1u << kSimulcastBitsSize) - 1) - << kSimulcastShift; // 0b00000110 - -static constexpr uint8_t kExperimentShift = 3; -static constexpr uint8_t kExperimentBitsSize = 3; -static constexpr uint8_t kExperimentBitsMask = - ((1u << kExperimentBitsSize) - 1) << kExperimentShift; // 0b00111000 - -static constexpr uint8_t kTotalBitsSize = - kScreenshareBitsSize + kSimulcastBitsSize + kExperimentBitsSize; } // namespace -bool SetExperimentId(VideoContentType* content_type, uint8_t experiment_id) { - // Store in bits 2-4. - if (experiment_id >= (1 << kExperimentBitsSize)) - return false; - *content_type = static_cast( - (static_cast(*content_type) & ~kExperimentBitsMask) | - ((experiment_id << kExperimentShift) & kExperimentBitsMask)); - return true; -} - -bool SetSimulcastId(VideoContentType* content_type, uint8_t simulcast_id) { - // Store in bits 5-6. - if (simulcast_id >= (1 << kSimulcastBitsSize)) - return false; - *content_type = static_cast( - (static_cast(*content_type) & ~kSimulcastBitsMask) | - ((simulcast_id << kSimulcastShift) & kSimulcastBitsMask)); - return true; -} - -uint8_t GetExperimentId(const VideoContentType& content_type) { - return (static_cast(content_type) & kExperimentBitsMask) >> - kExperimentShift; -} -uint8_t GetSimulcastId(const VideoContentType& content_type) { - return (static_cast(content_type) & kSimulcastBitsMask) >> - kSimulcastShift; -} - bool IsScreenshare(const VideoContentType& content_type) { + // Ensure no bits apart from the screenshare bit is set. + // This CHECK is a temporary measure to detect code that introduces + // values according to old versions. + RTC_CHECK((static_cast(content_type) & !kScreenshareBitsMask) == 0); return (static_cast(content_type) & kScreenshareBitsMask) > 0; } bool IsValidContentType(uint8_t value) { - // Any 6-bit value is allowed. - return value < (1 << kTotalBitsSize); + // Only the screenshare bit is allowed. + // However, due to previous usage of the next 5 bits, we allow + // the lower 6 bits to be set. + return value < (1 << 6); } const char* ToString(const VideoContentType& content_type) { diff --git a/webrtc/api/video/video_content_type.h b/webrtc/api/video/video_content_type.h index 2d38a62..b574201 100644 --- a/webrtc/api/video/video_content_type.h +++ b/webrtc/api/video/video_content_type.h @@ -15,18 +15,15 @@ namespace webrtc { +// VideoContentType stored as a single byte, which is sent over the network +// in the rtp-hdrext/video-content-type extension. +// Only the lowest bit is used, per the enum. enum class VideoContentType : uint8_t { UNSPECIFIED = 0, SCREENSHARE = 1, }; namespace videocontenttypehelpers { -bool SetExperimentId(VideoContentType* content_type, uint8_t experiment_id); -bool SetSimulcastId(VideoContentType* content_type, uint8_t simulcast_id); - -uint8_t GetExperimentId(const VideoContentType& content_type); -uint8_t GetSimulcastId(const VideoContentType& content_type); - bool IsScreenshare(const VideoContentType& content_type); bool IsValidContentType(uint8_t value); diff --git a/webrtc/api/video/video_timing.cc b/webrtc/api/video/video_timing.cc index df1bc48..d16911f 100644 --- a/webrtc/api/video/video_timing.cc +++ b/webrtc/api/video/video_timing.cc @@ -10,7 +10,10 @@ #include "api/video/video_timing.h" +#include + #include "api/array_view.h" +#include "api/units/time_delta.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/strings/string_builder.h" @@ -25,6 +28,14 @@ uint16_t VideoSendTiming::GetDeltaCappedMs(int64_t base_ms, int64_t time_ms) { return rtc::saturated_cast(time_ms - base_ms); } +uint16_t VideoSendTiming::GetDeltaCappedMs(TimeDelta delta) { + if (delta < TimeDelta::Zero()) { + RTC_DLOG(LS_ERROR) << "Delta " << delta.ms() + << "ms expected to be positive"; + } + return rtc::saturated_cast(delta.ms()); +} + TimingFrameInfo::TimingFrameInfo() : rtp_timestamp(0), capture_time_ms(-1), @@ -89,4 +100,23 @@ std::string TimingFrameInfo::ToString() const { return sb.str(); } +VideoPlayoutDelay::VideoPlayoutDelay(TimeDelta min, TimeDelta max) + : min_(std::clamp(min, TimeDelta::Zero(), kMax)), + max_(std::clamp(max, min_, kMax)) { + if (!(TimeDelta::Zero() <= min && min <= max && max <= kMax)) { + RTC_LOG(LS_ERROR) << "Invalid video playout delay: [" << min << "," << max + << "]. Clamped to [" << this->min() << "," << this->max() + << "]"; + } +} + +bool VideoPlayoutDelay::Set(TimeDelta min, TimeDelta max) { + if (TimeDelta::Zero() <= min && min <= max && max <= kMax) { + min_ = min; + max_ = max; + return true; + } + return false; +} + } // namespace webrtc diff --git a/webrtc/api/video/video_timing.h b/webrtc/api/video/video_timing.h index fbd9225..0a450cd 100644 --- a/webrtc/api/video/video_timing.h +++ b/webrtc/api/video/video_timing.h @@ -16,11 +16,14 @@ #include #include +#include "api/units/time_delta.h" +#include "rtc_base/system/rtc_export.h" + namespace webrtc { // Video timing timestamps in ms counted from capture_time_ms of a frame. // This structure represents data sent in video-timing RTP header extension. -struct VideoSendTiming { +struct RTC_EXPORT VideoSendTiming { enum TimingFrameFlags : uint8_t { kNotTriggered = 0, // Timing info valid, but not to be transmitted. // Used on send-side only. @@ -34,6 +37,7 @@ struct VideoSendTiming { // https://webrtc.org/experiments/rtp-hdrext/video-timing/ extension stores // 16-bit deltas of timestamps from packet capture time. static uint16_t GetDeltaCappedMs(int64_t base_ms, int64_t time_ms); + static uint16_t GetDeltaCappedMs(TimeDelta delta); uint16_t encode_start_delta_ms; uint16_t encode_finish_delta_ms; @@ -41,21 +45,21 @@ struct VideoSendTiming { uint16_t pacer_exit_delta_ms; uint16_t network_timestamp_delta_ms; uint16_t network2_timestamp_delta_ms; - uint8_t flags; + uint8_t flags = TimingFrameFlags::kInvalid; }; // Used to report precise timings of a 'timing frames'. Contains all important // timestamps for a lifetime of that specific frame. Reported as a string via // GetStats(). Only frame which took the longest between two GetStats calls is // reported. -struct TimingFrameInfo { +struct RTC_EXPORT TimingFrameInfo { TimingFrameInfo(); // Returns end-to-end delay of a frame, if sender and receiver timestamps are // synchronized, -1 otherwise. int64_t EndToEndDelay() const; - // Returns true if current frame took longer to process than |other| frame. + // Returns true if current frame took longer to process than `other` frame. // If other frame's clocks are not synchronized, current frame is always // preferred. bool IsLongerThan(const TimingFrameInfo& other) const; @@ -103,26 +107,43 @@ struct TimingFrameInfo { // Minimum and maximum playout delay values from capture to render. // These are best effort values. // -// A value < 0 indicates no change from previous valid value. -// // min = max = 0 indicates that the receiver should try and render // frame as soon as possible. // // min = x, max = y indicates that the receiver is free to adapt // in the range (x, y) based on network jitter. -struct VideoPlayoutDelay { - VideoPlayoutDelay() = default; - VideoPlayoutDelay(int min_ms, int max_ms) : min_ms(min_ms), max_ms(max_ms) {} - int min_ms = -1; - int max_ms = -1; +// This class ensures invariant 0 <= min <= max <= kMax. +class RTC_EXPORT VideoPlayoutDelay { + public: + // Maximum supported value for the delay limit. + static constexpr TimeDelta kMax = TimeDelta::Millis(10) * 0xFFF; - bool operator==(const VideoPlayoutDelay& rhs) const { - return min_ms == rhs.min_ms && max_ms == rhs.max_ms; + // Creates delay limits that indicates receiver should try to render frame + // as soon as possible. + static VideoPlayoutDelay Minimal() { + return VideoPlayoutDelay(TimeDelta::Zero(), TimeDelta::Zero()); } -}; -// TODO(bugs.webrtc.org/7660): Old name, delete after downstream use is updated. -using PlayoutDelay = VideoPlayoutDelay; + // Creates valid, but unspecified limits. + VideoPlayoutDelay() = default; + VideoPlayoutDelay(const VideoPlayoutDelay&) = default; + VideoPlayoutDelay& operator=(const VideoPlayoutDelay&) = default; + VideoPlayoutDelay(TimeDelta min, TimeDelta max); + + bool Set(TimeDelta min, TimeDelta max); + + TimeDelta min() const { return min_; } + TimeDelta max() const { return max_; } + + friend bool operator==(const VideoPlayoutDelay& lhs, + const VideoPlayoutDelay& rhs) { + return lhs.min_ == rhs.min_ && lhs.max_ == rhs.max_; + } + + private: + TimeDelta min_ = TimeDelta::Zero(); + TimeDelta max_ = kMax; +}; } // namespace webrtc diff --git a/webrtc/audio/utility/BUILD.gn b/webrtc/audio/utility/BUILD.gn index 54ca046..983b628 100644 --- a/webrtc/audio/utility/BUILD.gn +++ b/webrtc/audio/utility/BUILD.gn @@ -26,10 +26,11 @@ rtc_library("audio_frame_operations") { "../../api/audio:audio_frame_api", "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:deprecation", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:safe_conversions", "../../system_wrappers:field_trial", ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] } if (rtc_include_tests) { @@ -44,7 +45,9 @@ if (rtc_include_tests) { ":audio_frame_operations", "../../api/audio:audio_frame_api", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:stringutils", "../../test:field_trial", "../../test:test_support", "//testing/gtest", diff --git a/webrtc/audio/utility/audio_frame_operations.cc b/webrtc/audio/utility/audio_frame_operations.cc index a9d2cf1..1b936c2 100644 --- a/webrtc/audio/utility/audio_frame_operations.cc +++ b/webrtc/audio/utility/audio_frame_operations.cc @@ -131,8 +131,8 @@ void AudioFrameOperations::DownmixChannels(const int16_t* src_audio, return; } - RTC_NOTREACHED() << "src_channels: " << src_channels - << ", dst_channels: " << dst_channels; + RTC_DCHECK_NOTREACHED() << "src_channels: " << src_channels + << ", dst_channels: " << dst_channels; } void AudioFrameOperations::DownmixChannels(size_t dst_channels, @@ -149,8 +149,8 @@ void AudioFrameOperations::DownmixChannels(size_t dst_channels, int err = QuadToStereo(frame); RTC_DCHECK_EQ(err, 0); } else { - RTC_NOTREACHED() << "src_channels: " << frame->num_channels_ - << ", dst_channels: " << dst_channels; + RTC_DCHECK_NOTREACHED() << "src_channels: " << frame->num_channels_ + << ", dst_channels: " << dst_channels; } } @@ -169,10 +169,10 @@ void AudioFrameOperations::UpmixChannels(size_t target_number_of_channels, if (!frame->muted()) { // Up-mixing done in place. Going backwards through the frame ensure nothing // is irrevocably overwritten. + int16_t* frame_data = frame->mutable_data(); for (int i = frame->samples_per_channel_ - 1; i >= 0; i--) { for (size_t j = 0; j < target_number_of_channels; ++j) { - frame->mutable_data()[target_number_of_channels * i + j] = - frame->data()[i]; + frame_data[target_number_of_channels * i + j] = frame_data[i]; } } } @@ -222,14 +222,14 @@ void AudioFrameOperations::Mute(AudioFrame* frame, size_t end = count; float start_g = 0.0f; if (current_frame_muted) { - // Fade out the last |count| samples of frame. + // Fade out the last `count` samples of frame. RTC_DCHECK(!previous_frame_muted); start = frame->samples_per_channel_ - count; end = frame->samples_per_channel_; start_g = 1.0f; inc = -inc; } else { - // Fade in the first |count| samples of frame. + // Fade in the first `count` samples of frame. RTC_DCHECK(previous_frame_muted); } diff --git a/webrtc/audio/utility/audio_frame_operations.h b/webrtc/audio/utility/audio_frame_operations.h index 65c310c..2a5f29f 100644 --- a/webrtc/audio/utility/audio_frame_operations.h +++ b/webrtc/audio/utility/audio_frame_operations.h @@ -14,8 +14,8 @@ #include #include +#include "absl/base/attributes.h" #include "api/audio/audio_frame.h" -#include "rtc_base/deprecation.h" namespace webrtc { @@ -24,38 +24,40 @@ namespace webrtc { // than a class. class AudioFrameOperations { public: - // Add samples in |frame_to_add| with samples in |result_frame| - // putting the results in |results_frame|. The fields - // |vad_activity_| and |speech_type_| of the result frame are - // updated. If |result_frame| is empty (|samples_per_channel_|==0), - // the samples in |frame_to_add| are added to it. The number of + // Add samples in `frame_to_add` with samples in `result_frame` + // putting the results in `results_frame`. The fields + // `vad_activity_` and `speech_type_` of the result frame are + // updated. If `result_frame` is empty (`samples_per_channel_`==0), + // the samples in `frame_to_add` are added to it. The number of // channels and number of samples per channel must match except when - // |result_frame| is empty. + // `result_frame` is empty. static void Add(const AudioFrame& frame_to_add, AudioFrame* result_frame); - // |frame.num_channels_| will be updated. This version checks for sufficient - // buffer size and that |num_channels_| is mono. Use UpmixChannels + // `frame.num_channels_` will be updated. This version checks for sufficient + // buffer size and that `num_channels_` is mono. Use UpmixChannels // instead. TODO(bugs.webrtc.org/8649): remove. - RTC_DEPRECATED static int MonoToStereo(AudioFrame* frame); + ABSL_DEPRECATED("bugs.webrtc.org/8649") + static int MonoToStereo(AudioFrame* frame); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| is stereo. Use DownmixChannels + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` is stereo. Use DownmixChannels // instead. TODO(bugs.webrtc.org/8649): remove. - RTC_DEPRECATED static int StereoToMono(AudioFrame* frame); + ABSL_DEPRECATED("bugs.webrtc.org/8649") + static int StereoToMono(AudioFrame* frame); - // Downmixes 4 channels |src_audio| to stereo |dst_audio|. This is an in-place - // operation, meaning |src_audio| and |dst_audio| may point to the same + // Downmixes 4 channels `src_audio` to stereo `dst_audio`. This is an in-place + // operation, meaning `src_audio` and `dst_audio` may point to the same // buffer. static void QuadToStereo(const int16_t* src_audio, size_t samples_per_channel, int16_t* dst_audio); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| is 4 channels. + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` is 4 channels. static int QuadToStereo(AudioFrame* frame); - // Downmixes |src_channels| |src_audio| to |dst_channels| |dst_audio|. - // This is an in-place operation, meaning |src_audio| and |dst_audio| + // Downmixes `src_channels` `src_audio` to `dst_channels` `dst_audio`. + // This is an in-place operation, meaning `src_audio` and `dst_audio` // may point to the same buffer. Supported channel combinations are // Stereo to Mono, Quad to Mono, and Quad to Stereo. static void DownmixChannels(const int16_t* src_audio, @@ -64,27 +66,27 @@ class AudioFrameOperations { size_t dst_channels, int16_t* dst_audio); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| and |dst_channels| are valid and performs relevant downmix. + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` and `dst_channels` are valid and performs relevant downmix. // Supported channel combinations are N channels to Mono, and Quad to Stereo. static void DownmixChannels(size_t dst_channels, AudioFrame* frame); - // |frame.num_channels_| will be updated. This version checks that - // |num_channels_| and |dst_channels| are valid and performs relevant + // `frame.num_channels_` will be updated. This version checks that + // `num_channels_` and `dst_channels` are valid and performs relevant // downmix. Supported channel combinations are Mono to N // channels. The single channel is replicated. static void UpmixChannels(size_t target_number_of_channels, AudioFrame* frame); - // Swap the left and right channels of |frame|. Fails silently if |frame| is + // Swap the left and right channels of `frame`. Fails silently if `frame` is // not stereo. static void SwapStereoChannels(AudioFrame* frame); - // Conditionally zero out contents of |frame| for implementing audio mute: - // |previous_frame_muted| && |current_frame_muted| - Zero out whole frame. - // |previous_frame_muted| && !|current_frame_muted| - Fade-in at frame start. - // !|previous_frame_muted| && |current_frame_muted| - Fade-out at frame end. - // !|previous_frame_muted| && !|current_frame_muted| - Leave frame untouched. + // Conditionally zero out contents of `frame` for implementing audio mute: + // `previous_frame_muted` && `current_frame_muted` - Zero out whole frame. + // `previous_frame_muted` && !`current_frame_muted` - Fade-in at frame start. + // !`previous_frame_muted` && `current_frame_muted` - Fade-out at frame end. + // !`previous_frame_muted` && !`current_frame_muted` - Leave frame untouched. static void Mute(AudioFrame* frame, bool previous_frame_muted, bool current_frame_muted); @@ -92,7 +94,7 @@ class AudioFrameOperations { // Zero out contents of frame. static void Mute(AudioFrame* frame); - // Halve samples in |frame|. + // Halve samples in `frame`. static void ApplyHalfGain(AudioFrame* frame); static int Scale(float left, float right, AudioFrame* frame); diff --git a/webrtc/common_audio/BUILD.gn b/webrtc/common_audio/BUILD.gn index a03e9ab..2ae6d32 100644 --- a/webrtc/common_audio/BUILD.gn +++ b/webrtc/common_audio/BUILD.gn @@ -48,8 +48,10 @@ rtc_library("common_audio") { "../api:array_view", "../rtc_base:checks", "../rtc_base:gtest_prod", - "../rtc_base:rtc_base_approved", + "../rtc_base:logging", + "../rtc_base:safe_conversions", "../rtc_base:sanitizer", + "../rtc_base:timeutils", "../rtc_base/memory:aligned_malloc", "../rtc_base/system:arch", "../rtc_base/system:file_wrapper", @@ -180,7 +182,6 @@ rtc_library("common_audio_c") { ":common_audio_cc", "../rtc_base:checks", "../rtc_base:compile_assert_c", - "../rtc_base:rtc_base_approved", "../rtc_base:sanitizer", "../rtc_base/system:arch", "../system_wrappers", @@ -196,7 +197,7 @@ rtc_library("common_audio_cc") { ] deps = [ - "../rtc_base:rtc_base_approved", + "../rtc_base:safe_conversions", "../system_wrappers", ] } @@ -205,7 +206,6 @@ rtc_source_set("sinc_resampler") { sources = [ "resampler/sinc_resampler.h" ] deps = [ "../rtc_base:gtest_prod", - "../rtc_base:rtc_base_approved", "../rtc_base/memory:aligned_malloc", "../rtc_base/system:arch", "../system_wrappers", @@ -228,7 +228,6 @@ rtc_library("fir_filter_factory") { deps = [ ":fir_filter", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", "../rtc_base/system:arch", "../system_wrappers", ] @@ -257,7 +256,6 @@ if (current_cpu == "x86" || current_cpu == "x64") { ":fir_filter", ":sinc_resampler", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", "../rtc_base/memory:aligned_malloc", ] } @@ -282,7 +280,6 @@ if (current_cpu == "x86" || current_cpu == "x64") { ":fir_filter", ":sinc_resampler", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", "../rtc_base/memory:aligned_malloc", ] } @@ -307,7 +304,6 @@ if (rtc_build_with_neon) { ":fir_filter", ":sinc_resampler", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", "../rtc_base/memory:aligned_malloc", ] } @@ -329,13 +325,12 @@ if (rtc_build_with_neon) { deps = [ ":common_audio_c", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", "../rtc_base/system:arch", ] } } -if (rtc_include_tests) { +if (rtc_include_tests && !build_with_chromium) { rtc_test("common_audio_unittests") { visibility += webrtc_default_visibility testonly = true @@ -378,8 +373,10 @@ if (rtc_include_tests) { ":fir_filter_factory", ":sinc_resampler", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", + "../rtc_base:macromagic", "../rtc_base:rtc_base_tests_utils", + "../rtc_base:stringutils", + "../rtc_base:timeutils", "../rtc_base/system:arch", "../system_wrappers", "../test:fileutils", diff --git a/webrtc/common_audio/audio_converter.h b/webrtc/common_audio/audio_converter.h index 481ac08..4afbb6d 100644 --- a/webrtc/common_audio/audio_converter.h +++ b/webrtc/common_audio/audio_converter.h @@ -15,12 +15,10 @@ #include -#include "rtc_base/constructor_magic.h" - namespace webrtc { // Format conversion (remixing and resampling) for audio. Only simple remixing -// conversions are supported: downmix to mono (i.e. |dst_channels| == 1) or +// conversions are supported: downmix to mono (i.e. `dst_channels` == 1) or // upmix from mono (i.e. |src_channels == 1|). // // The source and destination chunks have the same duration in time; specifying @@ -35,8 +33,11 @@ class AudioConverter { size_t dst_frames); virtual ~AudioConverter() {} - // Convert |src|, containing |src_size| samples, to |dst|, having a sample - // capacity of |dst_capacity|. Both point to a series of buffers containing + AudioConverter(const AudioConverter&) = delete; + AudioConverter& operator=(const AudioConverter&) = delete; + + // Convert `src`, containing `src_size` samples, to `dst`, having a sample + // capacity of `dst_capacity`. Both point to a series of buffers containing // the samples for each channel. The sizes must correspond to the format // passed to Create(). virtual void Convert(const float* const* src, @@ -64,8 +65,6 @@ class AudioConverter { const size_t src_frames_; const size_t dst_channels_; const size_t dst_frames_; - - RTC_DISALLOW_COPY_AND_ASSIGN(AudioConverter); }; } // namespace webrtc diff --git a/webrtc/common_audio/channel_buffer.h b/webrtc/common_audio/channel_buffer.h index f027080..9f08d60 100644 --- a/webrtc/common_audio/channel_buffer.h +++ b/webrtc/common_audio/channel_buffer.h @@ -29,15 +29,15 @@ namespace webrtc { // // The buffer structure is showed below for a 2 channel and 2 bands case: // -// |data_|: +// `data_`: // { [ --- b1ch1 --- ] [ --- b2ch1 --- ] [ --- b1ch2 --- ] [ --- b2ch2 --- ] } // // The pointer arrays for the same example are as follows: // -// |channels_|: +// `channels_`: // { [ b1ch1* ] [ b1ch2* ] [ b2ch1* ] [ b2ch2* ] } // -// |bands_|: +// `bands_`: // { [ b1ch1* ] [ b2ch1* ] [ b1ch2* ] [ b2ch2* ] } template class ChannelBuffer { @@ -81,15 +81,15 @@ class ChannelBuffer { // If band is explicitly specificed, the channels for a specific band are // returned and the usage becomes: channels(band)[channel][sample]. // Where: - // 0 <= band < |num_bands_| - // 0 <= channel < |num_allocated_channels_| - // 0 <= sample < |num_frames_per_band_| + // 0 <= band < `num_bands_` + // 0 <= channel < `num_allocated_channels_` + // 0 <= sample < `num_frames_per_band_` // If band is not explicitly specified, the full-band channels (or lower band // channels) are returned and the usage becomes: channels()[channel][sample]. // Where: - // 0 <= channel < |num_allocated_channels_| - // 0 <= sample < |num_frames_| + // 0 <= channel < `num_allocated_channels_` + // 0 <= sample < `num_frames_` const T* const* channels(size_t band = 0) const { RTC_DCHECK_LT(band, num_bands_); return &channels_[band * num_allocated_channels_]; @@ -109,9 +109,9 @@ class ChannelBuffer { // Usage: // bands(channel)[band][sample]. // Where: - // 0 <= channel < |num_channels_| - // 0 <= band < |num_bands_| - // 0 <= sample < |num_frames_per_band_| + // 0 <= channel < `num_channels_` + // 0 <= band < `num_bands_` + // 0 <= sample < `num_frames_per_band_` const T* const* bands(size_t channel) const { RTC_DCHECK_LT(channel, num_channels_); RTC_DCHECK_GE(channel, 0); @@ -129,8 +129,8 @@ class ChannelBuffer { return bands_view_[channel]; } - // Sets the |slice| pointers to the |start_frame| position for each channel. - // Returns |slice| for convenience. + // Sets the `slice` pointers to the `start_frame` position for each channel. + // Returns `slice` for convenience. const T* const* Slice(T** slice, size_t start_frame) const { RTC_DCHECK_LT(start_frame, num_frames_); for (size_t i = 0; i < num_channels_; ++i) diff --git a/webrtc/common_audio/fir_filter.h b/webrtc/common_audio/fir_filter.h index a76e936..e0b18ca 100644 --- a/webrtc/common_audio/fir_filter.h +++ b/webrtc/common_audio/fir_filter.h @@ -20,8 +20,8 @@ class FIRFilter { public: virtual ~FIRFilter() {} - // Filters the |in| data supplied. - // |out| must be previously allocated and it must be at least of |length|. + // Filters the `in` data supplied. + // `out` must be previously allocated and it must be at least of `length`. virtual void Filter(const float* in, size_t length, float* out) = 0; }; diff --git a/webrtc/common_audio/fir_filter_avx2.cc b/webrtc/common_audio/fir_filter_avx2.cc index 26468e2..9cb0f77 100644 --- a/webrtc/common_audio/fir_filter_avx2.cc +++ b/webrtc/common_audio/fir_filter_avx2.cc @@ -52,7 +52,7 @@ void FIRFilterAVX2::Filter(const float* in, size_t length, float* out) { memcpy(&state_[state_length_], in, length * sizeof(*in)); - // Convolves the input signal |in| with the filter kernel |coefficients_| + // Convolves the input signal `in` with the filter kernel `coefficients_` // taking into account the previous state. for (size_t i = 0; i < length; ++i) { float* in_ptr = &state_[i]; diff --git a/webrtc/common_audio/fir_filter_c.cc b/webrtc/common_audio/fir_filter_c.cc index 3f1fa09..dc1c8e0 100644 --- a/webrtc/common_audio/fir_filter_c.cc +++ b/webrtc/common_audio/fir_filter_c.cc @@ -34,7 +34,7 @@ FIRFilterC::FIRFilterC(const float* coefficients, size_t coefficients_length) void FIRFilterC::Filter(const float* in, size_t length, float* out) { RTC_DCHECK_GT(length, 0); - // Convolves the input signal |in| with the filter kernel |coefficients_| + // Convolves the input signal `in` with the filter kernel `coefficients_` // taking into account the previous state. for (size_t i = 0; i < length; ++i) { out[i] = 0.f; diff --git a/webrtc/common_audio/fir_filter_factory.cc b/webrtc/common_audio/fir_filter_factory.cc index 4bcf052..2ecef65 100644 --- a/webrtc/common_audio/fir_filter_factory.cc +++ b/webrtc/common_audio/fir_filter_factory.cc @@ -28,7 +28,7 @@ FIRFilter* CreateFirFilter(const float* coefficients, size_t coefficients_length, size_t max_input_length) { if (!coefficients || coefficients_length <= 0 || max_input_length <= 0) { - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return nullptr; } diff --git a/webrtc/common_audio/fir_filter_factory.h b/webrtc/common_audio/fir_filter_factory.h index a952541..e76c3ae 100644 --- a/webrtc/common_audio/fir_filter_factory.h +++ b/webrtc/common_audio/fir_filter_factory.h @@ -20,7 +20,7 @@ class FIRFilter; // Creates a filter with the given coefficients. All initial state values will // be zeros. // The length of the chunks fed to the filter should never be greater than -// |max_input_length|. This is needed because, when vectorizing it is +// `max_input_length`. This is needed because, when vectorizing it is // necessary to concatenate the input after the state, and resizing this array // dynamically is expensive. FIRFilter* CreateFirFilter(const float* coefficients, diff --git a/webrtc/common_audio/fir_filter_neon.cc b/webrtc/common_audio/fir_filter_neon.cc index f668841..346cb69 100644 --- a/webrtc/common_audio/fir_filter_neon.cc +++ b/webrtc/common_audio/fir_filter_neon.cc @@ -48,7 +48,7 @@ void FIRFilterNEON::Filter(const float* in, size_t length, float* out) { memcpy(&state_[state_length_], in, length * sizeof(*in)); - // Convolves the input signal |in| with the filter kernel |coefficients_| + // Convolves the input signal `in` with the filter kernel `coefficients_` // taking into account the previous state. for (size_t i = 0; i < length; ++i) { float* in_ptr = &state_[i]; diff --git a/webrtc/common_audio/fir_filter_sse.cc b/webrtc/common_audio/fir_filter_sse.cc index ee75fb3..0e45994 100644 --- a/webrtc/common_audio/fir_filter_sse.cc +++ b/webrtc/common_audio/fir_filter_sse.cc @@ -49,7 +49,7 @@ void FIRFilterSSE2::Filter(const float* in, size_t length, float* out) { memcpy(&state_[state_length_], in, length * sizeof(*in)); - // Convolves the input signal |in| with the filter kernel |coefficients_| + // Convolves the input signal `in` with the filter kernel `coefficients_` // taking into account the previous state. for (size_t i = 0; i < length; ++i) { float* in_ptr = &state_[i]; diff --git a/webrtc/common_audio/include/audio_util.h b/webrtc/common_audio/include/audio_util.h index f6b6bfd..4ce4680 100644 --- a/webrtc/common_audio/include/audio_util.h +++ b/webrtc/common_audio/include/audio_util.h @@ -91,9 +91,9 @@ inline float FloatS16ToDbfs(float v) { return 20.0f * std::log10(v) + kMinDbfs; } -// Copy audio from |src| channels to |dest| channels unless |src| and |dest| -// point to the same address. |src| and |dest| must have the same number of -// channels, and there must be sufficient space allocated in |dest|. +// Copy audio from `src` channels to `dest` channels unless `src` and `dest` +// point to the same address. `src` and `dest` must have the same number of +// channels, and there must be sufficient space allocated in `dest`. template void CopyAudioIfNeeded(const T* const* src, int num_frames, @@ -106,9 +106,9 @@ void CopyAudioIfNeeded(const T* const* src, } } -// Deinterleave audio from |interleaved| to the channel buffers pointed to -// by |deinterleaved|. There must be sufficient space allocated in the -// |deinterleaved| buffers (|num_channel| buffers with |samples_per_channel| +// Deinterleave audio from `interleaved` to the channel buffers pointed to +// by `deinterleaved`. There must be sufficient space allocated in the +// `deinterleaved` buffers (`num_channel` buffers with `samples_per_channel` // per buffer). template void Deinterleave(const T* interleaved, @@ -125,9 +125,9 @@ void Deinterleave(const T* interleaved, } } -// Interleave audio from the channel buffers pointed to by |deinterleaved| to -// |interleaved|. There must be sufficient space allocated in |interleaved| -// (|samples_per_channel| * |num_channels|). +// Interleave audio from the channel buffers pointed to by `deinterleaved` to +// `interleaved`. There must be sufficient space allocated in `interleaved` +// (`samples_per_channel` * `num_channels`). template void Interleave(const T* const* deinterleaved, size_t samples_per_channel, @@ -143,9 +143,9 @@ void Interleave(const T* const* deinterleaved, } } -// Copies audio from a single channel buffer pointed to by |mono| to each -// channel of |interleaved|. There must be sufficient space allocated in -// |interleaved| (|samples_per_channel| * |num_channels|). +// Copies audio from a single channel buffer pointed to by `mono` to each +// channel of `interleaved`. There must be sufficient space allocated in +// `interleaved` (`samples_per_channel` * `num_channels`). template void UpmixMonoToInterleaved(const T* mono, int num_frames, diff --git a/webrtc/common_audio/real_fourier.h b/webrtc/common_audio/real_fourier.h index 4881fb7..78a4fc6 100644 --- a/webrtc/common_audio/real_fourier.h +++ b/webrtc/common_audio/real_fourier.h @@ -50,7 +50,7 @@ class RealFourier { // output (i.e. |2^order / 2 + 1|). static size_t ComplexLength(int order); - // Buffer allocation helpers. The buffers are large enough to hold |count| + // Buffer allocation helpers. The buffers are large enough to hold `count` // floats/complexes and suitably aligned for use by the implementation. // The returned scopers are set up with proper deleters; the caller owns // the allocated memory. diff --git a/webrtc/common_audio/resampler/push_resampler.cc b/webrtc/common_audio/resampler/push_resampler.cc index d7aa8d7..810d778 100644 --- a/webrtc/common_audio/resampler/push_resampler.cc +++ b/webrtc/common_audio/resampler/push_resampler.cc @@ -20,42 +20,6 @@ #include "rtc_base/checks.h" namespace webrtc { -namespace { -// These checks were factored out into a non-templatized function -// due to problems with clang on Windows in debug builds. -// For some reason having the DCHECKs inline in the template code -// caused the compiler to generate code that threw off the linker. -// TODO(tommi): Re-enable when we've figured out what the problem is. -// http://crbug.com/615050 -void CheckValidInitParams(int src_sample_rate_hz, - int dst_sample_rate_hz, - size_t num_channels) { -// The below checks are temporarily disabled on WEBRTC_WIN due to problems -// with clang debug builds. -#if !defined(WEBRTC_WIN) && defined(__clang__) - RTC_DCHECK_GT(src_sample_rate_hz, 0); - RTC_DCHECK_GT(dst_sample_rate_hz, 0); - RTC_DCHECK_GT(num_channels, 0); -#endif -} - -void CheckExpectedBufferSizes(size_t src_length, - size_t dst_capacity, - size_t num_channels, - int src_sample_rate, - int dst_sample_rate) { -// The below checks are temporarily disabled on WEBRTC_WIN due to problems -// with clang debug builds. -// TODO(tommi): Re-enable when we've figured out what the problem is. -// http://crbug.com/615050 -#if !defined(WEBRTC_WIN) && defined(__clang__) - const size_t src_size_10ms = src_sample_rate * num_channels / 100; - const size_t dst_size_10ms = dst_sample_rate * num_channels / 100; - RTC_DCHECK_EQ(src_length, src_size_10ms); - RTC_DCHECK_GE(dst_capacity, dst_size_10ms); -#endif -} -} // namespace template PushResampler::PushResampler() @@ -68,7 +32,11 @@ template int PushResampler::InitializeIfNeeded(int src_sample_rate_hz, int dst_sample_rate_hz, size_t num_channels) { - CheckValidInitParams(src_sample_rate_hz, dst_sample_rate_hz, num_channels); + // These checks used to be factored out of this template function due to + // Windows debug build issues with clang. http://crbug.com/615050 + RTC_DCHECK_GT(src_sample_rate_hz, 0); + RTC_DCHECK_GT(dst_sample_rate_hz, 0); + RTC_DCHECK_GT(num_channels, 0); if (src_sample_rate_hz == src_sample_rate_hz_ && dst_sample_rate_hz == dst_sample_rate_hz_ && @@ -109,8 +77,12 @@ int PushResampler::Resample(const T* src, size_t src_length, T* dst, size_t dst_capacity) { - CheckExpectedBufferSizes(src_length, dst_capacity, num_channels_, - src_sample_rate_hz_, dst_sample_rate_hz_); + // These checks used to be factored out of this template function due to + // Windows debug build issues with clang. http://crbug.com/615050 + const size_t src_size_10ms = (src_sample_rate_hz_ / 100) * num_channels_; + const size_t dst_size_10ms = (dst_sample_rate_hz_ / 100) * num_channels_; + RTC_DCHECK_EQ(src_length, src_size_10ms); + RTC_DCHECK_GE(dst_capacity, dst_size_10ms); if (src_sample_rate_hz_ == dst_sample_rate_hz_) { // The old resampler provides this memcpy facility in the case of matching diff --git a/webrtc/common_audio/resampler/push_sinc_resampler.cc b/webrtc/common_audio/resampler/push_sinc_resampler.cc index 3bfead2..d4b7eed 100644 --- a/webrtc/common_audio/resampler/push_sinc_resampler.cc +++ b/webrtc/common_audio/resampler/push_sinc_resampler.cc @@ -63,12 +63,12 @@ size_t PushSincResampler::Resample(const float* source, // request through Run(). // // If this wasn't done, SincResampler would call Run() twice on the first - // pass, and we'd have to introduce an entire |source_frames| of delay, rather + // pass, and we'd have to introduce an entire `source_frames` of delay, rather // than the minimum half kernel. // // It works out that ChunkSize() is exactly the amount of output we need to // request in order to prime the buffer with a single Run() request for - // |source_frames|. + // `source_frames`. if (first_pass_) resampler_->Resample(resampler_->ChunkSize(), destination); diff --git a/webrtc/common_audio/resampler/push_sinc_resampler.h b/webrtc/common_audio/resampler/push_sinc_resampler.h index bd609c4..7946ef8 100644 --- a/webrtc/common_audio/resampler/push_sinc_resampler.h +++ b/webrtc/common_audio/resampler/push_sinc_resampler.h @@ -17,7 +17,6 @@ #include #include "common_audio/resampler/sinc_resampler.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -33,11 +32,14 @@ class PushSincResampler : public SincResamplerCallback { PushSincResampler(size_t source_frames, size_t destination_frames); ~PushSincResampler() override; - // Perform the resampling. |source_frames| must always equal the - // |source_frames| provided at construction. |destination_capacity| must be - // at least as large as |destination_frames|. Returns the number of samples + PushSincResampler(const PushSincResampler&) = delete; + PushSincResampler& operator=(const PushSincResampler&) = delete; + + // Perform the resampling. `source_frames` must always equal the + // `source_frames` provided at construction. `destination_capacity` must be + // at least as large as `destination_frames`. Returns the number of samples // provided in destination (for convenience, since this will always be equal - // to |destination_frames|). + // to `destination_frames`). size_t Resample(const int16_t* source, size_t source_frames, int16_t* destination, @@ -72,8 +74,6 @@ class PushSincResampler : public SincResamplerCallback { // Used to assert we are only requested for as much data as is available. size_t source_available_; - - RTC_DISALLOW_COPY_AND_ASSIGN(PushSincResampler); }; } // namespace webrtc diff --git a/webrtc/common_audio/resampler/resampler.cc b/webrtc/common_audio/resampler/resampler.cc index ccfed5a..0fdb249 100644 --- a/webrtc/common_audio/resampler/resampler.cc +++ b/webrtc/common_audio/resampler/resampler.cc @@ -916,7 +916,6 @@ int Resampler::Push(const int16_t* samplesIn, outLen = (lengthIn * 8) / 11; free(tmp_mem); return 0; - break; } return 0; } diff --git a/webrtc/common_audio/resampler/sinc_resampler.cc b/webrtc/common_audio/resampler/sinc_resampler.cc index 4fa78c5..66a99b6 100644 --- a/webrtc/common_audio/resampler/sinc_resampler.cc +++ b/webrtc/common_audio/resampler/sinc_resampler.cc @@ -80,7 +80,7 @@ // 8) Else, if we're not on the second load, goto (4). // // Note: we're glossing over how the sub-sample handling works with -// |virtual_source_idx_|, etc. +// `virtual_source_idx_`, etc. // MSVC++ requires this to be set before any other includes to get M_PI. #define _USE_MATH_DEFINES @@ -102,7 +102,7 @@ namespace webrtc { namespace { double SincScaleFactor(double io_ratio) { - // |sinc_scale_factor| is basically the normalized cutoff frequency of the + // `sinc_scale_factor` is basically the normalized cutoff frequency of the // low-pass filter. double sinc_scale_factor = io_ratio > 1.0 ? 1.0 / io_ratio : 1.0; @@ -126,8 +126,8 @@ void SincResampler::InitializeCPUSpecificFeatures() { #if defined(WEBRTC_HAS_NEON) convolve_proc_ = Convolve_NEON; #elif defined(WEBRTC_ARCH_X86_FAMILY) - // Using AVX2 instead of SSE2 when AVX2 supported. - if (GetCPUInfo(kAVX2)) + // Using AVX2 instead of SSE2 when AVX2/FMA3 supported. + if (GetCPUInfo(kAVX2) && GetCPUInfo(kFMA3)) convolve_proc_ = Convolve_AVX2; else if (GetCPUInfo(kSSE2)) convolve_proc_ = Convolve_SSE; @@ -238,7 +238,7 @@ void SincResampler::SetRatio(double io_sample_rate_ratio) { io_sample_rate_ratio_ = io_sample_rate_ratio; // Optimize reinitialization by reusing values which are independent of - // |sinc_scale_factor|. Provides a 3x speedup. + // `sinc_scale_factor`. Provides a 3x speedup. const double sinc_scale_factor = SincScaleFactor(io_sample_rate_ratio_); for (size_t offset_idx = 0; offset_idx <= kKernelOffsetCount; ++offset_idx) { for (size_t i = 0; i < kKernelSize; ++i) { @@ -268,8 +268,8 @@ void SincResampler::Resample(size_t frames, float* destination) { const double current_io_ratio = io_sample_rate_ratio_; const float* const kernel_ptr = kernel_storage_.get(); while (remaining_frames) { - // |i| may be negative if the last Resample() call ended on an iteration - // that put |virtual_source_idx_| over the limit. + // `i` may be negative if the last Resample() call ended on an iteration + // that put `virtual_source_idx_` over the limit. // // Note: The loop construct here can severely impact performance on ARM // or when built with clang. See https://codereview.chromium.org/18566009/ @@ -278,7 +278,7 @@ void SincResampler::Resample(size_t frames, float* destination) { i > 0; --i) { RTC_DCHECK_LT(virtual_source_idx_, block_size_); - // |virtual_source_idx_| lies in between two kernel offsets so figure out + // `virtual_source_idx_` lies in between two kernel offsets so figure out // what they are. const int source_idx = static_cast(virtual_source_idx_); const double subsample_remainder = virtual_source_idx_ - source_idx; @@ -288,16 +288,16 @@ void SincResampler::Resample(size_t frames, float* destination) { const int offset_idx = static_cast(virtual_offset_idx); // We'll compute "convolutions" for the two kernels which straddle - // |virtual_source_idx_|. + // `virtual_source_idx_`. const float* const k1 = kernel_ptr + offset_idx * kKernelSize; const float* const k2 = k1 + kKernelSize; - // Ensure |k1|, |k2| are 32-byte aligned for SIMD usage. Should always be + // Ensure `k1`, `k2` are 32-byte aligned for SIMD usage. Should always be // true so long as kKernelSize is a multiple of 32. RTC_DCHECK_EQ(0, reinterpret_cast(k1) % 32); RTC_DCHECK_EQ(0, reinterpret_cast(k2) % 32); - // Initialize input pointer based on quantized |virtual_source_idx_|. + // Initialize input pointer based on quantized `virtual_source_idx_`. const float* const input_ptr = r1_ + source_idx; // Figure out how much to weight each kernel's "convolution". diff --git a/webrtc/common_audio/resampler/sinc_resampler.h b/webrtc/common_audio/resampler/sinc_resampler.h index a72a0c6..c6a43ab 100644 --- a/webrtc/common_audio/resampler/sinc_resampler.h +++ b/webrtc/common_audio/resampler/sinc_resampler.h @@ -18,15 +18,14 @@ #include -#include "rtc_base/constructor_magic.h" #include "rtc_base/gtest_prod_util.h" #include "rtc_base/memory/aligned_malloc.h" #include "rtc_base/system/arch.h" namespace webrtc { -// Callback class for providing more data into the resampler. Expects |frames| -// of data to be rendered into |destination|; zero padded if not enough frames +// Callback class for providing more data into the resampler. Expects `frames` +// of data to be rendered into `destination`; zero padded if not enough frames // are available to satisfy the request. class SincResamplerCallback { public: @@ -53,10 +52,10 @@ class SincResampler { static const size_t kKernelStorageSize = kKernelSize * (kKernelOffsetCount + 1); - // Constructs a SincResampler with the specified |read_cb|, which is used to - // acquire audio data for resampling. |io_sample_rate_ratio| is the ratio - // of input / output sample rates. |request_frames| controls the size in - // frames of the buffer requested by each |read_cb| call. The value must be + // Constructs a SincResampler with the specified `read_cb`, which is used to + // acquire audio data for resampling. `io_sample_rate_ratio` is the ratio + // of input / output sample rates. `request_frames` controls the size in + // frames of the buffer requested by each `read_cb` call. The value must be // greater than kKernelSize. Specify kDefaultRequestSize if there are no // request size constraints. SincResampler(double io_sample_rate_ratio, @@ -64,11 +63,14 @@ class SincResampler { SincResamplerCallback* read_cb); virtual ~SincResampler(); - // Resample |frames| of data from |read_cb_| into |destination|. + SincResampler(const SincResampler&) = delete; + SincResampler& operator=(const SincResampler&) = delete; + + // Resample `frames` of data from `read_cb_` into `destination`. void Resample(size_t frames, float* destination); // The maximum size in frames that guarantees Resample() will only make a - // single call to |read_cb_| for more data. + // single call to `read_cb_` for more data. size_t ChunkSize() const; size_t request_frames() const { return request_frames_; } @@ -77,12 +79,12 @@ class SincResampler { // not call while Resample() is in progress. void Flush(); - // Update |io_sample_rate_ratio_|. SetRatio() will cause a reconstruction of + // Update `io_sample_rate_ratio_`. SetRatio() will cause a reconstruction of // the kernels used for resampling. Not thread safe, do not call while // Resample() is in progress. // // TODO(ajm): Use this in PushSincResampler rather than reconstructing - // SincResampler. We would also need a way to update |request_frames_|. + // SincResampler. We would also need a way to update `request_frames_`. void SetRatio(double io_sample_rate_ratio); float* get_kernel_for_testing() { return kernel_storage_.get(); } @@ -97,11 +99,11 @@ class SincResampler { // Selects runtime specific CPU features like SSE. Must be called before // using SincResampler. // TODO(ajm): Currently managed by the class internally. See the note with - // |convolve_proc_| below. + // `convolve_proc_` below. void InitializeCPUSpecificFeatures(); - // Compute convolution of |k1| and |k2| over |input_ptr|, resultant sums are - // linearly interpolated using |kernel_interpolation_factor|. On x86 and ARM + // Compute convolution of `k1` and `k2` over `input_ptr`, resultant sums are + // linearly interpolated using `kernel_interpolation_factor`. On x86 and ARM // the underlying implementation is chosen at run time. static float Convolve_C(const float* input_ptr, const float* k1, @@ -136,7 +138,7 @@ class SincResampler { // Source of data for resampling. SincResamplerCallback* read_cb_; - // The size (in samples) to request from each |read_cb_| execution. + // The size (in samples) to request from each `read_cb_` execution. const size_t request_frames_; // The number of source frames processed per pass. @@ -155,25 +157,23 @@ class SincResampler { // Data from the source is copied into this buffer for each processing pass. std::unique_ptr input_buffer_; -// Stores the runtime selection of which Convolve function to use. -// TODO(ajm): Move to using a global static which must only be initialized -// once by the user. We're not doing this initially, because we don't have -// e.g. a LazyInstance helper in webrtc. + // Stores the runtime selection of which Convolve function to use. + // TODO(ajm): Move to using a global static which must only be initialized + // once by the user. We're not doing this initially, because we don't have + // e.g. a LazyInstance helper in webrtc. typedef float (*ConvolveProc)(const float*, const float*, const float*, double); ConvolveProc convolve_proc_; - // Pointers to the various regions inside |input_buffer_|. See the diagram at + // Pointers to the various regions inside `input_buffer_`. See the diagram at // the top of the .cc file for more information. float* r0_; float* const r1_; float* const r2_; float* r3_; float* r4_; - - RTC_DISALLOW_COPY_AND_ASSIGN(SincResampler); }; } // namespace webrtc diff --git a/webrtc/common_audio/resampler/sinc_resampler_avx2.cc b/webrtc/common_audio/resampler/sinc_resampler_avx2.cc index 3eb5d4a..d945a10 100644 --- a/webrtc/common_audio/resampler/sinc_resampler_avx2.cc +++ b/webrtc/common_audio/resampler/sinc_resampler_avx2.cc @@ -25,7 +25,7 @@ float SincResampler::Convolve_AVX2(const float* input_ptr, __m256 m_sums1 = _mm256_setzero_ps(); __m256 m_sums2 = _mm256_setzero_ps(); - // Based on |input_ptr| alignment, we need to use loadu or load. Unrolling + // Based on `input_ptr` alignment, we need to use loadu or load. Unrolling // these loops has not been tested or benchmarked. bool aligned_input = (reinterpret_cast(input_ptr) & 0x1F) == 0; if (!aligned_input) { diff --git a/webrtc/common_audio/resampler/sinc_resampler_sse.cc b/webrtc/common_audio/resampler/sinc_resampler_sse.cc index f6a24d0..30a8d1b 100644 --- a/webrtc/common_audio/resampler/sinc_resampler_sse.cc +++ b/webrtc/common_audio/resampler/sinc_resampler_sse.cc @@ -27,7 +27,7 @@ float SincResampler::Convolve_SSE(const float* input_ptr, __m128 m_sums1 = _mm_setzero_ps(); __m128 m_sums2 = _mm_setzero_ps(); - // Based on |input_ptr| alignment, we need to use loadu or load. Unrolling + // Based on `input_ptr` alignment, we need to use loadu or load. Unrolling // these loops hurt performance in local testing. if (reinterpret_cast(input_ptr) & 0x0F) { for (size_t i = 0; i < kKernelSize; i += 4) { diff --git a/webrtc/common_audio/resampler/sinusoidal_linear_chirp_source.h b/webrtc/common_audio/resampler/sinusoidal_linear_chirp_source.h index 81f6a24..ccd11bb 100644 --- a/webrtc/common_audio/resampler/sinusoidal_linear_chirp_source.h +++ b/webrtc/common_audio/resampler/sinusoidal_linear_chirp_source.h @@ -15,7 +15,6 @@ #define COMMON_AUDIO_RESAMPLER_SINUSOIDAL_LINEAR_CHIRP_SOURCE_H_ #include "common_audio/resampler/sinc_resampler.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -24,7 +23,7 @@ namespace webrtc { // resampler for the specific sample rate conversion being used. class SinusoidalLinearChirpSource : public SincResamplerCallback { public: - // |delay_samples| can be used to insert a fractional sample delay into the + // `delay_samples` can be used to insert a fractional sample delay into the // source. It will produce zeros until non-negative time is reached. SinusoidalLinearChirpSource(int sample_rate, size_t samples, @@ -33,12 +32,16 @@ class SinusoidalLinearChirpSource : public SincResamplerCallback { ~SinusoidalLinearChirpSource() override {} + SinusoidalLinearChirpSource(const SinusoidalLinearChirpSource&) = delete; + SinusoidalLinearChirpSource& operator=(const SinusoidalLinearChirpSource&) = + delete; + void Run(size_t frames, float* destination) override; double Frequency(size_t position); private: - enum { kMinFrequency = 5 }; + static constexpr int kMinFrequency = 5; int sample_rate_; size_t total_samples_; @@ -46,8 +49,6 @@ class SinusoidalLinearChirpSource : public SincResamplerCallback { double k_; size_t current_index_; double delay_samples_; - - RTC_DISALLOW_COPY_AND_ASSIGN(SinusoidalLinearChirpSource); }; } // namespace webrtc diff --git a/webrtc/common_audio/ring_buffer.c b/webrtc/common_audio/ring_buffer.c index a20ada5..590f5f9 100644 --- a/webrtc/common_audio/ring_buffer.c +++ b/webrtc/common_audio/ring_buffer.c @@ -18,9 +18,9 @@ #include // Get address of region(s) from which we can read data. -// If the region is contiguous, |data_ptr_bytes_2| will be zero. -// If non-contiguous, |data_ptr_bytes_2| will be the size in bytes of the second -// region. Returns room available to be read or |element_count|, whichever is +// If the region is contiguous, `data_ptr_bytes_2` will be zero. +// If non-contiguous, `data_ptr_bytes_2` will be the size in bytes of the second +// region. Returns room available to be read or `element_count`, whichever is // smaller. static size_t GetBufferReadRegions(RingBuffer* buf, size_t element_count, @@ -120,7 +120,7 @@ size_t WebRtc_ReadBuffer(RingBuffer* self, &buf_ptr_bytes_2); if (buf_ptr_bytes_2 > 0) { // We have a wrap around when reading the buffer. Copy the buffer data to - // |data| and point to it. + // `data` and point to it. memcpy(data, buf_ptr_1, buf_ptr_bytes_1); memcpy(((char*) data) + buf_ptr_bytes_1, buf_ptr_2, buf_ptr_bytes_2); buf_ptr_1 = data; @@ -129,7 +129,7 @@ size_t WebRtc_ReadBuffer(RingBuffer* self, memcpy(data, buf_ptr_1, buf_ptr_bytes_1); } if (data_ptr) { - // |buf_ptr_1| == |data| in the case of a wrap. + // `buf_ptr_1` == `data` in the case of a wrap. *data_ptr = read_count == 0 ? NULL : buf_ptr_1; } diff --git a/webrtc/common_audio/ring_buffer.h b/webrtc/common_audio/ring_buffer.h index bcc40e1..de0b4fe 100644 --- a/webrtc/common_audio/ring_buffer.h +++ b/webrtc/common_audio/ring_buffer.h @@ -39,14 +39,14 @@ void WebRtc_InitBuffer(RingBuffer* handle); void WebRtc_FreeBuffer(void* handle); // Reads data from the buffer. Returns the number of elements that were read. -// The |data_ptr| will point to the address where the read data is located. -// If no data can be read, |data_ptr| is set to |NULL|. If all data can be read -// without buffer wrap around then |data_ptr| will point to the location in the -// buffer. Otherwise, the data will be copied to |data| (memory allocation done -// by the user) and |data_ptr| points to the address of |data|. |data_ptr| is +// The `data_ptr` will point to the address where the read data is located. +// If no data can be read, `data_ptr` is set to `NULL`. If all data can be read +// without buffer wrap around then `data_ptr` will point to the location in the +// buffer. Otherwise, the data will be copied to `data` (memory allocation done +// by the user) and `data_ptr` points to the address of `data`. `data_ptr` is // only guaranteed to be valid until the next call to WebRtc_WriteBuffer(). // -// To force a copying to |data|, pass a null |data_ptr|. +// To force a copying to `data`, pass a null `data_ptr`. // // Returns number of elements read. size_t WebRtc_ReadBuffer(RingBuffer* handle, @@ -54,14 +54,14 @@ size_t WebRtc_ReadBuffer(RingBuffer* handle, void* data, size_t element_count); -// Writes |data| to buffer and returns the number of elements written. +// Writes `data` to buffer and returns the number of elements written. size_t WebRtc_WriteBuffer(RingBuffer* handle, const void* data, size_t element_count); // Moves the buffer read position and returns the number of elements moved. -// Positive |element_count| moves the read position towards the write position, -// that is, flushing the buffer. Negative |element_count| moves the read +// Positive `element_count` moves the read position towards the write position, +// that is, flushing the buffer. Negative `element_count` moves the read // position away from the the write position, that is, stuffing the buffer. // Returns number of elements moved. int WebRtc_MoveReadPtr(RingBuffer* handle, int element_count); diff --git a/webrtc/common_audio/signal_processing/cross_correlation_neon.c b/webrtc/common_audio/signal_processing/cross_correlation_neon.c index f2afbdf..d3ecf13 100644 --- a/webrtc/common_audio/signal_processing/cross_correlation_neon.c +++ b/webrtc/common_audio/signal_processing/cross_correlation_neon.c @@ -72,9 +72,9 @@ void WebRtcSpl_CrossCorrelationNeon(int32_t* cross_correlation, size_t dim_cross_correlation, int right_shifts, int step_seq2) { - size_t i = 0; + int i = 0; - for (i = 0; i < dim_cross_correlation; i++) { + for (i = 0; i < (int)dim_cross_correlation; i++) { const int16_t* seq1_ptr = seq1; const int16_t* seq2_ptr = seq2 + (step_seq2 * i); diff --git a/webrtc/common_audio/signal_processing/division_operations.c b/webrtc/common_audio/signal_processing/division_operations.c index c6195e7..4764ddf 100644 --- a/webrtc/common_audio/signal_processing/division_operations.c +++ b/webrtc/common_audio/signal_processing/division_operations.c @@ -98,8 +98,7 @@ int32_t WebRtcSpl_DivResultInQ31(int32_t num, int32_t den) return div; } -int32_t RTC_NO_SANITIZE("signed-integer-overflow") // bugs.webrtc.org/5486 -WebRtcSpl_DivW32HiLow(int32_t num, int16_t den_hi, int16_t den_low) +int32_t WebRtcSpl_DivW32HiLow(int32_t num, int16_t den_hi, int16_t den_low) { int16_t approx, tmp_hi, tmp_low, num_hi, num_low; int32_t tmpW32; @@ -111,8 +110,8 @@ WebRtcSpl_DivW32HiLow(int32_t num, int16_t den_hi, int16_t den_low) tmpW32 = (den_hi * approx << 1) + ((den_low * approx >> 15) << 1); // tmpW32 = den * approx - tmpW32 = (int32_t)0x7fffffffL - tmpW32; // result in Q30 (tmpW32 = 2.0-(den*approx)) - // UBSan: 2147483647 - -2 cannot be represented in type 'int' + // result in Q30 (tmpW32 = 2.0-(den*approx)) + tmpW32 = (int32_t)((int64_t)0x7fffffffL - tmpW32); // Store tmpW32 in hi and low format tmp_hi = (int16_t)(tmpW32 >> 16); diff --git a/webrtc/common_audio/signal_processing/dot_product_with_scale.h b/webrtc/common_audio/signal_processing/dot_product_with_scale.h index bb892d4..9f0d922 100644 --- a/webrtc/common_audio/signal_processing/dot_product_with_scale.h +++ b/webrtc/common_audio/signal_processing/dot_product_with_scale.h @@ -26,7 +26,7 @@ extern "C" { // - vector_length : Number of samples used in the dot product // - scaling : The number of right bit shifts to apply on each term // during calculation to avoid overflow, i.e., the -// output will be in Q(-|scaling|) +// output will be in Q(-`scaling`) // // Return value : The dot product in Q(-scaling) int32_t WebRtcSpl_DotProductWithScale(const int16_t* vector1, diff --git a/webrtc/common_audio/signal_processing/downsample_fast_neon.c b/webrtc/common_audio/signal_processing/downsample_fast_neon.c index 36fc0c8..f1b754b 100644 --- a/webrtc/common_audio/signal_processing/downsample_fast_neon.c +++ b/webrtc/common_audio/signal_processing/downsample_fast_neon.c @@ -8,9 +8,11 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "common_audio/signal_processing/include/signal_processing_library.h" -#include +#include "rtc_base/checks.h" // NEON intrinsics version of WebRtcSpl_DownsampleFast() // for ARM 32-bit/64-bit platforms. @@ -22,19 +24,24 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, size_t coefficients_length, int factor, size_t delay) { - size_t i = 0; - size_t j = 0; + // Using signed indexes to be able to compute negative i-j that + // is used to index data_in. + int i = 0; + int j = 0; int32_t out_s32 = 0; - size_t endpos = delay + factor * (data_out_length - 1) + 1; + int endpos = delay + factor * (data_out_length - 1) + 1; size_t res = data_out_length & 0x7; - size_t endpos1 = endpos - factor * res; + int endpos1 = endpos - factor * res; // Return error if any of the running conditions doesn't meet. if (data_out_length == 0 || coefficients_length == 0 - || data_in_length < endpos) { + || (int)data_in_length < endpos) { return -1; } + RTC_DCHECK_GE(endpos, 0); + RTC_DCHECK_GE(endpos1, 0); + // First part, unroll the loop 8 times, with 3 subcases // (factor == 2, 4, others). switch (factor) { @@ -46,7 +53,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, #if defined(WEBRTC_ARCH_ARM64) // Unroll the loop 2 times. - for (j = 0; j < coefficients_length - 1; j += 2) { + for (j = 0; j < (int)coefficients_length - 1; j += 2) { int32x2_t coeff32 = vld1_dup_s32((int32_t*)&coefficients[j]); int16x4_t coeff16x4 = vreinterpret_s16_s32(coeff32); int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j - 1]); @@ -68,7 +75,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_3, coeff16x4, 0); } - for (; j < coefficients_length; j++) { + for (; j < (int)coefficients_length; j++) { int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]); int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j]); @@ -87,7 +94,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, #else // On ARMv7, the loop unrolling 2 times results in performance // regression. - for (j = 0; j < coefficients_length; j++) { + for (j = 0; j < (int)coefficients_length; j++) { int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]); int16x8x2_t in16x8x2 = vld2q_s16(&data_in[i - j]); @@ -114,7 +121,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, int32x4_t out32x4_1 = vdupq_n_s32(2048); // Unroll the loop 4 times. - for (j = 0; j < coefficients_length - 3; j += 4) { + for (j = 0; j < (int)coefficients_length - 3; j += 4) { int16x4_t coeff16x4 = vld1_s16(&coefficients[j]); int16x8x4_t in16x8x4 = vld4q_s16(&data_in[i - j - 3]); @@ -143,7 +150,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, out32x4_1 = vmlal_lane_s16(out32x4_1, in16x4_7, coeff16x4, 0); } - for (; j < coefficients_length; j++) { + for (; j < (int)coefficients_length; j++) { int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]); int16x8x4_t in16x8x4 = vld4q_s16(&data_in[i - j]); @@ -174,7 +181,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, int32x4_t out32x4_0 = vdupq_n_s32(2048); int32x4_t out32x4_1 = vdupq_n_s32(2048); - for (j = 0; j < coefficients_length; j++) { + for (j = 0; j < (int)coefficients_length; j++) { int16x4_t coeff16x4 = vld1_dup_s16(&coefficients[j]); int16x4_t in16x4_0 = vld1_dup_s16(&data_in[i - j]); in16x4_0 = vld1_lane_s16(&data_in[i + factor - j], in16x4_0, 1); @@ -204,7 +211,7 @@ int WebRtcSpl_DownsampleFastNeon(const int16_t* data_in, for (; i < endpos; i += factor) { out_s32 = 2048; // Round value, 0.5 in Q12. - for (j = 0; j < coefficients_length; j++) { + for (j = 0; j < (int)coefficients_length; j++) { out_s32 = WebRtc_MulAccumW16(coefficients[j], data_in[i - j], out_s32); } diff --git a/webrtc/common_audio/signal_processing/include/real_fft.h b/webrtc/common_audio/signal_processing/include/real_fft.h index 8445066..a0da509 100644 --- a/webrtc/common_audio/signal_processing/include/real_fft.h +++ b/webrtc/common_audio/signal_processing/include/real_fft.h @@ -81,7 +81,7 @@ int WebRtcSpl_RealForwardFFT(struct RealFFT* self, // boundary. // // Return Value: -// 0 or a positive number - a value that the elements in the |real_data_out| +// 0 or a positive number - a value that the elements in the `real_data_out` // should be shifted left with in order to get // correct physical values. // -1 - Error with bad arguments (null pointers). diff --git a/webrtc/common_audio/signal_processing/include/signal_processing_library.h b/webrtc/common_audio/signal_processing/include/signal_processing_library.h index 4ad92c4..48c9b30 100644 --- a/webrtc/common_audio/signal_processing/include/signal_processing_library.h +++ b/webrtc/common_audio/signal_processing/include/signal_processing_library.h @@ -166,7 +166,7 @@ int32_t WebRtcSpl_MaxAbsValueW32_mips(const int32_t* vector, size_t length); // - vector : 16-bit input vector. // - length : Number of samples in vector. // -// Return value : Maximum sample value in |vector|. +// Return value : Maximum sample value in `vector`. typedef int16_t (*MaxValueW16)(const int16_t* vector, size_t length); extern const MaxValueW16 WebRtcSpl_MaxValueW16; int16_t WebRtcSpl_MaxValueW16C(const int16_t* vector, size_t length); @@ -183,7 +183,7 @@ int16_t WebRtcSpl_MaxValueW16_mips(const int16_t* vector, size_t length); // - vector : 32-bit input vector. // - length : Number of samples in vector. // -// Return value : Maximum sample value in |vector|. +// Return value : Maximum sample value in `vector`. typedef int32_t (*MaxValueW32)(const int32_t* vector, size_t length); extern const MaxValueW32 WebRtcSpl_MaxValueW32; int32_t WebRtcSpl_MaxValueW32C(const int32_t* vector, size_t length); @@ -200,7 +200,7 @@ int32_t WebRtcSpl_MaxValueW32_mips(const int32_t* vector, size_t length); // - vector : 16-bit input vector. // - length : Number of samples in vector. // -// Return value : Minimum sample value in |vector|. +// Return value : Minimum sample value in `vector`. typedef int16_t (*MinValueW16)(const int16_t* vector, size_t length); extern const MinValueW16 WebRtcSpl_MinValueW16; int16_t WebRtcSpl_MinValueW16C(const int16_t* vector, size_t length); @@ -217,7 +217,7 @@ int16_t WebRtcSpl_MinValueW16_mips(const int16_t* vector, size_t length); // - vector : 32-bit input vector. // - length : Number of samples in vector. // -// Return value : Minimum sample value in |vector|. +// Return value : Minimum sample value in `vector`. typedef int32_t (*MinValueW32)(const int32_t* vector, size_t length); extern const MinValueW32 WebRtcSpl_MinValueW32; int32_t WebRtcSpl_MinValueW32C(const int32_t* vector, size_t length); @@ -228,6 +228,25 @@ int32_t WebRtcSpl_MinValueW32Neon(const int32_t* vector, size_t length); int32_t WebRtcSpl_MinValueW32_mips(const int32_t* vector, size_t length); #endif +// Returns both the minimum and maximum values of a 16-bit vector. +// +// Input: +// - vector : 16-bit input vector. +// - length : Number of samples in vector. +// Ouput: +// - max_val : Maximum sample value in `vector`. +// - min_val : Minimum sample value in `vector`. +void WebRtcSpl_MinMaxW16(const int16_t* vector, + size_t length, + int16_t* min_val, + int16_t* max_val); +#if defined(WEBRTC_HAS_NEON) +void WebRtcSpl_MinMaxW16Neon(const int16_t* vector, + size_t length, + int16_t* min_val, + int16_t* max_val); +#endif + // Returns the vector index to the largest absolute value of a 16-bit vector. // // Input: @@ -240,6 +259,17 @@ int32_t WebRtcSpl_MinValueW32_mips(const int32_t* vector, size_t length); // -32768 presenting an int16 absolute value of 32767). size_t WebRtcSpl_MaxAbsIndexW16(const int16_t* vector, size_t length); +// Returns the element with the largest absolute value of a 16-bit vector. Note +// that this function can return a negative value. +// +// Input: +// - vector : 16-bit input vector. +// - length : Number of samples in vector. +// +// Return value : The element with the largest absolute value. Note that this +// may be a negative value. +int16_t WebRtcSpl_MaxAbsElementW16(const int16_t* vector, size_t length); + // Returns the vector index to the maximum sample value of a 16-bit vector. // // Input: @@ -396,7 +426,7 @@ void WebRtcSpl_AffineTransformVector(int16_t* out_vector, // // Input: // - in_vector : Vector to calculate autocorrelation upon -// - in_vector_length : Length (in samples) of |vector| +// - in_vector_length : Length (in samples) of `vector` // - order : The order up to which the autocorrelation should be // calculated // @@ -408,7 +438,7 @@ void WebRtcSpl_AffineTransformVector(int16_t* out_vector, // - scale : The number of left shifts required to obtain the // auto-correlation in Q0 // -// Return value : Number of samples in |result|, i.e. (order+1) +// Return value : Number of samples in `result`, i.e. (order+1) size_t WebRtcSpl_AutoCorrelation(const int16_t* in_vector, size_t in_vector_length, size_t order, @@ -419,7 +449,7 @@ size_t WebRtcSpl_AutoCorrelation(const int16_t* in_vector, // does NOT use the 64 bit class // // Input: -// - auto_corr : Vector with autocorrelation values of length >= |order|+1 +// - auto_corr : Vector with autocorrelation values of length >= `order`+1 // - order : The LPC filter order (support up to order 20) // // Output: @@ -432,7 +462,7 @@ int16_t WebRtcSpl_LevinsonDurbin(const int32_t* auto_corr, int16_t* refl_coef, size_t order); -// Converts reflection coefficients |refl_coef| to LPC coefficients |lpc_coef|. +// Converts reflection coefficients `refl_coef` to LPC coefficients `lpc_coef`. // This version is a 16 bit operation. // // NOTE: The 16 bit refl_coef -> lpc_coef conversion might result in a @@ -442,7 +472,7 @@ int16_t WebRtcSpl_LevinsonDurbin(const int32_t* auto_corr, // Input: // - refl_coef : Reflection coefficients in Q15 that should be converted // to LPC coefficients -// - use_order : Number of coefficients in |refl_coef| +// - use_order : Number of coefficients in `refl_coef` // // Output: // - lpc_coef : LPC coefficients in Q12 @@ -450,14 +480,14 @@ void WebRtcSpl_ReflCoefToLpc(const int16_t* refl_coef, int use_order, int16_t* lpc_coef); -// Converts LPC coefficients |lpc_coef| to reflection coefficients |refl_coef|. +// Converts LPC coefficients `lpc_coef` to reflection coefficients `refl_coef`. // This version is a 16 bit operation. // The conversion is implemented by the step-down algorithm. // // Input: // - lpc_coef : LPC coefficients in Q12, that should be converted to // reflection coefficients -// - use_order : Number of coefficients in |lpc_coef| +// - use_order : Number of coefficients in `lpc_coef` // // Output: // - refl_coef : Reflection coefficients in Q15. @@ -478,24 +508,24 @@ void WebRtcSpl_AutoCorrToReflCoef(const int32_t* auto_corr, int16_t* refl_coef); // The functions (with related pointer) calculate the cross-correlation between -// two sequences |seq1| and |seq2|. -// |seq1| is fixed and |seq2| slides as the pointer is increased with the -// amount |step_seq2|. Note the arguments should obey the relationship: -// |dim_seq| - 1 + |step_seq2| * (|dim_cross_correlation| - 1) < -// buffer size of |seq2| +// two sequences `seq1` and `seq2`. +// `seq1` is fixed and `seq2` slides as the pointer is increased with the +// amount `step_seq2`. Note the arguments should obey the relationship: +// `dim_seq` - 1 + `step_seq2` * (`dim_cross_correlation` - 1) < +// buffer size of `seq2` // // Input: // - seq1 : First sequence (fixed throughout the correlation) -// - seq2 : Second sequence (slides |step_vector2| for each +// - seq2 : Second sequence (slides `step_vector2` for each // new correlation) // - dim_seq : Number of samples to use in the cross-correlation // - dim_cross_correlation : Number of cross-correlations to calculate (the -// start position for |vector2| is updated for each +// start position for `vector2` is updated for each // new one) // - right_shifts : Number of right bit shifts to use. This will // become the output Q-domain. // - step_seq2 : How many (positive or negative) steps the -// |vector2| pointer should be updated for each new +// `vector2` pointer should be updated for each new // cross-correlation value. // // Output: @@ -545,11 +575,11 @@ void WebRtcSpl_CrossCorrelation_mips(int32_t* cross_correlation, void WebRtcSpl_GetHanningWindow(int16_t* window, size_t size); // Calculates y[k] = sqrt(1 - x[k]^2) for each element of the input vector -// |in_vector|. Input and output values are in Q15. +// `in_vector`. Input and output values are in Q15. // // Inputs: // - in_vector : Values to calculate sqrt(1 - x^2) of -// - vector_length : Length of vector |in_vector| +// - vector_length : Length of vector `in_vector` // // Output: // - out_vector : Output values in Q15 @@ -637,9 +667,9 @@ void WebRtcSpl_FilterARFastQ12(const int16_t* data_in, // Input: // - data_in : Input samples (state in positions // data_in[-order] .. data_in[-1]) -// - data_in_length : Number of samples in |data_in| to be filtered. +// - data_in_length : Number of samples in `data_in` to be filtered. // This must be at least -// |delay| + |factor|*(|out_vector_length|-1) + 1) +// `delay` + `factor`*(`out_vector_length`-1) + 1) // - data_out_length : Number of down sampled samples desired // - coefficients : Filter coefficients (in Q12) // - coefficients_length: Number of coefficients (order+1) @@ -647,7 +677,7 @@ void WebRtcSpl_FilterARFastQ12(const int16_t* data_in, // - delay : Delay of filter (compensated for in out_vector) // Output: // - data_out : Filtered samples -// Return value : 0 if OK, -1 if |in_vector| is too short +// Return value : 0 if OK, -1 if `in_vector` is too short typedef int (*DownsampleFast)(const int16_t* data_in, size_t data_in_length, int16_t* data_out, @@ -693,12 +723,12 @@ int WebRtcSpl_DownsampleFast_mips(const int16_t* data_in, int WebRtcSpl_ComplexFFT(int16_t vector[], int stages, int mode); int WebRtcSpl_ComplexIFFT(int16_t vector[], int stages, int mode); -// Treat a 16-bit complex data buffer |complex_data| as an array of 32-bit +// Treat a 16-bit complex data buffer `complex_data` as an array of 32-bit // values, and swap elements whose indexes are bit-reverses of each other. // // Input: -// - complex_data : Complex data buffer containing 2^|stages| real -// elements interleaved with 2^|stages| imaginary +// - complex_data : Complex data buffer containing 2^`stages` real +// elements interleaved with 2^`stages` imaginary // elements: [Re Im Re Im Re Im....] // - stages : Number of FFT stages. Must be at least 3 and at most // 10, since the table WebRtcSpl_kSinTable1024[] is 1024 @@ -908,7 +938,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // WebRtcSpl_AddSatW32(...) // // Returns the result of a saturated 16-bit, respectively 32-bit, addition of -// the numbers specified by the |var1| and |var2| parameters. +// the numbers specified by the `var1` and `var2` parameters. // // Input: // - var1 : Input variable 1 @@ -922,7 +952,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // WebRtcSpl_SubSatW32(...) // // Returns the result of a saturated 16-bit, respectively 32-bit, subtraction -// of the numbers specified by the |var1| and |var2| parameters. +// of the numbers specified by the `var1` and `var2` parameters. // // Input: // - var1 : Input variable 1 @@ -935,61 +965,61 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // WebRtcSpl_GetSizeInBits(...) // // Returns the # of bits that are needed at the most to represent the number -// specified by the |value| parameter. +// specified by the `value` parameter. // // Input: // - value : Input value // -// Return value : Number of bits needed to represent |value| +// Return value : Number of bits needed to represent `value` // // // WebRtcSpl_NormW32(...) // // Norm returns the # of left shifts required to 32-bit normalize the 32-bit -// signed number specified by the |value| parameter. +// signed number specified by the `value` parameter. // // Input: // - value : Input value // -// Return value : Number of bit shifts needed to 32-bit normalize |value| +// Return value : Number of bit shifts needed to 32-bit normalize `value` // // // WebRtcSpl_NormW16(...) // // Norm returns the # of left shifts required to 16-bit normalize the 16-bit -// signed number specified by the |value| parameter. +// signed number specified by the `value` parameter. // // Input: // - value : Input value // -// Return value : Number of bit shifts needed to 32-bit normalize |value| +// Return value : Number of bit shifts needed to 32-bit normalize `value` // // // WebRtcSpl_NormU32(...) // // Norm returns the # of left shifts required to 32-bit normalize the unsigned -// 32-bit number specified by the |value| parameter. +// 32-bit number specified by the `value` parameter. // // Input: // - value : Input value // -// Return value : Number of bit shifts needed to 32-bit normalize |value| +// Return value : Number of bit shifts needed to 32-bit normalize `value` // // // WebRtcSpl_GetScalingSquare(...) // // Returns the # of bits required to scale the samples specified in the -// |in_vector| parameter so that, if the squares of the samples are added the -// # of times specified by the |times| parameter, the 32-bit addition will not +// `in_vector` parameter so that, if the squares of the samples are added the +// # of times specified by the `times` parameter, the 32-bit addition will not // overflow (result in int32_t). // // Input: // - in_vector : Input vector to check scaling on -// - in_vector_length : Samples in |in_vector| +// - in_vector_length : Samples in `in_vector` // - times : Number of additions to be performed // // Return value : Number of right bit shifts needed to avoid @@ -999,8 +1029,8 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_MemSetW16(...) // -// Sets all the values in the int16_t vector |vector| of length -// |vector_length| to the specified value |set_value| +// Sets all the values in the int16_t vector `vector` of length +// `vector_length` to the specified value `set_value` // // Input: // - vector : Pointer to the int16_t vector @@ -1011,8 +1041,8 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_MemSetW32(...) // -// Sets all the values in the int32_t vector |vector| of length -// |vector_length| to the specified value |set_value| +// Sets all the values in the int32_t vector `vector` of length +// `vector_length` to the specified value `set_value` // // Input: // - vector : Pointer to the int16_t vector @@ -1023,34 +1053,34 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_MemCpyReversedOrder(...) // -// Copies all the values from the source int16_t vector |in_vector| to a -// destination int16_t vector |out_vector|. It is done in reversed order, -// meaning that the first sample of |in_vector| is copied to the last sample of -// the |out_vector|. The procedure continues until the last sample of -// |in_vector| has been copied to the first sample of |out_vector|. This +// Copies all the values from the source int16_t vector `in_vector` to a +// destination int16_t vector `out_vector`. It is done in reversed order, +// meaning that the first sample of `in_vector` is copied to the last sample of +// the `out_vector`. The procedure continues until the last sample of +// `in_vector` has been copied to the first sample of `out_vector`. This // creates a reversed vector. Used in e.g. prediction in iLBC. // // Input: // - in_vector : Pointer to the first sample in a int16_t vector -// of length |length| +// of length `length` // - vector_length : Number of elements to copy // // Output: // - out_vector : Pointer to the last sample in a int16_t vector -// of length |length| +// of length `length` // // // WebRtcSpl_CopyFromEndW16(...) // -// Copies the rightmost |samples| of |in_vector| (of length |in_vector_length|) -// to the vector |out_vector|. +// Copies the rightmost `samples` of `in_vector` (of length `in_vector_length`) +// to the vector `out_vector`. // // Input: // - in_vector : Input vector -// - in_vector_length : Number of samples in |in_vector| +// - in_vector_length : Number of samples in `in_vector` // - samples : Number of samples to extract (from right side) -// from |in_vector| +// from `in_vector` // // Output: // - out_vector : Vector with the requested samples @@ -1085,7 +1115,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // Output: // - out_vector : Pointer to the result vector (can be the same as -// |in_vector|) +// `in_vector`) // // @@ -1103,7 +1133,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // Output: // - out_vector : Pointer to the result vector (can be the same as -// |in_vector|) +// `in_vector`) // // @@ -1115,11 +1145,11 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // Input: // - in_vector : Input vector // - gain : Scaling gain -// - vector_length : Elements in the |in_vector| +// - vector_length : Elements in the `in_vector` // - right_shifts : Number of right bit shifts applied // // Output: -// - out_vector : Output vector (can be the same as |in_vector|) +// - out_vector : Output vector (can be the same as `in_vector`) // // @@ -1131,11 +1161,11 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // Input: // - in_vector : Input vector // - gain : Scaling gain -// - vector_length : Elements in the |in_vector| +// - vector_length : Elements in the `in_vector` // - right_shifts : Number of right bit shifts applied // // Output: -// - out_vector : Output vector (can be the same as |in_vector|) +// - out_vector : Output vector (can be the same as `in_vector`) // // @@ -1170,10 +1200,10 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // should be set to the last value in the vector // - right_shifts : Number of right bit shift to be applied after the // multiplication -// - vector_length : Number of elements in |in_vector| +// - vector_length : Number of elements in `in_vector` // // Output: -// - out_vector : Output vector (can be same as |in_vector|) +// - out_vector : Output vector (can be same as `in_vector`) // // @@ -1187,10 +1217,10 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - window : Window vector. // - right_shifts : Number of right bit shift to be applied after the // multiplication -// - vector_length : Number of elements in |in_vector| +// - vector_length : Number of elements in `in_vector` // // Output: -// - out_vector : Output vector (can be same as |in_vector|) +// - out_vector : Output vector (can be same as `in_vector`) // // @@ -1204,16 +1234,16 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - in_vector2 : Input vector 2 // - right_shifts : Number of right bit shift to be applied after the // multiplication -// - vector_length : Number of elements in |in_vector1| and |in_vector2| +// - vector_length : Number of elements in `in_vector1` and `in_vector2` // // Output: -// - out_vector : Output vector (can be same as |in_vector1|) +// - out_vector : Output vector (can be same as `in_vector1`) // // // WebRtcSpl_AddAffineVectorToVector(...) // -// Adds an affine transformed vector to another vector |out_vector|, i.e, +// Adds an affine transformed vector to another vector `out_vector`, i.e, // performs // out_vector[k] += (in_vector[k]*gain+add_constant)>>right_shifts // @@ -1223,7 +1253,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - add_constant : Constant value to add (usually 1<<(right_shifts-1), // but others can be used as well // - right_shifts : Number of right bit shifts (0-16) -// - vector_length : Number of samples in |in_vector| and |out_vector| +// - vector_length : Number of samples in `in_vector` and `out_vector` // // Output: // - out_vector : Vector with the output @@ -1241,7 +1271,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - add_constant : Constant value to add (usually 1<<(right_shifts-1), // but others can be used as well // - right_shifts : Number of right bit shifts (0-16) -// - vector_length : Number of samples in |in_vector| and |out_vector| +// - vector_length : Number of samples in `in_vector` and `out_vector` // // Output: // - out_vector : Vector with the output @@ -1304,15 +1334,15 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - vector : Vector with the uniform values // - seed : Updated seed value // -// Return value : Number of samples in vector, i.e., |vector_length| +// Return value : Number of samples in vector, i.e., `vector_length` // // // WebRtcSpl_Sqrt(...) // -// Returns the square root of the input value |value|. The precision of this +// Returns the square root of the input value `value`. The precision of this // function is integer precision, i.e., sqrt(8) gives 2 as answer. -// If |value| is a negative number then 0 is returned. +// If `value` is a negative number then 0 is returned. // // Algorithm: // @@ -1332,9 +1362,9 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_DivU32U16(...) // -// Divides a uint32_t |num| by a uint16_t |den|. +// Divides a uint32_t `num` by a uint16_t `den`. // -// If |den|==0, (uint32_t)0xFFFFFFFF is returned. +// If `den`==0, (uint32_t)0xFFFFFFFF is returned. // // Input: // - num : Numerator @@ -1347,9 +1377,9 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_DivW32W16(...) // -// Divides a int32_t |num| by a int16_t |den|. +// Divides a int32_t `num` by a int16_t `den`. // -// If |den|==0, (int32_t)0x7FFFFFFF is returned. +// If `den`==0, (int32_t)0x7FFFFFFF is returned. // // Input: // - num : Numerator @@ -1362,10 +1392,10 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_DivW32W16ResW16(...) // -// Divides a int32_t |num| by a int16_t |den|, assuming that the +// Divides a int32_t `num` by a int16_t `den`, assuming that the // result is less than 32768, otherwise an unpredictable result will occur. // -// If |den|==0, (int16_t)0x7FFF is returned. +// If `den`==0, (int16_t)0x7FFF is returned. // // Input: // - num : Numerator @@ -1378,7 +1408,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_DivResultInQ31(...) // -// Divides a int32_t |num| by a int16_t |den|, assuming that the +// Divides a int32_t `num` by a int16_t `den`, assuming that the // absolute value of the denominator is larger than the numerator, otherwise // an unpredictable result will occur. // @@ -1392,7 +1422,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // WebRtcSpl_DivW32HiLow(...) // -// Divides a int32_t |num| by a denominator in hi, low format. The +// Divides a int32_t `num` by a denominator in hi, low format. The // absolute value of the denominator has to be larger (or equal to) the // numerator. // @@ -1417,7 +1447,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - scale_factor : Number of left bit shifts needed to get the physical // energy value, i.e, to get the Q0 value // -// Return value : Energy value in Q(-|scale_factor|) +// Return value : Energy value in Q(-`scale_factor`) // // @@ -1428,15 +1458,15 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // Input: // - ar_coef : AR-coefficient vector (values in Q12), // ar_coef[0] must be 4096. -// - ar_coef_length : Number of coefficients in |ar_coef|. +// - ar_coef_length : Number of coefficients in `ar_coef`. // - in_vector : Vector to be filtered. -// - in_vector_length : Number of samples in |in_vector|. +// - in_vector_length : Number of samples in `in_vector`. // - filter_state : Current state (higher part) of the filter. -// - filter_state_length : Length (in samples) of |filter_state|. +// - filter_state_length : Length (in samples) of `filter_state`. // - filter_state_low : Current state (lower part) of the filter. -// - filter_state_low_length : Length (in samples) of |filter_state_low|. +// - filter_state_low_length : Length (in samples) of `filter_state_low`. // - out_vector_low_length : Maximum length (in samples) of -// |out_vector_low|. +// `out_vector_low`. // // Output: // - filter_state : Updated state (upper part) vector. @@ -1446,7 +1476,7 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - out_vector_low : Vector containing the lower part of the // filtered values. // -// Return value : Number of samples in the |out_vector|. +// Return value : Number of samples in the `out_vector`. // // @@ -1454,11 +1484,11 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // Complex Inverse FFT // -// Computes an inverse complex 2^|stages|-point FFT on the input vector, which +// Computes an inverse complex 2^`stages`-point FFT on the input vector, which // is in bit-reversed order. The original content of the vector is destroyed in // the process, since the input is overwritten by the output, normal-ordered, // FFT vector. With X as the input complex vector, y as the output complex -// vector and with M = 2^|stages|, the following is computed: +// vector and with M = 2^`stages`, the following is computed: // // M-1 // y(k) = sum[X(i)*[cos(2*pi*i*k/M) + j*sin(2*pi*i*k/M)]] @@ -1468,8 +1498,8 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // decimation-in-time algorithm with radix-2 butterfly technique. // // Input: -// - vector : In pointer to complex vector containing 2^|stages| -// real elements interleaved with 2^|stages| imaginary +// - vector : In pointer to complex vector containing 2^`stages` +// real elements interleaved with 2^`stages` imaginary // elements. // [ReImReImReIm....] // The elements are in Q(-scale) domain, see more on Return @@ -1488,10 +1518,10 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // - vector : Out pointer to the FFT vector (the same as input). // // Return Value : The scale value that tells the number of left bit shifts -// that the elements in the |vector| should be shifted with +// that the elements in the `vector` should be shifted with // in order to get Q0 values, i.e. the physically correct // values. The scale parameter is always 0 or positive, -// except if N>1024 (|stages|>10), which returns a scale +// except if N>1024 (`stages`>10), which returns a scale // value of -1, indicating error. // @@ -1500,11 +1530,11 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // // Complex FFT // -// Computes a complex 2^|stages|-point FFT on the input vector, which is in +// Computes a complex 2^`stages`-point FFT on the input vector, which is in // bit-reversed order. The original content of the vector is destroyed in // the process, since the input is overwritten by the output, normal-ordered, // FFT vector. With x as the input complex vector, Y as the output complex -// vector and with M = 2^|stages|, the following is computed: +// vector and with M = 2^`stages`, the following is computed: // // M-1 // Y(k) = 1/M * sum[x(i)*[cos(2*pi*i*k/M) + j*sin(2*pi*i*k/M)]] @@ -1519,8 +1549,8 @@ void WebRtcSpl_SynthesisQMF(const int16_t* low_band, // accuracy. // // Input: -// - vector : In pointer to complex vector containing 2^|stages| real -// elements interleaved with 2^|stages| imaginary elements. +// - vector : In pointer to complex vector containing 2^`stages` real +// elements interleaved with 2^`stages` imaginary elements. // [ReImReImReIm....] // The output is in the Q0 domain. // diff --git a/webrtc/common_audio/signal_processing/include/spl_inl.h b/webrtc/common_audio/signal_processing/include/spl_inl.h index 656a312..2b09958 100644 --- a/webrtc/common_audio/signal_processing/include/spl_inl.h +++ b/webrtc/common_audio/signal_processing/include/spl_inl.h @@ -14,6 +14,8 @@ #ifndef COMMON_AUDIO_SIGNAL_PROCESSING_INCLUDE_SPL_INL_H_ #define COMMON_AUDIO_SIGNAL_PROCESSING_INCLUDE_SPL_INL_H_ +#include + #include "rtc_base/compile_assert_c.h" extern const int8_t kWebRtcSpl_CountLeadingZeros32_Table[64]; diff --git a/webrtc/common_audio/signal_processing/include/spl_inl_armv7.h b/webrtc/common_audio/signal_processing/include/spl_inl_armv7.h index 930e91e..6fc3e7c 100644 --- a/webrtc/common_audio/signal_processing/include/spl_inl_armv7.h +++ b/webrtc/common_audio/signal_processing/include/spl_inl_armv7.h @@ -15,6 +15,8 @@ #ifndef COMMON_AUDIO_SIGNAL_PROCESSING_INCLUDE_SPL_INL_ARMV7_H_ #define COMMON_AUDIO_SIGNAL_PROCESSING_INCLUDE_SPL_INL_ARMV7_H_ +#include + /* TODO(kma): Replace some assembly code with GCC intrinsics * (e.g. __builtin_clz). */ diff --git a/webrtc/common_audio/signal_processing/min_max_operations.c b/webrtc/common_audio/signal_processing/min_max_operations.c index d249a02..6acf882 100644 --- a/webrtc/common_audio/signal_processing/min_max_operations.c +++ b/webrtc/common_audio/signal_processing/min_max_operations.c @@ -25,6 +25,7 @@ */ #include +#include #include "rtc_base/checks.h" #include "common_audio/signal_processing/include/signal_processing_library.h" @@ -67,7 +68,8 @@ int32_t WebRtcSpl_MaxAbsValueW32C(const int32_t* vector, size_t length) { RTC_DCHECK_GT(length, 0); for (i = 0; i < length; i++) { - absolute = abs((int)vector[i]); + absolute = + (vector[i] != INT_MIN) ? abs((int)vector[i]) : INT_MAX + (uint32_t)1; if (absolute > maximum) { maximum = absolute; } @@ -155,6 +157,15 @@ size_t WebRtcSpl_MaxAbsIndexW16(const int16_t* vector, size_t length) { return index; } +int16_t WebRtcSpl_MaxAbsElementW16(const int16_t* vector, size_t length) { + int16_t min_val, max_val; + WebRtcSpl_MinMaxW16(vector, length, &min_val, &max_val); + if (min_val == max_val || min_val < -max_val) { + return min_val; + } + return max_val; +} + // Index of maximum value in a word16 vector. size_t WebRtcSpl_MaxIndexW16(const int16_t* vector, size_t length) { size_t i = 0, index = 0; @@ -222,3 +233,26 @@ size_t WebRtcSpl_MinIndexW32(const int32_t* vector, size_t length) { return index; } + +// Finds both the minimum and maximum elements in an array of 16-bit integers. +void WebRtcSpl_MinMaxW16(const int16_t* vector, size_t length, + int16_t* min_val, int16_t* max_val) { +#if defined(WEBRTC_HAS_NEON) + return WebRtcSpl_MinMaxW16Neon(vector, length, min_val, max_val); +#else + int16_t minimum = WEBRTC_SPL_WORD16_MAX; + int16_t maximum = WEBRTC_SPL_WORD16_MIN; + size_t i = 0; + + RTC_DCHECK_GT(length, 0); + + for (i = 0; i < length; i++) { + if (vector[i] < minimum) + minimum = vector[i]; + if (vector[i] > maximum) + maximum = vector[i]; + } + *min_val = minimum; + *max_val = maximum; +#endif +} diff --git a/webrtc/common_audio/signal_processing/min_max_operations_neon.c b/webrtc/common_audio/signal_processing/min_max_operations_neon.c index 53217df..e5b4b7c 100644 --- a/webrtc/common_audio/signal_processing/min_max_operations_neon.c +++ b/webrtc/common_audio/signal_processing/min_max_operations_neon.c @@ -281,3 +281,53 @@ int32_t WebRtcSpl_MinValueW32Neon(const int32_t* vector, size_t length) { return minimum; } +// Finds both the minimum and maximum elements in an array of 16-bit integers. +void WebRtcSpl_MinMaxW16Neon(const int16_t* vector, size_t length, + int16_t* min_val, int16_t* max_val) { + int16_t minimum = WEBRTC_SPL_WORD16_MAX; + int16_t maximum = WEBRTC_SPL_WORD16_MIN; + size_t i = 0; + size_t residual = length & 0x7; + + RTC_DCHECK_GT(length, 0); + + const int16_t* p_start = vector; + int16x8_t min16x8 = vdupq_n_s16(WEBRTC_SPL_WORD16_MAX); + int16x8_t max16x8 = vdupq_n_s16(WEBRTC_SPL_WORD16_MIN); + + // First part, unroll the loop 8 times. + for (i = 0; i < length - residual; i += 8) { + int16x8_t in16x8 = vld1q_s16(p_start); + min16x8 = vminq_s16(min16x8, in16x8); + max16x8 = vmaxq_s16(max16x8, in16x8); + p_start += 8; + } + +#if defined(WEBRTC_ARCH_ARM64) + minimum = vminvq_s16(min16x8); + maximum = vmaxvq_s16(max16x8); +#else + int16x4_t min16x4 = vmin_s16(vget_low_s16(min16x8), vget_high_s16(min16x8)); + min16x4 = vpmin_s16(min16x4, min16x4); + min16x4 = vpmin_s16(min16x4, min16x4); + + minimum = vget_lane_s16(min16x4, 0); + + int16x4_t max16x4 = vmax_s16(vget_low_s16(max16x8), vget_high_s16(max16x8)); + max16x4 = vpmax_s16(max16x4, max16x4); + max16x4 = vpmax_s16(max16x4, max16x4); + + maximum = vget_lane_s16(max16x4, 0); +#endif + + // Second part, do the remaining iterations (if any). + for (i = residual; i > 0; i--) { + if (*p_start < minimum) + minimum = *p_start; + if (*p_start > maximum) + maximum = *p_start; + p_start++; + } + *min_val = minimum; + *max_val = maximum; +} diff --git a/webrtc/common_audio/signal_processing/splitting_filter.c b/webrtc/common_audio/signal_processing/splitting_filter.c index 399433f..27a0a2a 100644 --- a/webrtc/common_audio/signal_processing/splitting_filter.c +++ b/webrtc/common_audio/signal_processing/splitting_filter.c @@ -41,35 +41,39 @@ static const uint16_t WebRtcSpl_kAllPassFilter2[3] = {21333, 49062, 63010}; // // Output: // - out_data : Output data sequence (Q10), length equal to -// |data_length| +// `data_length` // -void WebRtcSpl_AllPassQMF(int32_t* in_data, size_t data_length, - int32_t* out_data, const uint16_t* filter_coefficients, - int32_t* filter_state) +static void WebRtcSpl_AllPassQMF(int32_t* in_data, + size_t data_length, + int32_t* out_data, + const uint16_t* filter_coefficients, + int32_t* filter_state) { - // The procedure is to filter the input with three first order all pass filters - // (cascade operations). + // The procedure is to filter the input with three first order all pass + // filters (cascade operations). // // a_3 + q^-1 a_2 + q^-1 a_1 + q^-1 // y[n] = ----------- ----------- ----------- x[n] // 1 + a_3q^-1 1 + a_2q^-1 1 + a_1q^-1 // - // The input vector |filter_coefficients| includes these three filter coefficients. - // The filter state contains the in_data state, in_data[-1], followed by - // the out_data state, out_data[-1]. This is repeated for each cascade. - // The first cascade filter will filter the |in_data| and store the output in - // |out_data|. The second will the take the |out_data| as input and make an - // intermediate storage in |in_data|, to save memory. The third, and final, cascade - // filter operation takes the |in_data| (which is the output from the previous cascade - // filter) and store the output in |out_data|. - // Note that the input vector values are changed during the process. + // The input vector `filter_coefficients` includes these three filter + // coefficients. The filter state contains the in_data state, in_data[-1], + // followed by the out_data state, out_data[-1]. This is repeated for each + // cascade. The first cascade filter will filter the `in_data` and store + // the output in `out_data`. The second will the take the `out_data` as + // input and make an intermediate storage in `in_data`, to save memory. The + // third, and final, cascade filter operation takes the `in_data` (which is + // the output from the previous cascade filter) and store the output in + // `out_data`. Note that the input vector values are changed during the + // process. size_t k; int32_t diff; // First all-pass cascade; filter from in_data to out_data. - // Let y_i[n] indicate the output of cascade filter i (with filter coefficient a_i) at - // vector position n. Then the final output will be y[n] = y_3[n] + // Let y_i[n] indicate the output of cascade filter i (with filter + // coefficient a_i) at vector position n. Then the final output will be + // y[n] = y_3[n] // First loop, use the states stored in memory. // "diff" should be safe from wrap around since max values are 2^25 diff --git a/webrtc/common_audio/smoothing_filter.cc b/webrtc/common_audio/smoothing_filter.cc index 961f4a1..eaaf3a0 100644 --- a/webrtc/common_audio/smoothing_filter.cc +++ b/webrtc/common_audio/smoothing_filter.cc @@ -23,12 +23,12 @@ SmoothingFilterImpl::SmoothingFilterImpl(int init_time_ms) : init_time_ms_(init_time_ms), // Duing the initalization time, we use an increasing alpha. Specifically, // alpha(n) = exp(-powf(init_factor_, n)), - // where |init_factor_| is chosen such that + // where `init_factor_` is chosen such that // alpha(init_time_ms_) = exp(-1.0f / init_time_ms_), init_factor_(init_time_ms_ == 0 ? 0.0f : powf(init_time_ms_, -1.0f / init_time_ms_)), - // |init_const_| is to a factor to help the calculation during + // `init_const_` is to a factor to help the calculation during // initialization phase. init_const_(init_time_ms_ == 0 ? 0.0f @@ -57,7 +57,7 @@ void SmoothingFilterImpl::AddSample(float sample) { absl::optional SmoothingFilterImpl::GetAverage() { if (!init_end_time_ms_) { - // |init_end_time_ms_| undefined since we have not received any sample. + // `init_end_time_ms_` undefined since we have not received any sample. return absl::nullopt; } ExtrapolateLastSample(rtc::TimeMillis()); @@ -84,17 +84,17 @@ void SmoothingFilterImpl::ExtrapolateLastSample(int64_t time_ms) { if (time_ms <= *init_end_time_ms_) { // Current update is to be made during initialization phase. - // We update the state as if the |alpha| has been increased according + // We update the state as if the `alpha` has been increased according // alpha(n) = exp(-powf(init_factor_, n)), // where n is the time (in millisecond) since the first sample received. // With algebraic derivation as shown in the Appendix, we can find that the // state can be updated in a similar manner as if alpha is a constant, // except for a different multiplier. if (init_time_ms_ == 0) { - // This means |init_factor_| = 0. + // This means `init_factor_` = 0. multiplier = 0.0f; } else if (init_time_ms_ == 1) { - // This means |init_factor_| = 1. + // This means `init_factor_` = 1. multiplier = std::exp(last_state_time_ms_ - time_ms); } else { multiplier = std::exp( diff --git a/webrtc/common_audio/smoothing_filter.h b/webrtc/common_audio/smoothing_filter.h index e96d52a..3419de7 100644 --- a/webrtc/common_audio/smoothing_filter.h +++ b/webrtc/common_audio/smoothing_filter.h @@ -33,13 +33,13 @@ class SmoothingFilter { // assumed to equal the last received sample. class SmoothingFilterImpl final : public SmoothingFilter { public: - // |init_time_ms| is initialization time. It defines a period starting from + // `init_time_ms` is initialization time. It defines a period starting from // the arriving time of the first sample. During this period, the exponential // filter uses a varying time constant so that a smaller time constant will be // applied to the earlier samples. This is to allow the the filter to adapt to // earlier samples quickly. After the initialization period, the time constant - // will be set to |init_time_ms| first and can be changed through - // |SetTimeConstantMs|. + // will be set to `init_time_ms` first and can be changed through + // `SetTimeConstantMs`. explicit SmoothingFilterImpl(int init_time_ms); SmoothingFilterImpl() = delete; diff --git a/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.cc b/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.cc index d2f7c1c..2573f23 100644 --- a/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.cc +++ b/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.cc @@ -286,11 +286,11 @@ Appendix : w[] and ip[] are compatible with all routines. */ +#include "common_audio/third_party/ooura/fft_size_256/fft4g.h" + #include #include -#include "common_audio/third_party/ooura/fft_size_256/fft4g.h" - namespace webrtc { namespace { diff --git a/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.h b/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.h index d41d2c6..5a465a3 100644 --- a/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.h +++ b/webrtc/common_audio/third_party/ooura/fft_size_256/fft4g.h @@ -11,6 +11,8 @@ #ifndef COMMON_AUDIO_THIRD_PARTY_OOURA_FFT_SIZE_256_FFT4G_H_ #define COMMON_AUDIO_THIRD_PARTY_OOURA_FFT_SIZE_256_FFT4G_H_ +#include + namespace webrtc { // Refer to fft4g.c for documentation. diff --git a/webrtc/common_audio/third_party/spl_sqrt_floor/spl_sqrt_floor.h b/webrtc/common_audio/third_party/spl_sqrt_floor/spl_sqrt_floor.h index eaa58e3..718a18f 100644 --- a/webrtc/common_audio/third_party/spl_sqrt_floor/spl_sqrt_floor.h +++ b/webrtc/common_audio/third_party/spl_sqrt_floor/spl_sqrt_floor.h @@ -13,9 +13,9 @@ // // WebRtcSpl_SqrtFloor(...) // -// Returns the square root of the input value |value|. The precision of this +// Returns the square root of the input value `value`. The precision of this // function is rounding down integer precision, i.e., sqrt(8) gives 2 as answer. -// If |value| is a negative number then 0 is returned. +// If `value` is a negative number then 0 is returned. // // Algorithm: // diff --git a/webrtc/common_audio/vad/include/webrtc_vad.h b/webrtc/common_audio/vad/include/webrtc_vad.h index f5bbadf..31e628f 100644 --- a/webrtc/common_audio/vad/include/webrtc_vad.h +++ b/webrtc/common_audio/vad/include/webrtc_vad.h @@ -54,7 +54,7 @@ int WebRtcVad_Init(VadInst* handle); // has not been initialized). int WebRtcVad_set_mode(VadInst* handle, int mode); -// Calculates a VAD decision for the |audio_frame|. For valid sampling rates +// Calculates a VAD decision for the `audio_frame`. For valid sampling rates // frame lengths, see the description of WebRtcVad_ValidRatesAndFrameLengths(). // // - handle [i/o] : VAD Instance. Needs to be initialized by @@ -71,7 +71,7 @@ int WebRtcVad_Process(VadInst* handle, const int16_t* audio_frame, size_t frame_length); -// Checks for valid combinations of |rate| and |frame_length|. We support 10, +// Checks for valid combinations of `rate` and `frame_length`. We support 10, // 20 and 30 ms frames and the rates 8000, 16000 and 32000 Hz. // // - rate [i] : Sampling frequency (Hz). diff --git a/webrtc/common_audio/vad/vad.cc b/webrtc/common_audio/vad/vad.cc index 987ed52..1647246 100644 --- a/webrtc/common_audio/vad/vad.cc +++ b/webrtc/common_audio/vad/vad.cc @@ -38,7 +38,7 @@ class VadImpl final : public Vad { case 1: return kActive; default: - RTC_NOTREACHED() << "WebRtcVad_Process returned an error."; + RTC_DCHECK_NOTREACHED() << "WebRtcVad_Process returned an error."; return kError; } } diff --git a/webrtc/common_audio/vad/vad_core.c b/webrtc/common_audio/vad/vad_core.c index 55927ce..0872449 100644 --- a/webrtc/common_audio/vad/vad_core.c +++ b/webrtc/common_audio/vad/vad_core.c @@ -90,11 +90,11 @@ static const int16_t kOverHangMax2VAG[3] = { 9, 5, 3 }; static const int16_t kLocalThresholdVAG[3] = { 94, 94, 94 }; static const int16_t kGlobalThresholdVAG[3] = { 1100, 1050, 1100 }; -// Calculates the weighted average w.r.t. number of Gaussians. The |data| are -// updated with an |offset| before averaging. +// Calculates the weighted average w.r.t. number of Gaussians. The `data` are +// updated with an `offset` before averaging. // // - data [i/o] : Data to average. -// - offset [i] : An offset added to |data|. +// - offset [i] : An offset added to `data`. // - weights [i] : Weights used for averaging. // // returns : The weighted average. @@ -124,7 +124,7 @@ static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow") // type of signal is most probable. // // - self [i/o] : Pointer to VAD instance -// - features [i] : Feature vector of length |kNumChannels| +// - features [i] : Feature vector of length `kNumChannels` // = log10(energy in frequency band) // - total_power [i] : Total power in audio frame. // - frame_length [i] : Number of input samples @@ -183,10 +183,10 @@ static int16_t GmmProbability(VadInstT* self, int16_t* features, // H1: Speech // // We combine a global LRT with local tests, for each frequency sub-band, - // here defined as |channel|. + // here defined as `channel`. for (channel = 0; channel < kNumChannels; channel++) { // For each channel we model the probability with a GMM consisting of - // |kNumGaussians|, with different means and standard deviations depending + // `kNumGaussians`, with different means and standard deviations depending // on H0 or H1. h0_test = 0; h1_test = 0; @@ -234,7 +234,7 @@ static int16_t GmmProbability(VadInstT* self, int16_t* features, } log_likelihood_ratio = shifts_h0 - shifts_h1; - // Update |sum_log_likelihood_ratios| with spectrum weighting. This is + // Update `sum_log_likelihood_ratios` with spectrum weighting. This is // used for the global VAD decision. sum_log_likelihood_ratios += (int32_t) (log_likelihood_ratio * kSpectrumWeight[channel]); @@ -298,8 +298,8 @@ static int16_t GmmProbability(VadInstT* self, int16_t* features, nmk2 = nmk; if (!vadflag) { // deltaN = (x-mu)/sigma^2 - // ngprvec[k] = |noise_probability[k]| / - // (|noise_probability[0]| + |noise_probability[1]|) + // ngprvec[k] = `noise_probability[k]` / + // (`noise_probability[0]` + `noise_probability[1]`) // (Q14 * Q11 >> 11) = Q14. delt = (int16_t)((ngprvec[gaussian] * deltaN[gaussian]) >> 11); @@ -326,9 +326,9 @@ static int16_t GmmProbability(VadInstT* self, int16_t* features, if (vadflag) { // Update speech mean vector: - // |deltaS| = (x-mu)/sigma^2 - // sgprvec[k] = |speech_probability[k]| / - // (|speech_probability[0]| + |speech_probability[1]|) + // `deltaS` = (x-mu)/sigma^2 + // sgprvec[k] = `speech_probability[k]` / + // (`speech_probability[0]` + `speech_probability[1]`) // (Q14 * Q11) >> 11 = Q14. delt = (int16_t)((sgprvec[gaussian] * deltaS[gaussian]) >> 11); @@ -409,35 +409,35 @@ static int16_t GmmProbability(VadInstT* self, int16_t* features, } // Separate models if they are too close. - // |noise_global_mean| in Q14 (= Q7 * Q7). + // `noise_global_mean` in Q14 (= Q7 * Q7). noise_global_mean = WeightedAverage(&self->noise_means[channel], 0, &kNoiseDataWeights[channel]); - // |speech_global_mean| in Q14 (= Q7 * Q7). + // `speech_global_mean` in Q14 (= Q7 * Q7). speech_global_mean = WeightedAverage(&self->speech_means[channel], 0, &kSpeechDataWeights[channel]); - // |diff| = "global" speech mean - "global" noise mean. + // `diff` = "global" speech mean - "global" noise mean. // (Q14 >> 9) - (Q14 >> 9) = Q5. diff = (int16_t) (speech_global_mean >> 9) - (int16_t) (noise_global_mean >> 9); if (diff < kMinimumDifference[channel]) { tmp_s16 = kMinimumDifference[channel] - diff; - // |tmp1_s16| = ~0.8 * (kMinimumDifference - diff) in Q7. - // |tmp2_s16| = ~0.2 * (kMinimumDifference - diff) in Q7. + // `tmp1_s16` = ~0.8 * (kMinimumDifference - diff) in Q7. + // `tmp2_s16` = ~0.2 * (kMinimumDifference - diff) in Q7. tmp1_s16 = (int16_t)((13 * tmp_s16) >> 2); tmp2_s16 = (int16_t)((3 * tmp_s16) >> 2); - // Move Gaussian means for speech model by |tmp1_s16| and update - // |speech_global_mean|. Note that |self->speech_means[channel]| is + // Move Gaussian means for speech model by `tmp1_s16` and update + // `speech_global_mean`. Note that `self->speech_means[channel]` is // changed after the call. speech_global_mean = WeightedAverage(&self->speech_means[channel], tmp1_s16, &kSpeechDataWeights[channel]); - // Move Gaussian means for noise model by -|tmp2_s16| and update - // |noise_global_mean|. Note that |self->noise_means[channel]| is + // Move Gaussian means for noise model by -`tmp2_s16` and update + // `noise_global_mean`. Note that `self->noise_means[channel]` is // changed after the call. noise_global_mean = WeightedAverage(&self->noise_means[channel], -tmp2_s16, @@ -534,7 +534,7 @@ int WebRtcVad_InitCore(VadInstT* self) { self->mean_value[i] = 1600; } - // Set aggressiveness mode to default (=|kDefaultMode|). + // Set aggressiveness mode to default (=`kDefaultMode`). if (WebRtcVad_set_mode_core(self, kDefaultMode) != 0) { return -1; } @@ -609,7 +609,7 @@ int WebRtcVad_CalcVad48khz(VadInstT* inst, const int16_t* speech_frame, int vad; size_t i; int16_t speech_nb[240]; // 30 ms in 8 kHz. - // |tmp_mem| is a temporary memory used by resample function, length is + // `tmp_mem` is a temporary memory used by resample function, length is // frame length in 10 ms (480 samples) + 256 extra. int32_t tmp_mem[480 + 256] = { 0 }; const size_t kFrameLen10ms48khz = 480; diff --git a/webrtc/common_audio/vad/vad_core.h b/webrtc/common_audio/vad/vad_core.h index e79696c..fbaf970 100644 --- a/webrtc/common_audio/vad/vad_core.h +++ b/webrtc/common_audio/vad/vad_core.h @@ -17,10 +17,19 @@ #include "common_audio/signal_processing/include/signal_processing_library.h" -enum { kNumChannels = 6 }; // Number of frequency bands (named channels). -enum { kNumGaussians = 2 }; // Number of Gaussians per channel in the GMM. -enum { kTableSize = kNumChannels * kNumGaussians }; -enum { kMinEnergy = 10 }; // Minimum energy required to trigger audio signal. +// TODO(https://bugs.webrtc.org/14476): When converted to C++, remove the macro. +#if defined(__cplusplus) +#define CONSTEXPR_INT(x) constexpr int x +#else +#define CONSTEXPR_INT(x) enum { x } +#endif + +CONSTEXPR_INT(kNumChannels = 6); // Number of frequency bands (named channels). +CONSTEXPR_INT( + kNumGaussians = 2); // Number of Gaussians per channel in the GMM. +CONSTEXPR_INT(kTableSize = kNumChannels * kNumGaussians); +CONSTEXPR_INT( + kMinEnergy = 10); // Minimum energy required to trigger audio signal. typedef struct VadInstT_ { int vad; @@ -30,14 +39,14 @@ typedef struct VadInstT_ { int16_t speech_means[kTableSize]; int16_t noise_stds[kTableSize]; int16_t speech_stds[kTableSize]; - // TODO(bjornv): Change to |frame_count|. + // TODO(bjornv): Change to `frame_count`. int32_t frame_counter; int16_t over_hang; // Over Hang int16_t num_of_speech; - // TODO(bjornv): Change to |age_vector|. + // TODO(bjornv): Change to `age_vector`. int16_t index_vector[16 * kNumChannels]; int16_t low_value_vector[16 * kNumChannels]; - // TODO(bjornv): Change to |median|. + // TODO(bjornv): Change to `median`. int16_t mean_value[kNumChannels]; int16_t upper_state[5]; int16_t lower_state[5]; @@ -51,7 +60,7 @@ typedef struct VadInstT_ { } VadInstT; // Initializes the core VAD component. The default aggressiveness mode is -// controlled by |kDefaultMode| in vad_core.c. +// controlled by `kDefaultMode` in vad_core.c. // // - self [i/o] : Instance that should be initialized // diff --git a/webrtc/common_audio/vad/vad_filterbank.c b/webrtc/common_audio/vad/vad_filterbank.c index 1513153..aff63f7 100644 --- a/webrtc/common_audio/vad/vad_filterbank.c +++ b/webrtc/common_audio/vad/vad_filterbank.c @@ -28,7 +28,7 @@ static const int16_t kAllPassCoefsQ15[2] = { 20972, 5571 }; // Adjustment for division with two in SplitFilter. static const int16_t kOffsetVector[6] = { 368, 368, 272, 176, 176, 176 }; -// High pass filtering, with a cut-off frequency at 80 Hz, if the |data_in| is +// High pass filtering, with a cut-off frequency at 80 Hz, if the `data_in` is // sampled at 500 Hz. // // - data_in [i] : Input audio data sampled at 500 Hz. @@ -69,9 +69,9 @@ static void HighPassFilter(const int16_t* data_in, size_t data_length, } } -// All pass filtering of |data_in|, used before splitting the signal into two +// All pass filtering of `data_in`, used before splitting the signal into two // frequency bands (low pass vs high pass). -// Note that |data_in| and |data_out| can NOT correspond to the same address. +// Note that `data_in` and `data_out` can NOT correspond to the same address. // // - data_in [i] : Input audio signal given in Q0. // - data_length [i] : Length of input and output data. @@ -104,17 +104,17 @@ static void AllPassFilter(const int16_t* data_in, size_t data_length, *filter_state = (int16_t) (state32 >> 16); // Q(-1) } -// Splits |data_in| into |hp_data_out| and |lp_data_out| corresponding to +// Splits `data_in` into `hp_data_out` and `lp_data_out` corresponding to // an upper (high pass) part and a lower (low pass) part respectively. // // - data_in [i] : Input audio data to be split into two frequency bands. -// - data_length [i] : Length of |data_in|. +// - data_length [i] : Length of `data_in`. // - upper_state [i/o] : State of the upper filter, given in Q(-1). // - lower_state [i/o] : State of the lower filter, given in Q(-1). // - hp_data_out [o] : Output audio data of the upper half of the spectrum. -// The length is |data_length| / 2. +// The length is `data_length` / 2. // - lp_data_out [o] : Output audio data of the lower half of the spectrum. -// The length is |data_length| / 2. +// The length is `data_length` / 2. static void SplitFilter(const int16_t* data_in, size_t data_length, int16_t* upper_state, int16_t* lower_state, int16_t* hp_data_out, int16_t* lp_data_out) { @@ -138,23 +138,23 @@ static void SplitFilter(const int16_t* data_in, size_t data_length, } } -// Calculates the energy of |data_in| in dB, and also updates an overall -// |total_energy| if necessary. +// Calculates the energy of `data_in` in dB, and also updates an overall +// `total_energy` if necessary. // // - data_in [i] : Input audio data for energy calculation. // - data_length [i] : Length of input data. -// - offset [i] : Offset value added to |log_energy|. +// - offset [i] : Offset value added to `log_energy`. // - total_energy [i/o] : An external energy updated with the energy of -// |data_in|. -// NOTE: |total_energy| is only updated if -// |total_energy| <= |kMinEnergy|. -// - log_energy [o] : 10 * log10("energy of |data_in|") given in Q4. +// `data_in`. +// NOTE: `total_energy` is only updated if +// `total_energy` <= `kMinEnergy`. +// - log_energy [o] : 10 * log10("energy of `data_in`") given in Q4. static void LogOfEnergy(const int16_t* data_in, size_t data_length, int16_t offset, int16_t* total_energy, int16_t* log_energy) { - // |tot_rshifts| accumulates the number of right shifts performed on |energy|. + // `tot_rshifts` accumulates the number of right shifts performed on `energy`. int tot_rshifts = 0; - // The |energy| will be normalized to 15 bits. We use unsigned integer because + // The `energy` will be normalized to 15 bits. We use unsigned integer because // we eventually will mask out the fractional part. uint32_t energy = 0; @@ -169,14 +169,14 @@ static void LogOfEnergy(const int16_t* data_in, size_t data_length, // zeros of an unsigned 32 bit value. int normalizing_rshifts = 17 - WebRtcSpl_NormU32(energy); // In a 15 bit representation the leading bit is 2^14. log2(2^14) in Q10 is - // (14 << 10), which is what we initialize |log2_energy| with. For a more + // (14 << 10), which is what we initialize `log2_energy` with. For a more // detailed derivations, see below. int16_t log2_energy = kLogEnergyIntPart; tot_rshifts += normalizing_rshifts; - // Normalize |energy| to 15 bits. - // |tot_rshifts| is now the total number of right shifts performed on - // |energy| after normalization. This means that |energy| is in + // Normalize `energy` to 15 bits. + // `tot_rshifts` is now the total number of right shifts performed on + // `energy` after normalization. This means that `energy` is in // Q(-tot_rshifts). if (normalizing_rshifts < 0) { energy <<= -normalizing_rshifts; @@ -184,30 +184,30 @@ static void LogOfEnergy(const int16_t* data_in, size_t data_length, energy >>= normalizing_rshifts; } - // Calculate the energy of |data_in| in dB, in Q4. + // Calculate the energy of `data_in` in dB, in Q4. // // 10 * log10("true energy") in Q4 = 2^4 * 10 * log10("true energy") = - // 160 * log10(|energy| * 2^|tot_rshifts|) = - // 160 * log10(2) * log2(|energy| * 2^|tot_rshifts|) = - // 160 * log10(2) * (log2(|energy|) + log2(2^|tot_rshifts|)) = - // (160 * log10(2)) * (log2(|energy|) + |tot_rshifts|) = - // |kLogConst| * (|log2_energy| + |tot_rshifts|) + // 160 * log10(`energy` * 2^`tot_rshifts`) = + // 160 * log10(2) * log2(`energy` * 2^`tot_rshifts`) = + // 160 * log10(2) * (log2(`energy`) + log2(2^`tot_rshifts`)) = + // (160 * log10(2)) * (log2(`energy`) + `tot_rshifts`) = + // `kLogConst` * (`log2_energy` + `tot_rshifts`) // - // We know by construction that |energy| is normalized to 15 bits. Hence, - // |energy| = 2^14 + frac_Q15, where frac_Q15 is a fractional part in Q15. - // Further, we'd like |log2_energy| in Q10 - // log2(|energy|) in Q10 = 2^10 * log2(2^14 + frac_Q15) = + // We know by construction that `energy` is normalized to 15 bits. Hence, + // `energy` = 2^14 + frac_Q15, where frac_Q15 is a fractional part in Q15. + // Further, we'd like `log2_energy` in Q10 + // log2(`energy`) in Q10 = 2^10 * log2(2^14 + frac_Q15) = // 2^10 * log2(2^14 * (1 + frac_Q15 * 2^-14)) = // 2^10 * (14 + log2(1 + frac_Q15 * 2^-14)) ~= // (14 << 10) + 2^10 * (frac_Q15 * 2^-14) = // (14 << 10) + (frac_Q15 * 2^-4) = (14 << 10) + (frac_Q15 >> 4) // - // Note that frac_Q15 = (|energy| & 0x00003FFF) + // Note that frac_Q15 = (`energy` & 0x00003FFF) - // Calculate and add the fractional part to |log2_energy|. + // Calculate and add the fractional part to `log2_energy`. log2_energy += (int16_t) ((energy & 0x00003FFF) >> 4); - // |kLogConst| is in Q9, |log2_energy| in Q10 and |tot_rshifts| in Q0. + // `kLogConst` is in Q9, `log2_energy` in Q10 and `tot_rshifts` in Q0. // Note that we in our derivation above have accounted for an output in Q4. *log_energy = (int16_t)(((kLogConst * log2_energy) >> 19) + ((tot_rshifts * kLogConst) >> 9)); @@ -222,19 +222,19 @@ static void LogOfEnergy(const int16_t* data_in, size_t data_length, *log_energy += offset; - // Update the approximate |total_energy| with the energy of |data_in|, if - // |total_energy| has not exceeded |kMinEnergy|. |total_energy| is used as an + // Update the approximate `total_energy` with the energy of `data_in`, if + // `total_energy` has not exceeded `kMinEnergy`. `total_energy` is used as an // energy indicator in WebRtcVad_GmmProbability() in vad_core.c. if (*total_energy <= kMinEnergy) { if (tot_rshifts >= 0) { - // We know by construction that the |energy| > |kMinEnergy| in Q0, so add - // an arbitrary value such that |total_energy| exceeds |kMinEnergy|. + // We know by construction that the `energy` > `kMinEnergy` in Q0, so add + // an arbitrary value such that `total_energy` exceeds `kMinEnergy`. *total_energy += kMinEnergy + 1; } else { - // By construction |energy| is represented by 15 bits, hence any number of - // right shifted |energy| will fit in an int16_t. In addition, adding the - // value to |total_energy| is wrap around safe as long as - // |kMinEnergy| < 8192. + // By construction `energy` is represented by 15 bits, hence any number of + // right shifted `energy` will fit in an int16_t. In addition, adding the + // value to `total_energy` is wrap around safe as long as + // `kMinEnergy` < 8192. *total_energy += (int16_t) (energy >> -tot_rshifts); // Q0. } } @@ -243,14 +243,14 @@ static void LogOfEnergy(const int16_t* data_in, size_t data_length, int16_t WebRtcVad_CalculateFeatures(VadInstT* self, const int16_t* data_in, size_t data_length, int16_t* features) { int16_t total_energy = 0; - // We expect |data_length| to be 80, 160 or 240 samples, which corresponds to + // We expect `data_length` to be 80, 160 or 240 samples, which corresponds to // 10, 20 or 30 ms in 8 kHz. Therefore, the intermediate downsampled data will // have at most 120 samples after the first split and at most 60 samples after // the second split. int16_t hp_120[120], lp_120[120]; int16_t hp_60[60], lp_60[60]; const size_t half_data_length = data_length >> 1; - size_t length = half_data_length; // |data_length| / 2, corresponds to + size_t length = half_data_length; // `data_length` / 2, corresponds to // bandwidth = 2000 Hz after downsampling. // Initialize variables for the first SplitFilter(). @@ -260,7 +260,7 @@ int16_t WebRtcVad_CalculateFeatures(VadInstT* self, const int16_t* data_in, int16_t* lp_out_ptr = lp_120; // [0 - 2000] Hz. RTC_DCHECK_LE(data_length, 240); - RTC_DCHECK_LT(4, kNumChannels - 1); // Checking maximum |frequency_band|. + RTC_DCHECK_LT(4, kNumChannels - 1); // Checking maximum `frequency_band`. // Split at 2000 Hz and downsample. SplitFilter(in_ptr, data_length, &self->upper_state[frequency_band], @@ -275,7 +275,7 @@ int16_t WebRtcVad_CalculateFeatures(VadInstT* self, const int16_t* data_in, &self->lower_state[frequency_band], hp_out_ptr, lp_out_ptr); // Energy in 3000 Hz - 4000 Hz. - length >>= 1; // |data_length| / 4 <=> bandwidth = 1000 Hz. + length >>= 1; // `data_length` / 4 <=> bandwidth = 1000 Hz. LogOfEnergy(hp_60, length, kOffsetVector[5], &total_energy, &features[5]); @@ -287,12 +287,12 @@ int16_t WebRtcVad_CalculateFeatures(VadInstT* self, const int16_t* data_in, in_ptr = lp_120; // [0 - 2000] Hz. hp_out_ptr = hp_60; // [1000 - 2000] Hz. lp_out_ptr = lp_60; // [0 - 1000] Hz. - length = half_data_length; // |data_length| / 2 <=> bandwidth = 2000 Hz. + length = half_data_length; // `data_length` / 2 <=> bandwidth = 2000 Hz. SplitFilter(in_ptr, length, &self->upper_state[frequency_band], &self->lower_state[frequency_band], hp_out_ptr, lp_out_ptr); // Energy in 1000 Hz - 2000 Hz. - length >>= 1; // |data_length| / 4 <=> bandwidth = 1000 Hz. + length >>= 1; // `data_length` / 4 <=> bandwidth = 1000 Hz. LogOfEnergy(hp_60, length, kOffsetVector[3], &total_energy, &features[3]); // For the lower band (0 Hz - 1000 Hz) split at 500 Hz and downsample. @@ -304,7 +304,7 @@ int16_t WebRtcVad_CalculateFeatures(VadInstT* self, const int16_t* data_in, &self->lower_state[frequency_band], hp_out_ptr, lp_out_ptr); // Energy in 500 Hz - 1000 Hz. - length >>= 1; // |data_length| / 8 <=> bandwidth = 500 Hz. + length >>= 1; // `data_length` / 8 <=> bandwidth = 500 Hz. LogOfEnergy(hp_120, length, kOffsetVector[2], &total_energy, &features[2]); // For the lower band (0 Hz - 500 Hz) split at 250 Hz and downsample. @@ -316,7 +316,7 @@ int16_t WebRtcVad_CalculateFeatures(VadInstT* self, const int16_t* data_in, &self->lower_state[frequency_band], hp_out_ptr, lp_out_ptr); // Energy in 250 Hz - 500 Hz. - length >>= 1; // |data_length| / 16 <=> bandwidth = 250 Hz. + length >>= 1; // `data_length` / 16 <=> bandwidth = 250 Hz. LogOfEnergy(hp_60, length, kOffsetVector[1], &total_energy, &features[1]); // Remove 0 Hz - 80 Hz, by high pass filtering the lower band. diff --git a/webrtc/common_audio/vad/vad_filterbank.h b/webrtc/common_audio/vad/vad_filterbank.h index 53bbbe1..205eac8 100644 --- a/webrtc/common_audio/vad/vad_filterbank.h +++ b/webrtc/common_audio/vad/vad_filterbank.h @@ -17,8 +17,8 @@ #include "common_audio/vad/vad_core.h" -// Takes |data_length| samples of |data_in| and calculates the logarithm of the -// energy of each of the |kNumChannels| = 6 frequency bands used by the VAD: +// Takes `data_length` samples of `data_in` and calculates the logarithm of the +// energy of each of the `kNumChannels` = 6 frequency bands used by the VAD: // 80 Hz - 250 Hz // 250 Hz - 500 Hz // 500 Hz - 1000 Hz @@ -26,10 +26,10 @@ // 2000 Hz - 3000 Hz // 3000 Hz - 4000 Hz // -// The values are given in Q4 and written to |features|. Further, an approximate +// The values are given in Q4 and written to `features`. Further, an approximate // overall energy is returned. The return value is used in // WebRtcVad_GmmProbability() as a signal indicator, hence it is arbitrary above -// the threshold |kMinEnergy|. +// the threshold `kMinEnergy`. // // - self [i/o] : State information of the VAD. // - data_in [i] : Input audio data, for feature extraction. diff --git a/webrtc/common_audio/vad/vad_gmm.c b/webrtc/common_audio/vad/vad_gmm.c index ddc87b6..4a7fe67 100644 --- a/webrtc/common_audio/vad/vad_gmm.c +++ b/webrtc/common_audio/vad/vad_gmm.c @@ -15,16 +15,16 @@ static const int32_t kCompVar = 22005; static const int16_t kLog2Exp = 5909; // log2(exp(1)) in Q12. -// For a normal distribution, the probability of |input| is calculated and +// For a normal distribution, the probability of `input` is calculated and // returned (in Q20). The formula for normal distributed probability is // // 1 / s * exp(-(x - m)^2 / (2 * s^2)) // // where the parameters are given in the following Q domains: -// m = |mean| (Q7) -// s = |std| (Q7) -// x = |input| (Q4) -// in addition to the probability we output |delta| (in Q11) used when updating +// m = `mean` (Q7) +// s = `std` (Q7) +// x = `input` (Q4) +// in addition to the probability we output `delta` (in Q11) used when updating // the noise/speech model. int32_t WebRtcVad_GaussianProbability(int16_t input, int16_t mean, @@ -33,13 +33,13 @@ int32_t WebRtcVad_GaussianProbability(int16_t input, int16_t tmp16, inv_std, inv_std2, exp_value = 0; int32_t tmp32; - // Calculate |inv_std| = 1 / s, in Q10. - // 131072 = 1 in Q17, and (|std| >> 1) is for rounding instead of truncation. + // Calculate `inv_std` = 1 / s, in Q10. + // 131072 = 1 in Q17, and (`std` >> 1) is for rounding instead of truncation. // Q-domain: Q17 / Q7 = Q10. tmp32 = (int32_t) 131072 + (int32_t) (std >> 1); inv_std = (int16_t) WebRtcSpl_DivW32W16(tmp32, std); - // Calculate |inv_std2| = 1 / s^2, in Q14. + // Calculate `inv_std2` = 1 / s^2, in Q14. tmp16 = (inv_std >> 2); // Q10 -> Q8. // Q-domain: (Q8 * Q8) >> 2 = Q14. inv_std2 = (int16_t)((tmp16 * tmp16) >> 2); @@ -51,20 +51,20 @@ int32_t WebRtcVad_GaussianProbability(int16_t input, tmp16 = tmp16 - mean; // Q7 - Q7 = Q7 // To be used later, when updating noise/speech model. - // |delta| = (x - m) / s^2, in Q11. + // `delta` = (x - m) / s^2, in Q11. // Q-domain: (Q14 * Q7) >> 10 = Q11. *delta = (int16_t)((inv_std2 * tmp16) >> 10); - // Calculate the exponent |tmp32| = (x - m)^2 / (2 * s^2), in Q10. Replacing + // Calculate the exponent `tmp32` = (x - m)^2 / (2 * s^2), in Q10. Replacing // division by two with one shift. // Q-domain: (Q11 * Q7) >> 8 = Q10. tmp32 = (*delta * tmp16) >> 9; // If the exponent is small enough to give a non-zero probability we calculate - // |exp_value| ~= exp(-(x - m)^2 / (2 * s^2)) - // ~= exp2(-log2(exp(1)) * |tmp32|). + // `exp_value` ~= exp(-(x - m)^2 / (2 * s^2)) + // ~= exp2(-log2(exp(1)) * `tmp32`). if (tmp32 < kCompVar) { - // Calculate |tmp16| = log2(exp(1)) * |tmp32|, in Q10. + // Calculate `tmp16` = log2(exp(1)) * `tmp32`, in Q10. // Q-domain: (Q12 * Q10) >> 12 = Q10. tmp16 = (int16_t)((kLog2Exp * tmp32) >> 12); tmp16 = -tmp16; @@ -72,7 +72,7 @@ int32_t WebRtcVad_GaussianProbability(int16_t input, tmp16 ^= 0xFFFF; tmp16 >>= 10; tmp16 += 1; - // Get |exp_value| = exp(-|tmp32|) in Q10. + // Get `exp_value` = exp(-`tmp32`) in Q10. exp_value >>= tmp16; } diff --git a/webrtc/common_audio/vad/vad_gmm.h b/webrtc/common_audio/vad/vad_gmm.h index 6b2d11b..ada5189 100644 --- a/webrtc/common_audio/vad/vad_gmm.h +++ b/webrtc/common_audio/vad/vad_gmm.h @@ -15,8 +15,8 @@ #include -// Calculates the probability for |input|, given that |input| comes from a -// normal distribution with mean and standard deviation (|mean|, |std|). +// Calculates the probability for `input`, given that `input` comes from a +// normal distribution with mean and standard deviation (`mean`, `std`). // // Inputs: // - input : input sample in Q4. @@ -26,11 +26,11 @@ // Output: // // - delta : input used when updating the model, Q11. -// |delta| = (|input| - |mean|) / |std|^2. +// `delta` = (`input` - `mean`) / `std`^2. // // Return: -// (probability for |input|) = -// 1 / |std| * exp(-(|input| - |mean|)^2 / (2 * |std|^2)); +// (probability for `input`) = +// 1 / `std` * exp(-(`input` - `mean`)^2 / (2 * `std`^2)); int32_t WebRtcVad_GaussianProbability(int16_t input, int16_t mean, int16_t std, diff --git a/webrtc/common_audio/vad/vad_sp.c b/webrtc/common_audio/vad/vad_sp.c index d710a37..3d24cf6 100644 --- a/webrtc/common_audio/vad/vad_sp.c +++ b/webrtc/common_audio/vad/vad_sp.c @@ -52,7 +52,7 @@ void WebRtcVad_Downsampling(const int16_t* signal_in, filter_state[1] = tmp32_2; } -// Inserts |feature_value| into |low_value_vector|, if it is one of the 16 +// Inserts `feature_value` into `low_value_vector`, if it is one of the 16 // smallest values the last 100 frames. Then calculates and returns the median // of the five smallest values. int16_t WebRtcVad_FindMinimum(VadInstT* self, @@ -66,13 +66,13 @@ int16_t WebRtcVad_FindMinimum(VadInstT* self, int16_t alpha = 0; int32_t tmp32 = 0; // Pointer to memory for the 16 minimum values and the age of each value of - // the |channel|. + // the `channel`. int16_t* age = &self->index_vector[offset]; int16_t* smallest_values = &self->low_value_vector[offset]; RTC_DCHECK_LT(channel, kNumChannels); - // Each value in |smallest_values| is getting 1 loop older. Update |age|, and + // Each value in `smallest_values` is getting 1 loop older. Update `age`, and // remove old values. for (i = 0; i < 16; i++) { if (age[i] != 100) { @@ -88,9 +88,9 @@ int16_t WebRtcVad_FindMinimum(VadInstT* self, } } - // Check if |feature_value| is smaller than any of the values in - // |smallest_values|. If so, find the |position| where to insert the new value - // (|feature_value|). + // Check if `feature_value` is smaller than any of the values in + // `smallest_values`. If so, find the `position` where to insert the new value + // (`feature_value`). if (feature_value < smallest_values[7]) { if (feature_value < smallest_values[3]) { if (feature_value < smallest_values[1]) { @@ -152,7 +152,7 @@ int16_t WebRtcVad_FindMinimum(VadInstT* self, age[position] = 1; } - // Get |current_median|. + // Get `current_median`. if (self->frame_counter > 2) { current_median = smallest_values[2]; } else if (self->frame_counter > 0) { diff --git a/webrtc/common_audio/vad/vad_sp.h b/webrtc/common_audio/vad/vad_sp.h index ff36760..89138c5 100644 --- a/webrtc/common_audio/vad/vad_sp.h +++ b/webrtc/common_audio/vad/vad_sp.h @@ -23,11 +23,11 @@ // // Input & Output: // - filter_state : Current filter states of the two all-pass filters. The -// |filter_state| is updated after all samples have been +// `filter_state` is updated after all samples have been // processed. // // Output: -// - signal_out : Downsampled signal (of length |in_length| / 2). +// - signal_out : Downsampled signal (of length `in_length` / 2). void WebRtcVad_Downsampling(const int16_t* signal_in, int16_t* signal_out, int32_t* filter_state, @@ -35,7 +35,7 @@ void WebRtcVad_Downsampling(const int16_t* signal_in, // Updates and returns the smoothed feature minimum. As minimum we use the // median of the five smallest feature values in a 100 frames long window. -// As long as |handle->frame_counter| is zero, that is, we haven't received any +// As long as `handle->frame_counter` is zero, that is, we haven't received any // "valid" data, FindMinimum() outputs the default value of 1600. // // Inputs: diff --git a/webrtc/common_audio/vad/webrtc_vad.c b/webrtc/common_audio/vad/webrtc_vad.c index 49e7682..6dd14d8 100644 --- a/webrtc/common_audio/vad/webrtc_vad.c +++ b/webrtc/common_audio/vad/webrtc_vad.c @@ -21,7 +21,7 @@ static const int kValidRates[] = { 8000, 16000, 32000, 48000 }; static const size_t kRatesSize = sizeof(kValidRates) / sizeof(*kValidRates); static const int kMaxFrameLengthMs = 30; -VadInst* WebRtcVad_Create() { +VadInst* WebRtcVad_Create(void) { VadInstT* self = (VadInstT*)malloc(sizeof(VadInstT)); self->init_flag = 0; diff --git a/webrtc/common_audio/wav_file.cc b/webrtc/common_audio/wav_file.cc index e49126f..127c9c0 100644 --- a/webrtc/common_audio/wav_file.cc +++ b/webrtc/common_audio/wav_file.cc @@ -65,7 +65,7 @@ constexpr size_t kMaxChunksize = 4096; } // namespace -WavReader::WavReader(const std::string& filename) +WavReader::WavReader(absl::string_view filename) : WavReader(FileWrapper::OpenReadOnly(filename)) {} WavReader::WavReader(FileWrapper file) : file_(std::move(file)) { @@ -178,7 +178,7 @@ void WavReader::Close() { file_.Close(); } -WavWriter::WavWriter(const std::string& filename, +WavWriter::WavWriter(absl::string_view filename, int sample_rate, size_t num_channels, SampleFormat sample_format) diff --git a/webrtc/common_audio/wav_file.h b/webrtc/common_audio/wav_file.h index dda611b..72a4db7 100644 --- a/webrtc/common_audio/wav_file.h +++ b/webrtc/common_audio/wav_file.h @@ -39,7 +39,7 @@ class WavFile { class WavWriter final : public WavFile { public: // Opens a new WAV file for writing. - WavWriter(const std::string& filename, + WavWriter(absl::string_view filename, int sample_rate, size_t num_channels, SampleFormat sample_format = SampleFormat::kInt16); @@ -77,7 +77,7 @@ class WavWriter final : public WavFile { class WavReader final : public WavFile { public: // Opens an existing WAV file for reading. - explicit WavReader(const std::string& filename); + explicit WavReader(absl::string_view filename); explicit WavReader(FileWrapper file); // Close the WAV file. diff --git a/webrtc/common_audio/wav_header.cc b/webrtc/common_audio/wav_header.cc index d3dca90..bca209a 100644 --- a/webrtc/common_audio/wav_header.cc +++ b/webrtc/common_audio/wav_header.cc @@ -80,8 +80,6 @@ const uint32_t kFmtIeeeFloatSubchunkSize = // read audio samples. #pragma pack(2) struct WavHeaderPcm { - WavHeaderPcm(const WavHeaderPcm&) = default; - WavHeaderPcm& operator=(const WavHeaderPcm&) = default; RiffHeader riff; FmtPcmSubchunk fmt; struct { @@ -95,8 +93,6 @@ static_assert(sizeof(WavHeaderPcm) == kPcmWavHeaderSize, // WAV implementation. #pragma pack(2) struct WavHeaderIeeeFloat { - WavHeaderIeeeFloat(const WavHeaderIeeeFloat&) = default; - WavHeaderIeeeFloat& operator=(const WavHeaderIeeeFloat&) = default; RiffHeader riff; FmtIeeeFloatSubchunk fmt; struct { @@ -132,7 +128,7 @@ uint16_t MapWavFormatToHeaderField(WavFormat format) { case WavFormat::kWavFormatMuLaw: return 7; } - RTC_CHECK(false); + RTC_CHECK_NOTREACHED(); } WavFormat MapHeaderFieldToWavFormat(uint16_t format_header_value) { @@ -161,7 +157,7 @@ uint16_t BlockAlign(size_t num_channels, size_t bytes_per_sample) { return static_cast(num_channels * bytes_per_sample); } -// Finds a chunk having the sought ID. If found, then |readable| points to the +// Finds a chunk having the sought ID. If found, then `readable` points to the // first byte of the sought chunk data. If not found, the end of the file is // reached. bool FindWaveChunk(ChunkHeader* chunk_header, @@ -278,10 +274,8 @@ size_t GetFormatBytesPerSample(WavFormat format) { return 1; case WavFormat::kWavFormatIeeeFloat: return 4; - default: - RTC_CHECK(false); - return 2; } + RTC_CHECK_NOTREACHED(); } bool CheckWavParameters(size_t num_channels, diff --git a/webrtc/common_audio/wav_header.h b/webrtc/common_audio/wav_header.h index 2cccd7d..a1aa942 100644 --- a/webrtc/common_audio/wav_header.h +++ b/webrtc/common_audio/wav_header.h @@ -13,6 +13,7 @@ #include #include + #include #include "rtc_base/checks.h" diff --git a/webrtc/experiments/registered_field_trials.h b/webrtc/experiments/registered_field_trials.h new file mode 100644 index 0000000..811cd56 --- /dev/null +++ b/webrtc/experiments/registered_field_trials.h @@ -0,0 +1,297 @@ +// This file was automatically generated. Do not edit. + +#ifndef GEN_REGISTERED_FIELD_TRIALS_H_ +#define GEN_REGISTERED_FIELD_TRIALS_H_ + +#include "absl/strings/string_view.h" + +namespace webrtc { + +inline constexpr absl::string_view kRegisteredFieldTrials[] = { + "UseTwccPlrForAna", + "WebRTC-AddNetworkCostToVpn", + "WebRTC-AddPacingToCongestionWindowPushback", + "WebRTC-AdjustOpusBandwidth", + "WebRTC-Aec3AecStateFullResetKillSwitch", + "WebRTC-Aec3AecStateSubtractorAnalyzerResetKillSwitch", + "WebRTC-Aec3AntiHowlingMinimizationKillSwitch", + "WebRTC-Aec3ClampInstQualityToOneKillSwitch", + "WebRTC-Aec3ClampInstQualityToZeroKillSwitch", + "WebRTC-Aec3CoarseFilterResetHangoverKillSwitch", + "WebRTC-Aec3ConservativeTailFreqResponse", + "WebRTC-Aec3DeactivateInitialStateResetKillSwitch", + "WebRTC-Aec3DelayEstimateSmoothingDelayFoundOverride", + "WebRTC-Aec3DelayEstimateSmoothingOverride", + "WebRTC-Aec3DelayEstimatorDetectPreEcho", + "WebRTC-Aec3EchoSaturationDetectionKillSwitch", + "WebRTC-Aec3EnforceCaptureDelayEstimationDownmixing", + "WebRTC-Aec3EnforceCaptureDelayEstimationLeftRightPrioritization", + "WebRTC-Aec3EnforceConservativeHfSuppression", + "WebRTC-Aec3EnforceLowActiveRenderLimit", + "WebRTC-Aec3EnforceMoreTransparentNearendSuppressorHfTuning", + "WebRTC-Aec3EnforceMoreTransparentNearendSuppressorTuning", + "WebRTC-Aec3EnforceMoreTransparentNormalSuppressorHfTuning", + "WebRTC-Aec3EnforceMoreTransparentNormalSuppressorTuning", + "WebRTC-Aec3EnforceRapidlyAdjustingNearendSuppressorTunings", + "WebRTC-Aec3EnforceRapidlyAdjustingNormalSuppressorTunings", + "WebRTC-Aec3EnforceRenderDelayEstimationDownmixing", + "WebRTC-Aec3EnforceSlowlyAdjustingNearendSuppressorTunings", + "WebRTC-Aec3EnforceSlowlyAdjustingNormalSuppressorTunings", + "WebRTC-Aec3EnforceStationarityProperties", + "WebRTC-Aec3EnforceStationarityPropertiesAtInit", + "WebRTC-Aec3EnforceVeryLowActiveRenderLimit", + "WebRTC-Aec3HighPassFilterEchoReference", + "WebRTC-Aec3MinErleDuringOnsetsKillSwitch", + "WebRTC-Aec3NonlinearModeReverbKillSwitch", + "WebRTC-Aec3OnsetDetectionKillSwitch", + "WebRTC-Aec3PenalyzeHighDelaysInitialPhase", + "WebRTC-Aec3PreEchoConfiguration", + "WebRTC-Aec3RenderDelayEstimationLeftRightPrioritizationKillSwitch", + "WebRTC-Aec3SensitiveDominantNearendActivation", + "WebRTC-Aec3SetupSpecificDefaultConfigDefaultsKillSwitch", + "WebRTC-Aec3ShortHeadroomKillSwitch", + "WebRTC-Aec3StereoContentDetectionKillSwitch", + "WebRTC-Aec3SuppressorAntiHowlingGainOverride", + "WebRTC-Aec3SuppressorDominantNearendEnrExitThresholdOverride", + "WebRTC-Aec3SuppressorDominantNearendEnrThresholdOverride", + "WebRTC-Aec3SuppressorDominantNearendHoldDurationOverride", + "WebRTC-Aec3SuppressorDominantNearendSnrThresholdOverride", + "WebRTC-Aec3SuppressorDominantNearendTriggerThresholdOverride", + "WebRTC-Aec3SuppressorNearendHfMaskSuppressOverride", + "WebRTC-Aec3SuppressorNearendHfMaskTransparentOverride", + "WebRTC-Aec3SuppressorNearendLfMaskSuppressOverride", + "WebRTC-Aec3SuppressorNearendLfMaskTransparentOverride", + "WebRTC-Aec3SuppressorNearendMaxDecFactorLfOverride", + "WebRTC-Aec3SuppressorNearendMaxIncFactorOverride", + "WebRTC-Aec3SuppressorNormalHfMaskSuppressOverride", + "WebRTC-Aec3SuppressorNormalHfMaskTransparentOverride", + "WebRTC-Aec3SuppressorNormalLfMaskSuppressOverride", + "WebRTC-Aec3SuppressorNormalLfMaskTransparentOverride", + "WebRTC-Aec3SuppressorNormalMaxDecFactorLfOverride", + "WebRTC-Aec3SuppressorNormalMaxIncFactorOverride", + "WebRTC-Aec3SuppressorTuningOverride", + "WebRTC-Aec3TransparentAntiHowlingGain", + "WebRTC-Aec3TransparentModeHmm", + "WebRTC-Aec3TransparentModeKillSwitch", + "WebRTC-Aec3Use1Dot2SecondsInitialStateDuration", + "WebRTC-Aec3Use1Dot6SecondsInitialStateDuration", + "WebRTC-Aec3Use2Dot0SecondsInitialStateDuration", + "WebRTC-Aec3UseDot1SecondsInitialStateDuration", + "WebRTC-Aec3UseDot2SecondsInitialStateDuration", + "WebRTC-Aec3UseDot3SecondsInitialStateDuration", + "WebRTC-Aec3UseDot6SecondsInitialStateDuration", + "WebRTC-Aec3UseDot9SecondsInitialStateDuration", + "WebRTC-Aec3UseErleOnsetCompensationInDominantNearend", + "WebRTC-Aec3UseLowEarlyReflectionsDefaultGain", + "WebRTC-Aec3UseLowLateReflectionsDefaultGain", + "WebRTC-Aec3UseNearendReverbLen", + "WebRTC-Aec3UseShortConfigChangeDuration", + "WebRTC-Aec3UseZeroInitialStateDuration", + "WebRTC-Aec3VerySensitiveDominantNearendActivation", + "WebRTC-Agc2SimdAvx2KillSwitch", + "WebRTC-Agc2SimdNeonKillSwitch", + "WebRTC-Agc2SimdSse2KillSwitch", + "WebRTC-AllowMACBasedIPv6", + "WebRTC-AlrDetectorParameters", + "WebRTC-AndroidNetworkMonitor-IsAdapterAvailable", + "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch", + "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch", + "WebRTC-Audio-2ndAgcMinMicLevelExperiment", + "WebRTC-Audio-ABWENoTWCC", + "WebRTC-Audio-AdaptivePtime", + "WebRTC-Audio-Allocation", + "WebRTC-Audio-AlrProbing", + "WebRTC-Audio-FecAdaptation", + "WebRTC-Audio-GainController2", + "WebRTC-Audio-LegacyOverhead", + "WebRTC-Audio-MinimizeResamplingOnMobile", + "WebRTC-Audio-NetEqDecisionLogicConfig", + "WebRTC-Audio-NetEqDelayManagerConfig", + "WebRTC-Audio-NetEqNackTrackerConfig", + "WebRTC-Audio-NetEqSmartFlushing", + "WebRTC-Audio-OpusAvoidNoisePumpingDuringDtx", + "WebRTC-Audio-OpusBitrateMultipliers", + "WebRTC-Audio-OpusPlcUsePrevDecodedSamples", + "WebRTC-Audio-OpusSetSignalVoiceWithDtx", + "WebRTC-Audio-Red-For-Opus", + "WebRTC-Audio-StableTargetAdaptation", + "WebRTC-Audio-iOS-Holding", + "WebRTC-AudioDevicePlayoutBufferSizeFactor", + "WebRTC-AutomaticAnimationDetectionScreenshare", + "WebRTC-Av1-GetEncoderInfoOverride", + "WebRTC-Avx2SupportKillSwitch", + "WebRTC-BindUsingInterfaceName", + "WebRTC-BoostedScreenshareQp", + "WebRTC-BurstyPacer", + "WebRTC-Bwe-AllocationProbing", + "WebRTC-Bwe-AlrProbing", + "WebRTC-Bwe-EstimateBoundedIncrease", + "WebRTC-Bwe-ExponentialProbing", + "WebRTC-Bwe-IgnoreProbesLowerThanNetworkStateEstimate", + "WebRTC-Bwe-InitialProbing", + "WebRTC-Bwe-InjectedCongestionController", + "WebRTC-Bwe-LimitProbesLowerThanThroughputEstimate", + "WebRTC-Bwe-LinkCapacity", + "WebRTC-Bwe-LossBasedBweV2", + "WebRTC-Bwe-LossBasedControl", + "WebRTC-Bwe-MaxRttLimit", + "WebRTC-Bwe-MinAllocAsLowerBound", + "WebRTC-Bwe-NetworkRouteConstraints", + "WebRTC-Bwe-NoFeedbackReset", + "WebRTC-Bwe-PaceAtMaxOfBweAndLowerLinkCapacity", + "WebRTC-Bwe-ProbingBehavior", + "WebRTC-Bwe-ProbingConfiguration", + "WebRTC-Bwe-ReceiveTimeFix", + "WebRTC-Bwe-ReceiverLimitCapsOnly", + "WebRTC-Bwe-RobustThroughputEstimatorSettings", + "WebRTC-Bwe-SafeResetOnRouteChange", + "WebRTC-Bwe-SeparateAudioPackets", + "WebRTC-Bwe-SubtractAdditionalBackoffTerm", + "WebRTC-Bwe-TrendlineEstimatorSettings", + "WebRTC-BweBackOffFactor", + "WebRTC-BweLossExperiment", + "WebRTC-BweRapidRecoveryExperiment", + "WebRTC-BweThroughputWindowConfig", + "WebRTC-BweWindowSizeInPackets", + "WebRTC-CongestionWindow", + "WebRTC-CpuLoadEstimator", + "WebRTC-Debugging-RtpDump", + "WebRTC-DecoderDataDumpDirectory", + "WebRTC-DefaultBitrateLimitsKillSwitch", + "WebRTC-DependencyDescriptorAdvertised", + "WebRTC-DisablePacerEmergencyStop", + "WebRTC-DisableRtxRateLimiter", + "WebRTC-DisableUlpFecExperiment", + "WebRTC-DontIncreaseDelayBasedBweInAlr", + "WebRTC-DscpFieldTrial", + "WebRTC-EncoderDataDumpDirectory", + "WebRTC-ExtraICEPing", + "WebRTC-FakeNetworkReceiveConfig", + "WebRTC-FakeNetworkSendConfig", + "WebRTC-FilterAbsSendTimeExtension", + "WebRTC-FindNetworkHandleWithoutIpv6TemporaryPart", + "WebRTC-FlexFEC-03", + "WebRTC-FlexFEC-03-Advertised", + "WebRTC-ForcePlayoutDelay", + "WebRTC-ForceSendPlayoutDelay", + "WebRTC-ForceSimulatedOveruseIntervalMs", + "WebRTC-FrameDropper", + "WebRTC-FullBandHpfKillSwitch", + "WebRTC-GenericCodecDependencyDescriptor", + "WebRTC-GenericDescriptorAdvertised", + "WebRTC-GenericDescriptorAuth", + "WebRTC-GenericPictureId", + "WebRTC-GetEncoderInfoOverride", + "WebRTC-H264HighProfile", + "WebRTC-IPv6Default", + "WebRTC-IPv6NetworkResolutionFixes", + "WebRTC-IceControllerFieldTrials", + "WebRTC-IceFieldTrials", + "WebRTC-IncomingTimestampOnMarkerBitOnly", + "WebRTC-IncreaseIceCandidatePriorityHostSrflx", + "WebRTC-JitterEstimatorConfig", + "WebRTC-KeyframeInterval", + "WebRTC-LegacyFrameIdJumpBehavior", + "WebRTC-LegacySimulcastLayerLimit", + "WebRTC-LegacyTlsProtocols", + "WebRTC-LibaomAv1Encoder-DisableFrameDropping", + "WebRTC-LowresSimulcastBitrateInterpolation", + "WebRTC-MutedStateKillSwitch", + "WebRTC-Network-UseNWPathMonitor", + "WebRTC-NetworkMonitorAutoDetect", + "WebRTC-NormalizeSimulcastResolution", + "WebRTC-Pacer-BlockAudio", + "WebRTC-Pacer-DrainQueue", + "WebRTC-Pacer-FastRetransmissions", + "WebRTC-Pacer-IgnoreTransportOverhead", + "WebRTC-Pacer-KeyframeFlushing", + "WebRTC-Pacer-PadInSilence", + "WebRTC-PacketBufferMaxSize", + "WebRTC-PaddingMode-RecentLargePacket", + "WebRTC-PcFactoryDefaultBitrates", + "WebRTC-PermuteTlsClientHello", + "WebRTC-PiggybackIceCheckAcknowledgement", + "WebRTC-PixelLimitResource", + "WebRTC-PreventSsrcGroupsWithUnexpectedSize", + "WebRTC-ProbingScreenshareBwe", + "WebRTC-ProtectionOverheadRateThreshold", + "WebRTC-QpParsingKillSwitch", + "WebRTC-ReceiveBufferSize", + "WebRTC-RtcEventLogEncodeDependencyDescriptor", + "WebRTC-RtcEventLogEncodeNetEqSetMinimumDelayKillSwitch", + "WebRTC-RtcEventLogKillSwitch", + "WebRTC-RtcEventLogNewFormat", + "WebRTC-RtcpLossNotification", + "WebRTC-RttMult", + "WebRTC-SCM-Timestamp", + "WebRTC-SendBufferSizeBytes", + "WebRTC-SendNackDelayMs", + "WebRTC-SendPacketsOnWorkerThread", + "WebRTC-SetSocketReceiveBuffer", + "WebRTC-SignalNetworkPreferenceChange", + "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride", + "WebRTC-SimulcastLayerLimitRoundUp", + "WebRTC-SpsPpsIdrIsH264Keyframe", + "WebRTC-StableTargetRate", + "WebRTC-Stats-RtxReceiveStats", + "WebRTC-StrictPacingAndProbing", + "WebRTC-StunInterPacketDelay", + "WebRTC-SurfaceCellularTypes", + "WebRTC-SwitchEncoderOnInitializationFailures", + "WebRTC-Target-Bitrate-Rtcp", + "WebRTC-TaskQueue-ReplaceLibeventWithStdlib", + "WebRTC-TransientSuppressorForcedOff", + "WebRTC-UseBaseHeavyVP8TL3RateAllocation", + "WebRTC-UseDifferentiatedCellularCosts", + "WebRTC-UseShortVP8TL2Pattern", + "WebRTC-UseShortVP8TL3Pattern", + "WebRTC-UseStandardBytesStats", + "WebRTC-UseTurnServerAsStunServer", + "WebRTC-VP8-CpuSpeed-Arm", + "WebRTC-VP8-ForcePartitionResilience", + "WebRTC-VP8-Forced-Fallback-Encoder-v2", + "WebRTC-VP8-GetEncoderInfoOverride", + "WebRTC-VP8-MaxFrameInterval", + "WebRTC-VP8-Postproc-Config", + "WebRTC-VP8-Postproc-Config-Arm", + "WebRTC-VP8ConferenceTemporalLayers", + "WebRTC-VP8IosMaxNumberOfThread", + "WebRTC-VP8VariableFramerateScreenshare", + "WebRTC-VP9-GetEncoderInfoOverride", + "WebRTC-VP9-LowTierOptimizations", + "WebRTC-VP9-PerformanceFlags", + "WebRTC-VP9QualityScaler", + "WebRTC-VP9VariableFramerateScreenshare", + "WebRTC-Video-BalancedDegradation", + "WebRTC-Video-BalancedDegradationSettings", + "WebRTC-Video-BandwidthQualityScalerSettings", + "WebRTC-Video-DisableAutomaticResize", + "WebRTC-Video-DiscardPacketsWithUnknownSsrc", + "WebRTC-Video-EnableRetransmitAllLayers", + "WebRTC-Video-EncoderFallbackSettings", + "WebRTC-Video-ForcedSwDecoderFallback", + "WebRTC-Video-InitialDecoderResolution", + "WebRTC-Video-MinVideoBitrate", + "WebRTC-Video-Pacing", + "WebRTC-Video-PreferTemporalSupportOnBaseLayer", + "WebRTC-Video-QualityRampupSettings", + "WebRTC-Video-QualityScalerSettings", + "WebRTC-Video-QualityScaling", + "WebRTC-Video-RequestedResolutionOverrideOutputFormatRequest", + "WebRTC-Video-UseFrameRateForOverhead", + "WebRTC-Video-VariableStartScaleFactor", + "WebRTC-VideoEncoderSettings", + "WebRTC-VideoFrameTrackingIdAdvertised", + "WebRTC-VideoLayersAllocationAdvertised", + "WebRTC-VideoRateControl", + "WebRTC-VoIPChannelRemixingAdjustmentKillSwitch", + "WebRTC-Vp9ExternalRefCtrl", + "WebRTC-Vp9InterLayerPred", + "WebRTC-Vp9IssueKeyFrameOnLayerDeactivation", + "WebRTC-ZeroHertzScreenshare", + "WebRTC-ZeroPlayoutDelay", +}; + +} // namespace webrtc + +#endif // GEN_REGISTERED_FIELD_TRIALS_H_ diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index bb6b7cc..4870cb3 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -21,7 +21,6 @@ group("modules") { "rtp_rtcp", "utility", "video_coding", - "video_processing", ] if (rtc_desktop_capture_supported) { @@ -36,10 +35,7 @@ rtc_source_set("module_api_public") { rtc_source_set("module_api") { visibility = [ "*" ] - sources = [ - "include/module.h", - "include/module_common_types.h", - ] + sources = [ "include/module_common_types.h" ] } rtc_source_set("module_fec_api") { @@ -47,7 +43,7 @@ rtc_source_set("module_fec_api") { sources = [ "include/module_fec_types.h" ] } -if (rtc_include_tests) { +if (rtc_include_tests && !build_with_chromium) { modules_tests_resources = [ "../resources/audio_coding/testfile16kHz.pcm", "../resources/audio_coding/testfile32kHz.pcm", @@ -82,12 +78,14 @@ if (rtc_include_tests) { data = modules_tests_resources if (is_android) { + use_default_launcher = false deps += [ # NOTE(brandtr): Including Java classes seems only to be possible from # rtc_test targets. Therefore we include this target here, instead of # in video_coding_modules_tests, where it is actually used. "../sdk/android:libjingle_peerconnection_java", - "//testing/android/native_test:native_test_native_code", + "//sdk/android:native_test_jni_onload", + "//testing/android/native_test:native_test_support", ] shard_timeout = 900 } @@ -147,18 +145,28 @@ if (rtc_include_tests) { "../resources/audio_processing/transient/wpd7.dat", "../resources/deflicker_before_cif_short.yuv", "../resources/far16_stereo.pcm", + "../resources/far176_stereo.pcm", + "../resources/far192_stereo.pcm", + "../resources/far22_stereo.pcm", "../resources/far32_stereo.pcm", "../resources/far44_stereo.pcm", "../resources/far48_stereo.pcm", + "../resources/far88_stereo.pcm", "../resources/far8_stereo.pcm", + "../resources/far96_stereo.pcm", "../resources/foremanColorEnhanced_cif_short.yuv", "../resources/foreman_cif.yuv", "../resources/foreman_cif_short.yuv", "../resources/near16_stereo.pcm", + "../resources/near176_stereo.pcm", + "../resources/near192_stereo.pcm", + "../resources/near22_stereo.pcm", "../resources/near32_stereo.pcm", "../resources/near44_stereo.pcm", "../resources/near48_stereo.pcm", + "../resources/near88_stereo.pcm", "../resources/near8_stereo.pcm", + "../resources/near96_stereo.pcm", "../resources/ref03.aecdump", "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin", "../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin", @@ -219,9 +227,9 @@ if (rtc_include_tests) { "pacing:pacing_unittests", "remote_bitrate_estimator:remote_bitrate_estimator_unittests", "rtp_rtcp:rtp_rtcp_unittests", - "utility:utility_unittests", "video_coding:video_coding_unittests", - "video_processing:video_processing_unittests", + "video_coding/deprecated:deprecated_unittests", + "video_coding/timing:timing_unittests", ] if (rtc_desktop_capture_supported) { @@ -231,6 +239,7 @@ if (rtc_include_tests) { data = modules_unittests_resources if (is_android) { + use_default_launcher = false deps += [ "../sdk/android:libjingle_peerconnection_java", "//testing/android/native_test:native_test_support", diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index cdf7821..61fecea 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -17,7 +17,6 @@ visibility = [ ":*" ] rtc_source_set("audio_coding_module_typedefs") { visibility += [ "*" ] sources = [ "include/audio_coding_module_typedefs.h" ] - deps = [ "../../rtc_base:deprecation" ] } rtc_library("audio_coding") { @@ -51,9 +50,11 @@ rtc_library("audio_coding") { "../../common_audio", "../../common_audio:common_audio_c", "../../rtc_base:audio_format_to_string", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:deprecation", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:safe_conversions", "../../rtc_base/synchronization:mutex", "../../system_wrappers", "../../system_wrappers:metrics", @@ -72,8 +73,8 @@ rtc_library("legacy_encoded_audio_frame") { deps = [ "../../api:array_view", "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } @@ -88,8 +89,8 @@ rtc_library("webrtc_cng") { deps = [ "../../api:array_view", "../../common_audio:common_audio_c", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", "../../rtc_base:safe_conversions", ] } @@ -120,13 +121,19 @@ rtc_library("red") { deps = [ "../../api:array_view", + "../../api:field_trials_view", "../../api/audio_codecs:audio_codecs_api", "../../api/units:time_delta", "../../common_audio", + "../../rtc_base:buffer", + "../../rtc_base:byte_order", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("g711") { @@ -144,11 +151,11 @@ rtc_library("g711") { "../../api:array_view", "../../api/audio_codecs:audio_codecs_api", "../../api/units:time_delta", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - public_deps = [ ":g711_c" ] # no-presubmit-check TODO(webrtc:8603) + public_deps += [ ":g711_c" ] # no-presubmit-check TODO(webrtc:8603) } rtc_library("g711_c") { @@ -176,11 +183,12 @@ rtc_library("g722") { "../../api/audio_codecs:audio_codecs_api", "../../api/audio_codecs/g722:audio_encoder_g722_config", "../../api/units:time_delta", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:safe_conversions", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - public_deps = [ ":g722_c" ] # no-presubmit-check TODO(webrtc:8603) + public_deps += [ ":g722_c" ] # no-presubmit-check TODO(webrtc:8603) } rtc_library("g722_c") { @@ -209,11 +217,13 @@ rtc_library("ilbc") { "../../api/audio_codecs/ilbc:audio_encoder_ilbc_config", "../../api/units:time_delta", "../../common_audio", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:safe_conversions", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - public_deps = [ ":ilbc_c" ] # no-presubmit-check TODO(webrtc:8603) + public_deps += [ ":ilbc_c" ] # no-presubmit-check TODO(webrtc:8603) } rtc_library("ilbc_c") { @@ -365,58 +375,14 @@ rtc_library("ilbc_c") { "../../common_audio", "../../common_audio:common_audio_c", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", "../../rtc_base:sanitizer", "../../rtc_base/system:arch", - "../../rtc_base/system:unused", ] -} - -rtc_source_set("isac_common") { - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/audio_decoder_isac_t.h", - "codecs/isac/audio_decoder_isac_t_impl.h", - "codecs/isac/audio_encoder_isac_t.h", - "codecs/isac/audio_encoder_isac_t_impl.h", - ] - deps = [ - ":isac_bwinfo", - "../../api:scoped_refptr", - "../../api/audio_codecs:audio_codecs_api", - "../../api/units:time_delta", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../rtc_base:safe_minmax", - "../../system_wrappers:field_trial", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] -} - -rtc_library("isac") { - visibility += [ "*" ] - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/main/include/audio_decoder_isac.h", - "codecs/isac/main/include/audio_encoder_isac.h", - "codecs/isac/main/source/audio_decoder_isac.cc", - "codecs/isac/main/source/audio_encoder_isac.cc", - ] - - deps = [ - ":isac_common", - "../../api/audio_codecs:audio_codecs_api", - ] - public_deps = [ ":isac_c" ] # no-presubmit-check TODO(webrtc:8603) -} - -rtc_source_set("isac_bwinfo") { - sources = [ "codecs/isac/bandwidth_info.h" ] - deps = [] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] } rtc_library("isac_vad") { - visibility += webrtc_default_visibility + visibility += [ "../audio_processing/vad:*" ] sources = [ "codecs/isac/main/source/filter_functions.c", "codecs/isac/main/source/filter_functions.h", @@ -439,251 +405,9 @@ rtc_library("isac_vad") { ] } -rtc_library("isac_c") { - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/main/include/isac.h", - "codecs/isac/main/source/arith_routines.c", - "codecs/isac/main/source/arith_routines.h", - "codecs/isac/main/source/arith_routines_hist.c", - "codecs/isac/main/source/arith_routines_logist.c", - "codecs/isac/main/source/bandwidth_estimator.c", - "codecs/isac/main/source/bandwidth_estimator.h", - "codecs/isac/main/source/codec.h", - "codecs/isac/main/source/crc.c", - "codecs/isac/main/source/crc.h", - "codecs/isac/main/source/decode.c", - "codecs/isac/main/source/decode_bwe.c", - "codecs/isac/main/source/encode.c", - "codecs/isac/main/source/encode_lpc_swb.c", - "codecs/isac/main/source/encode_lpc_swb.h", - "codecs/isac/main/source/entropy_coding.c", - "codecs/isac/main/source/entropy_coding.h", - "codecs/isac/main/source/filterbanks.c", - "codecs/isac/main/source/intialize.c", - "codecs/isac/main/source/isac.c", - "codecs/isac/main/source/isac_float_type.h", - "codecs/isac/main/source/lattice.c", - "codecs/isac/main/source/lpc_analysis.c", - "codecs/isac/main/source/lpc_analysis.h", - "codecs/isac/main/source/lpc_gain_swb_tables.c", - "codecs/isac/main/source/lpc_gain_swb_tables.h", - "codecs/isac/main/source/lpc_shape_swb12_tables.c", - "codecs/isac/main/source/lpc_shape_swb12_tables.h", - "codecs/isac/main/source/lpc_shape_swb16_tables.c", - "codecs/isac/main/source/lpc_shape_swb16_tables.h", - "codecs/isac/main/source/lpc_tables.c", - "codecs/isac/main/source/lpc_tables.h", - "codecs/isac/main/source/pitch_gain_tables.c", - "codecs/isac/main/source/pitch_gain_tables.h", - "codecs/isac/main/source/pitch_lag_tables.c", - "codecs/isac/main/source/pitch_lag_tables.h", - "codecs/isac/main/source/spectrum_ar_model_tables.c", - "codecs/isac/main/source/spectrum_ar_model_tables.h", - "codecs/isac/main/source/transform.c", - ] - - if (is_linux || is_chromeos) { - libs = [ "m" ] - } - - deps = [ - ":isac_bwinfo", - ":isac_vad", - "../../common_audio", - "../../common_audio:common_audio_c", - "../../rtc_base:checks", - "../../rtc_base:compile_assert_c", - "../../rtc_base:rtc_base_approved", - "../../rtc_base/system:arch", - "../third_party/fft", - ] -} - -rtc_library("isac_fix") { - visibility += [ "*" ] - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/fix/source/audio_decoder_isacfix.cc", - "codecs/isac/fix/source/audio_encoder_isacfix.cc", - ] - - deps = [ - ":isac_common", - "../../api/audio_codecs:audio_codecs_api", - "../../common_audio", - "../../system_wrappers", - ] - public_deps = [ ":isac_fix_c" ] # no-presubmit-check TODO(webrtc:8603) - - if (rtc_build_with_neon) { - deps += [ ":isac_neon" ] - } -} - -rtc_library("isac_fix_common") { - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/fix/source/codec.h", - "codecs/isac/fix/source/entropy_coding.h", - "codecs/isac/fix/source/fft.c", - "codecs/isac/fix/source/fft.h", - "codecs/isac/fix/source/filterbank_internal.h", - "codecs/isac/fix/source/settings.h", - "codecs/isac/fix/source/structs.h", - "codecs/isac/fix/source/transform_tables.c", - ] - deps = [ - ":isac_bwinfo", - "../../common_audio", - "../../common_audio:common_audio_c", - ] -} - -rtc_source_set("isac_fix_c_arm_asm") { - poisonous = [ "audio_codecs" ] - sources = [] - if (current_cpu == "arm" && arm_version >= 7) { - sources += [ - "codecs/isac/fix/source/lattice_armv7.S", - "codecs/isac/fix/source/pitch_filter_armv6.S", - ] - deps = [ - ":isac_fix_common", - "../../rtc_base/system:asm_defines", - ] - } -} - -rtc_library("isac_fix_c") { - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/fix/include/audio_decoder_isacfix.h", - "codecs/isac/fix/include/audio_encoder_isacfix.h", - "codecs/isac/fix/include/isacfix.h", - "codecs/isac/fix/source/arith_routines.c", - "codecs/isac/fix/source/arith_routines_hist.c", - "codecs/isac/fix/source/arith_routines_logist.c", - "codecs/isac/fix/source/arith_routins.h", - "codecs/isac/fix/source/bandwidth_estimator.c", - "codecs/isac/fix/source/bandwidth_estimator.h", - "codecs/isac/fix/source/decode.c", - "codecs/isac/fix/source/decode_bwe.c", - "codecs/isac/fix/source/decode_plc.c", - "codecs/isac/fix/source/encode.c", - "codecs/isac/fix/source/entropy_coding.c", - "codecs/isac/fix/source/filterbank_tables.c", - "codecs/isac/fix/source/filterbank_tables.h", - "codecs/isac/fix/source/filterbanks.c", - "codecs/isac/fix/source/filters.c", - "codecs/isac/fix/source/initialize.c", - "codecs/isac/fix/source/isac_fix_type.h", - "codecs/isac/fix/source/isacfix.c", - "codecs/isac/fix/source/lattice.c", - "codecs/isac/fix/source/lattice_c.c", - "codecs/isac/fix/source/lpc_masking_model.c", - "codecs/isac/fix/source/lpc_masking_model.h", - "codecs/isac/fix/source/lpc_tables.c", - "codecs/isac/fix/source/lpc_tables.h", - "codecs/isac/fix/source/pitch_estimator.c", - "codecs/isac/fix/source/pitch_estimator.h", - "codecs/isac/fix/source/pitch_estimator_c.c", - "codecs/isac/fix/source/pitch_filter.c", - "codecs/isac/fix/source/pitch_filter_c.c", - "codecs/isac/fix/source/pitch_gain_tables.c", - "codecs/isac/fix/source/pitch_gain_tables.h", - "codecs/isac/fix/source/pitch_lag_tables.c", - "codecs/isac/fix/source/pitch_lag_tables.h", - "codecs/isac/fix/source/spectrum_ar_model_tables.c", - "codecs/isac/fix/source/spectrum_ar_model_tables.h", - "codecs/isac/fix/source/transform.c", - ] - - deps = [ - ":isac_bwinfo", - ":isac_common", - "../../api/audio_codecs:audio_codecs_api", - "../../common_audio", - "../../common_audio:common_audio_c", - "../../rtc_base:checks", - "../../rtc_base:compile_assert_c", - "../../rtc_base:rtc_base_approved", - "../../rtc_base:sanitizer", - "../../system_wrappers", - "../third_party/fft", - ] - - public_deps = [ ":isac_fix_common" ] # no-presubmit-check TODO(webrtc:8603) - - if (rtc_build_with_neon) { - deps += [ ":isac_neon" ] - - # TODO(bugs.webrtc.org/9579): Consider moving the usage of NEON from - # pitch_estimator_c.c into the "isac_neon" target and delete this flag: - if (current_cpu != "arm64") { - suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] - cflags = [ "-mfpu=neon" ] - } - } - - if (current_cpu == "arm" && arm_version >= 7) { - sources -= [ - "codecs/isac/fix/source/lattice_c.c", - "codecs/isac/fix/source/pitch_filter_c.c", - ] - deps += [ ":isac_fix_c_arm_asm" ] - } - - if (current_cpu == "mipsel") { - sources += [ - "codecs/isac/fix/source/entropy_coding_mips.c", - "codecs/isac/fix/source/filters_mips.c", - "codecs/isac/fix/source/lattice_mips.c", - "codecs/isac/fix/source/pitch_estimator_mips.c", - "codecs/isac/fix/source/transform_mips.c", - ] - sources -= [ - "codecs/isac/fix/source/lattice_c.c", - "codecs/isac/fix/source/pitch_estimator_c.c", - ] - if (mips_dsp_rev > 0) { - sources += [ "codecs/isac/fix/source/filterbanks_mips.c" ] - } - if (mips_dsp_rev > 1) { - sources += [ - "codecs/isac/fix/source/lpc_masking_model_mips.c", - "codecs/isac/fix/source/pitch_filter_mips.c", - ] - sources -= [ "codecs/isac/fix/source/pitch_filter_c.c" ] - } - } -} - -if (rtc_build_with_neon) { - rtc_library("isac_neon") { - poisonous = [ "audio_codecs" ] - sources = [ - "codecs/isac/fix/source/entropy_coding_neon.c", - "codecs/isac/fix/source/filterbanks_neon.c", - "codecs/isac/fix/source/filters_neon.c", - "codecs/isac/fix/source/lattice_neon.c", - "codecs/isac/fix/source/transform_neon.c", - ] - - if (current_cpu != "arm64") { - # Enable compilation for the NEON instruction set. - suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] - cflags = [ "-mfpu=neon" ] - } - - deps = [ - ":isac_fix_common", - "../../common_audio", - "../../common_audio:common_audio_c", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - ] - } +rtc_source_set("isac_bwinfo") { + sources = [ "codecs/isac/bandwidth_info.h" ] + deps = [] } rtc_library("pcm16b") { @@ -703,10 +427,10 @@ rtc_library("pcm16b") { ":legacy_encoded_audio_frame", "../../api:array_view", "../../api/audio_codecs:audio_codecs_api", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", ] - public_deps = [ ":pcm16b_c" ] # no-presubmit-check TODO(webrtc:8603) + public_deps += [ ":pcm16b_c" ] # no-presubmit-check TODO(webrtc:8603) } rtc_library("pcm16b_c") { @@ -752,27 +476,26 @@ rtc_library("webrtc_opus") { "../../api/audio_codecs:audio_codecs_api", "../../api/audio_codecs/opus:audio_encoder_opus_config", "../../common_audio", + "../../rtc_base:buffer", "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", "../../rtc_base:protobuf_utils", - "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", "../../rtc_base:safe_minmax", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", "../../system_wrappers:field_trial", ] absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] - public_deps = # no-presubmit-check TODO(webrtc:8603) + public_deps += # no-presubmit-check TODO(webrtc:8603) [ ":webrtc_opus_wrapper" ] defines = audio_codec_defines - - if (rtc_build_opus) { - public_deps += [ rtc_opus_dir ] # no-presubmit-check TODO(webrtc:8603) - } else if (build_with_mozilla) { - include_dirs = [ "/media/libopus/include" ] - } } rtc_library("webrtc_multiopus") { @@ -791,9 +514,10 @@ rtc_library("webrtc_multiopus") { "../../api/audio_codecs/opus:audio_decoder_opus_config", "../../api/audio_codecs/opus:audio_encoder_opus_config", "../../api/units:time_delta", + "../../rtc_base:buffer", "../../rtc_base:checks", "../../rtc_base:logging", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:macromagic", "../../rtc_base:safe_minmax", "../../rtc_base:stringutils", ] @@ -802,16 +526,10 @@ rtc_library("webrtc_multiopus") { "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] - public_deps = # no-presubmit-check TODO(webrtc:8603) + public_deps += # no-presubmit-check TODO(webrtc:8603) [ ":webrtc_opus_wrapper" ] defines = audio_codec_defines - - if (rtc_build_opus) { - public_deps += [ rtc_opus_dir ] - } else if (build_with_mozilla) { - include_dirs = [ "/media/libopus/include" ] - } } rtc_library("webrtc_opus_wrapper") { @@ -824,18 +542,19 @@ rtc_library("webrtc_opus_wrapper") { defines = audio_coding_defines + deps = [ + "../../api:array_view", + "../../rtc_base:checks", + "../../rtc_base:ignore_wundef", + "../../system_wrappers:field_trial", + ] + if (rtc_build_opus) { - public_deps = [ rtc_opus_dir ] # no-presubmit-check TODO(webrtc:8603) + deps += [ rtc_opus_dir ] + public_configs = [ "//third_party/opus:opus_config" ] } else if (build_with_mozilla) { include_dirs = [ getenv("DIST") + "/include/opus" ] } - - deps = [ - "../../rtc_base:checks", - "../../rtc_base:ignore_wundef", - "../../rtc_base:rtc_base_approved", - "../../system_wrappers:field_trial", - ] } if (rtc_enable_protobuf) { @@ -890,7 +609,7 @@ rtc_library("audio_network_adaptor") { "audio_network_adaptor/util/threshold_curve.h", ] - public_deps = # no-presubmit-check TODO(webrtc:8603) + public_deps += # no-presubmit-check TODO(webrtc:8603) [ ":audio_network_adaptor_config" ] deps = [ @@ -900,14 +619,17 @@ rtc_library("audio_network_adaptor") { "../../logging:rtc_event_audio", "../../rtc_base:checks", "../../rtc_base:ignore_wundef", + "../../rtc_base:logging", "../../rtc_base:protobuf_utils", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:safe_conversions", + "../../rtc_base:timeutils", "../../rtc_base/system:file_wrapper", "../../system_wrappers", "../../system_wrappers:field_trial", ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -964,6 +686,8 @@ rtc_library("neteq") { "neteq/normal.h", "neteq/packet.cc", "neteq/packet.h", + "neteq/packet_arrival_history.cc", + "neteq/packet_arrival_history.h", "neteq/packet_buffer.cc", "neteq/packet_buffer.h", "neteq/post_decode_vad.cc", @@ -974,6 +698,8 @@ rtc_library("neteq") { "neteq/random_vector.h", "neteq/red_payload_splitter.cc", "neteq/red_payload_splitter.h", + "neteq/reorder_optimizer.cc", + "neteq/reorder_optimizer.h", "neteq/statistics_calculator.cc", "neteq/statistics_calculator.h", "neteq/sync_buffer.cc", @@ -982,12 +708,13 @@ rtc_library("neteq") { "neteq/time_stretch.h", "neteq/timestamp_scaler.cc", "neteq/timestamp_scaler.h", + "neteq/underrun_optimizer.cc", + "neteq/underrun_optimizer.h", ] deps = [ ":audio_coding_module_typedefs", ":webrtc_cng", - "..:module_api", "..:module_api_public", "../../api:array_view", "../../api:rtp_headers", @@ -1001,9 +728,14 @@ rtc_library("neteq") { "../../common_audio", "../../common_audio:common_audio_c", "../../rtc_base:audio_format_to_string", + "../../rtc_base:buffer", "../../rtc_base:checks", + "../../rtc_base:event_tracer", "../../rtc_base:gtest_prod", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", "../../rtc_base:safe_minmax", "../../rtc_base:sanitizer", "../../rtc_base/experiments:field_trial_parser", @@ -1058,6 +790,7 @@ rtc_library("neteq_tools_minimal") { deps = [ ":default_neteq_factory", ":neteq", + "../../api:array_view", "../../api:neteq_simulator_api", "../../api:rtp_headers", "../../api/audio:audio_frame_api", @@ -1065,10 +798,12 @@ rtc_library("neteq_tools_minimal") { "../../api/neteq:custom_neteq_factory", "../../api/neteq:default_neteq_controller_factory", "../../api/neteq:neteq_api", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:copy_on_write_buffer", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", "../../system_wrappers", - "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] @@ -1086,8 +821,8 @@ rtc_library("neteq_test_tools") { "neteq/tools/constant_pcm_packet_source.h", "neteq/tools/initial_packet_inserter_neteq_input.cc", "neteq/tools/initial_packet_inserter_neteq_input.h", - "neteq/tools/neteq_packet_source_input.cc", - "neteq/tools/neteq_packet_source_input.h", + "neteq/tools/neteq_rtp_dump_input.cc", + "neteq/tools/neteq_rtp_dump_input.h", "neteq/tools/output_audio_file.h", "neteq/tools/output_wav_file.h", "neteq/tools/rtp_file_source.cc", @@ -1097,32 +832,24 @@ rtc_library("neteq_test_tools") { ] deps = [ + ":neteq_tools", + ":neteq_tools_minimal", ":pcm16b", "../../api:array_view", "../../api:rtp_headers", "../../common_audio", - "../../rtc_base", + "../../rtc_base:buffer", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:ssl", + "../../rtc_base:stringutils", "../../rtc_base/system:arch", "../../test:rtp_test_utils", - "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - - public_deps = [ - ":neteq_tools", - ":neteq_tools_minimal", + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] - - if (rtc_enable_protobuf) { - sources += [ - "neteq/tools/neteq_event_log_input.cc", - "neteq/tools/neteq_event_log_input.h", - ] - deps += [ ":rtc_event_log_source" ] - } } rtc_library("neteq_tools") { @@ -1141,11 +868,16 @@ rtc_library("neteq_tools") { ] deps = [ + ":neteq_input_audio_tools", + ":neteq_tools_minimal", "..:module_api_public", "../../api:array_view", "../../api/audio_codecs:audio_codecs_api", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", ] @@ -1154,10 +886,15 @@ rtc_library("neteq_tools") { "//third_party/abseil-cpp/absl/types:optional", ] - public_deps = [ - ":neteq_input_audio_tools", - ":neteq_tools_minimal", - ] + if (rtc_enable_protobuf) { + sources += [ + "neteq/tools/neteq_event_log_input.cc", + "neteq/tools/neteq_event_log_input.h", + ] + deps += [ "../../logging:rtc_event_log_parser" ] + public_deps += # no-presubmit-check TODO(webrtc:8603) + [ "../../logging:rtc_event_log_proto" ] + } } rtc_library("neteq_input_audio_tools") { @@ -1172,32 +909,11 @@ rtc_library("neteq_input_audio_tools") { deps = [ "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } if (rtc_enable_protobuf) { - rtc_library("rtc_event_log_source") { - testonly = true - - sources = [ - "neteq/tools/rtc_event_log_source.cc", - "neteq/tools/rtc_event_log_source.h", - ] - - deps = [ - ":neteq_tools_minimal", - "../../logging:rtc_event_log_parser", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../rtp_rtcp", - "../rtp_rtcp:rtp_rtcp_format", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - public_deps = # no-presubmit-check TODO(webrtc:8603) - [ "../../logging:rtc_event_log_proto" ] - } - # Only used for test purpose. Since we want to use it from chromium # (see audio_coding_modules_tests_shared below), we cannot guard it # under rtc_include_tests. @@ -1240,15 +956,13 @@ rtc_library("audio_coding_modules_tests_shared") { "../../api/audio_codecs:builtin_audio_decoder_factory", "../../api/audio_codecs:builtin_audio_encoder_factory", "../../api/neteq:neteq_api", - "../../rtc_base", "../../rtc_base:checks", "../../rtc_base:ignore_wundef", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:ssl", "../../rtc_base:stringutils", "../../system_wrappers", "../../test:fileutils", "../../test:test_support", - "../rtp_rtcp:rtp_rtcp_format", "//testing/gtest", ] absl_deps = [ @@ -1278,11 +992,6 @@ if (rtc_include_tests) { if (rtc_include_opus) { audio_coding_deps += [ ":webrtc_opus" ] } - if (current_cpu == "arm") { - audio_coding_deps += [ ":isac_fix" ] - } else { - audio_coding_deps += [ ":isac" ] - } if (!build_with_mozilla && !build_with_chromium) { audio_coding_deps += [ ":red" ] } @@ -1301,36 +1010,34 @@ if (rtc_include_tests) { ] } - group("audio_coding_tests") { - visibility += webrtc_default_visibility - testonly = true - public_deps = [ - ":acm_receive_test", - ":acm_send_test", - ":audio_codec_speed_tests", - ":audio_decoder_unittests", - ":audio_decoder_unittests", - ":g711_test", - ":g722_test", - ":ilbc_test", - ":isac_api_test", - ":isac_fix_test", - ":isac_switch_samprate_test", - ":isac_test", - ":neteq_ilbc_quality_test", - ":neteq_isac_quality_test", - ":neteq_opus_quality_test", - ":neteq_pcm16b_quality_test", - ":neteq_pcmu_quality_test", - ":neteq_speed_test", - ":rtp_analyze", - ":rtp_encode", - ":rtp_jitter", - ":rtpcat", - ":webrtc_opus_fec_test", - ] - if (rtc_enable_protobuf) { - public_deps += [ ":neteq_rtpplay" ] + if (!build_with_chromium) { + group("audio_coding_tests") { + visibility += webrtc_default_visibility + testonly = true + public_deps = [ # no-presubmit-check TODO(webrtc:8603) + ":acm_receive_test", + ":acm_send_test", + ":audio_codec_speed_tests", + ":audio_decoder_unittests", + ":audio_decoder_unittests", + ":g711_test", + ":g722_test", + ":ilbc_test", + ":neteq_ilbc_quality_test", + ":neteq_opus_quality_test", + ":neteq_pcm16b_quality_test", + ":neteq_pcmu_quality_test", + ":neteq_speed_test", + ":rtp_analyze", + ":rtp_encode", + ":rtp_jitter", + ":rtpcat", + ":webrtc_opus_fec_test", + ] + if (rtc_enable_protobuf) { + public_deps += # no-presubmit-check TODO(webrtc:8603) + [ ":neteq_rtpplay" ] + } } } @@ -1354,10 +1061,6 @@ if (rtc_include_tests) { "test/TestVADDTX.cc", "test/TestVADDTX.h", "test/Tester.cc", - "test/TwoWayCommunication.cc", - "test/TwoWayCommunication.h", - "test/iSACTest.cc", - "test/iSACTest.h", "test/target_delay_unittest.cc", ] deps = [ @@ -1382,17 +1085,17 @@ if (rtc_include_tests) { "../../api/audio_codecs/g722:audio_encoder_g722", "../../api/audio_codecs/ilbc:audio_decoder_ilbc", "../../api/audio_codecs/ilbc:audio_encoder_ilbc", - "../../api/audio_codecs/isac:audio_decoder_isac_float", - "../../api/audio_codecs/isac:audio_encoder_isac_float", "../../api/audio_codecs/opus:audio_decoder_opus", "../../api/audio_codecs/opus:audio_encoder_opus", "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", - "../../rtc_base/synchronization:rw_lock_wrapper", - "../../system_wrappers", "../../test:fileutils", + "../../test:scoped_key_value_config", "../../test:test_support", ] absl_deps = [ @@ -1414,13 +1117,16 @@ if (rtc_include_tests) { ":neteq_test_support", ":neteq_test_tools", "../../api/audio_codecs/opus:audio_encoder_opus", - "../../rtc_base:rtc_base_approved", + "../../api/test/metrics:global_metrics_logger_and_exporter", + "../../api/test/metrics:metric", + "../../rtc_base:macromagic", + "../../rtc_base:timeutils", "../../system_wrappers", - "../../system_wrappers:field_trial", "../../test:fileutils", - "../../test:perf_test", + "../../test:test_flags", "../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag" ] } rtc_library("acm_receive_test") { @@ -1432,17 +1138,18 @@ if (rtc_include_tests) { defines = audio_coding_defines - deps = audio_coding_deps + [ - "../../api:scoped_refptr", - "..:module_api", - ":audio_coding", - "../../api/audio_codecs:audio_codecs_api", - "../../api/audio_codecs:builtin_audio_decoder_factory", - ":neteq_tools", - "../../rtc_base:rtc_base_approved", - "../../test:test_support", - "//testing/gtest", - ] + deps = [ + ":audio_coding", + ":neteq_tools", + ":neteq_tools_minimal", + "../../api:scoped_refptr", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../test:test_support", + "//testing/gtest", + ] + + deps += audio_coding_deps absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } @@ -1455,62 +1162,71 @@ if (rtc_include_tests) { defines = audio_coding_defines - deps = audio_coding_deps + [ - "//third_party/abseil-cpp/absl/strings", - "../../api/audio:audio_frame_api", - "../../rtc_base:checks", - ":audio_coding", - ":neteq_tools", - "../../api/audio_codecs:builtin_audio_decoder_factory", - "../../api/audio_codecs:builtin_audio_encoder_factory", - "../../api/audio_codecs:audio_codecs_api", - "../../rtc_base:rtc_base_approved", - "../../test:test_support", - "//testing/gtest", - ] - } - - audio_decoder_unittests_resources = - [ "../../resources/audio_coding/testfile32kHz.pcm" ] - - if (is_ios) { - bundle_data("audio_decoder_unittests_bundle_data") { - testonly = true - sources = audio_decoder_unittests_resources - outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] - } - } - - rtc_test("audio_decoder_unittests") { - testonly = true - sources = [ "neteq/audio_decoder_unittest.cc" ] - - defines = neteq_defines - deps = [ - ":ilbc", - ":isac", - ":isac_fix", - ":neteq", - ":neteq_tools", - "../../test:fileutils", - "../../api/audio_codecs:audio_codecs_api", - "../../api/audio_codecs/opus:audio_encoder_opus", - "../../common_audio", - "../../rtc_base/system:arch", - "../../test:test_main", - "//testing/gtest", - "../../test:test_support", - ] + audio_coding_deps + ":audio_coding", + ":neteq_input_audio_tools", + ":neteq_tools", + ":neteq_tools_minimal", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../rtc_base:checks", + "../../rtc_base:stringutils", + "../../test:test_support", + "//testing/gtest", + ] + deps += audio_coding_deps + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } - data = audio_decoder_unittests_resources + if (!build_with_chromium) { + audio_decoder_unittests_resources = + [ "../../resources/audio_coding/testfile32kHz.pcm" ] - if (is_android) { - deps += [ "//testing/android/native_test:native_test_native_code" ] - shard_timeout = 900 - } if (is_ios) { - deps += [ ":audio_decoder_unittests_bundle_data" ] + bundle_data("audio_decoder_unittests_bundle_data") { + testonly = true + sources = audio_decoder_unittests_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + } + + rtc_test("audio_decoder_unittests") { + testonly = true + sources = [ "neteq/audio_decoder_unittest.cc" ] + + defines = neteq_defines + + deps = [ + ":ilbc", + ":neteq", + ":neteq_input_audio_tools", + ":neteq_tools", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../common_audio", + "../../rtc_base/system:arch", + "../../test:fileutils", + "../../test:test_main", + "../../test:test_support", + "//testing/gtest", + ] + audio_coding_deps + + data = audio_decoder_unittests_resources + + if (is_android) { + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] + shard_timeout = 900 + } + if (is_ios) { + deps += [ ":audio_decoder_unittests_bundle_data" ] + } } } @@ -1520,10 +1236,17 @@ if (rtc_include_tests) { visibility += webrtc_default_visibility defines = audio_codec_defines deps = [ + ":neteq_input_audio_tools", + ":neteq_tools", + ":neteq_tools_minimal", "../../rtc_base:checks", + "../../rtc_base:refcount", "../../test:fileutils", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] sources = [ "neteq/tools/neteq_test_factory.cc", "neteq/tools/neteq_test_factory.h", @@ -1534,13 +1257,14 @@ if (rtc_include_tests) { ":neteq_test_tools", "../../api/audio_codecs:builtin_audio_decoder_factory", "../../api/neteq:neteq_api", - "../../rtc_base:rtc_base_approved", - "../../test:audio_codec_mocks", + "../../test:audio_test_common", "../../test:field_trial", "../../test:test_support", ] } + } + if (rtc_enable_protobuf && !build_with_chromium) { rtc_executable("neteq_rtpplay") { testonly = true visibility += [ "*" ] @@ -1548,7 +1272,7 @@ if (rtc_include_tests) { deps = [ ":neteq_test_factory", ":neteq_test_tools", - "../../rtc_base:rtc_base_approved", + ":neteq_tools_minimal", "../../rtc_base:stringutils", "../../system_wrappers:field_trial", "../../test:field_trial", @@ -1561,51 +1285,59 @@ if (rtc_include_tests) { } } - audio_codec_speed_tests_resources = [ - "//resources/audio_coding/music_stereo_48kHz.pcm", - "//resources/audio_coding/speech_mono_16kHz.pcm", - "//resources/audio_coding/speech_mono_32_48kHz.pcm", - ] - - if (is_ios) { - bundle_data("audio_codec_speed_tests_data") { - testonly = true - sources = audio_codec_speed_tests_resources - outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] - } - } - - rtc_test("audio_codec_speed_tests") { - testonly = true - defines = [] - deps = [ "../../test:fileutils" ] - sources = [ - "codecs/isac/fix/test/isac_speed_test.cc", - "codecs/opus/opus_speed_test.cc", - "codecs/tools/audio_codec_speed_test.cc", - "codecs/tools/audio_codec_speed_test.h", + if (!build_with_chromium) { + audio_codec_speed_tests_resources = [ + "//resources/audio_coding/music_stereo_48kHz.pcm", + "//resources/audio_coding/speech_mono_16kHz.pcm", + "//resources/audio_coding/speech_mono_32_48kHz.pcm", ] - data = audio_codec_speed_tests_resources - - if (is_android) { - deps += [ "//testing/android/native_test:native_test_native_code" ] - shard_timeout = 900 - } - if (is_ios) { - deps += [ ":audio_codec_speed_tests_data" ] + bundle_data("audio_codec_speed_tests_data") { + testonly = true + sources = audio_codec_speed_tests_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } } - deps += [ - ":isac_fix", - ":webrtc_opus", - "../../rtc_base:rtc_base_approved", - "../../test:test_main", - "../../test:test_support", - "../audio_processing", - "//testing/gtest", - ] + rtc_test("audio_codec_speed_tests") { + testonly = true + defines = [] + deps = [ + "../../rtc_base:macromagic", + "../../test:fileutils", + ] + sources = [ + "codecs/opus/opus_speed_test.cc", + "codecs/tools/audio_codec_speed_test.cc", + "codecs/tools/audio_codec_speed_test.h", + ] + + data = audio_codec_speed_tests_resources + + if (is_android) { + use_default_launcher = false + deps += [ + "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", + "//testing/android/native_test:native_test_java", + "//testing/android/native_test:native_test_support", + ] + shard_timeout = 900 + } + + if (is_ios) { + deps += [ ":audio_codec_speed_tests_data" ] + } + + deps += [ + ":webrtc_opus", + "../../rtc_base:checks", + "../../test:test_main", + "../../test:test_support", + "../audio_processing", + "//testing/gtest", + ] + } } rtc_library("neteq_test_support") { @@ -1625,7 +1357,6 @@ if (rtc_include_tests) { "../../api/audio_codecs:builtin_audio_decoder_factory", "../../api/neteq:neteq_api", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", "../../system_wrappers", "../../test:fileutils", "../../test:test_support", @@ -1633,230 +1364,190 @@ if (rtc_include_tests) { ] } - rtc_library("neteq_quality_test_support") { - testonly = true - sources = [ - "neteq/tools/neteq_quality_test.cc", - "neteq/tools/neteq_quality_test.h", - ] + if (!build_with_chromium) { + rtc_library("neteq_quality_test_support") { + testonly = true + sources = [ + "neteq/tools/neteq_quality_test.cc", + "neteq/tools/neteq_quality_test.h", + ] - deps = [ - ":default_neteq_factory", - ":neteq", - ":neteq_test_tools", - "../../api/audio_codecs:builtin_audio_decoder_factory", - "../../api/neteq:neteq_api", - "../../rtc_base:checks", - "../../system_wrappers", - "../../test:fileutils", - "../../test:test_support", - "//testing/gtest", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag" ] - } + deps = [ + ":default_neteq_factory", + ":neteq", + ":neteq_input_audio_tools", + ":neteq_test_tools", + ":neteq_tools_minimal", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/neteq:neteq_api", + "../../rtc_base:checks", + "../../rtc_base:stringutils", + "../../system_wrappers", + "../../test:fileutils", + "../../test:test_support", + "//testing/gtest", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/strings", + ] + } - rtc_executable("rtp_encode") { - testonly = true + rtc_executable("rtp_encode") { + testonly = true - deps = audio_coding_deps + [ - "//third_party/abseil-cpp/absl/flags:flag", - "//third_party/abseil-cpp/absl/flags:parse", - ":audio_coding", - ":audio_encoder_cng", - ":neteq_input_audio_tools", - "../../api/audio:audio_frame_api", - "../../api/audio_codecs/g711:audio_encoder_g711", - "../../api/audio_codecs/L16:audio_encoder_L16", - "../../api/audio_codecs/g722:audio_encoder_g722", - "../../api/audio_codecs/ilbc:audio_encoder_ilbc", - "../../api/audio_codecs/isac:audio_encoder_isac", - "../../api/audio_codecs/opus:audio_encoder_opus", - "../../rtc_base:safe_conversions", - "//third_party/abseil-cpp/absl/memory", - ] + deps = [ + ":audio_coding", + ":audio_encoder_cng", + ":neteq_input_audio_tools", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs/L16:audio_encoder_L16", + "../../api/audio_codecs/g711:audio_encoder_g711", + "../../api/audio_codecs/g722:audio_encoder_g722", + "../../api/audio_codecs/ilbc:audio_encoder_ilbc", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../rtc_base:safe_conversions", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + "//third_party/abseil-cpp/absl/memory", + ] - sources = [ "neteq/tools/rtp_encode.cc" ] + deps += audio_coding_deps - defines = audio_coding_defines - } + sources = [ "neteq/tools/rtp_encode.cc" ] - rtc_executable("rtp_jitter") { - testonly = true + defines = audio_coding_defines + } - deps = audio_coding_deps + [ - "../rtp_rtcp:rtp_rtcp_format", - "../../api:array_view", - "../../rtc_base:rtc_base_approved", - ] + rtc_executable("rtp_jitter") { + testonly = true - sources = [ "neteq/tools/rtp_jitter.cc" ] + deps = [ + "../../api:array_view", + "../../rtc_base:buffer", + "../rtp_rtcp:rtp_rtcp_format", + ] - defines = audio_coding_defines - } + deps += audio_coding_deps - rtc_executable("rtpcat") { - testonly = true + sources = [ "neteq/tools/rtp_jitter.cc" ] - sources = [ "neteq/tools/rtpcat.cc" ] + defines = audio_coding_defines + } - deps = [ - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../test:rtp_test_utils", - "//testing/gtest", - ] - } + rtc_executable("rtpcat") { + testonly = true - rtc_executable("rtp_analyze") { - testonly = true + sources = [ "neteq/tools/rtpcat.cc" ] - sources = [ "neteq/tools/rtp_analyze.cc" ] + deps = [ + "../../rtc_base:checks", + "../../test:rtp_test_utils", + "//testing/gtest", + ] + } - deps = [ - ":neteq", - ":neteq_test_tools", - ":pcm16b", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - "//third_party/abseil-cpp/absl/flags:parse", - ] - } + rtc_executable("rtp_analyze") { + testonly = true - rtc_executable("neteq_opus_quality_test") { - testonly = true + sources = [ "neteq/tools/rtp_analyze.cc" ] - sources = [ "neteq/test/neteq_opus_quality_test.cc" ] + deps = [ + ":neteq", + ":neteq_test_tools", + ":neteq_tools_minimal", + ":pcm16b", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + ] + } - deps = [ - ":neteq", - ":neteq_quality_test_support", - ":neteq_tools", - ":webrtc_opus", - "../../rtc_base:rtc_base_approved", - "../../test:test_main", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - ] - } + rtc_executable("neteq_opus_quality_test") { + testonly = true - rtc_executable("neteq_speed_test") { - testonly = true + sources = [ "neteq/test/neteq_opus_quality_test.cc" ] - sources = [ "neteq/test/neteq_speed_test.cc" ] + deps = [ + ":neteq", + ":neteq_quality_test_support", + ":neteq_tools", + ":webrtc_opus", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } - deps = [ - ":neteq", - ":neteq_test_support", - "../../rtc_base:checks", - "../../test:test_support", - "//third_party/abseil-cpp/absl/flags:flag", - "//third_party/abseil-cpp/absl/flags:parse", - ] - } + rtc_executable("neteq_speed_test") { + testonly = true - rtc_executable("neteq_ilbc_quality_test") { - testonly = true + sources = [ "neteq/test/neteq_speed_test.cc" ] - sources = [ "neteq/test/neteq_ilbc_quality_test.cc" ] + deps = [ + ":neteq", + ":neteq_test_support", + "../../rtc_base:checks", + "../../test:test_support", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + ] + } - deps = [ - ":ilbc", - ":neteq", - ":neteq_quality_test_support", - ":neteq_tools", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../test:fileutils", - "../../test:test_main", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - ] - } + rtc_executable("neteq_ilbc_quality_test") { + testonly = true - rtc_executable("neteq_isac_quality_test") { - testonly = true + sources = [ "neteq/test/neteq_ilbc_quality_test.cc" ] - sources = [ "neteq/test/neteq_isac_quality_test.cc" ] + deps = [ + ":ilbc", + ":neteq", + ":neteq_quality_test_support", + ":neteq_tools", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + "../../test:fileutils", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } - deps = [ - ":isac_fix", - ":neteq", - ":neteq_quality_test_support", - "../../rtc_base:rtc_base_approved", - "../../test:test_main", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - ] - } + rtc_executable("neteq_pcmu_quality_test") { + testonly = true - rtc_executable("neteq_pcmu_quality_test") { - testonly = true + sources = [ "neteq/test/neteq_pcmu_quality_test.cc" ] - sources = [ "neteq/test/neteq_pcmu_quality_test.cc" ] + deps = [ + ":g711", + ":neteq", + ":neteq_quality_test_support", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + "../../test:fileutils", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } - deps = [ - ":g711", - ":neteq", - ":neteq_quality_test_support", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../test:fileutils", - "../../test:test_main", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - ] - } + rtc_executable("neteq_pcm16b_quality_test") { + testonly = true - rtc_executable("neteq_pcm16b_quality_test") { - testonly = true + sources = [ "neteq/test/neteq_pcm16b_quality_test.cc" ] - sources = [ "neteq/test/neteq_pcm16b_quality_test.cc" ] - - deps = [ - ":neteq", - ":neteq_quality_test_support", - ":pcm16b", - "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", - "../../test:fileutils", - "../../test:test_main", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - ] - } - - rtc_executable("isac_fix_test") { - testonly = true - - sources = [ "codecs/isac/fix/test/kenny.cc" ] - - deps = [ - ":isac_fix", - "../../test:perf_test", - "../../test:test_support", - ] - - data = [ "../../resources/speech_and_misc_wb.pcm" ] - } - - rtc_library("isac_test_util") { - testonly = true - sources = [ - "codecs/isac/main/util/utility.c", - "codecs/isac/main/util/utility.h", - ] - } - - rtc_executable("isac_test") { - testonly = true - - sources = [ "codecs/isac/main/test/simpleKenny.c" ] - - deps = [ - ":isac", - ":isac_test_util", - "../../rtc_base:rtc_base_approved", - ] + deps = [ + ":neteq", + ":neteq_quality_test_support", + ":pcm16b", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + "../../test:fileutils", + "../../test:test_main", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + ] + } } rtc_executable("g711_test") { @@ -1875,224 +1566,201 @@ if (rtc_include_tests) { deps = [ ":g722" ] } - rtc_executable("isac_api_test") { - testonly = true + if (!build_with_chromium) { + rtc_executable("ilbc_test") { + testonly = true - sources = [ "codecs/isac/main/test/ReleaseTest-API/ReleaseTest-API.cc" ] + sources = [ "codecs/ilbc/test/iLBC_test.c" ] - deps = [ - ":isac", - ":isac_test_util", - "../../rtc_base:rtc_base_approved", - ] - } + deps = [ ":ilbc" ] + } - rtc_executable("isac_switch_samprate_test") { - testonly = true + rtc_executable("webrtc_opus_fec_test") { + testonly = true - sources = [ "codecs/isac/main/test/SwitchingSampRate/SwitchingSampRate.cc" ] + sources = [ "codecs/opus/opus_fec_test.cc" ] - deps = [ - ":isac", - ":isac_test_util", - "../../common_audio", - "../../common_audio:common_audio_c", - ] - } - - rtc_executable("ilbc_test") { - testonly = true - - sources = [ "codecs/ilbc/test/iLBC_test.c" ] - - deps = [ ":ilbc" ] - } - - rtc_executable("webrtc_opus_fec_test") { - testonly = true - - sources = [ "codecs/opus/opus_fec_test.cc" ] - - deps = [ - ":webrtc_opus", - "../../common_audio", - "../../rtc_base:rtc_base_approved", - "../../test:fileutils", - "../../test:test_main", - "../../test:test_support", - "//testing/gtest", - ] - } - - rtc_library("audio_coding_unittests") { - testonly = true - visibility += webrtc_default_visibility - - sources = [ - "acm2/acm_receiver_unittest.cc", - "acm2/acm_remixing_unittest.cc", - "acm2/audio_coding_module_unittest.cc", - "acm2/call_statistics_unittest.cc", - "audio_network_adaptor/audio_network_adaptor_impl_unittest.cc", - "audio_network_adaptor/bitrate_controller_unittest.cc", - "audio_network_adaptor/channel_controller_unittest.cc", - "audio_network_adaptor/controller_manager_unittest.cc", - "audio_network_adaptor/dtx_controller_unittest.cc", - "audio_network_adaptor/event_log_writer_unittest.cc", - "audio_network_adaptor/fec_controller_plr_based_unittest.cc", - "audio_network_adaptor/frame_length_controller_unittest.cc", - "audio_network_adaptor/frame_length_controller_v2_unittest.cc", - "audio_network_adaptor/util/threshold_curve_unittest.cc", - "codecs/builtin_audio_decoder_factory_unittest.cc", - "codecs/builtin_audio_encoder_factory_unittest.cc", - "codecs/cng/audio_encoder_cng_unittest.cc", - "codecs/cng/cng_unittest.cc", - "codecs/ilbc/ilbc_unittest.cc", - "codecs/isac/fix/source/filterbanks_unittest.cc", - "codecs/isac/fix/source/filters_unittest.cc", - "codecs/isac/fix/source/lpc_masking_model_unittest.cc", - "codecs/isac/fix/source/transform_unittest.cc", - "codecs/isac/isac_webrtc_api_test.cc", - "codecs/isac/main/source/audio_encoder_isac_unittest.cc", - "codecs/isac/main/source/isac_unittest.cc", - "codecs/legacy_encoded_audio_frame_unittest.cc", - "codecs/opus/audio_decoder_multi_channel_opus_unittest.cc", - "codecs/opus/audio_encoder_multi_channel_opus_unittest.cc", - "codecs/opus/audio_encoder_opus_unittest.cc", - "codecs/opus/opus_bandwidth_unittest.cc", - "codecs/opus/opus_unittest.cc", - "codecs/red/audio_encoder_copy_red_unittest.cc", - "neteq/audio_multi_vector_unittest.cc", - "neteq/audio_vector_unittest.cc", - "neteq/background_noise_unittest.cc", - "neteq/buffer_level_filter_unittest.cc", - "neteq/comfort_noise_unittest.cc", - "neteq/decision_logic_unittest.cc", - "neteq/decoder_database_unittest.cc", - "neteq/delay_manager_unittest.cc", - "neteq/dsp_helper_unittest.cc", - "neteq/dtmf_buffer_unittest.cc", - "neteq/dtmf_tone_generator_unittest.cc", - "neteq/expand_unittest.cc", - "neteq/histogram_unittest.cc", - "neteq/merge_unittest.cc", - "neteq/mock/mock_decoder_database.h", - "neteq/mock/mock_dtmf_buffer.h", - "neteq/mock/mock_dtmf_tone_generator.h", - "neteq/mock/mock_expand.h", - "neteq/mock/mock_histogram.h", - "neteq/mock/mock_neteq_controller.h", - "neteq/mock/mock_packet_buffer.h", - "neteq/mock/mock_red_payload_splitter.h", - "neteq/mock/mock_statistics_calculator.h", - "neteq/nack_tracker_unittest.cc", - "neteq/neteq_decoder_plc_unittest.cc", - "neteq/neteq_impl_unittest.cc", - "neteq/neteq_network_stats_unittest.cc", - "neteq/neteq_stereo_unittest.cc", - "neteq/neteq_unittest.cc", - "neteq/normal_unittest.cc", - "neteq/packet_buffer_unittest.cc", - "neteq/post_decode_vad_unittest.cc", - "neteq/random_vector_unittest.cc", - "neteq/red_payload_splitter_unittest.cc", - "neteq/statistics_calculator_unittest.cc", - "neteq/sync_buffer_unittest.cc", - "neteq/time_stretch_unittest.cc", - "neteq/timestamp_scaler_unittest.cc", - "neteq/tools/input_audio_file_unittest.cc", - "neteq/tools/packet_unittest.cc", - ] - - deps = [ - ":acm_receive_test", - ":acm_send_test", - ":audio_coding", - ":audio_coding_module_typedefs", - ":audio_coding_modules_tests_shared", - ":audio_coding_opus_common", - ":audio_encoder_cng", - ":audio_network_adaptor", - ":default_neteq_factory", - ":g711", - ":ilbc", - ":isac", - ":isac_c", - ":isac_common", - ":isac_fix", - ":legacy_encoded_audio_frame", - ":mocks", - ":neteq", - ":neteq_test_support", - ":neteq_test_tools", - ":pcm16b", - ":red", - ":webrtc_cng", - ":webrtc_opus", - "..:module_api", - "..:module_api_public", - "../../api:array_view", - "../../api/audio:audio_frame_api", - "../../api/audio_codecs:audio_codecs_api", - "../../api/audio_codecs:builtin_audio_decoder_factory", - "../../api/audio_codecs:builtin_audio_encoder_factory", - "../../api/audio_codecs/isac:audio_decoder_isac_fix", - "../../api/audio_codecs/isac:audio_decoder_isac_float", - "../../api/audio_codecs/isac:audio_encoder_isac_fix", - "../../api/audio_codecs/isac:audio_encoder_isac_float", - "../../api/audio_codecs/opus:audio_decoder_multiopus", - "../../api/audio_codecs/opus:audio_decoder_opus", - "../../api/audio_codecs/opus:audio_encoder_multiopus", - "../../api/audio_codecs/opus:audio_encoder_opus", - "../../api/neteq:default_neteq_controller_factory", - "../../api/neteq:neteq_api", - "../../api/neteq:neteq_controller_api", - "../../api/neteq:tick_timer", - "../../api/neteq:tick_timer_unittest", - "../../api/rtc_event_log", - "../../common_audio", - "../../common_audio:common_audio_c", - "../../common_audio:mock_common_audio", - "../../logging:mocks", - "../../logging:rtc_event_audio", - "../../modules/rtp_rtcp:rtp_rtcp_format", - "../../rtc_base", - "../../rtc_base:checks", - "../../rtc_base:ignore_wundef", - "../../rtc_base:rtc_base_approved", - "../../rtc_base:rtc_base_tests_utils", - "../../rtc_base:sanitizer", - "../../rtc_base:timeutils", - "../../rtc_base/synchronization:mutex", - "../../rtc_base/system:arch", - "../../system_wrappers", - "../../test:audio_codec_mocks", - "../../test:field_trial", - "../../test:fileutils", - "../../test:rtc_expect_death", - "../../test:rtp_test_utils", - "../../test:test_common", - "../../test:test_support", - "codecs/opus/test", - "codecs/opus/test:test_unittest", - "//testing/gtest", - ] - absl_deps = [ - "//third_party/abseil-cpp/absl/flags:flag", - "//third_party/abseil-cpp/absl/memory", - "//third_party/abseil-cpp/absl/strings", - "//third_party/abseil-cpp/absl/types:optional", - ] - - defines = audio_coding_defines - - if (rtc_enable_protobuf) { - defines += [ "WEBRTC_NETEQ_UNITTEST_BITEXACT" ] - deps += [ - ":ana_config_proto", - ":neteq_unittest_proto", + deps = [ + ":webrtc_opus", + "../../common_audio", + "../../rtc_base:macromagic", + "../../test:fileutils", + "../../test:test_main", + "../../test:test_support", + "//testing/gtest", ] } + + rtc_library("audio_coding_unittests") { + testonly = true + visibility += webrtc_default_visibility + + sources = [ + "acm2/acm_receiver_unittest.cc", + "acm2/acm_remixing_unittest.cc", + "acm2/audio_coding_module_unittest.cc", + "acm2/call_statistics_unittest.cc", + "audio_network_adaptor/audio_network_adaptor_impl_unittest.cc", + "audio_network_adaptor/bitrate_controller_unittest.cc", + "audio_network_adaptor/channel_controller_unittest.cc", + "audio_network_adaptor/controller_manager_unittest.cc", + "audio_network_adaptor/dtx_controller_unittest.cc", + "audio_network_adaptor/event_log_writer_unittest.cc", + "audio_network_adaptor/fec_controller_plr_based_unittest.cc", + "audio_network_adaptor/frame_length_controller_unittest.cc", + "audio_network_adaptor/frame_length_controller_v2_unittest.cc", + "audio_network_adaptor/util/threshold_curve_unittest.cc", + "codecs/builtin_audio_decoder_factory_unittest.cc", + "codecs/builtin_audio_encoder_factory_unittest.cc", + "codecs/cng/audio_encoder_cng_unittest.cc", + "codecs/cng/cng_unittest.cc", + "codecs/ilbc/ilbc_unittest.cc", + "codecs/legacy_encoded_audio_frame_unittest.cc", + "codecs/opus/audio_decoder_multi_channel_opus_unittest.cc", + "codecs/opus/audio_encoder_multi_channel_opus_unittest.cc", + "codecs/opus/audio_encoder_opus_unittest.cc", + "codecs/opus/opus_bandwidth_unittest.cc", + "codecs/opus/opus_unittest.cc", + "codecs/red/audio_encoder_copy_red_unittest.cc", + "neteq/audio_multi_vector_unittest.cc", + "neteq/audio_vector_unittest.cc", + "neteq/background_noise_unittest.cc", + "neteq/buffer_level_filter_unittest.cc", + "neteq/comfort_noise_unittest.cc", + "neteq/decision_logic_unittest.cc", + "neteq/decoder_database_unittest.cc", + "neteq/delay_manager_unittest.cc", + "neteq/dsp_helper_unittest.cc", + "neteq/dtmf_buffer_unittest.cc", + "neteq/dtmf_tone_generator_unittest.cc", + "neteq/expand_unittest.cc", + "neteq/histogram_unittest.cc", + "neteq/merge_unittest.cc", + "neteq/mock/mock_buffer_level_filter.h", + "neteq/mock/mock_decoder_database.h", + "neteq/mock/mock_delay_manager.h", + "neteq/mock/mock_dtmf_buffer.h", + "neteq/mock/mock_dtmf_tone_generator.h", + "neteq/mock/mock_expand.h", + "neteq/mock/mock_histogram.h", + "neteq/mock/mock_neteq_controller.h", + "neteq/mock/mock_packet_buffer.h", + "neteq/mock/mock_red_payload_splitter.h", + "neteq/mock/mock_statistics_calculator.h", + "neteq/nack_tracker_unittest.cc", + "neteq/neteq_decoder_plc_unittest.cc", + "neteq/neteq_impl_unittest.cc", + "neteq/neteq_network_stats_unittest.cc", + "neteq/neteq_stereo_unittest.cc", + "neteq/neteq_unittest.cc", + "neteq/normal_unittest.cc", + "neteq/packet_arrival_history_unittest.cc", + "neteq/packet_buffer_unittest.cc", + "neteq/post_decode_vad_unittest.cc", + "neteq/random_vector_unittest.cc", + "neteq/red_payload_splitter_unittest.cc", + "neteq/reorder_optimizer_unittest.cc", + "neteq/statistics_calculator_unittest.cc", + "neteq/sync_buffer_unittest.cc", + "neteq/time_stretch_unittest.cc", + "neteq/timestamp_scaler_unittest.cc", + "neteq/tools/input_audio_file_unittest.cc", + "neteq/tools/packet_unittest.cc", + "neteq/underrun_optimizer_unittest.cc", + ] + + deps = [ + ":acm_receive_test", + ":acm_send_test", + ":audio_coding", + ":audio_coding_module_typedefs", + ":audio_coding_modules_tests_shared", + ":audio_coding_opus_common", + ":audio_encoder_cng", + ":audio_network_adaptor", + ":default_neteq_factory", + ":g711", + ":ilbc", + ":legacy_encoded_audio_frame", + ":mocks", + ":neteq", + ":neteq_input_audio_tools", + ":neteq_test_support", + ":neteq_test_tools", + ":neteq_tools", + ":neteq_tools_minimal", + ":pcm16b", + ":red", + ":webrtc_cng", + ":webrtc_opus", + "..:module_api", + "..:module_api_public", + "../../api:array_view", + "../../api/audio:audio_frame_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/audio_codecs:builtin_audio_decoder_factory", + "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/audio_codecs/opus:audio_decoder_multiopus", + "../../api/audio_codecs/opus:audio_decoder_opus", + "../../api/audio_codecs/opus:audio_encoder_multiopus", + "../../api/audio_codecs/opus:audio_encoder_opus", + "../../api/neteq:default_neteq_controller_factory", + "../../api/neteq:neteq_api", + "../../api/neteq:neteq_controller_api", + "../../api/neteq:tick_timer", + "../../api/neteq:tick_timer_unittest", + "../../api/rtc_event_log", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../common_audio:mock_common_audio", + "../../logging:mocks", + "../../logging:rtc_event_audio", + "../../modules/rtp_rtcp:rtp_rtcp_format", + "../../rtc_base:checks", + "../../rtc_base:ignore_wundef", + "../../rtc_base:macromagic", + "../../rtc_base:platform_thread", + "../../rtc_base:refcount", + "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_event", + "../../rtc_base:safe_conversions", + "../../rtc_base:sanitizer", + "../../rtc_base:ssl", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", + "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:arch", + "../../system_wrappers", + "../../test:audio_codec_mocks", + "../../test:audio_test_common", + "../../test:field_trial", + "../../test:fileutils", + "../../test:rtc_expect_death", + "../../test:rtp_test_utils", + "../../test:scoped_key_value_config", + "../../test:test_common", + "../../test:test_support", + "codecs/opus/test", + "codecs/opus/test:test_unittest", + "//testing/gtest", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + defines = audio_coding_defines + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_NETEQ_UNITTEST_BITEXACT" ] + deps += [ + ":ana_config_proto", + ":neteq_unittest_proto", + ] + } + } } } diff --git a/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h b/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h index 48a9b74..a747a7f 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h +++ b/webrtc/modules/audio_coding/codecs/isac/main/source/filter_functions.h @@ -11,6 +11,8 @@ #ifndef MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ #define MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_FILTER_FUNCTIONS_H_ +#include + #include "modules/audio_coding/codecs/isac/main/source/structs.h" void WebRtcIsac_AutoCorr(double* r, const double* x, size_t N, size_t order); diff --git a/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c b/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c index 61cd533..bf03dff 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c +++ b/webrtc/modules/audio_coding/codecs/isac/main/source/pitch_filter.c @@ -25,8 +25,8 @@ * Post-filtering: * y(z) = x(z) - damper(z) * gain * (x(z) + y(z)) * z ^ (-lag); * - * Note that |lag| is a floating number so we perform an interpolation to - * obtain the correct |lag|. + * Note that `lag` is a floating number so we perform an interpolation to + * obtain the correct `lag`. * */ @@ -86,7 +86,7 @@ typedef enum { * buffer : a buffer where the sum of previous inputs and outputs * are stored. * damper_state : the state of the damping filter. The filter is defined by - * |kDampFilter|. + * `kDampFilter`. * interpol_coeff : pointer to a set of coefficient which are used to utilize * fractional pitch by interpolation. * gain : pitch-gain to be applied to the current segment of input. @@ -140,9 +140,9 @@ static void FilterSegment(const double* in_data, PitchFilterParam* parameters, int j; double sum; double sum2; - /* Index of |parameters->buffer| where the output is written to. */ + /* Index of `parameters->buffer` where the output is written to. */ int pos = parameters->index + PITCH_BUFFSIZE; - /* Index of |parameters->buffer| where samples are read for fractional-lag + /* Index of `parameters->buffer` where samples are read for fractional-lag * computation. */ int pos_lag = pos - parameters->lag_offset; @@ -174,9 +174,9 @@ static void FilterSegment(const double* in_data, PitchFilterParam* parameters, /* Filter for fractional pitch. */ sum2 = 0.0; for (m = PITCH_FRACORDER-1; m >= m_tmp; --m) { - /* |lag_index + m| is always larger than or equal to zero, see how + /* `lag_index + m` is always larger than or equal to zero, see how * m_tmp is computed. This is equivalent to assume samples outside - * |out_dg[j]| are zero. */ + * `out_dg[j]` are zero. */ sum2 += out_dg[j][lag_index + m] * parameters->interpol_coeff[m]; } /* Add the contribution of differential gain change. */ @@ -353,7 +353,7 @@ static void FilterFrame(const double* in_data, PitchFiltstr* filter_state, if ((mode == kPitchFilterPreGain) || (mode == kPitchFilterPreLa)) { /* Filter the lookahead segment, this is treated as the last sub-frame. So - * set |pf_param| to last sub-frame. */ + * set `pf_param` to last sub-frame. */ filter_parameters.sub_frame = PITCH_SUBFRAMES - 1; filter_parameters.num_samples = QLOOKAHEAD; FilterSegment(in_data, &filter_parameters, out_data, out_dg); diff --git a/webrtc/modules/audio_processing/BUILD.gn b/webrtc/modules/audio_processing/BUILD.gn index dbb1882..2b81427 100644 --- a/webrtc/modules/audio_processing/BUILD.gn +++ b/webrtc/modules/audio_processing/BUILD.gn @@ -19,15 +19,6 @@ config("apm_debug_dump") { } } -rtc_library("config") { - visibility = [ ":*" ] - sources = [ - "include/config.cc", - "include/config.h", - ] - deps = [ "../../rtc_base/system:rtc_export" ] -} - rtc_library("api") { visibility = [ "*" ] sources = [ @@ -37,20 +28,23 @@ rtc_library("api") { deps = [ ":audio_frame_view", ":audio_processing_statistics", - ":config", "../../api:array_view", "../../api:scoped_refptr", "../../api/audio:aec3_config", "../../api/audio:audio_frame_api", "../../api/audio:echo_control", - "../../rtc_base:deprecation", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:macromagic", + "../../rtc_base:refcount", + "../../rtc_base:stringutils", "../../rtc_base/system:arch", "../../rtc_base/system:file_wrapper", "../../rtc_base/system:rtc_export", "agc:gain_control_interface", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] } rtc_library("audio_frame_proxies") { @@ -119,7 +113,41 @@ rtc_source_set("aec_dump_interface") { deps = [ ":api", ":audio_frame_view", - "../../rtc_base:deprecation", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("gain_controller2") { + configs += [ ":apm_debug_dump" ] + sources = [ + "gain_controller2.cc", + "gain_controller2.h", + ] + defines = [] + deps = [ + ":aec_dump_interface", + ":api", + ":apm_logging", + ":audio_buffer", + ":audio_frame_view", + "../../common_audio", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:stringutils", + "../../system_wrappers:field_trial", + "agc2:adaptive_digital_gain_controller", + "agc2:common", + "agc2:cpu_features", + "agc2:fixed_digital", + "agc2:gain_applier", + "agc2:input_volume_controller", + "agc2:noise_level_estimator", + "agc2:saturation_protector", + "agc2:speech_level_estimator", + "agc2:vad_wrapper", ] } @@ -130,28 +158,11 @@ rtc_library("audio_processing") { "audio_processing_builder_impl.cc", "audio_processing_impl.cc", "audio_processing_impl.h", - "common.h", "echo_control_mobile_impl.cc", "echo_control_mobile_impl.h", - "echo_detector/circular_buffer.cc", - "echo_detector/circular_buffer.h", - "echo_detector/mean_variance_estimator.cc", - "echo_detector/mean_variance_estimator.h", - "echo_detector/moving_max.cc", - "echo_detector/moving_max.h", - "echo_detector/normalized_covariance_estimator.cc", - "echo_detector/normalized_covariance_estimator.h", "gain_control_impl.cc", "gain_control_impl.h", - "gain_controller2.cc", - "gain_controller2.h", - "level_estimator.cc", - "level_estimator.h", "render_queue_item_verifier.h", - "residual_echo_detector.cc", - "residual_echo_detector.h", - "typing_detection.cc", - "typing_detection.h", ] defines = [] @@ -163,13 +174,13 @@ rtc_library("audio_processing") { ":audio_frame_proxies", ":audio_frame_view", ":audio_processing_statistics", - ":config", + ":gain_controller2", ":high_pass_filter", ":optionally_built_submodule_creators", ":rms_level", - ":voice_detection", "../../api:array_view", "../../api:function_view", + "../../api:make_ref_counted", "../../api/audio:aec3_config", "../../api/audio:audio_frame_api", "../../api/audio:echo_control", @@ -177,15 +188,20 @@ rtc_library("audio_processing") { "../../common_audio:common_audio_c", "../../common_audio/third_party/ooura:fft_size_256", "../../rtc_base:checks", - "../../rtc_base:deprecation", + "../../rtc_base:event_tracer", "../../rtc_base:gtest_prod", "../../rtc_base:ignore_wundef", - "../../rtc_base:refcount", + "../../rtc_base:logging", + "../../rtc_base:macromagic", "../../rtc_base:safe_minmax", "../../rtc_base:sanitizer", + "../../rtc_base:swap_queue", + "../../rtc_base:timeutils", + "../../rtc_base/experiments:field_trial_parser", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:rtc_export", "../../system_wrappers", + "../../system_wrappers:denormal_disabler", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", "aec3", @@ -194,20 +210,21 @@ rtc_library("audio_processing") { "agc", "agc:gain_control_interface", "agc:legacy_agc", - "agc2:adaptive_digital", - "agc2:fixed_digital", - "agc2:gain_applier", + "agc2:input_volume_stats_reporter", + "capture_levels_adjuster", "ns", "transient:transient_suppressor_api", "vad", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] deps += [ "../../common_audio", "../../common_audio:fir_filter", "../../common_audio:fir_filter_factory", - "../../rtc_base:rtc_base_approved", "../../system_wrappers", ] @@ -218,18 +235,30 @@ rtc_library("audio_processing") { } } -rtc_library("voice_detection") { +rtc_library("residual_echo_detector") { + poisonous = [ "default_echo_detector" ] + configs += [ ":apm_debug_dump" ] sources = [ - "voice_detection.cc", - "voice_detection.h", + "echo_detector/circular_buffer.cc", + "echo_detector/circular_buffer.h", + "echo_detector/mean_variance_estimator.cc", + "echo_detector/mean_variance_estimator.h", + "echo_detector/moving_max.cc", + "echo_detector/moving_max.h", + "echo_detector/normalized_covariance_estimator.cc", + "echo_detector/normalized_covariance_estimator.h", + "residual_echo_detector.cc", + "residual_echo_detector.h", ] deps = [ ":api", - ":audio_buffer", - "../../api/audio:audio_frame_api", - "../../common_audio:common_audio_c", + ":apm_logging", + "../../api:array_view", "../../rtc_base:checks", + "../../rtc_base:logging", + "../../system_wrappers:metrics", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("optionally_built_submodule_creators") { @@ -289,7 +318,11 @@ rtc_library("apm_logging") { "../../api:array_view", "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:stringutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] defines = [] } @@ -306,143 +339,165 @@ if (rtc_include_tests) { ":audio_processing_statistics", "../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } - group("audio_processing_tests") { - testonly = true - deps = [ - ":audioproc_test_utils", - "transient:click_annotate", - "transient:transient_suppression_test", - ] - - if (rtc_enable_protobuf) { - deps += [ - ":audioproc_unittest_proto", - "aec_dump:aec_dump_unittests", - "test/conversational_speech", - "test/py_quality_assessment", - ] - } - } - - rtc_library("audio_processing_unittests") { - testonly = true - - configs += [ ":apm_debug_dump" ] - sources = [ - "audio_buffer_unittest.cc", - "audio_frame_view_unittest.cc", - "config_unittest.cc", - "echo_control_mobile_unittest.cc", - "gain_controller2_unittest.cc", - "splitting_filter_unittest.cc", - "test/fake_recording_device_unittest.cc", - ] - - deps = [ - ":analog_mic_simulation", - ":api", - ":apm_logging", - ":audio_buffer", - ":audio_frame_view", - ":audio_processing", - ":audioproc_test_utils", - ":config", - ":high_pass_filter", - ":mocks", - ":voice_detection", - "../../api:array_view", - "../../api:scoped_refptr", - "../../api/audio:aec3_config", - "../../api/audio:aec3_factory", - "../../common_audio", - "../../common_audio:common_audio_c", - "../../rtc_base", - "../../rtc_base:checks", - "../../rtc_base:gtest_prod", - "../../rtc_base:ignore_wundef", - "../../rtc_base:protobuf_utils", - "../../rtc_base:rtc_base_approved", - "../../rtc_base:rtc_base_tests_utils", - "../../rtc_base:safe_minmax", - "../../rtc_base:task_queue_for_test", - "../../rtc_base/synchronization:mutex", - "../../rtc_base/system:arch", - "../../rtc_base/system:file_wrapper", - "../../system_wrappers", - "../../test:fileutils", - "../../test:rtc_expect_death", - "../../test:test_support", - "../audio_coding:neteq_input_audio_tools", - "aec_dump:mock_aec_dump_unittests", - "agc:agc_unittests", - "agc2:adaptive_digital_unittests", - "agc2:biquad_filter_unittests", - "agc2:fixed_digital_unittests", - "agc2:noise_estimator_unittests", - "agc2:rnn_vad_with_level_unittests", - "agc2:test_utils", - "agc2/rnn_vad:unittests", - "test/conversational_speech:unittest", - "transient:transient_suppression_unittests", - "utility:legacy_delay_estimator_unittest", - "utility:pffft_wrapper_unittest", - "vad:vad_unittests", - "//testing/gtest", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - - defines = [] - - if (rtc_prefer_fixed_point) { - defines += [ "WEBRTC_AUDIOPROC_FIXED_PROFILE" ] - } else { - defines += [ "WEBRTC_AUDIOPROC_FLOAT_PROFILE" ] - } - - if (rtc_enable_protobuf) { - defines += [ "WEBRTC_AUDIOPROC_DEBUG_DUMP" ] - deps += [ - ":audioproc_debug_proto", - ":audioproc_protobuf_utils", + if (!build_with_chromium) { + group("audio_processing_tests") { + testonly = true + deps = [ ":audioproc_test_utils", - ":audioproc_unittest_proto", - ":optionally_built_submodule_creators", - ":rms_level", - ":runtime_settings_protobuf_utils", - "../../api/audio:audio_frame_api", - "../../api/audio:echo_control", + "transient:click_annotate", + "transient:transient_suppression_test", + ] + + if (rtc_enable_protobuf) { + deps += [ + ":audioproc_unittest_proto", + "aec_dump:aec_dump_unittests", + "test/conversational_speech", + "test/py_quality_assessment", + ] + } + } + + rtc_library("audio_processing_unittests") { + testonly = true + + configs += [ ":apm_debug_dump" ] + sources = [ + "audio_buffer_unittest.cc", + "audio_frame_view_unittest.cc", + "echo_control_mobile_unittest.cc", + "gain_controller2_unittest.cc", + "splitting_filter_unittest.cc", + "test/echo_canceller3_config_json_unittest.cc", + "test/fake_recording_device_unittest.cc", + ] + + deps = [ + ":aec3_config_json", + ":analog_mic_simulation", + ":api", + ":apm_logging", + ":audio_buffer", + ":audio_frame_view", + ":audio_processing", + ":audioproc_test_utils", + ":gain_controller2", + ":high_pass_filter", + ":mocks", + "../../api:array_view", + "../../api:make_ref_counted", + "../../api:scoped_refptr", + "../../api/audio:aec3_config", + "../../api/audio:aec3_factory", + "../../api/audio:echo_detector_creator", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../rtc_base:checks", + "../../rtc_base:gtest_prod", + "../../rtc_base:ignore_wundef", + "../../rtc_base:macromagic", + "../../rtc_base:platform_thread", + "../../rtc_base:protobuf_utils", + "../../rtc_base:random", "../../rtc_base:rtc_base_tests_utils", - "../../rtc_base:rtc_task_queue", - "aec_dump", - "aec_dump:aec_dump_unittests", + "../../rtc_base:rtc_event", + "../../rtc_base:safe_conversions", + "../../rtc_base:safe_minmax", + "../../rtc_base:stringutils", + "../../rtc_base:swap_queue", + "../../rtc_base:task_queue_for_test", + "../../rtc_base:threading", + "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:arch", + "../../rtc_base/system:file_wrapper", + "../../system_wrappers", + "../../system_wrappers:denormal_disabler", + "../../test:field_trial", + "../../test:fileutils", + "../../test:rtc_expect_death", + "../../test:test_support", + "../audio_coding:neteq_input_audio_tools", + "aec_dump:mock_aec_dump_unittests", + "agc:agc_unittests", + "agc2:adaptive_digital_gain_controller_unittest", + "agc2:biquad_filter_unittests", + "agc2:fixed_digital_unittests", + "agc2:gain_applier_unittest", + "agc2:input_volume_controller_unittests", + "agc2:input_volume_stats_reporter_unittests", + "agc2:noise_estimator_unittests", + "agc2:saturation_protector_unittest", + "agc2:speech_level_estimator_unittest", + "agc2:test_utils", + "agc2:vad_wrapper_unittests", + "agc2/rnn_vad:unittests", + "capture_levels_adjuster", + "capture_levels_adjuster:capture_levels_adjuster_unittests", + "test/conversational_speech:unittest", + "transient:transient_suppression_unittests", + "utility:legacy_delay_estimator_unittest", + "utility:pffft_wrapper_unittest", + "vad:vad_unittests", + "//testing/gtest", ] - absl_deps += [ "//third_party/abseil-cpp/absl/flags:flag" ] - sources += [ - "audio_processing_impl_locking_unittest.cc", - "audio_processing_impl_unittest.cc", - "audio_processing_unittest.cc", - "echo_control_mobile_bit_exact_unittest.cc", - "echo_detector/circular_buffer_unittest.cc", - "echo_detector/mean_variance_estimator_unittest.cc", - "echo_detector/moving_max_unittest.cc", - "echo_detector/normalized_covariance_estimator_unittest.cc", - "gain_control_unittest.cc", - "high_pass_filter_unittest.cc", - "level_estimator_unittest.cc", - "residual_echo_detector_unittest.cc", - "rms_level_unittest.cc", - "test/debug_dump_replayer.cc", - "test/debug_dump_replayer.h", - "test/debug_dump_test.cc", - "test/echo_canceller_test_tools.cc", - "test/echo_canceller_test_tools.h", - "test/echo_canceller_test_tools_unittest.cc", - "test/echo_control_mock.h", - "test/test_utils.h", - "voice_detection_unittest.cc", + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] + + defines = [] + + if (rtc_prefer_fixed_point) { + defines += [ "WEBRTC_AUDIOPROC_FIXED_PROFILE" ] + } else { + defines += [ "WEBRTC_AUDIOPROC_FLOAT_PROFILE" ] + } + + if (rtc_enable_protobuf) { + defines += [ "WEBRTC_AUDIOPROC_DEBUG_DUMP" ] + deps += [ + ":audioproc_debug_proto", + ":audioproc_protobuf_utils", + ":audioproc_test_utils", + ":audioproc_unittest_proto", + ":optionally_built_submodule_creators", + ":residual_echo_detector", + ":rms_level", + ":runtime_settings_protobuf_utils", + "../../api/audio:audio_frame_api", + "../../api/audio:echo_control", + "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_task_queue", + "aec_dump", + "aec_dump:aec_dump_unittests", + ] + absl_deps += [ "//third_party/abseil-cpp/absl/flags:flag" ] + sources += [ + "audio_processing_impl_locking_unittest.cc", + "audio_processing_impl_unittest.cc", + "audio_processing_unittest.cc", + "echo_control_mobile_bit_exact_unittest.cc", + "echo_detector/circular_buffer_unittest.cc", + "echo_detector/mean_variance_estimator_unittest.cc", + "echo_detector/moving_max_unittest.cc", + "echo_detector/normalized_covariance_estimator_unittest.cc", + "gain_control_unittest.cc", + "high_pass_filter_unittest.cc", + "residual_echo_detector_unittest.cc", + "rms_level_unittest.cc", + "test/debug_dump_replayer.cc", + "test/debug_dump_replayer.h", + "test/debug_dump_test.cc", + "test/echo_canceller_test_tools.cc", + "test/echo_canceller_test_tools.h", + "test/echo_canceller_test_tools_unittest.cc", + "test/echo_control_mock.h", + "test/test_utils.h", + ] + } } } @@ -455,12 +510,18 @@ if (rtc_include_tests) { ":audio_processing", ":audioproc_test_utils", "../../api:array_view", + "../../api/numerics", + "../../api/test/metrics:global_metrics_logger_and_exporter", + "../../api/test/metrics:metric", + "../../rtc_base:platform_thread", "../../rtc_base:protobuf_utils", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:random", + "../../rtc_base:rtc_event", + "../../rtc_base:safe_conversions", "../../system_wrappers", - "../../test:perf_test", "../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("analog_mic_simulation") { @@ -473,14 +534,15 @@ if (rtc_include_tests) { "../../api/audio:audio_frame_api", "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:logging", + "../../rtc_base:safe_conversions", "../../rtc_base:safe_minmax", - "agc:gain_map", + "agc2:gain_map", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } - if (rtc_enable_protobuf) { + if (rtc_enable_protobuf && !build_with_chromium) { rtc_library("audioproc_f_impl") { testonly = true configs += [ ":apm_debug_dump" ] @@ -498,6 +560,7 @@ if (rtc_include_tests) { ] deps = [ + ":aec3_config_json", ":analog_mic_simulation", ":api", ":apm_logging", @@ -506,15 +569,18 @@ if (rtc_include_tests) { ":audioproc_protobuf_utils", ":audioproc_test_utils", ":runtime_settings_protobuf_utils", - "../../api/audio:aec3_config_json", "../../api/audio:aec3_factory", + "../../api/audio:echo_detector_creator", "../../common_audio", "../../rtc_base:checks", "../../rtc_base:ignore_wundef", + "../../rtc_base:logging", "../../rtc_base:protobuf_utils", - "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_json", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", "../../rtc_base:task_queue_for_test", + "../../rtc_base:timeutils", "../../rtc_base/system:file_wrapper", "../../system_wrappers", "../../system_wrappers:field_trial", @@ -549,7 +615,6 @@ if (rtc_include_tests) { "../../rtc_base:checks", "../../rtc_base:ignore_wundef", "../../rtc_base:protobuf_utils", - "../../rtc_base:rtc_base_approved", "../../rtc_base/system:arch", ] } @@ -599,7 +664,7 @@ rtc_library("audioproc_test_utils") { "../../api/audio:audio_frame_api", "../../common_audio", "../../rtc_base:checks", - "../../rtc_base:rtc_base_approved", + "../../rtc_base:random", "../../rtc_base/system:arch", "../../system_wrappers", "../../test:fileutils", @@ -607,5 +672,26 @@ rtc_library("audioproc_test_utils") { "../audio_coding:neteq_input_audio_tools", "//testing/gtest", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("aec3_config_json") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/echo_canceller3_config_json.cc", + "test/echo_canceller3_config_json.h", + ] + deps = [ + "../../api/audio:aec3_config", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:rtc_json", + "../../rtc_base:stringutils", + "../../rtc_base/system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } diff --git a/webrtc/modules/audio_processing/aec3/BUILD.gn b/webrtc/modules/audio_processing/aec3/BUILD.gn index c98fa4c..f5eb5d5 100644 --- a/webrtc/modules/audio_processing/aec3/BUILD.gn +++ b/webrtc/modules/audio_processing/aec3/BUILD.gn @@ -22,6 +22,7 @@ rtc_library("aec3") { "alignment_mixer.h", "api_call_jitter_metrics.cc", "api_call_jitter_metrics.h", + "block.h", "block_buffer.cc", "block_delay_buffer.cc", "block_delay_buffer.h", @@ -37,6 +38,8 @@ rtc_library("aec3") { "coarse_filter_update_gain.h", "comfort_noise_generator.cc", "comfort_noise_generator.h", + "config_selector.cc", + "config_selector.h", "decimator.cc", "decimator.h", "delay_estimate.h", @@ -72,6 +75,8 @@ rtc_library("aec3") { "matched_filter_lag_aggregator.h", "moving_average.cc", "moving_average.h", + "multi_channel_content_detector.cc", + "multi_channel_content_detector.h", "nearend_detector.h", "refined_filter_update_gain.cc", "refined_filter_update_gain.h", @@ -140,8 +145,11 @@ rtc_library("aec3") { "../../../api/audio:echo_control", "../../../common_audio:common_audio_c", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base:race_checker", "../../../rtc_base:safe_minmax", + "../../../rtc_base:swap_queue", "../../../rtc_base/experiments:field_trial_parser", "../../../rtc_base/system:arch", "../../../system_wrappers", @@ -149,7 +157,10 @@ rtc_library("aec3") { "../../../system_wrappers:metrics", "../utility:cascaded_biquad_filter", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] if (current_cpu == "x86" || current_cpu == "x64") { deps += [ ":aec3_avx2" ] @@ -168,13 +179,13 @@ rtc_source_set("aec3_fft") { "../../../api:array_view", "../../../common_audio/third_party/ooura:fft_size_128", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", "../../../rtc_base/system:arch", ] } rtc_source_set("render_buffer") { sources = [ + "block.h", "block_buffer.h", "fft_buffer.h", "render_buffer.h", @@ -185,7 +196,6 @@ rtc_source_set("render_buffer") { ":fft_data", "../../../api:array_view", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", "../../../rtc_base/system:arch", ] } @@ -201,6 +211,7 @@ rtc_source_set("adaptive_fir_filter") { "../../../api:array_view", "../../../rtc_base/system:arch", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_source_set("adaptive_fir_filter_erl") { @@ -217,9 +228,10 @@ rtc_source_set("matched_filter") { deps = [ ":aec3_common", "../../../api:array_view", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:gtest_prod", "../../../rtc_base/system:arch", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_source_set("vector_math") { @@ -302,15 +314,17 @@ if (rtc_include_tests) { "..:apm_logging", "..:audio_buffer", "..:audio_processing", - "..:audio_processing_unittests", "..:high_pass_filter", "../../../api:array_view", "../../../api/audio:aec3_config", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:macromagic", + "../../../rtc_base:random", "../../../rtc_base:safe_minmax", + "../../../rtc_base:stringutils", "../../../rtc_base/system:arch", "../../../system_wrappers", + "../../../system_wrappers:metrics", "../../../test:field_trial", "../../../test:test_support", "../utility:cascaded_biquad_filter", @@ -334,6 +348,7 @@ if (rtc_include_tests) { "clockdrift_detector_unittest.cc", "coarse_filter_update_gain_unittest.cc", "comfort_noise_generator_unittest.cc", + "config_selector_unittest.cc", "decimator_unittest.cc", "echo_canceller3_unittest.cc", "echo_path_delay_estimator_unittest.cc", @@ -348,6 +363,7 @@ if (rtc_include_tests) { "matched_filter_lag_aggregator_unittest.cc", "matched_filter_unittest.cc", "moving_average_unittest.cc", + "multi_channel_content_detector_unittest.cc", "refined_filter_update_gain_unittest.cc", "render_buffer_unittest.cc", "render_delay_buffer_unittest.cc", @@ -363,5 +379,9 @@ if (rtc_include_tests) { "vector_math_unittest.cc", ] } + + if (!build_with_chromium) { + deps += [ "..:audio_processing_unittests" ] + } } } diff --git a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc index bf3a780..917aa95 100644 --- a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc +++ b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.cc @@ -68,19 +68,21 @@ void ComputeFrequencyResponse_Neon( RTC_DCHECK_EQ(H.size(), H2->capacity()); for (size_t p = 0; p < num_partitions; ++p) { RTC_DCHECK_EQ(kFftLengthBy2Plus1, (*H2)[p].size()); + auto& H2_p = (*H2)[p]; for (size_t ch = 0; ch < num_render_channels; ++ch) { + const FftData& H_p_ch = H[p][ch]; for (size_t j = 0; j < kFftLengthBy2; j += 4) { - const float32x4_t re = vld1q_f32(&H[p][ch].re[j]); - const float32x4_t im = vld1q_f32(&H[p][ch].im[j]); + const float32x4_t re = vld1q_f32(&H_p_ch.re[j]); + const float32x4_t im = vld1q_f32(&H_p_ch.im[j]); float32x4_t H2_new = vmulq_f32(re, re); H2_new = vmlaq_f32(H2_new, im, im); - float32x4_t H2_p_j = vld1q_f32(&(*H2)[p][j]); + float32x4_t H2_p_j = vld1q_f32(&H2_p[j]); H2_p_j = vmaxq_f32(H2_p_j, H2_new); - vst1q_f32(&(*H2)[p][j], H2_p_j); + vst1q_f32(&H2_p[j], H2_p_j); } - float H2_new = H[p][ch].re[kFftLengthBy2] * H[p][ch].re[kFftLengthBy2] + - H[p][ch].im[kFftLengthBy2] * H[p][ch].im[kFftLengthBy2]; - (*H2)[p][kFftLengthBy2] = std::max((*H2)[p][kFftLengthBy2], H2_new); + float H2_new = H_p_ch.re[kFftLengthBy2] * H_p_ch.re[kFftLengthBy2] + + H_p_ch.im[kFftLengthBy2] * H_p_ch.im[kFftLengthBy2]; + H2_p[kFftLengthBy2] = std::max(H2_p[kFftLengthBy2], H2_new); } } } @@ -101,20 +103,22 @@ void ComputeFrequencyResponse_Sse2( // constexpr __mmmask8 kMaxMask = static_cast<__mmmask8>(256u); for (size_t p = 0; p < num_partitions; ++p) { RTC_DCHECK_EQ(kFftLengthBy2Plus1, (*H2)[p].size()); + auto& H2_p = (*H2)[p]; for (size_t ch = 0; ch < num_render_channels; ++ch) { + const FftData& H_p_ch = H[p][ch]; for (size_t j = 0; j < kFftLengthBy2; j += 4) { - const __m128 re = _mm_loadu_ps(&H[p][ch].re[j]); + const __m128 re = _mm_loadu_ps(&H_p_ch.re[j]); const __m128 re2 = _mm_mul_ps(re, re); - const __m128 im = _mm_loadu_ps(&H[p][ch].im[j]); + const __m128 im = _mm_loadu_ps(&H_p_ch.im[j]); const __m128 im2 = _mm_mul_ps(im, im); const __m128 H2_new = _mm_add_ps(re2, im2); - __m128 H2_k_j = _mm_loadu_ps(&(*H2)[p][j]); + __m128 H2_k_j = _mm_loadu_ps(&H2_p[j]); H2_k_j = _mm_max_ps(H2_k_j, H2_new); - _mm_storeu_ps(&(*H2)[p][j], H2_k_j); + _mm_storeu_ps(&H2_p[j], H2_k_j); } - float H2_new = H[p][ch].re[kFftLengthBy2] * H[p][ch].re[kFftLengthBy2] + - H[p][ch].im[kFftLengthBy2] * H[p][ch].im[kFftLengthBy2]; - (*H2)[p][kFftLengthBy2] = std::max((*H2)[p][kFftLengthBy2], H2_new); + float H2_new = H_p_ch.re[kFftLengthBy2] * H_p_ch.re[kFftLengthBy2] + + H_p_ch.im[kFftLengthBy2] * H_p_ch.im[kFftLengthBy2]; + H2_p[kFftLengthBy2] = std::max(H2_p[kFftLengthBy2], H2_new); } } } diff --git a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h index 7597709..34c06f4 100644 --- a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h +++ b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter.h @@ -16,6 +16,7 @@ #include #include +#include "absl/strings/string_view.h" #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec3_fft.h" @@ -141,7 +142,7 @@ class AdaptiveFirFilter { // Returns the maximum number of partitions for the filter. size_t max_filter_size_partitions() const { return max_size_partitions_; } - void DumpFilter(const char* name_frequency_domain) { + void DumpFilter(absl::string_view name_frequency_domain) { for (size_t p = 0; p < max_size_partitions_; ++p) { data_dumper_->DumpRaw(name_frequency_domain, H_[p][0].re); data_dumper_->DumpRaw(name_frequency_domain, H_[p][0].im); diff --git a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_avx2.cc b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_avx2.cc index 245b45a..b6eda9f 100644 --- a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_avx2.cc +++ b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_avx2.cc @@ -8,10 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/aec3/adaptive_fir_filter.h" - #include +#include "modules/audio_processing/aec3/adaptive_fir_filter.h" #include "rtc_base/checks.h" namespace webrtc { @@ -31,19 +30,21 @@ void ComputeFrequencyResponse_Avx2( RTC_DCHECK_EQ(H.size(), H2->capacity()); for (size_t p = 0; p < num_partitions; ++p) { RTC_DCHECK_EQ(kFftLengthBy2Plus1, (*H2)[p].size()); + auto& H2_p = (*H2)[p]; for (size_t ch = 0; ch < num_render_channels; ++ch) { + const FftData& H_p_ch = H[p][ch]; for (size_t j = 0; j < kFftLengthBy2; j += 8) { - __m256 re = _mm256_loadu_ps(&H[p][ch].re[j]); + __m256 re = _mm256_loadu_ps(&H_p_ch.re[j]); __m256 re2 = _mm256_mul_ps(re, re); - __m256 im = _mm256_loadu_ps(&H[p][ch].im[j]); + __m256 im = _mm256_loadu_ps(&H_p_ch.im[j]); re2 = _mm256_fmadd_ps(im, im, re2); - __m256 H2_k_j = _mm256_loadu_ps(&(*H2)[p][j]); + __m256 H2_k_j = _mm256_loadu_ps(&H2_p[j]); H2_k_j = _mm256_max_ps(H2_k_j, re2); - _mm256_storeu_ps(&(*H2)[p][j], H2_k_j); + _mm256_storeu_ps(&H2_p[j], H2_k_j); } - float H2_new = H[p][ch].re[kFftLengthBy2] * H[p][ch].re[kFftLengthBy2] + - H[p][ch].im[kFftLengthBy2] * H[p][ch].im[kFftLengthBy2]; - (*H2)[p][kFftLengthBy2] = std::max((*H2)[p][kFftLengthBy2], H2_new); + float H2_new = H_p_ch.re[kFftLengthBy2] * H_p_ch.re[kFftLengthBy2] + + H_p_ch.im[kFftLengthBy2] * H_p_ch.im[kFftLengthBy2]; + H2_p[kFftLengthBy2] = std::max(H2_p[kFftLengthBy2], H2_new); } } } diff --git a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_erl_avx2.cc b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_erl_avx2.cc index 5fe7514..1e63cf8 100644 --- a/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_erl_avx2.cc +++ b/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_erl_avx2.cc @@ -8,10 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/aec3/adaptive_fir_filter_erl.h" - #include +#include "modules/audio_processing/aec3/adaptive_fir_filter_erl.h" + namespace webrtc { namespace aec3 { diff --git a/webrtc/modules/audio_processing/aec3/aec3_common.cc b/webrtc/modules/audio_processing/aec3/aec3_common.cc index 7bd8d62..3ba10d5 100644 --- a/webrtc/modules/audio_processing/aec3/aec3_common.cc +++ b/webrtc/modules/audio_processing/aec3/aec3_common.cc @@ -29,9 +29,9 @@ Aec3Optimization DetectOptimization() { #if defined(WEBRTC_HAS_NEON) return Aec3Optimization::kNeon; -#endif - +#else return Aec3Optimization::kNone; +#endif } float FastApproxLog2f(const float in) { diff --git a/webrtc/modules/audio_processing/aec3/aec3_common.h b/webrtc/modules/audio_processing/aec3/aec3_common.h index a7e3121..32b564f 100644 --- a/webrtc/modules/audio_processing/aec3/aec3_common.h +++ b/webrtc/modules/audio_processing/aec3/aec3_common.h @@ -28,7 +28,7 @@ enum class Aec3Optimization { kNone, kSse2, kAvx2, kNeon }; constexpr int kNumBlocksPerSecond = 250; constexpr int kMetricsReportingIntervalBlocks = 10 * kNumBlocksPerSecond; -constexpr int kMetricsComputationBlocks = 7; +constexpr int kMetricsComputationBlocks = 3; constexpr int kMetricsCollectionBlocks = kMetricsReportingIntervalBlocks - kMetricsComputationBlocks; @@ -85,10 +85,10 @@ constexpr size_t GetRenderDelayBufferSize(size_t down_sampling_factor, Aec3Optimization DetectOptimization(); // Computes the log2 of the input in a fast an approximate manner. -float FastApproxLog2f(const float in); +float FastApproxLog2f(float in); // Returns dB from a power quantity expressed in log2. -float Log2TodB(const float in_log2); +float Log2TodB(float in_log2); static_assert(1 << kBlockSizeLog2 == kBlockSize, "Proper number of shifts for blocksize"); diff --git a/webrtc/modules/audio_processing/aec3/aec3_fft.cc b/webrtc/modules/audio_processing/aec3/aec3_fft.cc index 8dfa183..9cc8016 100644 --- a/webrtc/modules/audio_processing/aec3/aec3_fft.cc +++ b/webrtc/modules/audio_processing/aec3/aec3_fft.cc @@ -101,10 +101,10 @@ void Aec3Fft::ZeroPaddedFft(rtc::ArrayView x, [](float a, float b) { return a * b; }); break; case Window::kSqrtHanning: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; default: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } Fft(&fft, X); @@ -125,7 +125,7 @@ void Aec3Fft::PaddedFft(rtc::ArrayView x, std::copy(x.begin(), x.end(), fft.begin() + x_old.size()); break; case Window::kHanning: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; case Window::kSqrtHanning: std::transform(x_old.begin(), x_old.end(), std::begin(kSqrtHanning128), @@ -135,7 +135,7 @@ void Aec3Fft::PaddedFft(rtc::ArrayView x, fft.begin() + x_old.size(), std::multiplies()); break; default: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } Fft(&fft, X); diff --git a/webrtc/modules/audio_processing/aec3/aec3_fft.h b/webrtc/modules/audio_processing/aec3/aec3_fft.h index 6f7fbe4..c68de53 100644 --- a/webrtc/modules/audio_processing/aec3/aec3_fft.h +++ b/webrtc/modules/audio_processing/aec3/aec3_fft.h @@ -18,7 +18,6 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/fft_data.h" #include "rtc_base/checks.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -30,6 +29,9 @@ class Aec3Fft { Aec3Fft(); + Aec3Fft(const Aec3Fft&) = delete; + Aec3Fft& operator=(const Aec3Fft&) = delete; + // Computes the FFT. Note that both the input and output are modified. void Fft(std::array* x, FftData* X) const { RTC_DCHECK(x); @@ -66,8 +68,6 @@ class Aec3Fft { private: const OouraFft ooura_fft_; - - RTC_DISALLOW_COPY_AND_ASSIGN(Aec3Fft); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/aec_state.cc b/webrtc/modules/audio_processing/aec3/aec_state.cc index df56c3a..81fd91f 100644 --- a/webrtc/modules/audio_processing/aec3/aec_state.cc +++ b/webrtc/modules/audio_processing/aec3/aec_state.cc @@ -20,7 +20,6 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" #include "system_wrappers/include/field_trial.h" @@ -97,7 +96,7 @@ void ComputeAvgRenderReverb( } // namespace -int AecState::instance_count_ = 0; +std::atomic AecState::instance_count_(0); void AecState::GetResidualEchoScaling( rtc::ArrayView residual_scaling) const { @@ -113,18 +112,9 @@ void AecState::GetResidualEchoScaling( residual_scaling); } -absl::optional AecState::ErleUncertainty() const { - if (SaturatedEcho()) { - return 1.f; - } - - return absl::nullopt; -} - AecState::AecState(const EchoCanceller3Config& config, size_t num_capture_channels) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), config_(config), num_capture_channels_(num_capture_channels), deactivate_initial_state_reset_at_echo_path_change_( @@ -197,8 +187,10 @@ void AecState::Update( // Analyze the filter outputs and filters. bool any_filter_converged; + bool any_coarse_filter_converged; bool all_filters_diverged; subtractor_output_analyzer_.Update(subtractor_output, &any_filter_converged, + &any_coarse_filter_converged, &all_filters_diverged); bool any_filter_consistent; @@ -212,15 +204,16 @@ void AecState::Update( strong_not_saturated_render_blocks_); } - const std::vector>& aligned_render_block = - render_buffer.Block(-delay_state_.MinDirectPathFilterDelay())[0]; + const Block& aligned_render_block = + render_buffer.GetBlock(-delay_state_.MinDirectPathFilterDelay()); // Update render counters. bool active_render = false; - for (size_t ch = 0; ch < aligned_render_block.size(); ++ch) { - const float render_energy = std::inner_product( - aligned_render_block[ch].begin(), aligned_render_block[ch].end(), - aligned_render_block[ch].begin(), 0.f); + for (int ch = 0; ch < aligned_render_block.NumChannels(); ++ch) { + const float render_energy = + std::inner_product(aligned_render_block.begin(/*block=*/0, ch), + aligned_render_block.end(/*block=*/0, ch), + aligned_render_block.begin(/*block=*/0, ch), 0.f); if (render_energy > (config_.render_levels.active_render_limit * config_.render_levels.active_render_limit) * kFftLengthBy2) { @@ -235,8 +228,9 @@ void AecState::Update( std::array avg_render_spectrum_with_reverb; ComputeAvgRenderReverb(render_buffer.GetSpectrumBuffer(), - delay_state_.MinDirectPathFilterDelay(), ReverbDecay(), - &avg_render_reverb_, avg_render_spectrum_with_reverb); + delay_state_.MinDirectPathFilterDelay(), + ReverbDecay(/*mild=*/false), &avg_render_reverb_, + avg_render_spectrum_with_reverb); if (config_.echo_audibility.use_stationarity_properties) { // Update the echo audibility evaluator. @@ -272,10 +266,10 @@ void AecState::Update( // Detect whether the transparent mode should be activated. if (transparent_state_) { - transparent_state_->Update(delay_state_.MinDirectPathFilterDelay(), - any_filter_consistent, any_filter_converged, - all_filters_diverged, active_render, - SaturatedCapture()); + transparent_state_->Update( + delay_state_.MinDirectPathFilterDelay(), any_filter_consistent, + any_filter_converged, any_coarse_filter_converged, all_filters_diverged, + active_render, SaturatedCapture()); } // Analyze the quality of the filter. @@ -300,7 +294,9 @@ void AecState::Update( data_dumper_->DumpRaw("aec3_active_render", active_render); data_dumper_->DumpRaw("aec3_erl", Erl()); data_dumper_->DumpRaw("aec3_erl_time_domain", ErlTimeDomain()); - data_dumper_->DumpRaw("aec3_erle", Erle()[0]); + data_dumper_->DumpRaw("aec3_erle", Erle(/*onset_compensated=*/false)[0]); + data_dumper_->DumpRaw("aec3_erle_onset_compensated", + Erle(/*onset_compensated=*/true)[0]); data_dumper_->DumpRaw("aec3_usable_linear_estimate", UsableLinearEstimate()); data_dumper_->DumpRaw("aec3_transparent_mode", TransparentModeActive()); data_dumper_->DumpRaw("aec3_filter_delay", @@ -312,12 +308,19 @@ void AecState::Update( data_dumper_->DumpRaw("aec3_capture_saturation", SaturatedCapture()); data_dumper_->DumpRaw("aec3_echo_saturation", SaturatedEcho()); data_dumper_->DumpRaw("aec3_any_filter_converged", any_filter_converged); + data_dumper_->DumpRaw("aec3_any_coarse_filter_converged", + any_coarse_filter_converged); data_dumper_->DumpRaw("aec3_all_filters_diverged", all_filters_diverged); data_dumper_->DumpRaw("aec3_external_delay_avaliable", external_delay ? 1 : 0); data_dumper_->DumpRaw("aec3_filter_tail_freq_resp_est", GetReverbFrequencyResponse()); + data_dumper_->DumpRaw("aec3_subtractor_y2", subtractor_output[0].y2); + data_dumper_->DumpRaw("aec3_subtractor_e2_coarse", + subtractor_output[0].e2_coarse); + data_dumper_->DumpRaw("aec3_subtractor_e2_refined", + subtractor_output[0].e2_refined); } AecState::InitialState::InitialState(const EchoCanceller3Config& config) @@ -442,7 +445,7 @@ void AecState::FilteringQualityAnalyzer::Update( } void AecState::SaturationDetector::Update( - rtc::ArrayView> x, + const Block& x, bool saturated_capture, bool usable_linear_estimate, rtc::ArrayView subtractor_output, @@ -462,8 +465,9 @@ void AecState::SaturationDetector::Update( } } else { float max_sample = 0.f; - for (auto& channel : x) { - for (float sample : channel) { + for (int ch = 0; ch < x.NumChannels(); ++ch) { + rtc::ArrayView x_ch = x.View(/*band=*/0, ch); + for (float sample : x_ch) { max_sample = std::max(max_sample, fabsf(sample)); } } diff --git a/webrtc/modules/audio_processing/aec3/aec_state.h b/webrtc/modules/audio_processing/aec3/aec_state.h index 5b40e95..a39325c 100644 --- a/webrtc/modules/audio_processing/aec3/aec_state.h +++ b/webrtc/modules/audio_processing/aec3/aec_state.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -70,15 +71,16 @@ class AecState { } // Returns the ERLE. - rtc::ArrayView> Erle() const { - return erle_estimator_.Erle(); + rtc::ArrayView> Erle( + bool onset_compensated) const { + return erle_estimator_.Erle(onset_compensated); } - // Returns an offset to apply to the estimation of the residual echo - // computation. Returning nullopt means that no offset should be used, while - // any other value will be applied as a multiplier to the estimated residual - // echo. - absl::optional ErleUncertainty() const; + // Returns the non-capped ERLE. + rtc::ArrayView> ErleUnbounded() + const { + return erle_estimator_.ErleUnbounded(); + } // Returns the fullband ERLE estimate in log2 units. float FullBandErleLog2() const { return erle_estimator_.FullbandErleLog2(); } @@ -115,8 +117,12 @@ class AecState { // Takes appropriate action at an echo path change. void HandleEchoPathChange(const EchoPathVariability& echo_path_variability); - // Returns the decay factor for the echo reverberation. - float ReverbDecay() const { return reverb_model_estimator_.ReverbDecay(); } + // Returns the decay factor for the echo reverberation. The parameter `mild` + // indicates which exponential decay to return. The default one or a milder + // one that can be used during nearend regions. + float ReverbDecay(bool mild) const { + return reverb_model_estimator_.ReverbDecay(mild); + } // Return the frequency response of the reverberant echo. rtc::ArrayView GetReverbFrequencyResponse() const { @@ -149,7 +155,7 @@ class AecState { } private: - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const EchoCanceller3Config config_; const size_t num_capture_channels_; @@ -267,7 +273,7 @@ class AecState { bool SaturatedEcho() const { return saturated_echo_; } // Updates the detection decision based on new data. - void Update(rtc::ArrayView> x, + void Update(const Block& x, bool saturated_capture, bool usable_linear_estimate, rtc::ArrayView subtractor_output, diff --git a/webrtc/modules/audio_processing/aec3/alignment_mixer.cc b/webrtc/modules/audio_processing/aec3/alignment_mixer.cc index 87488d2..7f076de 100644 --- a/webrtc/modules/audio_processing/aec3/alignment_mixer.cc +++ b/webrtc/modules/audio_processing/aec3/alignment_mixer.cc @@ -63,9 +63,10 @@ AlignmentMixer::AlignmentMixer(size_t num_channels, } } -void AlignmentMixer::ProduceOutput(rtc::ArrayView> x, +void AlignmentMixer::ProduceOutput(const Block& x, rtc::ArrayView y) { - RTC_DCHECK_EQ(x.size(), num_channels_); + RTC_DCHECK_EQ(x.NumChannels(), num_channels_); + if (selection_variant_ == MixingVariant::kDownmix) { Downmix(x, y); return; @@ -73,18 +74,20 @@ void AlignmentMixer::ProduceOutput(rtc::ArrayView> x, int ch = selection_variant_ == MixingVariant::kFixed ? 0 : SelectChannel(x); - RTC_DCHECK_GE(x.size(), ch); - std::copy(x[ch].begin(), x[ch].end(), y.begin()); + RTC_DCHECK_GT(x.NumChannels(), ch); + std::copy(x.begin(/*band=*/0, ch), x.end(/*band=*/0, ch), y.begin()); } -void AlignmentMixer::Downmix(rtc::ArrayView> x, +void AlignmentMixer::Downmix(const Block& x, rtc::ArrayView y) const { - RTC_DCHECK_EQ(x.size(), num_channels_); + RTC_DCHECK_EQ(x.NumChannels(), num_channels_); RTC_DCHECK_GE(num_channels_, 2); - std::copy(x[0].begin(), x[0].end(), y.begin()); + std::memcpy(&y[0], x.View(/*band=*/0, /*channel=*/0).data(), + kBlockSize * sizeof(y[0])); for (size_t ch = 1; ch < num_channels_; ++ch) { + const auto x_ch = x.View(/*band=*/0, ch); for (size_t i = 0; i < kBlockSize; ++i) { - y[i] += x[ch][i]; + y[i] += x_ch[i]; } } @@ -93,8 +96,8 @@ void AlignmentMixer::Downmix(rtc::ArrayView> x, } } -int AlignmentMixer::SelectChannel(rtc::ArrayView> x) { - RTC_DCHECK_EQ(x.size(), num_channels_); +int AlignmentMixer::SelectChannel(const Block& x) { + RTC_DCHECK_EQ(x.NumChannels(), num_channels_); RTC_DCHECK_GE(num_channels_, 2); RTC_DCHECK_EQ(cumulative_energies_.size(), num_channels_); @@ -112,10 +115,10 @@ int AlignmentMixer::SelectChannel(rtc::ArrayView> x) { ++block_counter_; for (int ch = 0; ch < num_ch_to_analyze; ++ch) { - RTC_DCHECK_EQ(x[ch].size(), kBlockSize); float x2_sum = 0.f; + rtc::ArrayView x_ch = x.View(/*band=*/0, ch); for (size_t i = 0; i < kBlockSize; ++i) { - x2_sum += x[ch][i] * x[ch][i]; + x2_sum += x_ch[i] * x_ch[i]; } if (ch < 2 && x2_sum > excitation_energy_threshold_) { diff --git a/webrtc/modules/audio_processing/aec3/alignment_mixer.h b/webrtc/modules/audio_processing/aec3/alignment_mixer.h index 682aec9..b3ed047 100644 --- a/webrtc/modules/audio_processing/aec3/alignment_mixer.h +++ b/webrtc/modules/audio_processing/aec3/alignment_mixer.h @@ -16,6 +16,7 @@ #include "api/array_view.h" #include "api/audio/echo_canceller3_config.h" #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block.h" namespace webrtc { @@ -33,8 +34,7 @@ class AlignmentMixer { float excitation_limit, bool prefer_first_two_channels); - void ProduceOutput(rtc::ArrayView> x, - rtc::ArrayView y); + void ProduceOutput(const Block& x, rtc::ArrayView y); enum class MixingVariant { kDownmix, kAdaptive, kFixed }; @@ -49,9 +49,8 @@ class AlignmentMixer { int selected_channel_ = 0; size_t block_counter_ = 0; - void Downmix(const rtc::ArrayView> x, - rtc::ArrayView y) const; - int SelectChannel(rtc::ArrayView> x); + void Downmix(const Block& x, rtc::ArrayView y) const; + int SelectChannel(const Block& x); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/block.h b/webrtc/modules/audio_processing/aec3/block.h new file mode 100644 index 0000000..c1fc707 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/block.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_H_ + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +// Contains one or more channels of 4 milliseconds of audio data. +// The audio is split in one or more frequency bands, each with a sampling +// rate of 16 kHz. +class Block { + public: + Block(int num_bands, int num_channels, float default_value = 0.0f) + : num_bands_(num_bands), + num_channels_(num_channels), + data_(num_bands * num_channels * kBlockSize, default_value) {} + + // Returns the number of bands. + int NumBands() const { return num_bands_; } + + // Returns the number of channels. + int NumChannels() const { return num_channels_; } + + // Modifies the number of channels and sets all samples to zero. + void SetNumChannels(int num_channels) { + num_channels_ = num_channels; + data_.resize(num_bands_ * num_channels_ * kBlockSize); + std::fill(data_.begin(), data_.end(), 0.0f); + } + + // Iterators for accessing the data. + auto begin(int band, int channel) { + return data_.begin() + GetIndex(band, channel); + } + + auto begin(int band, int channel) const { + return data_.begin() + GetIndex(band, channel); + } + + auto end(int band, int channel) { return begin(band, channel) + kBlockSize; } + + auto end(int band, int channel) const { + return begin(band, channel) + kBlockSize; + } + + // Access data via ArrayView. + rtc::ArrayView View(int band, int channel) { + return rtc::ArrayView(&data_[GetIndex(band, channel)], + kBlockSize); + } + + rtc::ArrayView View(int band, int channel) const { + return rtc::ArrayView( + &data_[GetIndex(band, channel)], kBlockSize); + } + + // Lets two Blocks swap audio data. + void Swap(Block& b) { + std::swap(num_bands_, b.num_bands_); + std::swap(num_channels_, b.num_channels_); + data_.swap(b.data_); + } + + private: + // Returns the index of the first sample of the requested |band| and + // |channel|. + int GetIndex(int band, int channel) const { + return (band * num_channels_ + channel) * kBlockSize; + } + + int num_bands_; + int num_channels_; + std::vector data_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_PROCESSING_AEC3_BLOCK_H_ diff --git a/webrtc/modules/audio_processing/aec3/block_buffer.cc b/webrtc/modules/audio_processing/aec3/block_buffer.cc index 77ce3de..289c3f0 100644 --- a/webrtc/modules/audio_processing/aec3/block_buffer.cc +++ b/webrtc/modules/audio_processing/aec3/block_buffer.cc @@ -14,25 +14,9 @@ namespace webrtc { -BlockBuffer::BlockBuffer(size_t size, - size_t num_bands, - size_t num_channels, - size_t frame_length) +BlockBuffer::BlockBuffer(size_t size, size_t num_bands, size_t num_channels) : size(static_cast(size)), - buffer(size, - std::vector>>( - num_bands, - std::vector>( - num_channels, - std::vector(frame_length, 0.f)))) { - for (auto& block : buffer) { - for (auto& band : block) { - for (auto& channel : band) { - std::fill(channel.begin(), channel.end(), 0.f); - } - } - } -} + buffer(size, Block(num_bands, num_channels)) {} BlockBuffer::~BlockBuffer() = default; diff --git a/webrtc/modules/audio_processing/aec3/block_buffer.h b/webrtc/modules/audio_processing/aec3/block_buffer.h index b28d659..3489d51 100644 --- a/webrtc/modules/audio_processing/aec3/block_buffer.h +++ b/webrtc/modules/audio_processing/aec3/block_buffer.h @@ -15,6 +15,7 @@ #include +#include "modules/audio_processing/aec3/block.h" #include "rtc_base/checks.h" namespace webrtc { @@ -22,10 +23,7 @@ namespace webrtc { // Struct for bundling a circular buffer of two dimensional vector objects // together with the read and write indices. struct BlockBuffer { - BlockBuffer(size_t size, - size_t num_bands, - size_t num_channels, - size_t frame_length); + BlockBuffer(size_t size, size_t num_bands, size_t num_channels); ~BlockBuffer(); int IncIndex(int index) const { @@ -52,7 +50,7 @@ struct BlockBuffer { void DecReadIndex() { read = DecIndex(read); } const int size; - std::vector>>> buffer; + std::vector buffer; int write = 0; int read = 0; }; diff --git a/webrtc/modules/audio_processing/aec3/block_delay_buffer.cc b/webrtc/modules/audio_processing/aec3/block_delay_buffer.cc index b9eb3c9..059bbaf 100644 --- a/webrtc/modules/audio_processing/aec3/block_delay_buffer.cc +++ b/webrtc/modules/audio_processing/aec3/block_delay_buffer.cc @@ -41,17 +41,24 @@ void BlockDelayBuffer::DelaySignal(AudioBuffer* frame) { RTC_DCHECK_EQ(buf_[ch].size(), frame->num_bands()); RTC_DCHECK_EQ(buf_[ch].size(), num_bands); rtc::ArrayView frame_ch(frame->split_bands(ch), num_bands); + const size_t delay = delay_; for (size_t band = 0; band < num_bands; ++band) { RTC_DCHECK_EQ(delay_, buf_[ch][band].size()); i = i_start; - for (size_t k = 0; k < frame_length_; ++k) { - const float tmp = buf_[ch][band][i]; - buf_[ch][band][i] = frame_ch[band][k]; - frame_ch[band][k] = tmp; + // Offloading these pointers and class variables to local variables allows + // the compiler to optimize the below loop when compiling with + // '-fno-strict-aliasing'. + float* buf_ch_band = buf_[ch][band].data(); + float* frame_ch_band = frame_ch[band]; - i = i < delay_ - 1 ? i + 1 : 0; + for (size_t k = 0, frame_length = frame_length_; k < frame_length; ++k) { + const float tmp = buf_ch_band[i]; + buf_ch_band[i] = frame_ch_band[k]; + frame_ch_band[k] = tmp; + + i = i < delay - 1 ? i + 1 : 0; } } } diff --git a/webrtc/modules/audio_processing/aec3/block_framer.cc b/webrtc/modules/audio_processing/aec3/block_framer.cc index 8241ce6..4243dde 100644 --- a/webrtc/modules/audio_processing/aec3/block_framer.cc +++ b/webrtc/modules/audio_processing/aec3/block_framer.cc @@ -34,35 +34,32 @@ BlockFramer::~BlockFramer() = default; // samples for InsertBlockAndExtractSubFrame to produce a frame. In order to // achieve this, the InsertBlockAndExtractSubFrame and InsertBlock methods need // to be called in the correct order. -void BlockFramer::InsertBlock( - const std::vector>>& block) { - RTC_DCHECK_EQ(num_bands_, block.size()); +void BlockFramer::InsertBlock(const Block& block) { + RTC_DCHECK_EQ(num_bands_, block.NumBands()); + RTC_DCHECK_EQ(num_channels_, block.NumChannels()); for (size_t band = 0; band < num_bands_; ++band) { - RTC_DCHECK_EQ(num_channels_, block[band].size()); for (size_t channel = 0; channel < num_channels_; ++channel) { - RTC_DCHECK_EQ(kBlockSize, block[band][channel].size()); RTC_DCHECK_EQ(0, buffer_[band][channel].size()); buffer_[band][channel].insert(buffer_[band][channel].begin(), - block[band][channel].begin(), - block[band][channel].end()); + block.begin(band, channel), + block.end(band, channel)); } } } void BlockFramer::InsertBlockAndExtractSubFrame( - const std::vector>>& block, + const Block& block, std::vector>>* sub_frame) { RTC_DCHECK(sub_frame); - RTC_DCHECK_EQ(num_bands_, block.size()); + RTC_DCHECK_EQ(num_bands_, block.NumBands()); + RTC_DCHECK_EQ(num_channels_, block.NumChannels()); RTC_DCHECK_EQ(num_bands_, sub_frame->size()); for (size_t band = 0; band < num_bands_; ++band) { - RTC_DCHECK_EQ(num_channels_, block[band].size()); RTC_DCHECK_EQ(num_channels_, (*sub_frame)[0].size()); for (size_t channel = 0; channel < num_channels_; ++channel) { RTC_DCHECK_LE(kSubFrameLength, buffer_[band][channel].size() + kBlockSize); - RTC_DCHECK_EQ(kBlockSize, block[band][channel].size()); RTC_DCHECK_GE(kBlockSize, buffer_[band][channel].size()); RTC_DCHECK_EQ(kSubFrameLength, (*sub_frame)[band][channel].size()); @@ -71,14 +68,14 @@ void BlockFramer::InsertBlockAndExtractSubFrame( std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(), (*sub_frame)[band][channel].begin()); std::copy( - block[band][channel].begin(), - block[band][channel].begin() + samples_to_frame, + block.begin(band, channel), + block.begin(band, channel) + samples_to_frame, (*sub_frame)[band][channel].begin() + buffer_[band][channel].size()); buffer_[band][channel].clear(); buffer_[band][channel].insert( buffer_[band][channel].begin(), - block[band][channel].begin() + samples_to_frame, - block[band][channel].end()); + block.begin(band, channel) + samples_to_frame, + block.end(band, channel)); } } } diff --git a/webrtc/modules/audio_processing/aec3/block_framer.h b/webrtc/modules/audio_processing/aec3/block_framer.h index 1d37866..e2cdd5a 100644 --- a/webrtc/modules/audio_processing/aec3/block_framer.h +++ b/webrtc/modules/audio_processing/aec3/block_framer.h @@ -15,6 +15,7 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block.h" namespace webrtc { @@ -32,10 +33,10 @@ class BlockFramer { BlockFramer& operator=(const BlockFramer&) = delete; // Adds a 64 sample block into the data that will form the next output frame. - void InsertBlock(const std::vector>>& block); + void InsertBlock(const Block& block); // Adds a 64 sample block and extracts an 80 sample subframe. void InsertBlockAndExtractSubFrame( - const std::vector>>& block, + const Block& block, std::vector>>* sub_frame); private: diff --git a/webrtc/modules/audio_processing/aec3/block_processor.cc b/webrtc/modules/audio_processing/aec3/block_processor.cc index f2f3261..63e3d9c 100644 --- a/webrtc/modules/audio_processing/aec3/block_processor.cc +++ b/webrtc/modules/audio_processing/aec3/block_processor.cc @@ -11,6 +11,7 @@ #include +#include #include #include #include @@ -26,7 +27,6 @@ #include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/aec3/render_delay_controller.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -49,23 +49,22 @@ class BlockProcessorImpl final : public BlockProcessor { ~BlockProcessorImpl() override; - void ProcessCapture( - bool echo_path_gain_change, - bool capture_signal_saturation, - std::vector>>* linear_output, - std::vector>>* capture_block) override; + void ProcessCapture(bool echo_path_gain_change, + bool capture_signal_saturation, + Block* linear_output, + Block* capture_block) override; - void BufferRender( - const std::vector>>& block) override; + void BufferRender(const Block& block) override; void UpdateEchoLeakageStatus(bool leakage_detected) override; void GetMetrics(EchoControl::Metrics* metrics) const override; void SetAudioBufferDelay(int delay_ms) override; + void SetCaptureOutputUsage(bool capture_output_used) override; private: - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const EchoCanceller3Config config_; bool capture_properly_started_ = false; @@ -80,7 +79,7 @@ class BlockProcessorImpl final : public BlockProcessor { absl::optional estimated_delay_; }; -int BlockProcessorImpl::instance_count_ = 0; +std::atomic BlockProcessorImpl::instance_count_(0); BlockProcessorImpl::BlockProcessorImpl( const EchoCanceller3Config& config, @@ -90,8 +89,7 @@ BlockProcessorImpl::BlockProcessorImpl( std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), config_(config), sample_rate_hz_(sample_rate_hz), render_buffer_(std::move(render_buffer)), @@ -103,21 +101,20 @@ BlockProcessorImpl::BlockProcessorImpl( BlockProcessorImpl::~BlockProcessorImpl() = default; -void BlockProcessorImpl::ProcessCapture( - bool echo_path_gain_change, - bool capture_signal_saturation, - std::vector>>* linear_output, - std::vector>>* capture_block) { +void BlockProcessorImpl::ProcessCapture(bool echo_path_gain_change, + bool capture_signal_saturation, + Block* linear_output, + Block* capture_block) { RTC_DCHECK(capture_block); - RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size()); - RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0][0].size()); + RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->NumBands()); capture_call_counter_++; data_dumper_->DumpRaw("aec3_processblock_call_order", static_cast(BlockProcessorApiCall::kCapture)); - data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize, - &(*capture_block)[0][0][0], 16000, 1); + data_dumper_->DumpWav("aec3_processblock_capture_input", + capture_block->View(/*band=*/0, /*channel=*/0), 16000, + 1); if (render_properly_started_) { if (!capture_properly_started_) { @@ -158,8 +155,9 @@ void BlockProcessorImpl::ProcessCapture( delay_controller_->Reset(false); } - data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, - &(*capture_block)[0][0][0], 16000, 1); + data_dumper_->DumpWav("aec3_processblock_capture_input2", + capture_block->View(/*band=*/0, /*channel=*/0), 16000, + 1); bool has_delay_estimator = !config_.delay.use_external_delay_estimator; if (has_delay_estimator) { @@ -168,7 +166,7 @@ void BlockProcessorImpl::ProcessCapture( // alignment. estimated_delay_ = delay_controller_->GetDelay( render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(), - (*capture_block)[0]); + *capture_block); if (estimated_delay_) { bool delay_change = @@ -201,16 +199,12 @@ void BlockProcessorImpl::ProcessCapture( metrics_.UpdateCapture(false); } -void BlockProcessorImpl::BufferRender( - const std::vector>>& block) { - RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size()); - RTC_DCHECK_EQ(kBlockSize, block[0][0].size()); +void BlockProcessorImpl::BufferRender(const Block& block) { + RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.NumBands()); data_dumper_->DumpRaw("aec3_processblock_call_order", static_cast(BlockProcessorApiCall::kRender)); - data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, - &block[0][0][0], 16000, 1); - data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize, - &block[0][0][0], 16000, 1); + data_dumper_->DumpWav("aec3_processblock_render_input", + block.View(/*band=*/0, /*channel=*/0), 16000, 1); render_event_ = render_buffer_->Insert(block); @@ -237,6 +231,10 @@ void BlockProcessorImpl::SetAudioBufferDelay(int delay_ms) { render_buffer_->SetAudioBufferDelay(delay_ms); } +void BlockProcessorImpl::SetCaptureOutputUsage(bool capture_output_used) { + echo_remover_->SetCaptureOutputUsage(capture_output_used); +} + } // namespace BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, diff --git a/webrtc/modules/audio_processing/aec3/block_processor.h b/webrtc/modules/audio_processing/aec3/block_processor.h index 9bb0cf1..01a83ae 100644 --- a/webrtc/modules/audio_processing/aec3/block_processor.h +++ b/webrtc/modules/audio_processing/aec3/block_processor.h @@ -18,6 +18,7 @@ #include "api/audio/echo_canceller3_config.h" #include "api/audio/echo_control.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/echo_remover.h" #include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/aec3/render_delay_controller.h" @@ -56,19 +57,23 @@ class BlockProcessor { virtual void SetAudioBufferDelay(int delay_ms) = 0; // Processes a block of capture data. - virtual void ProcessCapture( - bool echo_path_gain_change, - bool capture_signal_saturation, - std::vector>>* linear_output, - std::vector>>* capture_block) = 0; + virtual void ProcessCapture(bool echo_path_gain_change, + bool capture_signal_saturation, + Block* linear_output, + Block* capture_block) = 0; // Buffers a block of render data supplied by a FrameBlocker object. - virtual void BufferRender( - const std::vector>>& render_block) = 0; + virtual void BufferRender(const Block& render_block) = 0; // Reports whether echo leakage has been detected in the echo canceller // output. virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; + + // Specifies whether the capture output will be used. The purpose of this is + // to allow the block processor to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + virtual void SetCaptureOutputUsage(bool capture_output_used) = 0; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/block_processor_metrics.h b/webrtc/modules/audio_processing/aec3/block_processor_metrics.h index 4ba0536..a70d0da 100644 --- a/webrtc/modules/audio_processing/aec3/block_processor_metrics.h +++ b/webrtc/modules/audio_processing/aec3/block_processor_metrics.h @@ -11,8 +11,6 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_METRICS_H_ #define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_PROCESSOR_METRICS_H_ -#include "rtc_base/constructor_magic.h" - namespace webrtc { // Handles the reporting of metrics for the block_processor. @@ -20,6 +18,9 @@ class BlockProcessorMetrics { public: BlockProcessorMetrics() = default; + BlockProcessorMetrics(const BlockProcessorMetrics&) = delete; + BlockProcessorMetrics& operator=(const BlockProcessorMetrics&) = delete; + // Updates the metric with new capture data. void UpdateCapture(bool underrun); @@ -38,8 +39,6 @@ class BlockProcessorMetrics { int render_buffer_underruns_ = 0; int render_buffer_overruns_ = 0; int buffer_render_calls_ = 0; - - RTC_DISALLOW_COPY_AND_ASSIGN(BlockProcessorMetrics); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h b/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h index 16eaf35..2785b76 100644 --- a/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h +++ b/webrtc/modules/audio_processing/aec3/comfort_noise_generator.h @@ -19,7 +19,6 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec_state.h" #include "modules/audio_processing/aec3/fft_data.h" -#include "rtc_base/constructor_magic.h" #include "rtc_base/system/arch.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aec3/config_selector.cc b/webrtc/modules/audio_processing/aec3/config_selector.cc new file mode 100644 index 0000000..c55344d --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/config_selector.cc @@ -0,0 +1,71 @@ + +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/config_selector.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +// Validates that the mono and the multichannel configs have compatible fields. +bool CompatibleConfigs(const EchoCanceller3Config& mono_config, + const EchoCanceller3Config& multichannel_config) { + if (mono_config.delay.fixed_capture_delay_samples != + multichannel_config.delay.fixed_capture_delay_samples) { + return false; + } + if (mono_config.filter.export_linear_aec_output != + multichannel_config.filter.export_linear_aec_output) { + return false; + } + if (mono_config.filter.high_pass_filter_echo_reference != + multichannel_config.filter.high_pass_filter_echo_reference) { + return false; + } + if (mono_config.multi_channel.detect_stereo_content != + multichannel_config.multi_channel.detect_stereo_content) { + return false; + } + if (mono_config.multi_channel.stereo_detection_timeout_threshold_seconds != + multichannel_config.multi_channel + .stereo_detection_timeout_threshold_seconds) { + return false; + } + return true; +} + +} // namespace + +ConfigSelector::ConfigSelector( + const EchoCanceller3Config& config, + const absl::optional& multichannel_config, + int num_render_input_channels) + : config_(config), multichannel_config_(multichannel_config) { + if (multichannel_config_.has_value()) { + RTC_DCHECK(CompatibleConfigs(config_, *multichannel_config_)); + } + + Update(!config_.multi_channel.detect_stereo_content && + num_render_input_channels > 1); + + RTC_DCHECK(active_config_); +} + +void ConfigSelector::Update(bool multichannel_content) { + if (multichannel_content && multichannel_config_.has_value()) { + active_config_ = &(*multichannel_config_); + } else { + active_config_ = &config_; + } +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/config_selector.h b/webrtc/modules/audio_processing/aec3/config_selector.h new file mode 100644 index 0000000..3b3f94e --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/config_selector.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_CONFIG_SELECTOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_CONFIG_SELECTOR_H_ + +#include "absl/types/optional.h" +#include "api/audio/echo_canceller3_config.h" + +namespace webrtc { + +// Selects the config to use. +class ConfigSelector { + public: + ConfigSelector( + const EchoCanceller3Config& config, + const absl::optional& multichannel_config, + int num_render_input_channels); + + // Updates the config selection based on the detection of multichannel + // content. + void Update(bool multichannel_content); + + const EchoCanceller3Config& active_config() const { return *active_config_; } + + private: + const EchoCanceller3Config config_; + const absl::optional multichannel_config_; + const EchoCanceller3Config* active_config_ = nullptr; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_CONFIG_SELECTOR_H_ diff --git a/webrtc/modules/audio_processing/aec3/decimator.h b/webrtc/modules/audio_processing/aec3/decimator.h index 3ccd292..dbff3d9 100644 --- a/webrtc/modules/audio_processing/aec3/decimator.h +++ b/webrtc/modules/audio_processing/aec3/decimator.h @@ -17,7 +17,6 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/utility/cascaded_biquad_filter.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -26,6 +25,9 @@ class Decimator { public: explicit Decimator(size_t down_sampling_factor); + Decimator(const Decimator&) = delete; + Decimator& operator=(const Decimator&) = delete; + // Downsamples the signal. void Decimate(rtc::ArrayView in, rtc::ArrayView out); @@ -33,8 +35,6 @@ class Decimator { const size_t down_sampling_factor_; CascadedBiQuadFilter anti_aliasing_filter_; CascadedBiQuadFilter noise_reduction_filter_; - - RTC_DISALLOW_COPY_AND_ASSIGN(Decimator); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/delay_estimate.h b/webrtc/modules/audio_processing/aec3/delay_estimate.h index ea5dd27..7838a0c 100644 --- a/webrtc/modules/audio_processing/aec3/delay_estimate.h +++ b/webrtc/modules/audio_processing/aec3/delay_estimate.h @@ -11,6 +11,8 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC3_DELAY_ESTIMATE_H_ #define MODULES_AUDIO_PROCESSING_AEC3_DELAY_ESTIMATE_H_ +#include + namespace webrtc { // Stores delay_estimates. diff --git a/webrtc/modules/audio_processing/aec3/echo_audibility.cc b/webrtc/modules/audio_processing/aec3/echo_audibility.cc index 6ae414e..142a33d 100644 --- a/webrtc/modules/audio_processing/aec3/echo_audibility.cc +++ b/webrtc/modules/audio_processing/aec3/echo_audibility.cc @@ -88,7 +88,7 @@ void EchoAudibility::UpdateRenderNoiseEstimator( bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) { const int num_render_channels = - static_cast(block_buffer.buffer[0][0].size()); + static_cast(block_buffer.buffer[0].NumChannels()); bool too_low = false; const int render_block_write_current = block_buffer.write; if (render_block_write_current == render_block_write_prev_) { @@ -98,7 +98,8 @@ bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) { idx = block_buffer.IncIndex(idx)) { float max_abs_over_channels = 0.f; for (int ch = 0; ch < num_render_channels; ++ch) { - auto block = block_buffer.buffer[idx][0][ch]; + rtc::ArrayView block = + block_buffer.buffer[idx].View(/*band=*/0, /*channel=*/ch); auto r = std::minmax_element(block.cbegin(), block.cend()); float max_abs_channel = std::max(std::fabs(*r.first), std::fabs(*r.second)); diff --git a/webrtc/modules/audio_processing/aec3/echo_audibility.h b/webrtc/modules/audio_processing/aec3/echo_audibility.h index 1ffc017..b9d6f87 100644 --- a/webrtc/modules/audio_processing/aec3/echo_audibility.h +++ b/webrtc/modules/audio_processing/aec3/echo_audibility.h @@ -19,7 +19,6 @@ #include "modules/audio_processing/aec3/render_buffer.h" #include "modules/audio_processing/aec3/spectrum_buffer.h" #include "modules/audio_processing/aec3/stationarity_estimator.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aec3/echo_canceller3.cc b/webrtc/modules/audio_processing/aec3/echo_canceller3.cc index d2847df..e8e2175 100644 --- a/webrtc/modules/audio_processing/aec3/echo_canceller3.cc +++ b/webrtc/modules/audio_processing/aec3/echo_canceller3.cc @@ -12,10 +12,10 @@ #include #include +#include "absl/strings/string_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/high_pass_filter.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" @@ -27,8 +27,8 @@ namespace { enum class EchoCanceller3ApiCall { kCapture, kRender }; bool DetectSaturation(rtc::ArrayView y) { - for (auto y_k : y) { - if (y_k >= 32700.0f || y_k <= -32700.0f) { + for (size_t k = 0; k < y.size(); ++k) { + if (y[k] >= 32700.0f || y[k] <= -32700.0f) { return true; } } @@ -38,7 +38,7 @@ bool DetectSaturation(rtc::ArrayView y) { // Retrieves a value from a field trial if it is available. If no value is // present, the default value is returned. If the retrieved value is beyond the // specified limits, the default value is returned instead. -void RetrieveFieldTrialValue(const char* trial_name, +void RetrieveFieldTrialValue(absl::string_view trial_name, float min, float max, float* value_to_update) { @@ -49,12 +49,16 @@ void RetrieveFieldTrialValue(const char* trial_name, ParseFieldTrial({&field_trial_param}, field_trial_str); float field_trial_value = static_cast(field_trial_param.Get()); - if (field_trial_value >= min && field_trial_value <= max) { + if (field_trial_value >= min && field_trial_value <= max && + field_trial_value != *value_to_update) { + RTC_LOG(LS_INFO) << "Key " << trial_name + << " changing AEC3 parameter value from " + << *value_to_update << " to " << field_trial_value; *value_to_update = field_trial_value; } } -void RetrieveFieldTrialValue(const char* trial_name, +void RetrieveFieldTrialValue(absl::string_view trial_name, int min, int max, int* value_to_update) { @@ -65,7 +69,11 @@ void RetrieveFieldTrialValue(const char* trial_name, ParseFieldTrial({&field_trial_param}, field_trial_str); float field_trial_value = field_trial_param.Get(); - if (field_trial_value >= min && field_trial_value <= max) { + if (field_trial_value >= min && field_trial_value <= max && + field_trial_value != *value_to_update) { + RTC_LOG(LS_INFO) << "Key " << trial_name + << " changing AEC3 parameter value from " + << *value_to_update << " to " << field_trial_value; *value_to_update = field_trial_value; } } @@ -88,18 +96,50 @@ void FillSubFrameView( } void FillSubFrameView( + bool proper_downmix_needed, std::vector>>* frame, size_t sub_frame_index, std::vector>>* sub_frame_view) { RTC_DCHECK_GE(1, sub_frame_index); RTC_DCHECK_EQ(frame->size(), sub_frame_view->size()); - RTC_DCHECK_EQ((*frame)[0].size(), (*sub_frame_view)[0].size()); - for (size_t band = 0; band < frame->size(); ++band) { - for (size_t channel = 0; channel < (*frame)[band].size(); ++channel) { - (*sub_frame_view)[band][channel] = rtc::ArrayView( - &(*frame)[band][channel][sub_frame_index * kSubFrameLength], + const size_t frame_num_channels = (*frame)[0].size(); + const size_t sub_frame_num_channels = (*sub_frame_view)[0].size(); + if (frame_num_channels > sub_frame_num_channels) { + RTC_DCHECK_EQ(sub_frame_num_channels, 1u); + if (proper_downmix_needed) { + // When a proper downmix is needed (which is the case when proper stereo + // is present in the echo reference signal but the echo canceller does the + // processing in mono) downmix the echo reference by averaging the channel + // content (otherwise downmixing is done by selecting channel 0). + for (size_t band = 0; band < frame->size(); ++band) { + for (size_t ch = 1; ch < frame_num_channels; ++ch) { + for (size_t k = 0; k < kSubFrameLength; ++k) { + (*frame)[band][/*channel=*/0] + [sub_frame_index * kSubFrameLength + k] += + (*frame)[band][ch][sub_frame_index * kSubFrameLength + k]; + } + } + const float one_by_num_channels = 1.0f / frame_num_channels; + for (size_t k = 0; k < kSubFrameLength; ++k) { + (*frame)[band][/*channel=*/0][sub_frame_index * kSubFrameLength + + k] *= one_by_num_channels; + } + } + } + for (size_t band = 0; band < frame->size(); ++band) { + (*sub_frame_view)[band][/*channel=*/0] = rtc::ArrayView( + &(*frame)[band][/*channel=*/0][sub_frame_index * kSubFrameLength], kSubFrameLength); } + } else { + RTC_DCHECK_EQ(frame_num_channels, sub_frame_num_channels); + for (size_t band = 0; band < frame->size(); ++band) { + for (size_t channel = 0; channel < (*frame)[band].size(); ++channel) { + (*sub_frame_view)[band][channel] = rtc::ArrayView( + &(*frame)[band][channel][sub_frame_index * kSubFrameLength], + kSubFrameLength); + } + } } } @@ -107,16 +147,17 @@ void ProcessCaptureFrameContent( AudioBuffer* linear_output, AudioBuffer* capture, bool level_change, + bool aec_reference_is_downmixed_stereo, bool saturated_microphone_signal, size_t sub_frame_index, FrameBlocker* capture_blocker, BlockFramer* linear_output_framer, BlockFramer* output_framer, BlockProcessor* block_processor, - std::vector>>* linear_output_block, + Block* linear_output_block, std::vector>>* linear_output_sub_frame_view, - std::vector>>* capture_block, + Block* capture_block, std::vector>>* capture_sub_frame_view) { FillSubFrameView(capture, sub_frame_index, capture_sub_frame_view); @@ -130,8 +171,10 @@ void ProcessCaptureFrameContent( capture_blocker->InsertSubFrameAndExtractBlock(*capture_sub_frame_view, capture_block); - block_processor->ProcessCapture(level_change, saturated_microphone_signal, - linear_output_block, capture_block); + block_processor->ProcessCapture( + /*echo_path_gain_change=*/level_change || + aec_reference_is_downmixed_stereo, + saturated_microphone_signal, linear_output_block, capture_block); output_framer->InsertBlockAndExtractSubFrame(*capture_block, capture_sub_frame_view); @@ -142,22 +185,24 @@ void ProcessCaptureFrameContent( } } -void ProcessRemainingCaptureFrameContent( - bool level_change, - bool saturated_microphone_signal, - FrameBlocker* capture_blocker, - BlockFramer* linear_output_framer, - BlockFramer* output_framer, - BlockProcessor* block_processor, - std::vector>>* linear_output_block, - std::vector>>* block) { +void ProcessRemainingCaptureFrameContent(bool level_change, + bool aec_reference_is_downmixed_stereo, + bool saturated_microphone_signal, + FrameBlocker* capture_blocker, + BlockFramer* linear_output_framer, + BlockFramer* output_framer, + BlockProcessor* block_processor, + Block* linear_output_block, + Block* block) { if (!capture_blocker->IsBlockAvailable()) { return; } capture_blocker->ExtractBlock(block); - block_processor->ProcessCapture(level_change, saturated_microphone_signal, - linear_output_block, block); + block_processor->ProcessCapture( + /*echo_path_gain_change=*/level_change || + aec_reference_is_downmixed_stereo, + saturated_microphone_signal, linear_output_block, block); output_framer->InsertBlock(*block); if (linear_output_framer) { @@ -167,21 +212,22 @@ void ProcessRemainingCaptureFrameContent( } void BufferRenderFrameContent( + bool proper_downmix_needed, std::vector>>* render_frame, size_t sub_frame_index, FrameBlocker* render_blocker, BlockProcessor* block_processor, - std::vector>>* block, + Block* block, std::vector>>* sub_frame_view) { - FillSubFrameView(render_frame, sub_frame_index, sub_frame_view); + FillSubFrameView(proper_downmix_needed, render_frame, sub_frame_index, + sub_frame_view); render_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); block_processor->BufferRender(*block); } -void BufferRemainingRenderFrameContent( - FrameBlocker* render_blocker, - BlockProcessor* block_processor, - std::vector>>* block) { +void BufferRemainingRenderFrameContent(FrameBlocker* render_blocker, + BlockProcessor* block_processor, + Block* block) { if (!render_blocker->IsBlockAvailable()) { return; } @@ -213,6 +259,10 @@ void CopyBufferIntoFrame(const AudioBuffer& buffer, EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { EchoCanceller3Config adjusted_cfg = config; + if (field_trial::IsEnabled("WebRTC-Aec3StereoContentDetectionKillSwitch")) { + adjusted_cfg.multi_channel.detect_stereo_content = false; + } + if (field_trial::IsEnabled("WebRTC-Aec3AntiHowlingMinimizationKillSwitch")) { adjusted_cfg.suppressor.high_bands_suppression .anti_howling_activation_threshold = 25.f; @@ -251,24 +301,39 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { adjusted_cfg.filter.initial_state_seconds = 2.0f; } + if (field_trial::IsEnabled("WebRTC-Aec3HighPassFilterEchoReference")) { + adjusted_cfg.filter.high_pass_filter_echo_reference = true; + } + if (field_trial::IsEnabled("WebRTC-Aec3EchoSaturationDetectionKillSwitch")) { adjusted_cfg.ep_strength.echo_can_saturate = false; } - if (field_trial::IsEnabled("WebRTC-Aec3UseDot2ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.2f; - } else if (field_trial::IsEnabled("WebRTC-Aec3UseDot3ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.3f; - } else if (field_trial::IsEnabled("WebRTC-Aec3UseDot4ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.4f; - } else if (field_trial::IsEnabled("WebRTC-Aec3UseDot5ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.5f; - } else if (field_trial::IsEnabled("WebRTC-Aec3UseDot6ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.6f; - } else if (field_trial::IsEnabled("WebRTC-Aec3UseDot7ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.7f; - } else if (field_trial::IsEnabled("WebRTC-Aec3UseDot8ReverbDefaultLen")) { - adjusted_cfg.ep_strength.default_len = 0.8f; + const std::string use_nearend_reverb_len_tunings = + field_trial::FindFullName("WebRTC-Aec3UseNearendReverbLen"); + FieldTrialParameter nearend_reverb_default_len( + "default_len", adjusted_cfg.ep_strength.default_len); + FieldTrialParameter nearend_reverb_nearend_len( + "nearend_len", adjusted_cfg.ep_strength.nearend_len); + + ParseFieldTrial({&nearend_reverb_default_len, &nearend_reverb_nearend_len}, + use_nearend_reverb_len_tunings); + float default_len = static_cast(nearend_reverb_default_len.Get()); + float nearend_len = static_cast(nearend_reverb_nearend_len.Get()); + if (default_len > -1 && default_len < 1 && nearend_len > -1 && + nearend_len < 1) { + adjusted_cfg.ep_strength.default_len = + static_cast(nearend_reverb_default_len.Get()); + adjusted_cfg.ep_strength.nearend_len = + static_cast(nearend_reverb_nearend_len.Get()); + } + + if (field_trial::IsEnabled("WebRTC-Aec3ConservativeTailFreqResponse")) { + adjusted_cfg.ep_strength.use_conservative_tail_frequency_response = true; + } + + if (field_trial::IsDisabled("WebRTC-Aec3ConservativeTailFreqResponse")) { + adjusted_cfg.ep_strength.use_conservative_tail_frequency_response = false; } if (field_trial::IsEnabled("WebRTC-Aec3ShortHeadroomKillSwitch")) { @@ -313,6 +378,14 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { false; } + if (field_trial::IsEnabled("WebRTC-Aec3DelayEstimatorDetectPreEcho")) { + adjusted_cfg.delay.detect_pre_echo = true; + } + + if (field_trial::IsDisabled("WebRTC-Aec3DelayEstimatorDetectPreEcho")) { + adjusted_cfg.delay.detect_pre_echo = false; + } + if (field_trial::IsEnabled("WebRTC-Aec3SensitiveDominantNearendActivation")) { adjusted_cfg.suppressor.dominant_nearend_detection.enr_threshold = 0.5f; } else if (field_trial::IsEnabled( @@ -368,6 +441,10 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { adjusted_cfg.suppressor.nearend_tuning.max_dec_factor_lf = .2f; } + if (field_trial::IsEnabled("WebRTC-Aec3EnforceConservativeHfSuppression")) { + adjusted_cfg.suppressor.conservative_hf_suppression = true; + } + if (field_trial::IsEnabled("WebRTC-Aec3EnforceStationarityProperties")) { adjusted_cfg.echo_audibility.use_stationarity_properties = true; } @@ -443,8 +520,6 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { FieldTrialParameter dominant_nearend_detection_trigger_threshold( "dominant_nearend_detection_trigger_threshold", adjusted_cfg.suppressor.dominant_nearend_detection.trigger_threshold); - FieldTrialParameter ep_strength_default_len( - "ep_strength_default_len", adjusted_cfg.ep_strength.default_len); ParseFieldTrial( {&nearend_tuning_mask_lf_enr_transparent, @@ -461,7 +536,7 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { &dominant_nearend_detection_enr_exit_threshold, &dominant_nearend_detection_snr_threshold, &dominant_nearend_detection_hold_duration, - &dominant_nearend_detection_trigger_threshold, &ep_strength_default_len}, + &dominant_nearend_detection_trigger_threshold}, suppressor_tuning_override_trial_name); adjusted_cfg.suppressor.nearend_tuning.mask_lf.enr_transparent = @@ -498,8 +573,6 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { dominant_nearend_detection_hold_duration.Get(); adjusted_cfg.suppressor.dominant_nearend_detection.trigger_threshold = dominant_nearend_detection_trigger_threshold.Get(); - adjusted_cfg.ep_strength.default_len = - static_cast(ep_strength_default_len.Get()); // Field trial-based overrides of individual suppressor parameters. RetrieveFieldTrialValue( @@ -561,8 +634,12 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { "WebRTC-Aec3SuppressorAntiHowlingGainOverride", 0.f, 10.f, &adjusted_cfg.suppressor.high_bands_suppression.anti_howling_gain); - RetrieveFieldTrialValue("WebRTC-Aec3SuppressorEpStrengthDefaultLenOverride", - -1.f, 1.f, &adjusted_cfg.ep_strength.default_len); + // Field trial-based overrides of individual delay estimator parameters. + RetrieveFieldTrialValue("WebRTC-Aec3DelayEstimateSmoothingOverride", 0.f, 1.f, + &adjusted_cfg.delay.delay_estimate_smoothing); + RetrieveFieldTrialValue( + "WebRTC-Aec3DelayEstimateSmoothingDelayFoundOverride", 0.f, 1.f, + &adjusted_cfg.delay.delay_estimate_smoothing_delay_found); return adjusted_cfg; } @@ -570,6 +647,7 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { class EchoCanceller3::RenderWriter { public: RenderWriter(ApmDataDumper* data_dumper, + const EchoCanceller3Config& config, SwapQueue>>, Aec3RenderQueueItemVerifier>* render_transfer_queue, size_t num_bands, @@ -586,7 +664,7 @@ class EchoCanceller3::RenderWriter { ApmDataDumper* data_dumper_; const size_t num_bands_; const size_t num_channels_; - HighPassFilter high_pass_filter_; + std::unique_ptr high_pass_filter_; std::vector>> render_queue_input_frame_; SwapQueue>>, Aec3RenderQueueItemVerifier>* render_transfer_queue_; @@ -594,6 +672,7 @@ class EchoCanceller3::RenderWriter { EchoCanceller3::RenderWriter::RenderWriter( ApmDataDumper* data_dumper, + const EchoCanceller3Config& config, SwapQueue>>, Aec3RenderQueueItemVerifier>* render_transfer_queue, size_t num_bands, @@ -601,7 +680,6 @@ EchoCanceller3::RenderWriter::RenderWriter( : data_dumper_(data_dumper), num_bands_(num_bands), num_channels_(num_channels), - high_pass_filter_(16000, num_channels), render_queue_input_frame_( num_bands_, std::vector>( @@ -609,6 +687,9 @@ EchoCanceller3::RenderWriter::RenderWriter( std::vector(AudioBuffer::kSplitBandSize, 0.f))), render_transfer_queue_(render_transfer_queue) { RTC_DCHECK(data_dumper); + if (config.filter.high_pass_filter_echo_reference) { + high_pass_filter_ = std::make_unique(16000, num_channels); + } } EchoCanceller3::RenderWriter::~RenderWriter() = default; @@ -627,104 +708,124 @@ void EchoCanceller3::RenderWriter::Insert(const AudioBuffer& input) { CopyBufferIntoFrame(input, num_bands_, num_channels_, &render_queue_input_frame_); - high_pass_filter_.Process(&render_queue_input_frame_[0]); + if (high_pass_filter_) { + high_pass_filter_->Process(&render_queue_input_frame_[0]); + } static_cast(render_transfer_queue_->Insert(&render_queue_input_frame_)); } -int EchoCanceller3::instance_count_ = 0; +std::atomic EchoCanceller3::instance_count_(0); -EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, - int sample_rate_hz, - size_t num_render_channels, - size_t num_capture_channels) - : EchoCanceller3(AdjustConfig(config), - sample_rate_hz, - num_render_channels, - num_capture_channels, - std::unique_ptr( - BlockProcessor::Create(AdjustConfig(config), - sample_rate_hz, - num_render_channels, - num_capture_channels))) {} -EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, - int sample_rate_hz, - size_t num_render_channels, - size_t num_capture_channels, - std::unique_ptr block_processor) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), - config_(config), +EchoCanceller3::EchoCanceller3( + const EchoCanceller3Config& config, + const absl::optional& multichannel_config, + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels) + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), + config_(AdjustConfig(config)), sample_rate_hz_(sample_rate_hz), num_bands_(NumBandsForRate(sample_rate_hz_)), - num_render_channels_(num_render_channels), + num_render_input_channels_(num_render_channels), num_capture_channels_(num_capture_channels), + config_selector_(AdjustConfig(config), + multichannel_config, + num_render_input_channels_), + multichannel_content_detector_( + config_selector_.active_config().multi_channel.detect_stereo_content, + num_render_input_channels_, + config_selector_.active_config() + .multi_channel.stereo_detection_threshold, + config_selector_.active_config() + .multi_channel.stereo_detection_timeout_threshold_seconds, + config_selector_.active_config() + .multi_channel.stereo_detection_hysteresis_seconds), output_framer_(num_bands_, num_capture_channels_), capture_blocker_(num_bands_, num_capture_channels_), - render_blocker_(num_bands_, num_render_channels_), render_transfer_queue_( kRenderTransferQueueSizeFrames, std::vector>>( num_bands_, std::vector>( - num_render_channels_, + num_render_input_channels_, std::vector(AudioBuffer::kSplitBandSize, 0.f))), Aec3RenderQueueItemVerifier(num_bands_, - num_render_channels_, + num_render_input_channels_, AudioBuffer::kSplitBandSize)), - block_processor_(std::move(block_processor)), render_queue_output_frame_( num_bands_, std::vector>( - num_render_channels_, + num_render_input_channels_, std::vector(AudioBuffer::kSplitBandSize, 0.f))), - render_block_( - num_bands_, - std::vector>(num_render_channels_, - std::vector(kBlockSize, 0.f))), - capture_block_( - num_bands_, - std::vector>(num_capture_channels_, - std::vector(kBlockSize, 0.f))), - render_sub_frame_view_( - num_bands_, - std::vector>(num_render_channels_)), + render_block_(num_bands_, num_render_input_channels_), + capture_block_(num_bands_, num_capture_channels_), capture_sub_frame_view_( num_bands_, std::vector>(num_capture_channels_)) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); - if (config_.delay.fixed_capture_delay_samples > 0) { + if (config_selector_.active_config().delay.fixed_capture_delay_samples > 0) { block_delay_buffer_.reset(new BlockDelayBuffer( num_capture_channels_, num_bands_, AudioBuffer::kSplitBandSize, config_.delay.fixed_capture_delay_samples)); } - render_writer_.reset(new RenderWriter(data_dumper_.get(), - &render_transfer_queue_, num_bands_, - num_render_channels_)); + render_writer_.reset(new RenderWriter( + data_dumper_.get(), config_selector_.active_config(), + &render_transfer_queue_, num_bands_, num_render_input_channels_)); RTC_DCHECK_EQ(num_bands_, std::max(sample_rate_hz_, 16000) / 16000); RTC_DCHECK_GE(kMaxNumBands, num_bands_); - if (config_.filter.export_linear_aec_output) { - linear_output_framer_.reset(new BlockFramer(1, num_capture_channels_)); + if (config_selector_.active_config().filter.export_linear_aec_output) { + linear_output_framer_.reset( + new BlockFramer(/*num_bands=*/1, num_capture_channels_)); linear_output_block_ = - std::make_unique>>>( - 1, std::vector>( - num_capture_channels_, std::vector(kBlockSize, 0.f))); + std::make_unique(/*num_bands=*/1, num_capture_channels_), linear_output_sub_frame_view_ = std::vector>>( 1, std::vector>(num_capture_channels_)); } + + Initialize(); + + RTC_LOG(LS_INFO) << "AEC3 created with sample rate: " << sample_rate_hz_ + << " Hz, num render channels: " << num_render_input_channels_ + << ", num capture channels: " << num_capture_channels_; } EchoCanceller3::~EchoCanceller3() = default; +void EchoCanceller3::Initialize() { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + + num_render_channels_to_aec_ = + multichannel_content_detector_.IsProperMultiChannelContentDetected() + ? num_render_input_channels_ + : 1; + + config_selector_.Update( + multichannel_content_detector_.IsProperMultiChannelContentDetected()); + + render_block_.SetNumChannels(num_render_channels_to_aec_); + + render_blocker_.reset( + new FrameBlocker(num_bands_, num_render_channels_to_aec_)); + + block_processor_.reset(BlockProcessor::Create( + config_selector_.active_config(), sample_rate_hz_, + num_render_channels_to_aec_, num_capture_channels_)); + + render_sub_frame_view_ = std::vector>>( + num_bands_, + std::vector>(num_render_channels_to_aec_)); +} + void EchoCanceller3::AnalyzeRender(const AudioBuffer& render) { RTC_DCHECK_RUNS_SERIALIZED(&render_race_checker_); - RTC_DCHECK_EQ(render.num_channels(), num_render_channels_); + RTC_DCHECK_EQ(render.num_channels(), num_render_input_channels_); data_dumper_->DumpRaw("aec3_call_order", static_cast(EchoCanceller3ApiCall::kRender)); @@ -764,7 +865,7 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, if (linear_output && !linear_output_framer_) { RTC_LOG(LS_ERROR) << "Trying to retrieve the linear AEC output without " "properly configuring AEC3."; - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } // Report capture call in the metrics and periodically update API call @@ -772,7 +873,7 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, api_call_metrics_.ReportCaptureCall(); // Optionally delay the capture signal. - if (config_.delay.fixed_capture_delay_samples > 0) { + if (config_selector_.active_config().delay.fixed_capture_delay_samples > 0) { RTC_DCHECK(block_delay_buffer_); block_delay_buffer_->DelaySignal(capture); } @@ -784,22 +885,26 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, EmptyRenderQueue(); - ProcessCaptureFrameContent(linear_output, capture, level_change, - saturated_microphone_signal_, 0, &capture_blocker_, - linear_output_framer_.get(), &output_framer_, - block_processor_.get(), linear_output_block_.get(), - &linear_output_sub_frame_view_, &capture_block_, - &capture_sub_frame_view_); + ProcessCaptureFrameContent( + linear_output, capture, level_change, + multichannel_content_detector_.IsTemporaryMultiChannelContentDetected(), + saturated_microphone_signal_, 0, &capture_blocker_, + linear_output_framer_.get(), &output_framer_, block_processor_.get(), + linear_output_block_.get(), &linear_output_sub_frame_view_, + &capture_block_, &capture_sub_frame_view_); - ProcessCaptureFrameContent(linear_output, capture, level_change, - saturated_microphone_signal_, 1, &capture_blocker_, - linear_output_framer_.get(), &output_framer_, - block_processor_.get(), linear_output_block_.get(), - &linear_output_sub_frame_view_, &capture_block_, - &capture_sub_frame_view_); + ProcessCaptureFrameContent( + linear_output, capture, level_change, + multichannel_content_detector_.IsTemporaryMultiChannelContentDetected(), + saturated_microphone_signal_, 1, &capture_blocker_, + linear_output_framer_.get(), &output_framer_, block_processor_.get(), + linear_output_block_.get(), &linear_output_sub_frame_view_, + &capture_block_, &capture_sub_frame_view_); ProcessRemainingCaptureFrameContent( - level_change, saturated_microphone_signal_, &capture_blocker_, + level_change, + multichannel_content_detector_.IsTemporaryMultiChannelContentDetected(), + saturated_microphone_signal_, &capture_blocker_, linear_output_framer_.get(), &output_framer_, block_processor_.get(), linear_output_block_.get(), &capture_block_); @@ -819,29 +924,37 @@ void EchoCanceller3::SetAudioBufferDelay(int delay_ms) { block_processor_->SetAudioBufferDelay(delay_ms); } +void EchoCanceller3::SetCaptureOutputUsage(bool capture_output_used) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + block_processor_->SetCaptureOutputUsage(capture_output_used); +} + bool EchoCanceller3::ActiveProcessing() const { return true; } -EchoCanceller3Config EchoCanceller3::CreateDefaultConfig( - size_t num_render_channels, - size_t num_capture_channels) { +EchoCanceller3Config EchoCanceller3::CreateDefaultMultichannelConfig() { EchoCanceller3Config cfg; - if (num_render_channels > 1) { - // Use shorter and more rapidly adapting coarse filter to compensate for - // thge increased number of total filter parameters to adapt. - cfg.filter.coarse.length_blocks = 11; - cfg.filter.coarse.rate = 0.95f; - cfg.filter.coarse_initial.length_blocks = 11; - cfg.filter.coarse_initial.rate = 0.95f; + // Use shorter and more rapidly adapting coarse filter to compensate for + // thge increased number of total filter parameters to adapt. + cfg.filter.coarse.length_blocks = 11; + cfg.filter.coarse.rate = 0.95f; + cfg.filter.coarse_initial.length_blocks = 11; + cfg.filter.coarse_initial.rate = 0.95f; - // Use more concervative suppressor behavior for non-nearend speech. - cfg.suppressor.normal_tuning.max_dec_factor_lf = 0.35f; - cfg.suppressor.normal_tuning.max_inc_factor = 1.5f; - } + // Use more concervative suppressor behavior for non-nearend speech. + cfg.suppressor.normal_tuning.max_dec_factor_lf = 0.35f; + cfg.suppressor.normal_tuning.max_inc_factor = 1.5f; return cfg; } +void EchoCanceller3::SetBlockProcessorForTesting( + std::unique_ptr block_processor) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + RTC_DCHECK(block_processor); + block_processor_ = std::move(block_processor); +} + void EchoCanceller3::EmptyRenderQueue() { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); bool frame_to_buffer = @@ -850,16 +963,27 @@ void EchoCanceller3::EmptyRenderQueue() { // Report render call in the metrics. api_call_metrics_.ReportRenderCall(); - BufferRenderFrameContent(&render_queue_output_frame_, 0, &render_blocker_, - block_processor_.get(), &render_block_, - &render_sub_frame_view_); + if (multichannel_content_detector_.UpdateDetection( + render_queue_output_frame_)) { + // Reinitialize the AEC when proper stereo is detected. + Initialize(); + } - BufferRenderFrameContent(&render_queue_output_frame_, 1, &render_blocker_, - block_processor_.get(), &render_block_, - &render_sub_frame_view_); + // Buffer frame content. + BufferRenderFrameContent( + /*proper_downmix_needed=*/multichannel_content_detector_ + .IsTemporaryMultiChannelContentDetected(), + &render_queue_output_frame_, 0, render_blocker_.get(), + block_processor_.get(), &render_block_, &render_sub_frame_view_); - BufferRemainingRenderFrameContent(&render_blocker_, block_processor_.get(), - &render_block_); + BufferRenderFrameContent( + /*proper_downmix_needed=*/multichannel_content_detector_ + .IsTemporaryMultiChannelContentDetected(), + &render_queue_output_frame_, 1, render_blocker_.get(), + block_processor_.get(), &render_block_, &render_sub_frame_view_); + + BufferRemainingRenderFrameContent(render_blocker_.get(), + block_processor_.get(), &render_block_); frame_to_buffer = render_transfer_queue_.Remove(&render_queue_output_frame_); diff --git a/webrtc/modules/audio_processing/aec3/echo_canceller3.h b/webrtc/modules/audio_processing/aec3/echo_canceller3.h index bacd5df..7bf8e51 100644 --- a/webrtc/modules/audio_processing/aec3/echo_canceller3.h +++ b/webrtc/modules/audio_processing/aec3/echo_canceller3.h @@ -13,9 +13,11 @@ #include +#include #include #include +#include "absl/types/optional.h" #include "api/array_view.h" #include "api/audio/echo_canceller3_config.h" #include "api/audio/echo_control.h" @@ -23,7 +25,9 @@ #include "modules/audio_processing/aec3/block_delay_buffer.h" #include "modules/audio_processing/aec3/block_framer.h" #include "modules/audio_processing/aec3/block_processor.h" +#include "modules/audio_processing/aec3/config_selector.h" #include "modules/audio_processing/aec3/frame_blocker.h" +#include "modules/audio_processing/aec3/multi_channel_content_detector.h" #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" @@ -84,18 +88,15 @@ class Aec3RenderQueueItemVerifier { // AnalyzeRender call which can be called concurrently with the other methods. class EchoCanceller3 : public EchoControl { public: - // Normal c-tor to use. - EchoCanceller3(const EchoCanceller3Config& config, - int sample_rate_hz, - size_t num_render_channels, - size_t num_capture_channels); - // Testing c-tor that is used only for testing purposes. - EchoCanceller3(const EchoCanceller3Config& config, - int sample_rate_hz, - size_t num_render_channels, - size_t num_capture_channels, - std::unique_ptr block_processor); + EchoCanceller3( + const EchoCanceller3Config& config, + const absl::optional& multichannel_config, + int sample_rate_hz, + size_t num_render_channels, + size_t num_capture_channels); + ~EchoCanceller3() override; + EchoCanceller3(const EchoCanceller3&) = delete; EchoCanceller3& operator=(const EchoCanceller3&) = delete; @@ -118,6 +119,12 @@ class EchoCanceller3 : public EchoControl { // Provides an optional external estimate of the audio buffer delay. void SetAudioBufferDelay(int delay_ms) override; + // Specifies whether the capture output will be used. The purpose of this is + // to allow the echo controller to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + void SetCaptureOutputUsage(bool capture_output_used) override; + bool ActiveProcessing() const override; // Signals whether an external detector has detected echo leakage from the @@ -129,14 +136,39 @@ class EchoCanceller3 : public EchoControl { block_processor_->UpdateEchoLeakageStatus(leakage_detected); } - // Produces a default configuration that is suitable for a certain combination - // of render and capture channels. - static EchoCanceller3Config CreateDefaultConfig(size_t num_render_channels, - size_t num_capture_channels); + // Produces a default configuration for multichannel. + static EchoCanceller3Config CreateDefaultMultichannelConfig(); private: + friend class EchoCanceller3Tester; + FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, DetectionOfProperStereo); + FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, + DetectionOfProperStereoUsingThreshold); + FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, + DetectionOfProperStereoUsingHysteresis); + FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, + StereoContentDetectionForMonoSignals); + class RenderWriter; + // (Re-)Initializes the selected subset of the EchoCanceller3 fields, at + // creation as well as during reconfiguration. + void Initialize(); + + // Only for testing. Replaces the internal block processor. + void SetBlockProcessorForTesting( + std::unique_ptr block_processor); + + // Only for testing. Returns whether stereo processing is active. + bool StereoRenderProcessingActiveForTesting() const { + return multichannel_content_detector_.IsProperMultiChannelContentDetected(); + } + + // Only for testing. + const EchoCanceller3Config& GetActiveConfigForTesting() const { + return config_selector_.active_config(); + } + // Empties the render SwapQueue. void EmptyRenderQueue(); @@ -154,18 +186,22 @@ class EchoCanceller3 : public EchoControl { RTC_GUARDED_BY(render_race_checker_); // State that may be accessed by the capture thread. - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const EchoCanceller3Config config_; const int sample_rate_hz_; const int num_bands_; - const size_t num_render_channels_; + const size_t num_render_input_channels_; + size_t num_render_channels_to_aec_; const size_t num_capture_channels_; + ConfigSelector config_selector_; + MultiChannelContentDetector multichannel_content_detector_; std::unique_ptr linear_output_framer_ RTC_GUARDED_BY(capture_race_checker_); BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_); FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_); - FrameBlocker render_blocker_ RTC_GUARDED_BY(capture_race_checker_); + std::unique_ptr render_blocker_ + RTC_GUARDED_BY(capture_race_checker_); SwapQueue>>, Aec3RenderQueueItemVerifier> render_transfer_queue_; @@ -175,12 +211,10 @@ class EchoCanceller3 : public EchoControl { RTC_GUARDED_BY(capture_race_checker_); bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) = false; - std::vector>> render_block_ - RTC_GUARDED_BY(capture_race_checker_); - std::unique_ptr>>> - linear_output_block_ RTC_GUARDED_BY(capture_race_checker_); - std::vector>> capture_block_ + Block render_block_ RTC_GUARDED_BY(capture_race_checker_); + std::unique_ptr linear_output_block_ RTC_GUARDED_BY(capture_race_checker_); + Block capture_block_ RTC_GUARDED_BY(capture_race_checker_); std::vector>> render_sub_frame_view_ RTC_GUARDED_BY(capture_race_checker_); std::vector>> linear_output_sub_frame_view_ diff --git a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc index 2c987f9..510e4b8 100644 --- a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -42,10 +42,12 @@ EchoPathDelayEstimator::EchoPathDelayEstimator( ? config.render_levels.poor_excitation_render_limit_ds8 : config.render_levels.poor_excitation_render_limit, config.delay.delay_estimate_smoothing, - config.delay.delay_candidate_detection_threshold), + config.delay.delay_estimate_smoothing_delay_found, + config.delay.delay_candidate_detection_threshold, + config.delay.detect_pre_echo), matched_filter_lag_aggregator_(data_dumper_, matched_filter_.GetMaxFilterLag(), - config.delay.delay_selection_thresholds) { + config.delay) { RTC_DCHECK(data_dumper); RTC_DCHECK(down_sampling_factor_ > 0); } @@ -58,9 +60,7 @@ void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) { absl::optional EchoPathDelayEstimator::EstimateDelay( const DownsampledRenderBuffer& render_buffer, - const std::vector>& capture) { - RTC_DCHECK_EQ(kBlockSize, capture[0].size()); - + const Block& capture) { std::array downsampled_capture_data; rtc::ArrayView downsampled_capture(downsampled_capture_data.data(), sub_block_size_); @@ -71,17 +71,19 @@ absl::optional EchoPathDelayEstimator::EstimateDelay( data_dumper_->DumpWav("aec3_capture_decimator_output", downsampled_capture.size(), downsampled_capture.data(), 16000 / down_sampling_factor_, 1); - matched_filter_.Update(render_buffer, downsampled_capture); + matched_filter_.Update(render_buffer, downsampled_capture, + matched_filter_lag_aggregator_.ReliableDelayFound()); absl::optional aggregated_matched_filter_lag = matched_filter_lag_aggregator_.Aggregate( - matched_filter_.GetLagEstimates()); + matched_filter_.GetBestLagEstimate()); // Run clockdrift detection. if (aggregated_matched_filter_lag && (*aggregated_matched_filter_lag).quality == DelayEstimate::Quality::kRefined) - clockdrift_detector_.Update((*aggregated_matched_filter_lag).delay); + clockdrift_detector_.Update( + matched_filter_lag_aggregator_.GetDelayAtHighestPeak()); // TODO(peah): Move this logging outside of this class once EchoCanceller3 // development is done. @@ -118,7 +120,7 @@ void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator, if (reset_lag_aggregator) { matched_filter_lag_aggregator_.Reset(reset_delay_confidence); } - matched_filter_.Reset(); + matched_filter_.Reset(/*full_reset=*/reset_lag_aggregator); old_aggregated_lag_ = absl::nullopt; consistent_estimate_counter_ = 0; } diff --git a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h index 6c8c212..b24d0a2 100644 --- a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h +++ b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -16,12 +16,12 @@ #include "absl/types/optional.h" #include "api/array_view.h" #include "modules/audio_processing/aec3/alignment_mixer.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/clockdrift_detector.h" #include "modules/audio_processing/aec3/decimator.h" #include "modules/audio_processing/aec3/delay_estimate.h" #include "modules/audio_processing/aec3/matched_filter.h" #include "modules/audio_processing/aec3/matched_filter_lag_aggregator.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -37,6 +37,9 @@ class EchoPathDelayEstimator { size_t num_capture_channels); ~EchoPathDelayEstimator(); + EchoPathDelayEstimator(const EchoPathDelayEstimator&) = delete; + EchoPathDelayEstimator& operator=(const EchoPathDelayEstimator&) = delete; + // Resets the estimation. If the delay confidence is reset, the reset behavior // is as if the call is restarted. void Reset(bool reset_delay_confidence); @@ -44,7 +47,7 @@ class EchoPathDelayEstimator { // Produce a delay estimate if such is avaliable. absl::optional EstimateDelay( const DownsampledRenderBuffer& render_buffer, - const std::vector>& capture); + const Block& capture); // Log delay estimator properties. void LogDelayEstimationProperties(int sample_rate_hz, size_t shift) const { @@ -71,8 +74,6 @@ class EchoPathDelayEstimator { // Internal reset method with more granularity. void Reset(bool reset_lag_aggregator, bool reset_delay_confidence); - - RTC_DISALLOW_COPY_AND_ASSIGN(EchoPathDelayEstimator); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/echo_path_variability.h b/webrtc/modules/audio_processing/aec3/echo_path_variability.h index 78e4f64..45b33c2 100644 --- a/webrtc/modules/audio_processing/aec3/echo_path_variability.h +++ b/webrtc/modules/audio_processing/aec3/echo_path_variability.h @@ -14,11 +14,7 @@ namespace webrtc { struct EchoPathVariability { - enum class DelayAdjustment { - kNone, - kBufferFlush, - kNewDetectedDelay - }; + enum class DelayAdjustment { kNone, kBufferFlush, kNewDetectedDelay }; EchoPathVariability(bool gain_change, DelayAdjustment delay_change, diff --git a/webrtc/modules/audio_processing/aec3/echo_remover.cc b/webrtc/modules/audio_processing/aec3/echo_remover.cc index a3cd22f..673d88a 100644 --- a/webrtc/modules/audio_processing/aec3/echo_remover.cc +++ b/webrtc/modules/audio_processing/aec3/echo_remover.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -33,7 +34,6 @@ #include "modules/audio_processing/aec3/suppression_filter.h" #include "modules/audio_processing/aec3/suppression_gain.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -118,13 +118,12 @@ class EchoRemoverImpl final : public EchoRemover { // Removes the echo from a block of samples from the capture signal. The // supplied render signal is assumed to be pre-aligned with the capture // signal. - void ProcessCapture( - EchoPathVariability echo_path_variability, - bool capture_signal_saturation, - const absl::optional& external_delay, - RenderBuffer* render_buffer, - std::vector>>* linear_output, - std::vector>>* capture) override; + void ProcessCapture(EchoPathVariability echo_path_variability, + bool capture_signal_saturation, + const absl::optional& external_delay, + RenderBuffer* render_buffer, + Block* linear_output, + Block* capture) override; // Updates the status on whether echo leakage is detected in the output of the // echo remover. @@ -132,6 +131,10 @@ class EchoRemoverImpl final : public EchoRemover { echo_leakage_detected_ = leakage_detected; } + void SetCaptureOutputUsage(bool capture_output_used) override { + capture_output_used_ = capture_output_used; + } + private: // Selects which of the coarse and refined linear filter outputs that is most // appropriate to pass to the suppressor and forms the linear filter output by @@ -139,7 +142,7 @@ class EchoRemoverImpl final : public EchoRemover { void FormLinearFilterOutput(const SubtractorOutput& subtractor_output, rtc::ArrayView output); - static int instance_count_; + static std::atomic instance_count_; const EchoCanceller3Config config_; const Aec3Fft fft_; std::unique_ptr data_dumper_; @@ -155,6 +158,7 @@ class EchoRemoverImpl final : public EchoRemover { RenderSignalAnalyzer render_signal_analyzer_; ResidualEchoEstimator residual_echo_estimator_; bool echo_leakage_detected_ = false; + bool capture_output_used_ = true; AecState aec_state_; EchoRemoverMetrics metrics_; std::vector> e_old_; @@ -167,6 +171,7 @@ class EchoRemoverImpl final : public EchoRemover { std::vector> Y2_heap_; std::vector> E2_heap_; std::vector> R2_heap_; + std::vector> R2_unbounded_heap_; std::vector> S2_linear_heap_; std::vector Y_heap_; std::vector E_heap_; @@ -175,7 +180,7 @@ class EchoRemoverImpl final : public EchoRemover { std::vector subtractor_output_heap_; }; -int EchoRemoverImpl::instance_count_ = 0; +std::atomic EchoRemoverImpl::instance_count_(0); EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, int sample_rate_hz, @@ -183,8 +188,7 @@ EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, size_t num_capture_channels) : config_(config), fft_(), - data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), optimization_(DetectOptimization()), sample_rate_hz_(sample_rate_hz), num_render_channels_(num_render_channels), @@ -213,6 +217,7 @@ EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, Y2_heap_(NumChannelsOnHeap(num_capture_channels_)), E2_heap_(NumChannelsOnHeap(num_capture_channels_)), R2_heap_(NumChannelsOnHeap(num_capture_channels_)), + R2_unbounded_heap_(NumChannelsOnHeap(num_capture_channels_)), S2_linear_heap_(NumChannelsOnHeap(num_capture_channels_)), Y_heap_(NumChannelsOnHeap(num_capture_channels_)), E_heap_(NumChannelsOnHeap(num_capture_channels_)), @@ -236,20 +241,17 @@ void EchoRemoverImpl::ProcessCapture( bool capture_signal_saturation, const absl::optional& external_delay, RenderBuffer* render_buffer, - std::vector>>* linear_output, - std::vector>>* capture) { + Block* linear_output, + Block* capture) { ++block_counter_; - const std::vector>>& x = - render_buffer->Block(0); - std::vector>>* y = capture; + const Block& x = render_buffer->GetBlock(0); + Block* y = capture; RTC_DCHECK(render_buffer); RTC_DCHECK(y); - RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_)); - RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_)); - RTC_DCHECK_EQ(x[0].size(), num_render_channels_); - RTC_DCHECK_EQ((*y)[0].size(), num_capture_channels_); - RTC_DCHECK_EQ(x[0][0].size(), kBlockSize); - RTC_DCHECK_EQ((*y)[0][0].size(), kBlockSize); + RTC_DCHECK_EQ(x.NumBands(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(y->NumBands(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(x.NumChannels(), num_render_channels_); + RTC_DCHECK_EQ(y->NumChannels(), num_capture_channels_); // Stack allocated data to use when the number of channels is low. std::array, kMaxNumChannelsOnStack> e_stack; @@ -259,6 +261,8 @@ void EchoRemoverImpl::ProcessCapture( E2_stack; std::array, kMaxNumChannelsOnStack> R2_stack; + std::array, kMaxNumChannelsOnStack> + R2_unbounded_stack; std::array, kMaxNumChannelsOnStack> S2_linear_stack; std::array Y_stack; @@ -275,6 +279,8 @@ void EchoRemoverImpl::ProcessCapture( E2_stack.data(), num_capture_channels_); rtc::ArrayView> R2( R2_stack.data(), num_capture_channels_); + rtc::ArrayView> R2_unbounded( + R2_unbounded_stack.data(), num_capture_channels_); rtc::ArrayView> S2_linear( S2_linear_stack.data(), num_capture_channels_); rtc::ArrayView Y(Y_stack.data(), num_capture_channels_); @@ -296,6 +302,8 @@ void EchoRemoverImpl::ProcessCapture( E2_heap_.data(), num_capture_channels_); R2 = rtc::ArrayView>( R2_heap_.data(), num_capture_channels_); + R2_unbounded = rtc::ArrayView>( + R2_unbounded_heap_.data(), num_capture_channels_); S2_linear = rtc::ArrayView>( S2_linear_heap_.data(), num_capture_channels_); Y = rtc::ArrayView(Y_heap_.data(), num_capture_channels_); @@ -308,12 +316,14 @@ void EchoRemoverImpl::ProcessCapture( subtractor_output_heap_.data(), num_capture_channels_); } - data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, - &(*y)[0][0][0], 16000, 1); - data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, - &x[0][0][0], 16000, 1); - data_dumper_->DumpRaw("aec3_echo_remover_capture_input", (*y)[0][0]); - data_dumper_->DumpRaw("aec3_echo_remover_render_input", x[0][0]); + data_dumper_->DumpWav("aec3_echo_remover_capture_input", + y->View(/*band=*/0, /*channel=*/0), 16000, 1); + data_dumper_->DumpWav("aec3_echo_remover_render_input", + x.View(/*band=*/0, /*channel=*/0), 16000, 1); + data_dumper_->DumpRaw("aec3_echo_remover_capture_input", + y->View(/*band=*/0, /*channel=*/0)); + data_dumper_->DumpRaw("aec3_echo_remover_render_input", + x.View(/*band=*/0, /*channel=*/0)); aec_state_.UpdateCaptureSaturation(capture_signal_saturation); @@ -356,13 +366,13 @@ void EchoRemoverImpl::ProcessCapture( } // Perform linear echo cancellation. - subtractor_.Process(*render_buffer, (*y)[0], render_signal_analyzer_, - aec_state_, subtractor_output); + subtractor_.Process(*render_buffer, *y, render_signal_analyzer_, aec_state_, + subtractor_output); // Compute spectra. for (size_t ch = 0; ch < num_capture_channels_; ++ch) { FormLinearFilterOutput(subtractor_output[ch], e[ch]); - WindowedPaddedFft(fft_, (*y)[0][ch], y_old_[ch], &Y[ch]); + WindowedPaddedFft(fft_, y->View(/*band=*/0, ch), y_old_[ch], &Y[ch]); WindowedPaddedFft(fft_, e[ch], e_old_[ch], &E[ch]); LinearEchoPower(E[ch], Y[ch], &S2_linear[ch]); Y[ch].Spectrum(optimization_, Y2[ch]); @@ -371,11 +381,11 @@ void EchoRemoverImpl::ProcessCapture( // Optionally return the linear filter output. if (linear_output) { - RTC_DCHECK_GE(1, linear_output->size()); - RTC_DCHECK_EQ(num_capture_channels_, linear_output[0].size()); + RTC_DCHECK_GE(1, linear_output->NumBands()); + RTC_DCHECK_EQ(num_capture_channels_, linear_output->NumChannels()); for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - RTC_DCHECK_EQ(kBlockSize, (*linear_output)[0][ch].size()); - std::copy(e[ch].begin(), e[ch].end(), (*linear_output)[0][ch].begin()); + std::copy(e[ch].begin(), e[ch].end(), + linear_output->begin(/*band=*/0, ch)); } } @@ -387,42 +397,54 @@ void EchoRemoverImpl::ProcessCapture( // Choose the linear output. const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y; - data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &(*y)[0][0][0], 16000, - 1); + data_dumper_->DumpWav("aec3_output_linear", + y->View(/*band=*/0, /*channel=*/0), 16000, 1); data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1); - // Estimate the residual echo power. - residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2, - R2); - // Estimate the comfort noise. cng_.Compute(aec_state_.SaturatedCapture(), Y2, comfort_noise, high_band_comfort_noise); - // Suppressor nearend estimate. - if (aec_state_.UsableLinearEstimate()) { - // E2 is bound by Y2. - for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(), - E2[ch].begin(), - [](float a, float b) { return std::min(a, b); }); - } - } - const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2; - - // Suppressor echo estimate. - const auto& echo_spectrum = - aec_state_.UsableLinearEstimate() ? S2_linear : R2; - - // Compute preferred gains. - float high_bands_gain; + // Only do the below processing if the output of the audio processing module + // is used. std::array G; - suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2, - cng_.NoiseSpectrum(), render_signal_analyzer_, - aec_state_, x, &high_bands_gain, &G); + if (capture_output_used_) { + // Estimate the residual echo power. + residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2, + suppression_gain_.IsDominantNearend(), R2, + R2_unbounded); - suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, - high_bands_gain, Y_fft, y); + // Suppressor nearend estimate. + if (aec_state_.UsableLinearEstimate()) { + // E2 is bound by Y2. + for (size_t ch = 0; ch < num_capture_channels_; ++ch) { + std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(), + E2[ch].begin(), + [](float a, float b) { return std::min(a, b); }); + } + } + const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2; + + // Suppressor echo estimate. + const auto& echo_spectrum = + aec_state_.UsableLinearEstimate() ? S2_linear : R2; + + // Determine if the suppressor should assume clock drift. + const bool clock_drift = config_.echo_removal_control.has_clock_drift || + echo_path_variability.clock_drift; + + // Compute preferred gains. + float high_bands_gain; + suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2, R2_unbounded, + cng_.NoiseSpectrum(), render_signal_analyzer_, + aec_state_, x, clock_drift, &high_bands_gain, &G); + + suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, + high_bands_gain, Y_fft, y); + + } else { + G.fill(0.f); + } // Update the metrics. metrics_.Update(aec_state_, cng_.NoiseSpectrum()[0], G); @@ -430,13 +452,12 @@ void EchoRemoverImpl::ProcessCapture( // Debug outputs for the purpose of development and analysis. data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize, &subtractor_output[0].s_refined[0], 16000, 1); - data_dumper_->DumpRaw("aec3_output", (*y)[0][0]); + data_dumper_->DumpRaw("aec3_output", y->View(/*band=*/0, /*channel=*/0)); data_dumper_->DumpRaw("aec3_narrow_render", render_signal_analyzer_.NarrowPeakBand() ? 1 : 0); data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()[0]); data_dumper_->DumpRaw("aec3_suppressor_gain", G); - data_dumper_->DumpWav("aec3_output", - rtc::ArrayView(&(*y)[0][0][0], kBlockSize), + data_dumper_->DumpWav("aec3_output", y->View(/*band=*/0, /*channel=*/0), 16000, 1); data_dumper_->DumpRaw("aec3_using_subtractor_output[0]", aec_state_.UseLinearFilterOutput() ? 1 : 0); diff --git a/webrtc/modules/audio_processing/aec3/echo_remover.h b/webrtc/modules/audio_processing/aec3/echo_remover.h index ef41646..f2f4f5e 100644 --- a/webrtc/modules/audio_processing/aec3/echo_remover.h +++ b/webrtc/modules/audio_processing/aec3/echo_remover.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "api/audio/echo_canceller3_config.h" #include "api/audio/echo_control.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/delay_estimate.h" #include "modules/audio_processing/aec3/echo_path_variability.h" #include "modules/audio_processing/aec3/render_buffer.h" @@ -42,12 +43,18 @@ class EchoRemover { bool capture_signal_saturation, const absl::optional& external_delay, RenderBuffer* render_buffer, - std::vector>>* linear_output, - std::vector>>* capture) = 0; + Block* linear_output, + Block* capture) = 0; // Updates the status on whether echo leakage is detected in the output of the // echo remover. virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; + + // Specifies whether the capture output will be used. The purpose of this is + // to allow the echo remover to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + virtual void SetCaptureOutputUsage(bool capture_output_used) = 0; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc b/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc index 4502f31..c3fc807 100644 --- a/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc +++ b/webrtc/modules/audio_processing/aec3/echo_remover_metrics.cc @@ -23,12 +23,6 @@ namespace webrtc { -namespace { - -constexpr float kOneByMetricsCollectionBlocks = 1.f / kMetricsCollectionBlocks; - -} // namespace - EchoRemoverMetrics::DbMetric::DbMetric() : DbMetric(0.f, 0.f, 0.f) {} EchoRemoverMetrics::DbMetric::DbMetric(float sum_value, float floor_value, @@ -52,11 +46,8 @@ EchoRemoverMetrics::EchoRemoverMetrics() { } void EchoRemoverMetrics::ResetMetrics() { - erl_.fill(DbMetric(0.f, 10000.f, 0.000f)); erl_time_domain_ = DbMetric(0.f, 10000.f, 0.000f); - erle_.fill(DbMetric(0.f, 0.f, 1000.f)); erle_time_domain_ = DbMetric(0.f, 0.f, 1000.f); - active_render_count_ = 0; saturated_capture_ = false; } @@ -66,104 +57,24 @@ void EchoRemoverMetrics::Update( const std::array& suppressor_gain) { metrics_reported_ = false; if (++block_counter_ <= kMetricsCollectionBlocks) { - aec3::UpdateDbMetric(aec_state.Erl(), &erl_); erl_time_domain_.UpdateInstant(aec_state.ErlTimeDomain()); - aec3::UpdateDbMetric(aec_state.Erle()[0], &erle_); erle_time_domain_.UpdateInstant(aec_state.FullBandErleLog2()); - active_render_count_ += (aec_state.ActiveRender() ? 1 : 0); saturated_capture_ = saturated_capture_ || aec_state.SaturatedCapture(); } else { // Report the metrics over several frames in order to lower the impact of // the logarithms involved on the computational complexity. - constexpr int kMetricsCollectionBlocksBy2 = kMetricsCollectionBlocks / 2; switch (block_counter_) { case kMetricsCollectionBlocks + 1: - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErleBand0.Average", - aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, - kOneByMetricsCollectionBlocks, - erle_[0].sum_value), - 0, 19, 20); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErleBand0.Max", - aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, - erle_[0].ceil_value), - 0, 19, 20); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErleBand0.Min", - aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, - erle_[0].floor_value), - 0, 19, 20); - break; - case kMetricsCollectionBlocks + 2: - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErleBand1.Average", - aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, - kOneByMetricsCollectionBlocks, - erle_[1].sum_value), - 0, 19, 20); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErleBand1.Max", - aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, - erle_[1].ceil_value), - 0, 19, 20); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErleBand1.Min", - aec3::TransformDbMetricForReporting(true, 0.f, 19.f, 0.f, 1.f, - erle_[1].floor_value), - 0, 19, 20); - break; - case kMetricsCollectionBlocks + 3: - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErlBand0.Average", - aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, - kOneByMetricsCollectionBlocks, - erl_[0].sum_value), - 0, 59, 30); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErlBand0.Max", - aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, - erl_[0].ceil_value), - 0, 59, 30); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErlBand0.Min", - aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, - erl_[0].floor_value), - 0, 59, 30); - break; - case kMetricsCollectionBlocks + 4: - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErlBand1.Average", - aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, - kOneByMetricsCollectionBlocks, - erl_[1].sum_value), - 0, 59, 30); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErlBand1.Max", - aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, - erl_[1].ceil_value), - 0, 59, 30); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.EchoCanceller.ErlBand1.Min", - aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, - erl_[1].floor_value), - 0, 59, 30); - break; - case kMetricsCollectionBlocks + 5: RTC_HISTOGRAM_BOOLEAN( "WebRTC.Audio.EchoCanceller.UsableLinearEstimate", static_cast(aec_state.UsableLinearEstimate() ? 1 : 0)); - RTC_HISTOGRAM_BOOLEAN( - "WebRTC.Audio.EchoCanceller.ActiveRender", - static_cast( - active_render_count_ > kMetricsCollectionBlocksBy2 ? 1 : 0)); RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.FilterDelay", aec_state.MinDirectPathFilterDelay(), 0, 30, 31); RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.EchoCanceller.CaptureSaturation", static_cast(saturated_capture_ ? 1 : 0)); break; - case kMetricsCollectionBlocks + 6: + case kMetricsCollectionBlocks + 2: RTC_HISTOGRAM_COUNTS_LINEAR( "WebRTC.Audio.EchoCanceller.Erl.Value", aec3::TransformDbMetricForReporting(true, 0.f, 59.f, 30.f, 1.f, @@ -180,7 +91,7 @@ void EchoRemoverMetrics::Update( erl_time_domain_.floor_value), 0, 59, 30); break; - case kMetricsCollectionBlocks + 7: + case kMetricsCollectionBlocks + 3: RTC_HISTOGRAM_COUNTS_LINEAR( "WebRTC.Audio.EchoCanceller.Erle.Value", aec3::TransformDbMetricForReporting(false, 0.f, 19.f, 0.f, 1.f, @@ -202,7 +113,7 @@ void EchoRemoverMetrics::Update( ResetMetrics(); break; default: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; } } diff --git a/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h b/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h index 77fd8cd..aec8084 100644 --- a/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h +++ b/webrtc/modules/audio_processing/aec3/echo_remover_metrics.h @@ -15,7 +15,6 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec_state.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -34,6 +33,9 @@ class EchoRemoverMetrics { EchoRemoverMetrics(); + EchoRemoverMetrics(const EchoRemoverMetrics&) = delete; + EchoRemoverMetrics& operator=(const EchoRemoverMetrics&) = delete; + // Updates the metric with new data. void Update( const AecState& aec_state, @@ -48,15 +50,10 @@ class EchoRemoverMetrics { void ResetMetrics(); int block_counter_ = 0; - std::array erl_; DbMetric erl_time_domain_; - std::array erle_; DbMetric erle_time_domain_; - int active_render_count_ = 0; bool saturated_capture_ = false; bool metrics_reported_ = false; - - RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverMetrics); }; namespace aec3 { diff --git a/webrtc/modules/audio_processing/aec3/erl_estimator.h b/webrtc/modules/audio_processing/aec3/erl_estimator.h index 89bf6ac..639a52c 100644 --- a/webrtc/modules/audio_processing/aec3/erl_estimator.h +++ b/webrtc/modules/audio_processing/aec3/erl_estimator.h @@ -18,7 +18,6 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -28,6 +27,9 @@ class ErlEstimator { explicit ErlEstimator(size_t startup_phase_length_blocks_); ~ErlEstimator(); + ErlEstimator(const ErlEstimator&) = delete; + ErlEstimator& operator=(const ErlEstimator&) = delete; + // Resets the ERL estimation. void Reset(); @@ -49,7 +51,6 @@ class ErlEstimator { float erl_time_domain_; int hold_counter_time_domain_; size_t blocks_since_reset_ = 0; - RTC_DISALLOW_COPY_AND_ASSIGN(ErlEstimator); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/erle_estimator.cc b/webrtc/modules/audio_processing/aec3/erle_estimator.cc index 4d84345..0e3d715 100644 --- a/webrtc/modules/audio_processing/aec3/erle_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/erle_estimator.cc @@ -52,8 +52,9 @@ void ErleEstimator::Update( rtc::ArrayView> subtractor_spectra, const std::vector& converged_filters) { - RTC_DCHECK_EQ(subband_erle_estimator_.Erle().size(), capture_spectra.size()); - RTC_DCHECK_EQ(subband_erle_estimator_.Erle().size(), + RTC_DCHECK_EQ(subband_erle_estimator_.Erle(/*onset_compensated=*/true).size(), + capture_spectra.size()); + RTC_DCHECK_EQ(subband_erle_estimator_.Erle(/*onset_compensated=*/true).size(), subtractor_spectra.size()); const auto& X2_reverb = avg_render_spectrum_with_reverb; const auto& Y2 = capture_spectra; @@ -68,7 +69,9 @@ void ErleEstimator::Update( if (signal_dependent_erle_estimator_) { signal_dependent_erle_estimator_->Update( render_buffer, filter_frequency_responses, X2_reverb, Y2, E2, - subband_erle_estimator_.Erle(), converged_filters); + subband_erle_estimator_.Erle(/*onset_compensated=*/false), + subband_erle_estimator_.Erle(/*onset_compensated=*/true), + converged_filters); } fullband_erle_estimator_.Update(X2_reverb, Y2, E2, converged_filters); diff --git a/webrtc/modules/audio_processing/aec3/erle_estimator.h b/webrtc/modules/audio_processing/aec3/erle_estimator.h index d741cff..5579759 100644 --- a/webrtc/modules/audio_processing/aec3/erle_estimator.h +++ b/webrtc/modules/audio_processing/aec3/erle_estimator.h @@ -55,17 +55,30 @@ class ErleEstimator { const std::vector& converged_filters); // Returns the most recent subband ERLE estimates. - rtc::ArrayView> Erle() const { + rtc::ArrayView> Erle( + bool onset_compensated) const { return signal_dependent_erle_estimator_ - ? signal_dependent_erle_estimator_->Erle() - : subband_erle_estimator_.Erle(); + ? signal_dependent_erle_estimator_->Erle(onset_compensated) + : subband_erle_estimator_.Erle(onset_compensated); + } + + // Returns the non-capped subband ERLE. + rtc::ArrayView> ErleUnbounded() + const { + // Unbounded ERLE is only used with the subband erle estimator where the + // ERLE is often capped at low values. When the signal dependent ERLE + // estimator is used the capped ERLE is returned. + return !signal_dependent_erle_estimator_ + ? subband_erle_estimator_.ErleUnbounded() + : signal_dependent_erle_estimator_->Erle( + /*onset_compensated=*/false); } // Returns the subband ERLE that are estimated during onsets (only used for // testing). - rtc::ArrayView> ErleOnsets() + rtc::ArrayView> ErleDuringOnsets() const { - return subband_erle_estimator_.ErleOnsets(); + return subband_erle_estimator_.ErleDuringOnsets(); } // Returns the fullband ERLE estimate. diff --git a/webrtc/modules/audio_processing/aec3/fft_data_avx2.cc b/webrtc/modules/audio_processing/aec3/fft_data_avx2.cc index 1fe4bd6..a4b3056 100644 --- a/webrtc/modules/audio_processing/aec3/fft_data_avx2.cc +++ b/webrtc/modules/audio_processing/aec3/fft_data_avx2.cc @@ -8,11 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/aec3/fft_data.h" - #include #include "api/array_view.h" +#include "modules/audio_processing/aec3/fft_data.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aec3/filter_analyzer.cc b/webrtc/modules/audio_processing/aec3/filter_analyzer.cc index be954d3..d8fd3aa 100644 --- a/webrtc/modules/audio_processing/aec3/filter_analyzer.cc +++ b/webrtc/modules/audio_processing/aec3/filter_analyzer.cc @@ -19,7 +19,6 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/render_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" namespace webrtc { @@ -45,12 +44,11 @@ size_t FindPeakIndex(rtc::ArrayView filter_time_domain, } // namespace -int FilterAnalyzer::instance_count_ = 0; +std::atomic FilterAnalyzer::instance_count_(0); FilterAnalyzer::FilterAnalyzer(const EchoCanceller3Config& config, size_t num_capture_channels) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), bounded_erl_(config.ep_strength.bounded_erl), default_gain_(config.ep_strength.default_gain), h_highpass_(num_capture_channels, @@ -131,7 +129,7 @@ void FilterAnalyzer::AnalyzeRegion( st_ch.consistent_estimate = st_ch.consistent_filter_detector.Detect( h_highpass_[ch], region_, - render_buffer.Block(-filter_delays_blocks_[ch])[0], st_ch.peak_index, + render_buffer.GetBlock(-filter_delays_blocks_[ch]), st_ch.peak_index, filter_delays_blocks_[ch]); } } @@ -170,11 +168,16 @@ void FilterAnalyzer::PreProcessFilters( std::fill(h_highpass_[ch].begin() + region_.start_sample_, h_highpass_[ch].begin() + region_.end_sample_ + 1, 0.f); + float* h_highpass_ch = h_highpass_[ch].data(); + const float* filters_time_domain_ch = filters_time_domain[ch].data(); + const size_t region_end = region_.end_sample_; for (size_t k = std::max(h.size() - 1, region_.start_sample_); - k <= region_.end_sample_; ++k) { + k <= region_end; ++k) { + float tmp = h_highpass_ch[k]; for (size_t j = 0; j < h.size(); ++j) { - h_highpass_[ch][k] += filters_time_domain[ch][k - j] * h[j]; + tmp += filters_time_domain_ch[k - j] * h[j]; } + h_highpass_ch[k] = tmp; } } } @@ -219,7 +222,7 @@ void FilterAnalyzer::ConsistentFilterDetector::Reset() { bool FilterAnalyzer::ConsistentFilterDetector::Detect( rtc::ArrayView filter_to_analyze, const FilterRegion& region, - rtc::ArrayView> x_block, + const Block& x_block, size_t peak_index, int delay_blocks) { if (region.start_sample_ == 0) { @@ -230,19 +233,23 @@ bool FilterAnalyzer::ConsistentFilterDetector::Detect( peak_index > filter_to_analyze.size() - 129 ? 0 : peak_index + 128; } + float filter_floor_accum = filter_floor_accum_; + float filter_secondary_peak = filter_secondary_peak_; for (size_t k = region.start_sample_; k < std::min(region.end_sample_ + 1, filter_floor_low_limit_); ++k) { float abs_h = fabsf(filter_to_analyze[k]); - filter_floor_accum_ += abs_h; - filter_secondary_peak_ = std::max(filter_secondary_peak_, abs_h); + filter_floor_accum += abs_h; + filter_secondary_peak = std::max(filter_secondary_peak, abs_h); } for (size_t k = std::max(filter_floor_high_limit_, region.start_sample_); k <= region.end_sample_; ++k) { float abs_h = fabsf(filter_to_analyze[k]); - filter_floor_accum_ += abs_h; - filter_secondary_peak_ = std::max(filter_secondary_peak_, abs_h); + filter_floor_accum += abs_h; + filter_secondary_peak = std::max(filter_secondary_peak, abs_h); } + filter_floor_accum_ = filter_floor_accum; + filter_secondary_peak_ = filter_secondary_peak; if (region.end_sample_ == filter_to_analyze.size() - 1) { float filter_floor = filter_floor_accum_ / @@ -256,7 +263,9 @@ bool FilterAnalyzer::ConsistentFilterDetector::Detect( if (significant_peak_) { bool active_render_block = false; - for (auto& x_channel : x_block) { + for (int ch = 0; ch < x_block.NumChannels(); ++ch) { + rtc::ArrayView x_channel = + x_block.View(/*band=*/0, ch); const float x_energy = std::inner_product( x_channel.begin(), x_channel.end(), x_channel.begin(), 0.f); if (x_energy > active_render_threshold_) { diff --git a/webrtc/modules/audio_processing/aec3/filter_analyzer.h b/webrtc/modules/audio_processing/aec3/filter_analyzer.h index b0b7070..9aec8b1 100644 --- a/webrtc/modules/audio_processing/aec3/filter_analyzer.h +++ b/webrtc/modules/audio_processing/aec3/filter_analyzer.h @@ -14,13 +14,14 @@ #include #include +#include #include #include #include "api/array_view.h" #include "api/audio/echo_canceller3_config.h" #include "modules/audio_processing/aec3/aec3_common.h" -#include "rtc_base/constructor_magic.h" +#include "modules/audio_processing/aec3/block.h" namespace webrtc { @@ -94,7 +95,7 @@ class FilterAnalyzer { void Reset(); bool Detect(rtc::ArrayView filter_to_analyze, const FilterRegion& region, - rtc::ArrayView> x_block, + const Block& x_block, size_t peak_index, int delay_blocks); @@ -129,7 +130,7 @@ class FilterAnalyzer { ConsistentFilterDetector consistent_filter_detector; }; - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const bool bounded_erl_; const float default_gain_; diff --git a/webrtc/modules/audio_processing/aec3/frame_blocker.cc b/webrtc/modules/audio_processing/aec3/frame_blocker.cc index 63aaf09..3039dcf 100644 --- a/webrtc/modules/audio_processing/aec3/frame_blocker.cc +++ b/webrtc/modules/audio_processing/aec3/frame_blocker.cc @@ -33,26 +33,22 @@ FrameBlocker::~FrameBlocker() = default; void FrameBlocker::InsertSubFrameAndExtractBlock( const std::vector>>& sub_frame, - std::vector>>* block) { + Block* block) { RTC_DCHECK(block); - RTC_DCHECK_EQ(num_bands_, block->size()); + RTC_DCHECK_EQ(num_bands_, block->NumBands()); RTC_DCHECK_EQ(num_bands_, sub_frame.size()); for (size_t band = 0; band < num_bands_; ++band) { - RTC_DCHECK_EQ(num_channels_, (*block)[band].size()); + RTC_DCHECK_EQ(num_channels_, block->NumChannels()); RTC_DCHECK_EQ(num_channels_, sub_frame[band].size()); for (size_t channel = 0; channel < num_channels_; ++channel) { RTC_DCHECK_GE(kBlockSize - 16, buffer_[band][channel].size()); - RTC_DCHECK_EQ(kBlockSize, (*block)[band][channel].size()); RTC_DCHECK_EQ(kSubFrameLength, sub_frame[band][channel].size()); const int samples_to_block = kBlockSize - buffer_[band][channel].size(); - (*block)[band][channel].clear(); - (*block)[band][channel].insert((*block)[band][channel].begin(), - buffer_[band][channel].begin(), - buffer_[band][channel].end()); - (*block)[band][channel].insert( - (*block)[band][channel].begin() + buffer_[band][channel].size(), - sub_frame[band][channel].begin(), - sub_frame[band][channel].begin() + samples_to_block); + std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(), + block->begin(band, channel)); + std::copy(sub_frame[band][channel].begin(), + sub_frame[band][channel].begin() + samples_to_block, + block->begin(band, channel) + kBlockSize - samples_to_block); buffer_[band][channel].clear(); buffer_[band][channel].insert( buffer_[band][channel].begin(), @@ -66,20 +62,16 @@ bool FrameBlocker::IsBlockAvailable() const { return kBlockSize == buffer_[0][0].size(); } -void FrameBlocker::ExtractBlock( - std::vector>>* block) { +void FrameBlocker::ExtractBlock(Block* block) { RTC_DCHECK(block); - RTC_DCHECK_EQ(num_bands_, block->size()); + RTC_DCHECK_EQ(num_bands_, block->NumBands()); + RTC_DCHECK_EQ(num_channels_, block->NumChannels()); RTC_DCHECK(IsBlockAvailable()); for (size_t band = 0; band < num_bands_; ++band) { - RTC_DCHECK_EQ(num_channels_, (*block)[band].size()); for (size_t channel = 0; channel < num_channels_; ++channel) { RTC_DCHECK_EQ(kBlockSize, buffer_[band][channel].size()); - RTC_DCHECK_EQ(kBlockSize, (*block)[band][channel].size()); - (*block)[band][channel].clear(); - (*block)[band][channel].insert((*block)[band][channel].begin(), - buffer_[band][channel].begin(), - buffer_[band][channel].end()); + std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(), + block->begin(band, channel)); buffer_[band][channel].clear(); } } diff --git a/webrtc/modules/audio_processing/aec3/frame_blocker.h b/webrtc/modules/audio_processing/aec3/frame_blocker.h index ebd6f77..623c812 100644 --- a/webrtc/modules/audio_processing/aec3/frame_blocker.h +++ b/webrtc/modules/audio_processing/aec3/frame_blocker.h @@ -17,6 +17,7 @@ #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/block.h" namespace webrtc { @@ -33,12 +34,12 @@ class FrameBlocker { // extracts one 64 sample multiband block. void InsertSubFrameAndExtractBlock( const std::vector>>& sub_frame, - std::vector>>* block); + Block* block); // Reports whether a multiband block of 64 samples is available for // extraction. bool IsBlockAvailable() const; // Extracts a multiband block of 64 samples. - void ExtractBlock(std::vector>>* block); + void ExtractBlock(Block* block); private: const size_t num_bands_; diff --git a/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.cc b/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.cc index e421214..e56674e 100644 --- a/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.cc @@ -34,8 +34,8 @@ FullBandErleEstimator::FullBandErleEstimator( const EchoCanceller3Config::Erle& config, size_t num_capture_channels) : min_erle_log2_(FastApproxLog2f(config.min + kEpsilon)), - max_erle_lf_log2(FastApproxLog2f(config.max_l + kEpsilon)), - hold_counters_time_domain_(num_capture_channels, 0), + max_erle_lf_log2_(FastApproxLog2f(config.max_l + kEpsilon)), + hold_counters_instantaneous_erle_(num_capture_channels, 0), erle_time_domain_log2_(num_capture_channels, min_erle_log2_), instantaneous_erle_(num_capture_channels, ErleInstantaneous(config)), linear_filters_qualities_(num_capture_channels) { @@ -52,8 +52,8 @@ void FullBandErleEstimator::Reset() { UpdateQualityEstimates(); std::fill(erle_time_domain_log2_.begin(), erle_time_domain_log2_.end(), min_erle_log2_); - std::fill(hold_counters_time_domain_.begin(), - hold_counters_time_domain_.end(), 0); + std::fill(hold_counters_instantaneous_erle_.begin(), + hold_counters_instantaneous_erle_.end(), 0); } void FullBandErleEstimator::Update( @@ -71,21 +71,17 @@ void FullBandErleEstimator::Update( const float E2_sum = std::accumulate(E2[ch].begin(), E2[ch].end(), 0.0f); if (instantaneous_erle_[ch].Update(Y2_sum, E2_sum)) { - hold_counters_time_domain_[ch] = kBlocksToHoldErle; + hold_counters_instantaneous_erle_[ch] = kBlocksToHoldErle; erle_time_domain_log2_[ch] += - 0.1f * ((instantaneous_erle_[ch].GetInstErleLog2().value()) - - erle_time_domain_log2_[ch]); - erle_time_domain_log2_[ch] = rtc::SafeClamp( - erle_time_domain_log2_[ch], min_erle_log2_, max_erle_lf_log2); + 0.05f * ((instantaneous_erle_[ch].GetInstErleLog2().value()) - + erle_time_domain_log2_[ch]); + erle_time_domain_log2_[ch] = + std::max(erle_time_domain_log2_[ch], min_erle_log2_); } } } - --hold_counters_time_domain_[ch]; - if (hold_counters_time_domain_[ch] <= 0) { - erle_time_domain_log2_[ch] = - std::max(min_erle_log2_, erle_time_domain_log2_[ch] - 0.044f); - } - if (hold_counters_time_domain_[ch] == 0) { + --hold_counters_instantaneous_erle_[ch]; + if (hold_counters_instantaneous_erle_[ch] == 0) { instantaneous_erle_[ch].ResetAccumulators(); } } @@ -166,17 +162,12 @@ void FullBandErleEstimator::ErleInstantaneous::Dump( void FullBandErleEstimator::ErleInstantaneous::UpdateMaxMin() { RTC_DCHECK(erle_log2_); - if (erle_log2_.value() > max_erle_log2_) { - max_erle_log2_ = erle_log2_.value(); - } else { - max_erle_log2_ -= 0.0004; // Forget factor, approx 1dB every 3 sec. - } - - if (erle_log2_.value() < min_erle_log2_) { - min_erle_log2_ = erle_log2_.value(); - } else { - min_erle_log2_ += 0.0004; // Forget factor, approx 1dB every 3 sec. - } + // Adding the forgetting factors for the maximum and minimum and capping the + // result to the incoming value. + max_erle_log2_ -= 0.0004f; // Forget factor, approx 1dB every 3 sec. + max_erle_log2_ = std::max(max_erle_log2_, erle_log2_.value()); + min_erle_log2_ += 0.0004f; // Forget factor, approx 1dB every 3 sec. + min_erle_log2_ = std::min(min_erle_log2_, erle_log2_.value()); } void FullBandErleEstimator::ErleInstantaneous::UpdateQualityEstimate() { diff --git a/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.h b/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.h index 1580f1a..7a08217 100644 --- a/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.h +++ b/webrtc/modules/audio_processing/aec3/fullband_erle_estimator.h @@ -67,7 +67,7 @@ class FullBandErleEstimator { // Updates the estimator with a new point, returns true // if the instantaneous ERLE was updated due to having enough // points for performing the estimate. - bool Update(const float Y2_sum, const float E2_sum); + bool Update(float Y2_sum, float E2_sum); // Resets the instantaneous ERLE estimator to its initial state. void Reset(); // Resets the members related with an instantaneous estimate. @@ -106,8 +106,8 @@ class FullBandErleEstimator { }; const float min_erle_log2_; - const float max_erle_lf_log2; - std::vector hold_counters_time_domain_; + const float max_erle_lf_log2_; + std::vector hold_counters_instantaneous_erle_; std::vector erle_time_domain_log2_; std::vector instantaneous_erle_; std::vector> linear_filters_qualities_; diff --git a/webrtc/modules/audio_processing/aec3/matched_filter.cc b/webrtc/modules/audio_processing/aec3/matched_filter.cc index 64b2d4e..edaa2a4 100644 --- a/webrtc/modules/audio_processing/aec3/matched_filter.cc +++ b/webrtc/modules/audio_processing/aec3/matched_filter.cc @@ -24,16 +24,220 @@ #include #include +#include "absl/types/optional.h" +#include "api/array_view.h" #include "modules/audio_processing/aec3/downsampled_render_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace { + +// Subsample rate used for computing the accumulated error. +// The implementation of some core functions depends on this constant being +// equal to 4. +constexpr int kAccumulatedErrorSubSampleRate = 4; + +void UpdateAccumulatedError( + const rtc::ArrayView instantaneous_accumulated_error, + const rtc::ArrayView accumulated_error, + float one_over_error_sum_anchor, + float smooth_constant_increases) { + for (size_t k = 0; k < instantaneous_accumulated_error.size(); ++k) { + float error_norm = + instantaneous_accumulated_error[k] * one_over_error_sum_anchor; + if (error_norm < accumulated_error[k]) { + accumulated_error[k] = error_norm; + } else { + accumulated_error[k] += + smooth_constant_increases * (error_norm - accumulated_error[k]); + } + } +} + +size_t ComputePreEchoLag( + const webrtc::MatchedFilter::PreEchoConfiguration& pre_echo_configuration, + const rtc::ArrayView accumulated_error, + size_t lag, + size_t alignment_shift_winner) { + RTC_DCHECK_GE(lag, alignment_shift_winner); + size_t pre_echo_lag_estimate = lag - alignment_shift_winner; + size_t maximum_pre_echo_lag = + std::min(pre_echo_lag_estimate / kAccumulatedErrorSubSampleRate, + accumulated_error.size()); + switch (pre_echo_configuration.mode) { + case 0: + // Mode 0: Pre echo lag is defined as the first coefficient with an error + // lower than a threshold with a certain decrease slope. + for (size_t k = 1; k < maximum_pre_echo_lag; ++k) { + if (accumulated_error[k] < + pre_echo_configuration.threshold * accumulated_error[k - 1] && + accumulated_error[k] < pre_echo_configuration.threshold) { + pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1; + break; + } + } + break; + case 1: + // Mode 1: Pre echo lag is defined as the first coefficient with an error + // lower than a certain threshold. + for (size_t k = 0; k < maximum_pre_echo_lag; ++k) { + if (accumulated_error[k] < pre_echo_configuration.threshold) { + pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1; + break; + } + } + break; + case 2: + case 3: + // Mode 2,3: Pre echo lag is defined as the closest coefficient to the lag + // with an error lower than a certain threshold. + for (int k = static_cast(maximum_pre_echo_lag) - 1; k >= 0; --k) { + if (accumulated_error[k] > pre_echo_configuration.threshold) { + break; + } + pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1; + } + break; + default: + RTC_DCHECK_NOTREACHED(); + break; + } + return pre_echo_lag_estimate + alignment_shift_winner; +} + +webrtc::MatchedFilter::PreEchoConfiguration FetchPreEchoConfiguration() { + constexpr float kDefaultThreshold = 0.5f; + constexpr int kDefaultMode = 3; + float threshold = kDefaultThreshold; + int mode = kDefaultMode; + const std::string pre_echo_configuration_field_trial = + webrtc::field_trial::FindFullName("WebRTC-Aec3PreEchoConfiguration"); + webrtc::FieldTrialParameter threshold_field_trial_parameter( + /*key=*/"threshold", /*default_value=*/kDefaultThreshold); + webrtc::FieldTrialParameter mode_field_trial_parameter( + /*key=*/"mode", /*default_value=*/kDefaultMode); + webrtc::ParseFieldTrial( + {&threshold_field_trial_parameter, &mode_field_trial_parameter}, + pre_echo_configuration_field_trial); + float threshold_read = + static_cast(threshold_field_trial_parameter.Get()); + int mode_read = mode_field_trial_parameter.Get(); + if (threshold_read < 1.0f && threshold_read > 0.0f) { + threshold = threshold_read; + } else { + RTC_LOG(LS_ERROR) + << "AEC3: Pre echo configuration: wrong input, threshold = " + << threshold_read << "."; + } + if (mode_read >= 0 && mode_read <= 3) { + mode = mode_read; + } else { + RTC_LOG(LS_ERROR) << "AEC3: Pre echo configuration: wrong input, mode = " + << mode_read << "."; + } + RTC_LOG(LS_INFO) << "AEC3: Pre echo configuration: threshold = " << threshold + << ", mode = " << mode << "."; + return {.threshold = threshold, .mode = mode}; +} + +} // namespace namespace webrtc { namespace aec3 { #if defined(WEBRTC_HAS_NEON) +inline float SumAllElements(float32x4_t elements) { + float32x2_t sum = vpadd_f32(vget_low_f32(elements), vget_high_f32(elements)); + sum = vpadd_f32(sum, sum); + return vget_lane_f32(sum, 0); +} + +void MatchedFilterCoreWithAccumulatedError_NEON( + size_t x_start_index, + float x2_sum_threshold, + float smoothing, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory) { + const int h_size = static_cast(h.size()); + const int x_size = static_cast(x.size()); + RTC_DCHECK_EQ(0, h_size % 4); + std::fill(accumulated_error.begin(), accumulated_error.end(), 0.0f); + // Process for all samples in the sub-block. + for (size_t i = 0; i < y.size(); ++i) { + // Apply the matched filter as filter * x, and compute x * x. + RTC_DCHECK_GT(x_size, x_start_index); + // Compute loop chunk sizes until, and after, the wraparound of the circular + // buffer for x. + const int chunk1 = + std::min(h_size, static_cast(x_size - x_start_index)); + if (chunk1 != h_size) { + const int chunk2 = h_size - chunk1; + std::copy(x.begin() + x_start_index, x.end(), scratch_memory.begin()); + std::copy(x.begin(), x.begin() + chunk2, scratch_memory.begin() + chunk1); + } + const float* x_p = + chunk1 != h_size ? scratch_memory.data() : &x[x_start_index]; + const float* h_p = &h[0]; + float* accumulated_error_p = &accumulated_error[0]; + // Initialize values for the accumulation. + float32x4_t x2_sum_128 = vdupq_n_f32(0); + float x2_sum = 0.f; + float s = 0; + // Perform 128 bit vector operations. + const int limit_by_4 = h_size >> 2; + for (int k = limit_by_4; k > 0; + --k, h_p += 4, x_p += 4, accumulated_error_p++) { + // Load the data into 128 bit vectors. + const float32x4_t x_k = vld1q_f32(x_p); + const float32x4_t h_k = vld1q_f32(h_p); + // Compute and accumulate x * x. + x2_sum_128 = vmlaq_f32(x2_sum_128, x_k, x_k); + // Compute x * h + float32x4_t hk_xk_128 = vmulq_f32(h_k, x_k); + s += SumAllElements(hk_xk_128); + const float e = s - y[i]; + accumulated_error_p[0] += e * e; + } + // Combine the accumulated vector and scalar values. + x2_sum += SumAllElements(x2_sum_128); + // Compute the matched filter error. + float e = y[i] - s; + const bool saturation = y[i] >= 32000.f || y[i] <= -32000.f; + (*error_sum) += e * e; + // Update the matched filter estimate in an NLMS manner. + if (x2_sum > x2_sum_threshold && !saturation) { + RTC_DCHECK_LT(0.f, x2_sum); + const float alpha = smoothing * e / x2_sum; + const float32x4_t alpha_128 = vmovq_n_f32(alpha); + // filter = filter + smoothing * (y - filter * x) * x / x * x. + float* h_p = &h[0]; + x_p = chunk1 != h_size ? scratch_memory.data() : &x[x_start_index]; + // Perform 128 bit vector operations. + const int limit_by_4 = h_size >> 2; + for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + // Load the data into 128 bit vectors. + float32x4_t h_k = vld1q_f32(h_p); + const float32x4_t x_k = vld1q_f32(x_p); + // Compute h = h + alpha * x. + h_k = vmlaq_f32(h_k, alpha_128, x_k); + // Store the result. + vst1q_f32(h_p, h_k); + } + *filters_updated = true; + } + x_start_index = x_start_index > 0 ? x_start_index - 1 : x_size - 1; + } +} + void MatchedFilterCore_NEON(size_t x_start_index, float x2_sum_threshold, float smoothing, @@ -41,11 +245,20 @@ void MatchedFilterCore_NEON(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum) { + float* error_sum, + bool compute_accumulated_error, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory) { const int h_size = static_cast(h.size()); const int x_size = static_cast(x.size()); RTC_DCHECK_EQ(0, h_size % 4); + if (compute_accumulated_error) { + return MatchedFilterCoreWithAccumulatedError_NEON( + x_start_index, x2_sum_threshold, smoothing, x, y, h, filters_updated, + error_sum, accumulated_error, scratch_memory); + } + // Process for all samples in the sub-block. for (size_t i = 0; i < y.size(); ++i) { // Apply the matched filter as filter * x, and compute x * x. @@ -90,10 +303,8 @@ void MatchedFilterCore_NEON(size_t x_start_index, } // Combine the accumulated vector and scalar values. - float* v = reinterpret_cast(&x2_sum_128); - x2_sum += v[0] + v[1] + v[2] + v[3]; - v = reinterpret_cast(&s_128); - s += v[0] + v[1] + v[2] + v[3]; + s += SumAllElements(s_128); + x2_sum += SumAllElements(x2_sum_128); // Compute the matched filter error. float e = y[i] - s; @@ -144,6 +355,103 @@ void MatchedFilterCore_NEON(size_t x_start_index, #if defined(WEBRTC_ARCH_X86_FAMILY) +void MatchedFilterCore_AccumulatedError_SSE2( + size_t x_start_index, + float x2_sum_threshold, + float smoothing, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory) { + const int h_size = static_cast(h.size()); + const int x_size = static_cast(x.size()); + RTC_DCHECK_EQ(0, h_size % 8); + std::fill(accumulated_error.begin(), accumulated_error.end(), 0.0f); + // Process for all samples in the sub-block. + for (size_t i = 0; i < y.size(); ++i) { + // Apply the matched filter as filter * x, and compute x * x. + RTC_DCHECK_GT(x_size, x_start_index); + const int chunk1 = + std::min(h_size, static_cast(x_size - x_start_index)); + if (chunk1 != h_size) { + const int chunk2 = h_size - chunk1; + std::copy(x.begin() + x_start_index, x.end(), scratch_memory.begin()); + std::copy(x.begin(), x.begin() + chunk2, scratch_memory.begin() + chunk1); + } + const float* x_p = + chunk1 != h_size ? scratch_memory.data() : &x[x_start_index]; + const float* h_p = &h[0]; + float* a_p = &accumulated_error[0]; + __m128 s_inst_128; + __m128 s_inst_128_4; + __m128 x2_sum_128 = _mm_set1_ps(0); + __m128 x2_sum_128_4 = _mm_set1_ps(0); + __m128 e_128; + float* const s_p = reinterpret_cast(&s_inst_128); + float* const s_4_p = reinterpret_cast(&s_inst_128_4); + float* const e_p = reinterpret_cast(&e_128); + float x2_sum = 0.0f; + float s_acum = 0; + // Perform 128 bit vector operations. + const int limit_by_8 = h_size >> 3; + for (int k = limit_by_8; k > 0; --k, h_p += 8, x_p += 8, a_p += 2) { + // Load the data into 128 bit vectors. + const __m128 x_k = _mm_loadu_ps(x_p); + const __m128 h_k = _mm_loadu_ps(h_p); + const __m128 x_k_4 = _mm_loadu_ps(x_p + 4); + const __m128 h_k_4 = _mm_loadu_ps(h_p + 4); + const __m128 xx = _mm_mul_ps(x_k, x_k); + const __m128 xx_4 = _mm_mul_ps(x_k_4, x_k_4); + // Compute and accumulate x * x and h * x. + x2_sum_128 = _mm_add_ps(x2_sum_128, xx); + x2_sum_128_4 = _mm_add_ps(x2_sum_128_4, xx_4); + s_inst_128 = _mm_mul_ps(h_k, x_k); + s_inst_128_4 = _mm_mul_ps(h_k_4, x_k_4); + s_acum += s_p[0] + s_p[1] + s_p[2] + s_p[3]; + e_p[0] = s_acum - y[i]; + s_acum += s_4_p[0] + s_4_p[1] + s_4_p[2] + s_4_p[3]; + e_p[1] = s_acum - y[i]; + a_p[0] += e_p[0] * e_p[0]; + a_p[1] += e_p[1] * e_p[1]; + } + // Combine the accumulated vector and scalar values. + x2_sum_128 = _mm_add_ps(x2_sum_128, x2_sum_128_4); + float* v = reinterpret_cast(&x2_sum_128); + x2_sum += v[0] + v[1] + v[2] + v[3]; + // Compute the matched filter error. + float e = y[i] - s_acum; + const bool saturation = y[i] >= 32000.f || y[i] <= -32000.f; + (*error_sum) += e * e; + // Update the matched filter estimate in an NLMS manner. + if (x2_sum > x2_sum_threshold && !saturation) { + RTC_DCHECK_LT(0.f, x2_sum); + const float alpha = smoothing * e / x2_sum; + const __m128 alpha_128 = _mm_set1_ps(alpha); + // filter = filter + smoothing * (y - filter * x) * x / x * x. + float* h_p = &h[0]; + const float* x_p = + chunk1 != h_size ? scratch_memory.data() : &x[x_start_index]; + // Perform 128 bit vector operations. + const int limit_by_4 = h_size >> 2; + for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + // Load the data into 128 bit vectors. + __m128 h_k = _mm_loadu_ps(h_p); + const __m128 x_k = _mm_loadu_ps(x_p); + // Compute h = h + alpha * x. + const __m128 alpha_x = _mm_mul_ps(alpha_128, x_k); + h_k = _mm_add_ps(h_k, alpha_x); + // Store the result. + _mm_storeu_ps(h_p, h_k); + } + *filters_updated = true; + } + x_start_index = x_start_index > 0 ? x_start_index - 1 : x_size - 1; + } +} + void MatchedFilterCore_SSE2(size_t x_start_index, float x2_sum_threshold, float smoothing, @@ -151,77 +459,83 @@ void MatchedFilterCore_SSE2(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum) { + float* error_sum, + bool compute_accumulated_error, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory) { + if (compute_accumulated_error) { + return MatchedFilterCore_AccumulatedError_SSE2( + x_start_index, x2_sum_threshold, smoothing, x, y, h, filters_updated, + error_sum, accumulated_error, scratch_memory); + } const int h_size = static_cast(h.size()); const int x_size = static_cast(x.size()); RTC_DCHECK_EQ(0, h_size % 4); - // Process for all samples in the sub-block. for (size_t i = 0; i < y.size(); ++i) { // Apply the matched filter as filter * x, and compute x * x. - RTC_DCHECK_GT(x_size, x_start_index); const float* x_p = &x[x_start_index]; const float* h_p = &h[0]; - // Initialize values for the accumulation. __m128 s_128 = _mm_set1_ps(0); + __m128 s_128_4 = _mm_set1_ps(0); __m128 x2_sum_128 = _mm_set1_ps(0); + __m128 x2_sum_128_4 = _mm_set1_ps(0); float x2_sum = 0.f; float s = 0; - // Compute loop chunk sizes until, and after, the wraparound of the circular // buffer for x. const int chunk1 = std::min(h_size, static_cast(x_size - x_start_index)); - // Perform the loop in two chunks. const int chunk2 = h_size - chunk1; for (int limit : {chunk1, chunk2}) { // Perform 128 bit vector operations. - const int limit_by_4 = limit >> 2; - for (int k = limit_by_4; k > 0; --k, h_p += 4, x_p += 4) { + const int limit_by_8 = limit >> 3; + for (int k = limit_by_8; k > 0; --k, h_p += 8, x_p += 8) { // Load the data into 128 bit vectors. const __m128 x_k = _mm_loadu_ps(x_p); const __m128 h_k = _mm_loadu_ps(h_p); + const __m128 x_k_4 = _mm_loadu_ps(x_p + 4); + const __m128 h_k_4 = _mm_loadu_ps(h_p + 4); const __m128 xx = _mm_mul_ps(x_k, x_k); + const __m128 xx_4 = _mm_mul_ps(x_k_4, x_k_4); // Compute and accumulate x * x and h * x. x2_sum_128 = _mm_add_ps(x2_sum_128, xx); + x2_sum_128_4 = _mm_add_ps(x2_sum_128_4, xx_4); const __m128 hx = _mm_mul_ps(h_k, x_k); + const __m128 hx_4 = _mm_mul_ps(h_k_4, x_k_4); s_128 = _mm_add_ps(s_128, hx); + s_128_4 = _mm_add_ps(s_128_4, hx_4); } - // Perform non-vector operations for any remaining items. - for (int k = limit - limit_by_4 * 4; k > 0; --k, ++h_p, ++x_p) { + for (int k = limit - limit_by_8 * 8; k > 0; --k, ++h_p, ++x_p) { const float x_k = *x_p; x2_sum += x_k * x_k; s += *h_p * x_k; } - x_p = &x[0]; } - // Combine the accumulated vector and scalar values. + x2_sum_128 = _mm_add_ps(x2_sum_128, x2_sum_128_4); float* v = reinterpret_cast(&x2_sum_128); x2_sum += v[0] + v[1] + v[2] + v[3]; + s_128 = _mm_add_ps(s_128, s_128_4); v = reinterpret_cast(&s_128); s += v[0] + v[1] + v[2] + v[3]; - // Compute the matched filter error. float e = y[i] - s; const bool saturation = y[i] >= 32000.f || y[i] <= -32000.f; (*error_sum) += e * e; - // Update the matched filter estimate in an NLMS manner. if (x2_sum > x2_sum_threshold && !saturation) { RTC_DCHECK_LT(0.f, x2_sum); const float alpha = smoothing * e / x2_sum; const __m128 alpha_128 = _mm_set1_ps(alpha); - // filter = filter + smoothing * (y - filter * x) * x / x * x. float* h_p = &h[0]; x_p = &x[x_start_index]; - // Perform the loop in two chunks. for (int limit : {chunk1, chunk2}) { // Perform 128 bit vector operations. @@ -234,22 +548,17 @@ void MatchedFilterCore_SSE2(size_t x_start_index, // Compute h = h + alpha * x. const __m128 alpha_x = _mm_mul_ps(alpha_128, x_k); h_k = _mm_add_ps(h_k, alpha_x); - // Store the result. _mm_storeu_ps(h_p, h_k); } - // Perform non-vector operations for any remaining items. for (int k = limit - limit_by_4 * 4; k > 0; --k, ++h_p, ++x_p) { *h_p += alpha * *x_p; } - x_p = &x[0]; } - *filters_updated = true; } - x_start_index = x_start_index > 0 ? x_start_index - 1 : x_size - 1; } } @@ -262,17 +571,35 @@ void MatchedFilterCore(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum) { + float* error_sum, + bool compute_accumulated_error, + rtc::ArrayView accumulated_error) { + if (compute_accumulated_error) { + std::fill(accumulated_error.begin(), accumulated_error.end(), 0.0f); + } + // Process for all samples in the sub-block. for (size_t i = 0; i < y.size(); ++i) { // Apply the matched filter as filter * x, and compute x * x. float x2_sum = 0.f; float s = 0; size_t x_index = x_start_index; - for (size_t k = 0; k < h.size(); ++k) { - x2_sum += x[x_index] * x[x_index]; - s += h[k] * x[x_index]; - x_index = x_index < (x.size() - 1) ? x_index + 1 : 0; + if (compute_accumulated_error) { + for (size_t k = 0; k < h.size(); ++k) { + x2_sum += x[x_index] * x[x_index]; + s += h[k] * x[x_index]; + x_index = x_index < (x.size() - 1) ? x_index + 1 : 0; + if ((k + 1 & 0b11) == 0) { + int idx = k >> 2; + accumulated_error[idx] += (y[i] - s) * (y[i] - s); + } + } + } else { + for (size_t k = 0; k < h.size(); ++k) { + x2_sum += x[x_index] * x[x_index]; + s += h[k] * x[x_index]; + x_index = x_index < (x.size() - 1) ? x_index + 1 : 0; + } } // Compute the matched filter error. @@ -298,6 +625,41 @@ void MatchedFilterCore(size_t x_start_index, } } +size_t MaxSquarePeakIndex(rtc::ArrayView h) { + if (h.size() < 2) { + return 0; + } + float max_element1 = h[0] * h[0]; + float max_element2 = h[1] * h[1]; + size_t lag_estimate1 = 0; + size_t lag_estimate2 = 1; + const size_t last_index = h.size() - 1; + // Keeping track of even & odd max elements separately typically allows the + // compiler to produce more efficient code. + for (size_t k = 2; k < last_index; k += 2) { + float element1 = h[k] * h[k]; + float element2 = h[k + 1] * h[k + 1]; + if (element1 > max_element1) { + max_element1 = element1; + lag_estimate1 = k; + } + if (element2 > max_element2) { + max_element2 = element2; + lag_estimate2 = k + 1; + } + } + if (max_element2 > max_element1) { + max_element1 = max_element2; + lag_estimate1 = lag_estimate2; + } + // In case of odd h size, we have not yet checked the last element. + float last_element = h[last_index] * h[last_index]; + if (last_element > max_element1) { + return last_index; + } + return lag_estimate1; +} + } // namespace aec3 MatchedFilter::MatchedFilter(ApmDataDumper* data_dumper, @@ -307,8 +669,10 @@ MatchedFilter::MatchedFilter(ApmDataDumper* data_dumper, int num_matched_filters, size_t alignment_shift_sub_blocks, float excitation_limit, - float smoothing, - float matching_filter_threshold) + float smoothing_fast, + float smoothing_slow, + float matching_filter_threshold, + bool detect_pre_echo) : data_dumper_(data_dumper), optimization_(optimization), sub_block_size_(sub_block_size), @@ -316,42 +680,82 @@ MatchedFilter::MatchedFilter(ApmDataDumper* data_dumper, filters_( num_matched_filters, std::vector(window_size_sub_blocks * sub_block_size_, 0.f)), - lag_estimates_(num_matched_filters), filters_offsets_(num_matched_filters, 0), excitation_limit_(excitation_limit), - smoothing_(smoothing), - matching_filter_threshold_(matching_filter_threshold) { + smoothing_fast_(smoothing_fast), + smoothing_slow_(smoothing_slow), + matching_filter_threshold_(matching_filter_threshold), + detect_pre_echo_(detect_pre_echo), + pre_echo_config_(FetchPreEchoConfiguration()) { RTC_DCHECK(data_dumper); RTC_DCHECK_LT(0, window_size_sub_blocks); RTC_DCHECK((kBlockSize % sub_block_size) == 0); RTC_DCHECK((sub_block_size % 4) == 0); + static_assert(kAccumulatedErrorSubSampleRate == 4); + if (detect_pre_echo_) { + accumulated_error_ = std::vector>( + num_matched_filters, + std::vector(window_size_sub_blocks * sub_block_size_ / + kAccumulatedErrorSubSampleRate, + 1.0f)); + + instantaneous_accumulated_error_ = + std::vector(window_size_sub_blocks * sub_block_size_ / + kAccumulatedErrorSubSampleRate, + 0.0f); + scratch_memory_ = + std::vector(window_size_sub_blocks * sub_block_size_); + } } MatchedFilter::~MatchedFilter() = default; -void MatchedFilter::Reset() { +void MatchedFilter::Reset(bool full_reset) { for (auto& f : filters_) { std::fill(f.begin(), f.end(), 0.f); } - for (auto& l : lag_estimates_) { - l = MatchedFilter::LagEstimate(); + winner_lag_ = absl::nullopt; + reported_lag_estimate_ = absl::nullopt; + if (pre_echo_config_.mode != 3 || full_reset) { + for (auto& e : accumulated_error_) { + std::fill(e.begin(), e.end(), 1.0f); + } + number_pre_echo_updates_ = 0; } } void MatchedFilter::Update(const DownsampledRenderBuffer& render_buffer, - rtc::ArrayView capture) { + rtc::ArrayView capture, + bool use_slow_smoothing) { RTC_DCHECK_EQ(sub_block_size_, capture.size()); auto& y = capture; + const float smoothing = + use_slow_smoothing ? smoothing_slow_ : smoothing_fast_; + const float x2_sum_threshold = filters_[0].size() * excitation_limit_ * excitation_limit_; + // Compute anchor for the matched filter error. + float error_sum_anchor = 0.0f; + for (size_t k = 0; k < y.size(); ++k) { + error_sum_anchor += y[k] * y[k]; + } + // Apply all matched filters. + float winner_error_sum = error_sum_anchor; + winner_lag_ = absl::nullopt; + reported_lag_estimate_ = absl::nullopt; size_t alignment_shift = 0; - for (size_t n = 0; n < filters_.size(); ++n) { + absl::optional previous_lag_estimate; + const int num_filters = static_cast(filters_.size()); + int winner_index = -1; + for (int n = 0; n < num_filters; ++n) { float error_sum = 0.f; bool filters_updated = false; + const bool compute_pre_echo = + detect_pre_echo_ && n == last_detected_best_lag_filter_; size_t x_start_index = (render_buffer.read + alignment_shift + sub_block_size_ - 1) % @@ -360,87 +764,94 @@ void MatchedFilter::Update(const DownsampledRenderBuffer& render_buffer, switch (optimization_) { #if defined(WEBRTC_ARCH_X86_FAMILY) case Aec3Optimization::kSse2: - aec3::MatchedFilterCore_SSE2(x_start_index, x2_sum_threshold, - smoothing_, render_buffer.buffer, y, - filters_[n], &filters_updated, &error_sum); + aec3::MatchedFilterCore_SSE2( + x_start_index, x2_sum_threshold, smoothing, render_buffer.buffer, y, + filters_[n], &filters_updated, &error_sum, compute_pre_echo, + instantaneous_accumulated_error_, scratch_memory_); break; case Aec3Optimization::kAvx2: - aec3::MatchedFilterCore_AVX2(x_start_index, x2_sum_threshold, - smoothing_, render_buffer.buffer, y, - filters_[n], &filters_updated, &error_sum); + aec3::MatchedFilterCore_AVX2( + x_start_index, x2_sum_threshold, smoothing, render_buffer.buffer, y, + filters_[n], &filters_updated, &error_sum, compute_pre_echo, + instantaneous_accumulated_error_, scratch_memory_); break; #endif #if defined(WEBRTC_HAS_NEON) case Aec3Optimization::kNeon: - aec3::MatchedFilterCore_NEON(x_start_index, x2_sum_threshold, - smoothing_, render_buffer.buffer, y, - filters_[n], &filters_updated, &error_sum); + aec3::MatchedFilterCore_NEON( + x_start_index, x2_sum_threshold, smoothing, render_buffer.buffer, y, + filters_[n], &filters_updated, &error_sum, compute_pre_echo, + instantaneous_accumulated_error_, scratch_memory_); break; #endif default: - aec3::MatchedFilterCore(x_start_index, x2_sum_threshold, smoothing_, + aec3::MatchedFilterCore(x_start_index, x2_sum_threshold, smoothing, render_buffer.buffer, y, filters_[n], - &filters_updated, &error_sum); + &filters_updated, &error_sum, compute_pre_echo, + instantaneous_accumulated_error_); } - // Compute anchor for the matched filter error. - const float error_sum_anchor = - std::inner_product(y.begin(), y.end(), y.begin(), 0.f); - // Estimate the lag in the matched filter as the distance to the portion in // the filter that contributes the most to the matched filter output. This // is detected as the peak of the matched filter. - const size_t lag_estimate = std::distance( - filters_[n].begin(), - std::max_element( - filters_[n].begin(), filters_[n].end(), - [](float a, float b) -> bool { return a * a < b * b; })); + const size_t lag_estimate = aec3::MaxSquarePeakIndex(filters_[n]); + const bool reliable = + lag_estimate > 2 && lag_estimate < (filters_[n].size() - 10) && + error_sum < matching_filter_threshold_ * error_sum_anchor; - // Update the lag estimates for the matched filter. - lag_estimates_[n] = LagEstimate( - error_sum_anchor - error_sum, - (lag_estimate > 2 && lag_estimate < (filters_[n].size() - 10) && - error_sum < matching_filter_threshold_ * error_sum_anchor), - lag_estimate + alignment_shift, filters_updated); - - RTC_DCHECK_GE(10, filters_.size()); - switch (n) { - case 0: - data_dumper_->DumpRaw("aec3_correlator_0_h", filters_[0]); - break; - case 1: - data_dumper_->DumpRaw("aec3_correlator_1_h", filters_[1]); - break; - case 2: - data_dumper_->DumpRaw("aec3_correlator_2_h", filters_[2]); - break; - case 3: - data_dumper_->DumpRaw("aec3_correlator_3_h", filters_[3]); - break; - case 4: - data_dumper_->DumpRaw("aec3_correlator_4_h", filters_[4]); - break; - case 5: - data_dumper_->DumpRaw("aec3_correlator_5_h", filters_[5]); - break; - case 6: - data_dumper_->DumpRaw("aec3_correlator_6_h", filters_[6]); - break; - case 7: - data_dumper_->DumpRaw("aec3_correlator_7_h", filters_[7]); - break; - case 8: - data_dumper_->DumpRaw("aec3_correlator_8_h", filters_[8]); - break; - case 9: - data_dumper_->DumpRaw("aec3_correlator_9_h", filters_[9]); - break; - default: - RTC_NOTREACHED(); + // Find the best estimate + const size_t lag = lag_estimate + alignment_shift; + if (filters_updated && reliable && error_sum < winner_error_sum) { + winner_error_sum = error_sum; + winner_index = n; + // In case that 2 matched filters return the same winner candidate + // (overlap region), the one with the smaller index is chosen in order + // to search for pre-echoes. + if (previous_lag_estimate && previous_lag_estimate == lag) { + winner_lag_ = previous_lag_estimate; + winner_index = n - 1; + } else { + winner_lag_ = lag; + } } - + previous_lag_estimate = lag; alignment_shift += filter_intra_lag_shift_; } + + if (winner_index != -1) { + RTC_DCHECK(winner_lag_.has_value()); + reported_lag_estimate_ = + LagEstimate(winner_lag_.value(), /*pre_echo_lag=*/winner_lag_.value()); + if (detect_pre_echo_ && last_detected_best_lag_filter_ == winner_index) { + const float energy_threshold = + pre_echo_config_.mode == 3 ? 1.0f : 30.0f * 30.0f * y.size(); + + if (error_sum_anchor > energy_threshold) { + const float smooth_constant_increases = + pre_echo_config_.mode != 3 ? 0.01f : 0.015f; + + UpdateAccumulatedError( + instantaneous_accumulated_error_, accumulated_error_[winner_index], + 1.0f / error_sum_anchor, smooth_constant_increases); + number_pre_echo_updates_++; + } + if (pre_echo_config_.mode != 3 || number_pre_echo_updates_ >= 50) { + reported_lag_estimate_->pre_echo_lag = ComputePreEchoLag( + pre_echo_config_, accumulated_error_[winner_index], + winner_lag_.value(), + winner_index * filter_intra_lag_shift_ /*alignment_shift_winner*/); + } else { + reported_lag_estimate_->pre_echo_lag = winner_lag_.value(); + } + } + last_detected_best_lag_filter_ = winner_index; + } + if (ApmDataDumper::IsAvailable()) { + Dump(); + data_dumper_->DumpRaw("error_sum_anchor", error_sum_anchor / y.size()); + data_dumper_->DumpRaw("number_pre_echo_updates", number_pre_echo_updates_); + data_dumper_->DumpRaw("filter_smoothing", smoothing); + } } void MatchedFilter::LogFilterProperties(int sample_rate_hz, @@ -461,4 +872,31 @@ void MatchedFilter::LogFilterProperties(int sample_rate_hz, } } +void MatchedFilter::Dump() { + for (size_t n = 0; n < filters_.size(); ++n) { + const size_t lag_estimate = aec3::MaxSquarePeakIndex(filters_[n]); + std::string dumper_filter = "aec3_correlator_" + std::to_string(n) + "_h"; + data_dumper_->DumpRaw(dumper_filter.c_str(), filters_[n]); + std::string dumper_lag = "aec3_correlator_lag_" + std::to_string(n); + data_dumper_->DumpRaw(dumper_lag.c_str(), + lag_estimate + n * filter_intra_lag_shift_); + if (detect_pre_echo_) { + std::string dumper_error = + "aec3_correlator_error_" + std::to_string(n) + "_h"; + data_dumper_->DumpRaw(dumper_error.c_str(), accumulated_error_[n]); + + size_t pre_echo_lag = + ComputePreEchoLag(pre_echo_config_, accumulated_error_[n], + lag_estimate + n * filter_intra_lag_shift_, + n * filter_intra_lag_shift_); + std::string dumper_pre_lag = + "aec3_correlator_pre_echo_lag_" + std::to_string(n); + data_dumper_->DumpRaw(dumper_pre_lag.c_str(), pre_echo_lag); + if (static_cast(n) == last_detected_best_lag_filter_) { + data_dumper_->DumpRaw("aec3_pre_echo_delay_winner_inst", pre_echo_lag); + } + } + } +} + } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/matched_filter.h b/webrtc/modules/audio_processing/aec3/matched_filter.h index fa44eb2..bb54fba 100644 --- a/webrtc/modules/audio_processing/aec3/matched_filter.h +++ b/webrtc/modules/audio_processing/aec3/matched_filter.h @@ -15,8 +15,10 @@ #include +#include "absl/types/optional.h" #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/gtest_prod_util.h" #include "rtc_base/system/arch.h" namespace webrtc { @@ -36,7 +38,10 @@ void MatchedFilterCore_NEON(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum); + float* error_sum, + bool compute_accumulation_error, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory); #endif @@ -50,7 +55,10 @@ void MatchedFilterCore_SSE2(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum); + float* error_sum, + bool compute_accumulated_error, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory); // Filter core for the matched filter that is optimized for AVX2. void MatchedFilterCore_AVX2(size_t x_start_index, @@ -60,7 +68,10 @@ void MatchedFilterCore_AVX2(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum); + float* error_sum, + bool compute_accumulated_error, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory); #endif @@ -72,7 +83,12 @@ void MatchedFilterCore(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum); + float* error_sum, + bool compute_accumulation_error, + rtc::ArrayView accumulated_error); + +// Find largest peak of squared values in array. +size_t MaxSquarePeakIndex(rtc::ArrayView h); } // namespace aec3 @@ -84,13 +100,15 @@ class MatchedFilter { // shift. struct LagEstimate { LagEstimate() = default; - LagEstimate(float accuracy, bool reliable, size_t lag, bool updated) - : accuracy(accuracy), reliable(reliable), lag(lag), updated(updated) {} - - float accuracy = 0.f; - bool reliable = false; + LagEstimate(size_t lag, size_t pre_echo_lag) + : lag(lag), pre_echo_lag(pre_echo_lag) {} size_t lag = 0; - bool updated = false; + size_t pre_echo_lag = 0; + }; + + struct PreEchoConfiguration { + const float threshold; + const int mode; }; MatchedFilter(ApmDataDumper* data_dumper, @@ -100,8 +118,10 @@ class MatchedFilter { int num_matched_filters, size_t alignment_shift_sub_blocks, float excitation_limit, - float smoothing, - float matching_filter_threshold); + float smoothing_fast, + float smoothing_slow, + float matching_filter_threshold, + bool detect_pre_echo); MatchedFilter() = delete; MatchedFilter(const MatchedFilter&) = delete; @@ -111,14 +131,15 @@ class MatchedFilter { // Updates the correlation with the values in the capture buffer. void Update(const DownsampledRenderBuffer& render_buffer, - rtc::ArrayView capture); + rtc::ArrayView capture, + bool use_slow_smoothing); // Resets the matched filter. - void Reset(); + void Reset(bool full_reset); // Returns the current lag estimates. - rtc::ArrayView GetLagEstimates() const { - return lag_estimates_; + absl::optional GetBestLagEstimate() const { + return reported_lag_estimate_; } // Returns the maximum filter lag. @@ -132,16 +153,36 @@ class MatchedFilter { size_t downsampling_factor) const; private: + FRIEND_TEST_ALL_PREFIXES(MatchedFilterFieldTrialTest, + PreEchoConfigurationTest); + FRIEND_TEST_ALL_PREFIXES(MatchedFilterFieldTrialTest, + WrongPreEchoConfigurationTest); + + // Only for testing. Gets the pre echo detection configuration. + const PreEchoConfiguration& GetPreEchoConfiguration() const { + return pre_echo_config_; + } + void Dump(); + ApmDataDumper* const data_dumper_; const Aec3Optimization optimization_; const size_t sub_block_size_; const size_t filter_intra_lag_shift_; std::vector> filters_; - std::vector lag_estimates_; + std::vector> accumulated_error_; + std::vector instantaneous_accumulated_error_; + std::vector scratch_memory_; + absl::optional reported_lag_estimate_; + absl::optional winner_lag_; + int last_detected_best_lag_filter_ = -1; std::vector filters_offsets_; + int number_pre_echo_updates_ = 0; const float excitation_limit_; - const float smoothing_; + const float smoothing_fast_; + const float smoothing_slow_; const float matching_filter_threshold_; + const bool detect_pre_echo_; + const PreEchoConfiguration pre_echo_config_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/matched_filter_avx2.cc b/webrtc/modules/audio_processing/aec3/matched_filter_avx2.cc index ed32102..8c2ffcb 100644 --- a/webrtc/modules/audio_processing/aec3/matched_filter_avx2.cc +++ b/webrtc/modules/audio_processing/aec3/matched_filter_avx2.cc @@ -8,15 +8,134 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/aec3/matched_filter.h" - #include +#include "modules/audio_processing/aec3/matched_filter.h" #include "rtc_base/checks.h" namespace webrtc { namespace aec3 { +// Let ha denote the horizontal of a, and hb the horizontal sum of b +// returns [ha, hb, ha, hb] +inline __m128 hsum_ab(__m256 a, __m256 b) { + __m256 s_256 = _mm256_hadd_ps(a, b); + const __m256i mask = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0); + s_256 = _mm256_permutevar8x32_ps(s_256, mask); + __m128 s = _mm_hadd_ps(_mm256_extractf128_ps(s_256, 0), + _mm256_extractf128_ps(s_256, 1)); + s = _mm_hadd_ps(s, s); + return s; +} + +void MatchedFilterCore_AccumulatedError_AVX2( + size_t x_start_index, + float x2_sum_threshold, + float smoothing, + rtc::ArrayView x, + rtc::ArrayView y, + rtc::ArrayView h, + bool* filters_updated, + float* error_sum, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory) { + const int h_size = static_cast(h.size()); + const int x_size = static_cast(x.size()); + RTC_DCHECK_EQ(0, h_size % 16); + std::fill(accumulated_error.begin(), accumulated_error.end(), 0.0f); + + // Process for all samples in the sub-block. + for (size_t i = 0; i < y.size(); ++i) { + // Apply the matched filter as filter * x, and compute x * x. + RTC_DCHECK_GT(x_size, x_start_index); + const int chunk1 = + std::min(h_size, static_cast(x_size - x_start_index)); + if (chunk1 != h_size) { + const int chunk2 = h_size - chunk1; + std::copy(x.begin() + x_start_index, x.end(), scratch_memory.begin()); + std::copy(x.begin(), x.begin() + chunk2, scratch_memory.begin() + chunk1); + } + const float* x_p = + chunk1 != h_size ? scratch_memory.data() : &x[x_start_index]; + const float* h_p = &h[0]; + float* a_p = &accumulated_error[0]; + __m256 s_inst_hadd_256; + __m256 s_inst_256; + __m256 s_inst_256_8; + __m256 x2_sum_256 = _mm256_set1_ps(0); + __m256 x2_sum_256_8 = _mm256_set1_ps(0); + __m128 e_128; + float x2_sum = 0.0f; + float s_acum = 0; + const int limit_by_16 = h_size >> 4; + for (int k = limit_by_16; k > 0; --k, h_p += 16, x_p += 16, a_p += 4) { + // Load the data into 256 bit vectors. + __m256 x_k = _mm256_loadu_ps(x_p); + __m256 h_k = _mm256_loadu_ps(h_p); + __m256 x_k_8 = _mm256_loadu_ps(x_p + 8); + __m256 h_k_8 = _mm256_loadu_ps(h_p + 8); + // Compute and accumulate x * x and h * x. + x2_sum_256 = _mm256_fmadd_ps(x_k, x_k, x2_sum_256); + x2_sum_256_8 = _mm256_fmadd_ps(x_k_8, x_k_8, x2_sum_256_8); + s_inst_256 = _mm256_mul_ps(h_k, x_k); + s_inst_256_8 = _mm256_mul_ps(h_k_8, x_k_8); + s_inst_hadd_256 = _mm256_hadd_ps(s_inst_256, s_inst_256_8); + s_inst_hadd_256 = _mm256_hadd_ps(s_inst_hadd_256, s_inst_hadd_256); + s_acum += s_inst_hadd_256[0]; + e_128[0] = s_acum - y[i]; + s_acum += s_inst_hadd_256[4]; + e_128[1] = s_acum - y[i]; + s_acum += s_inst_hadd_256[1]; + e_128[2] = s_acum - y[i]; + s_acum += s_inst_hadd_256[5]; + e_128[3] = s_acum - y[i]; + + __m128 accumulated_error = _mm_load_ps(a_p); + accumulated_error = _mm_fmadd_ps(e_128, e_128, accumulated_error); + _mm_storeu_ps(a_p, accumulated_error); + } + // Sum components together. + x2_sum_256 = _mm256_add_ps(x2_sum_256, x2_sum_256_8); + __m128 x2_sum_128 = _mm_add_ps(_mm256_extractf128_ps(x2_sum_256, 0), + _mm256_extractf128_ps(x2_sum_256, 1)); + // Combine the accumulated vector and scalar values. + float* v = reinterpret_cast(&x2_sum_128); + x2_sum += v[0] + v[1] + v[2] + v[3]; + + // Compute the matched filter error. + float e = y[i] - s_acum; + const bool saturation = y[i] >= 32000.f || y[i] <= -32000.f; + (*error_sum) += e * e; + + // Update the matched filter estimate in an NLMS manner. + if (x2_sum > x2_sum_threshold && !saturation) { + RTC_DCHECK_LT(0.f, x2_sum); + const float alpha = smoothing * e / x2_sum; + const __m256 alpha_256 = _mm256_set1_ps(alpha); + + // filter = filter + smoothing * (y - filter * x) * x / x * x. + float* h_p = &h[0]; + const float* x_p = + chunk1 != h_size ? scratch_memory.data() : &x[x_start_index]; + // Perform 256 bit vector operations. + const int limit_by_8 = h_size >> 3; + for (int k = limit_by_8; k > 0; --k, h_p += 8, x_p += 8) { + // Load the data into 256 bit vectors. + __m256 h_k = _mm256_loadu_ps(h_p); + __m256 x_k = _mm256_loadu_ps(x_p); + // Compute h = h + alpha * x. + h_k = _mm256_fmadd_ps(x_k, alpha_256, h_k); + + // Store the result. + _mm256_storeu_ps(h_p, h_k); + } + *filters_updated = true; + } + + x_start_index = x_start_index > 0 ? x_start_index - 1 : x_size - 1; + } +} + void MatchedFilterCore_AVX2(size_t x_start_index, float x2_sum_threshold, float smoothing, @@ -24,7 +143,15 @@ void MatchedFilterCore_AVX2(size_t x_start_index, rtc::ArrayView y, rtc::ArrayView h, bool* filters_updated, - float* error_sum) { + float* error_sum, + bool compute_accumulated_error, + rtc::ArrayView accumulated_error, + rtc::ArrayView scratch_memory) { + if (compute_accumulated_error) { + return MatchedFilterCore_AccumulatedError_AVX2( + x_start_index, x2_sum_threshold, smoothing, x, y, h, filters_updated, + error_sum, accumulated_error, scratch_memory); + } const int h_size = static_cast(h.size()); const int x_size = static_cast(x.size()); RTC_DCHECK_EQ(0, h_size % 8); @@ -39,7 +166,9 @@ void MatchedFilterCore_AVX2(size_t x_start_index, // Initialize values for the accumulation. __m256 s_256 = _mm256_set1_ps(0); + __m256 s_256_8 = _mm256_set1_ps(0); __m256 x2_sum_256 = _mm256_set1_ps(0); + __m256 x2_sum_256_8 = _mm256_set1_ps(0); float x2_sum = 0.f; float s = 0; @@ -52,18 +181,22 @@ void MatchedFilterCore_AVX2(size_t x_start_index, const int chunk2 = h_size - chunk1; for (int limit : {chunk1, chunk2}) { // Perform 256 bit vector operations. - const int limit_by_8 = limit >> 3; - for (int k = limit_by_8; k > 0; --k, h_p += 8, x_p += 8) { + const int limit_by_16 = limit >> 4; + for (int k = limit_by_16; k > 0; --k, h_p += 16, x_p += 16) { // Load the data into 256 bit vectors. __m256 x_k = _mm256_loadu_ps(x_p); __m256 h_k = _mm256_loadu_ps(h_p); + __m256 x_k_8 = _mm256_loadu_ps(x_p + 8); + __m256 h_k_8 = _mm256_loadu_ps(h_p + 8); // Compute and accumulate x * x and h * x. x2_sum_256 = _mm256_fmadd_ps(x_k, x_k, x2_sum_256); + x2_sum_256_8 = _mm256_fmadd_ps(x_k_8, x_k_8, x2_sum_256_8); s_256 = _mm256_fmadd_ps(h_k, x_k, s_256); + s_256_8 = _mm256_fmadd_ps(h_k_8, x_k_8, s_256_8); } // Perform non-vector operations for any remaining items. - for (int k = limit - limit_by_8 * 8; k > 0; --k, ++h_p, ++x_p) { + for (int k = limit - limit_by_16 * 16; k > 0; --k, ++h_p, ++x_p) { const float x_k = *x_p; x2_sum += x_k * x_k; s += *h_p * x_k; @@ -73,15 +206,11 @@ void MatchedFilterCore_AVX2(size_t x_start_index, } // Sum components together. - __m128 x2_sum_128 = _mm_add_ps(_mm256_extractf128_ps(x2_sum_256, 0), - _mm256_extractf128_ps(x2_sum_256, 1)); - __m128 s_128 = _mm_add_ps(_mm256_extractf128_ps(s_256, 0), - _mm256_extractf128_ps(s_256, 1)); - // Combine the accumulated vector and scalar values. - float* v = reinterpret_cast(&x2_sum_128); - x2_sum += v[0] + v[1] + v[2] + v[3]; - v = reinterpret_cast(&s_128); - s += v[0] + v[1] + v[2] + v[3]; + x2_sum_256 = _mm256_add_ps(x2_sum_256, x2_sum_256_8); + s_256 = _mm256_add_ps(s_256, s_256_8); + __m128 sum = hsum_ab(x2_sum_256, s_256); + x2_sum += sum[0]; + s += sum[1]; // Compute the matched filter error. float e = y[i] - s; diff --git a/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc b/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc index 603a864..c8ac417 100644 --- a/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc +++ b/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc @@ -14,84 +14,179 @@ #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { +namespace { +constexpr int kPreEchoHistogramDataNotUpdated = -1; + +int GetDownSamplingBlockSizeLog2(int down_sampling_factor) { + int down_sampling_factor_log2 = 0; + down_sampling_factor >>= 1; + while (down_sampling_factor > 0) { + down_sampling_factor_log2++; + down_sampling_factor >>= 1; + } + return static_cast(kBlockSizeLog2) > down_sampling_factor_log2 + ? static_cast(kBlockSizeLog2) - down_sampling_factor_log2 + : 0; +} +} // namespace MatchedFilterLagAggregator::MatchedFilterLagAggregator( ApmDataDumper* data_dumper, size_t max_filter_lag, - const EchoCanceller3Config::Delay::DelaySelectionThresholds& thresholds) + const EchoCanceller3Config::Delay& delay_config) : data_dumper_(data_dumper), - histogram_(max_filter_lag + 1, 0), - thresholds_(thresholds) { + thresholds_(delay_config.delay_selection_thresholds), + headroom_(static_cast(delay_config.delay_headroom_samples / + delay_config.down_sampling_factor)), + highest_peak_aggregator_(max_filter_lag) { + if (delay_config.detect_pre_echo) { + pre_echo_lag_aggregator_ = std::make_unique( + max_filter_lag, delay_config.down_sampling_factor); + } RTC_DCHECK(data_dumper); RTC_DCHECK_LE(thresholds_.initial, thresholds_.converged); - histogram_data_.fill(0); } MatchedFilterLagAggregator::~MatchedFilterLagAggregator() = default; void MatchedFilterLagAggregator::Reset(bool hard_reset) { - std::fill(histogram_.begin(), histogram_.end(), 0); - histogram_data_.fill(0); - histogram_data_index_ = 0; + highest_peak_aggregator_.Reset(); + if (pre_echo_lag_aggregator_ != nullptr) { + pre_echo_lag_aggregator_->Reset(); + } if (hard_reset) { significant_candidate_found_ = false; } } absl::optional MatchedFilterLagAggregator::Aggregate( - rtc::ArrayView lag_estimates) { - // Choose the strongest lag estimate as the best one. - float best_accuracy = 0.f; - int best_lag_estimate_index = -1; - for (size_t k = 0; k < lag_estimates.size(); ++k) { - if (lag_estimates[k].updated && lag_estimates[k].reliable) { - if (lag_estimates[k].accuracy > best_accuracy) { - best_accuracy = lag_estimates[k].accuracy; - best_lag_estimate_index = static_cast(k); - } - } + const absl::optional& lag_estimate) { + if (lag_estimate && pre_echo_lag_aggregator_) { + pre_echo_lag_aggregator_->Dump(data_dumper_); + pre_echo_lag_aggregator_->Aggregate( + std::max(0, static_cast(lag_estimate->pre_echo_lag) - headroom_)); } - // TODO(peah): Remove this logging once all development is done. - data_dumper_->DumpRaw("aec3_echo_path_delay_estimator_best_index", - best_lag_estimate_index); - data_dumper_->DumpRaw("aec3_echo_path_delay_estimator_histogram", histogram_); - - if (best_lag_estimate_index != -1) { - RTC_DCHECK_GT(histogram_.size(), histogram_data_[histogram_data_index_]); - RTC_DCHECK_LE(0, histogram_data_[histogram_data_index_]); - --histogram_[histogram_data_[histogram_data_index_]]; - - histogram_data_[histogram_data_index_] = - lag_estimates[best_lag_estimate_index].lag; - - RTC_DCHECK_GT(histogram_.size(), histogram_data_[histogram_data_index_]); - RTC_DCHECK_LE(0, histogram_data_[histogram_data_index_]); - ++histogram_[histogram_data_[histogram_data_index_]]; - - histogram_data_index_ = - (histogram_data_index_ + 1) % histogram_data_.size(); - - const int candidate = - std::distance(histogram_.begin(), - std::max_element(histogram_.begin(), histogram_.end())); - - significant_candidate_found_ = - significant_candidate_found_ || - histogram_[candidate] > thresholds_.converged; - if (histogram_[candidate] > thresholds_.converged || - (histogram_[candidate] > thresholds_.initial && + if (lag_estimate) { + highest_peak_aggregator_.Aggregate( + std::max(0, static_cast(lag_estimate->lag) - headroom_)); + rtc::ArrayView histogram = highest_peak_aggregator_.histogram(); + int candidate = highest_peak_aggregator_.candidate(); + significant_candidate_found_ = significant_candidate_found_ || + histogram[candidate] > thresholds_.converged; + if (histogram[candidate] > thresholds_.converged || + (histogram[candidate] > thresholds_.initial && !significant_candidate_found_)) { DelayEstimate::Quality quality = significant_candidate_found_ ? DelayEstimate::Quality::kRefined : DelayEstimate::Quality::kCoarse; - return DelayEstimate(quality, candidate); + int reported_delay = pre_echo_lag_aggregator_ != nullptr + ? pre_echo_lag_aggregator_->pre_echo_candidate() + : candidate; + return DelayEstimate(quality, reported_delay); } } return absl::nullopt; } +MatchedFilterLagAggregator::HighestPeakAggregator::HighestPeakAggregator( + size_t max_filter_lag) + : histogram_(max_filter_lag + 1, 0) { + histogram_data_.fill(0); +} + +void MatchedFilterLagAggregator::HighestPeakAggregator::Reset() { + std::fill(histogram_.begin(), histogram_.end(), 0); + histogram_data_.fill(0); + histogram_data_index_ = 0; +} + +void MatchedFilterLagAggregator::HighestPeakAggregator::Aggregate(int lag) { + RTC_DCHECK_GT(histogram_.size(), histogram_data_[histogram_data_index_]); + RTC_DCHECK_LE(0, histogram_data_[histogram_data_index_]); + --histogram_[histogram_data_[histogram_data_index_]]; + histogram_data_[histogram_data_index_] = lag; + RTC_DCHECK_GT(histogram_.size(), histogram_data_[histogram_data_index_]); + RTC_DCHECK_LE(0, histogram_data_[histogram_data_index_]); + ++histogram_[histogram_data_[histogram_data_index_]]; + histogram_data_index_ = (histogram_data_index_ + 1) % histogram_data_.size(); + candidate_ = + std::distance(histogram_.begin(), + std::max_element(histogram_.begin(), histogram_.end())); +} + +MatchedFilterLagAggregator::PreEchoLagAggregator::PreEchoLagAggregator( + size_t max_filter_lag, + size_t down_sampling_factor) + : block_size_log2_(GetDownSamplingBlockSizeLog2(down_sampling_factor)), + penalize_high_delays_initial_phase_(!field_trial::IsDisabled( + "WebRTC-Aec3PenalyzeHighDelaysInitialPhase")), + histogram_( + ((max_filter_lag + 1) * down_sampling_factor) >> kBlockSizeLog2, + 0) { + Reset(); +} + +void MatchedFilterLagAggregator::PreEchoLagAggregator::Reset() { + std::fill(histogram_.begin(), histogram_.end(), 0); + histogram_data_.fill(kPreEchoHistogramDataNotUpdated); + histogram_data_index_ = 0; + pre_echo_candidate_ = 0; +} + +void MatchedFilterLagAggregator::PreEchoLagAggregator::Aggregate( + int pre_echo_lag) { + int pre_echo_block_size = pre_echo_lag >> block_size_log2_; + RTC_DCHECK(pre_echo_block_size >= 0 && + pre_echo_block_size < static_cast(histogram_.size())); + pre_echo_block_size = + rtc::SafeClamp(pre_echo_block_size, 0, histogram_.size() - 1); + // Remove the oldest point from the `histogram_`, it ignores the initial + // points where no updates have been done to the `histogram_data_` array. + if (histogram_data_[histogram_data_index_] != + kPreEchoHistogramDataNotUpdated) { + --histogram_[histogram_data_[histogram_data_index_]]; + } + histogram_data_[histogram_data_index_] = pre_echo_block_size; + ++histogram_[histogram_data_[histogram_data_index_]]; + histogram_data_index_ = (histogram_data_index_ + 1) % histogram_data_.size(); + int pre_echo_candidate_block_size = 0; + if (penalize_high_delays_initial_phase_ && + number_updates_ < kNumBlocksPerSecond * 2) { + number_updates_++; + float penalization_per_delay = 1.0f; + float max_histogram_value = -1.0f; + for (auto it = histogram_.begin(); + std::distance(it, histogram_.end()) >= + static_cast(kMatchedFilterWindowSizeSubBlocks); + it = it + kMatchedFilterWindowSizeSubBlocks) { + auto it_max_element = + std::max_element(it, it + kMatchedFilterWindowSizeSubBlocks); + float weighted_max_value = + static_cast(*it_max_element) * penalization_per_delay; + if (weighted_max_value > max_histogram_value) { + max_histogram_value = weighted_max_value; + pre_echo_candidate_block_size = + std::distance(histogram_.begin(), it_max_element); + } + penalization_per_delay *= 0.7f; + } + } else { + pre_echo_candidate_block_size = + std::distance(histogram_.begin(), + std::max_element(histogram_.begin(), histogram_.end())); + } + pre_echo_candidate_ = (pre_echo_candidate_block_size << block_size_log2_); +} + +void MatchedFilterLagAggregator::PreEchoLagAggregator::Dump( + ApmDataDumper* const data_dumper) { + data_dumper->DumpRaw("aec3_pre_echo_delay_candidate", pre_echo_candidate_); +} + } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h b/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h index d48011e..1287b38 100644 --- a/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h +++ b/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h @@ -26,10 +26,9 @@ class ApmDataDumper; // reliable combined lag estimate. class MatchedFilterLagAggregator { public: - MatchedFilterLagAggregator( - ApmDataDumper* data_dumper, - size_t max_filter_lag, - const EchoCanceller3Config::Delay::DelaySelectionThresholds& thresholds); + MatchedFilterLagAggregator(ApmDataDumper* data_dumper, + size_t max_filter_lag, + const EchoCanceller3Config::Delay& delay_config); MatchedFilterLagAggregator() = delete; MatchedFilterLagAggregator(const MatchedFilterLagAggregator&) = delete; @@ -43,15 +42,57 @@ class MatchedFilterLagAggregator { // Aggregates the provided lag estimates. absl::optional Aggregate( - rtc::ArrayView lag_estimates); + const absl::optional& lag_estimate); + + // Returns whether a reliable delay estimate has been found. + bool ReliableDelayFound() const { return significant_candidate_found_; } + + // Returns the delay candidate that is computed by looking at the highest peak + // on the matched filters. + int GetDelayAtHighestPeak() const { + return highest_peak_aggregator_.candidate(); + } private: + class PreEchoLagAggregator { + public: + PreEchoLagAggregator(size_t max_filter_lag, size_t down_sampling_factor); + void Reset(); + void Aggregate(int pre_echo_lag); + int pre_echo_candidate() const { return pre_echo_candidate_; } + void Dump(ApmDataDumper* const data_dumper); + + private: + const int block_size_log2_; + const bool penalize_high_delays_initial_phase_; + std::array histogram_data_; + std::vector histogram_; + int histogram_data_index_ = 0; + int pre_echo_candidate_ = 0; + int number_updates_ = 0; + }; + + class HighestPeakAggregator { + public: + explicit HighestPeakAggregator(size_t max_filter_lag); + void Reset(); + void Aggregate(int lag); + int candidate() const { return candidate_; } + rtc::ArrayView histogram() const { return histogram_; } + + private: + std::vector histogram_; + std::array histogram_data_; + int histogram_data_index_ = 0; + int candidate_ = -1; + }; + ApmDataDumper* const data_dumper_; - std::vector histogram_; - std::array histogram_data_; - int histogram_data_index_ = 0; bool significant_candidate_found_ = false; const EchoCanceller3Config::Delay::DelaySelectionThresholds thresholds_; + const int headroom_; + HighestPeakAggregator highest_peak_aggregator_; + std::unique_ptr pre_echo_lag_aggregator_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/multi_channel_content_detector.cc b/webrtc/modules/audio_processing/aec3/multi_channel_content_detector.cc new file mode 100644 index 0000000..9806896 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/multi_channel_content_detector.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/multi_channel_content_detector.h" + +#include + +#include "rtc_base/checks.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +constexpr int kNumFramesPerSecond = 100; + +// Compares the left and right channels in the render `frame` to determine +// whether the signal is a proper stereo signal. To allow for differences +// introduced by hardware drivers, a threshold `detection_threshold` is used for +// the detection. +bool HasStereoContent(const std::vector>>& frame, + float detection_threshold) { + if (frame[0].size() < 2) { + return false; + } + + for (size_t band = 0; band < frame.size(); ++band) { + for (size_t k = 0; k < frame[band][0].size(); ++k) { + if (std::fabs(frame[band][0][k] - frame[band][1][k]) > + detection_threshold) { + return true; + } + } + } + return false; +} + +// In order to avoid logging metrics for very short lifetimes that are unlikely +// to reflect real calls and that may dilute the "real" data, logging is limited +// to lifetimes of at leats 5 seconds. +constexpr int kMinNumberOfFramesRequiredToLogMetrics = 500; + +// Continuous metrics are logged every 10 seconds. +constexpr int kFramesPer10Seconds = 1000; + +} // namespace + +MultiChannelContentDetector::MetricsLogger::MetricsLogger() {} + +MultiChannelContentDetector::MetricsLogger::~MetricsLogger() { + if (frame_counter_ < kMinNumberOfFramesRequiredToLogMetrics) + return; + + RTC_HISTOGRAM_BOOLEAN( + "WebRTC.Audio.EchoCanceller.PersistentMultichannelContentEverDetected", + any_multichannel_content_detected_ ? 1 : 0); +} + +void MultiChannelContentDetector::MetricsLogger::Update( + bool persistent_multichannel_content_detected) { + ++frame_counter_; + if (persistent_multichannel_content_detected) { + any_multichannel_content_detected_ = true; + ++persistent_multichannel_frame_counter_; + } + + if (frame_counter_ < kMinNumberOfFramesRequiredToLogMetrics) + return; + if (frame_counter_ % kFramesPer10Seconds != 0) + return; + const bool mostly_multichannel_last_10_seconds = + (persistent_multichannel_frame_counter_ >= kFramesPer10Seconds / 2); + RTC_HISTOGRAM_BOOLEAN( + "WebRTC.Audio.EchoCanceller.ProcessingPersistentMultichannelContent", + mostly_multichannel_last_10_seconds ? 1 : 0); + + persistent_multichannel_frame_counter_ = 0; +} + +MultiChannelContentDetector::MultiChannelContentDetector( + bool detect_stereo_content, + int num_render_input_channels, + float detection_threshold, + int stereo_detection_timeout_threshold_seconds, + float stereo_detection_hysteresis_seconds) + : detect_stereo_content_(detect_stereo_content), + detection_threshold_(detection_threshold), + detection_timeout_threshold_frames_( + stereo_detection_timeout_threshold_seconds > 0 + ? absl::make_optional(stereo_detection_timeout_threshold_seconds * + kNumFramesPerSecond) + : absl::nullopt), + stereo_detection_hysteresis_frames_(static_cast( + stereo_detection_hysteresis_seconds * kNumFramesPerSecond)), + metrics_logger_((detect_stereo_content && num_render_input_channels > 1) + ? std::make_unique() + : nullptr), + persistent_multichannel_content_detected_( + !detect_stereo_content && num_render_input_channels > 1) {} + +bool MultiChannelContentDetector::UpdateDetection( + const std::vector>>& frame) { + if (!detect_stereo_content_) { + RTC_DCHECK_EQ(frame[0].size() > 1, + persistent_multichannel_content_detected_); + return false; + } + + const bool previous_persistent_multichannel_content_detected = + persistent_multichannel_content_detected_; + const bool stereo_detected_in_frame = + HasStereoContent(frame, detection_threshold_); + + consecutive_frames_with_stereo_ = + stereo_detected_in_frame ? consecutive_frames_with_stereo_ + 1 : 0; + frames_since_stereo_detected_last_ = + stereo_detected_in_frame ? 0 : frames_since_stereo_detected_last_ + 1; + + // Detect persistent multichannel content. + if (consecutive_frames_with_stereo_ > stereo_detection_hysteresis_frames_) { + persistent_multichannel_content_detected_ = true; + } + if (detection_timeout_threshold_frames_.has_value() && + frames_since_stereo_detected_last_ >= + *detection_timeout_threshold_frames_) { + persistent_multichannel_content_detected_ = false; + } + + // Detect temporary multichannel content. + temporary_multichannel_content_detected_ = + persistent_multichannel_content_detected_ ? false + : stereo_detected_in_frame; + + if (metrics_logger_) + metrics_logger_->Update(persistent_multichannel_content_detected_); + + return previous_persistent_multichannel_content_detected != + persistent_multichannel_content_detected_; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/multi_channel_content_detector.h b/webrtc/modules/audio_processing/aec3/multi_channel_content_detector.h new file mode 100644 index 0000000..be8717f --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/multi_channel_content_detector.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MULTI_CHANNEL_CONTENT_DETECTOR_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MULTI_CHANNEL_CONTENT_DETECTOR_H_ + +#include + +#include +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +// Analyzes audio content to determine whether the contained audio is proper +// multichannel, or only upmixed mono. To allow for differences introduced by +// hardware drivers, a threshold `detection_threshold` is used for the +// detection. +// Logs metrics continously and upon destruction. +class MultiChannelContentDetector { + public: + // If |stereo_detection_timeout_threshold_seconds| <= 0, no timeout is + // applied: Once multichannel is detected, the detector remains in that state + // for its lifetime. + MultiChannelContentDetector(bool detect_stereo_content, + int num_render_input_channels, + float detection_threshold, + int stereo_detection_timeout_threshold_seconds, + float stereo_detection_hysteresis_seconds); + + // Compares the left and right channels in the render `frame` to determine + // whether the signal is a proper multichannel signal. Returns a bool + // indicating whether a change in the proper multichannel content was + // detected. + bool UpdateDetection( + const std::vector>>& frame); + + bool IsProperMultiChannelContentDetected() const { + return persistent_multichannel_content_detected_; + } + + bool IsTemporaryMultiChannelContentDetected() const { + return temporary_multichannel_content_detected_; + } + + private: + // Tracks and logs metrics for the amount of multichannel content detected. + class MetricsLogger { + public: + MetricsLogger(); + + // The destructor logs call summary statistics. + ~MetricsLogger(); + + // Updates and logs metrics. + void Update(bool persistent_multichannel_content_detected); + + private: + int frame_counter_ = 0; + + // Counts the number of frames of persistent multichannel audio observed + // during the current metrics collection interval. + int persistent_multichannel_frame_counter_ = 0; + + // Indicates whether persistent multichannel content has ever been detected. + bool any_multichannel_content_detected_ = false; + }; + + const bool detect_stereo_content_; + const float detection_threshold_; + const absl::optional detection_timeout_threshold_frames_; + const int stereo_detection_hysteresis_frames_; + + // Collects and reports metrics on the amount of multichannel content + // detected. Only created if |num_render_input_channels| > 1 and + // |detect_stereo_content_| is true. + const std::unique_ptr metrics_logger_; + + bool persistent_multichannel_content_detected_; + bool temporary_multichannel_content_detected_ = false; + int64_t frames_since_stereo_detected_last_ = 0; + int64_t consecutive_frames_with_stereo_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MULTI_CHANNEL_CONTENT_DETECTOR_H_ diff --git a/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.cc b/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.cc index 138329a..8e391d6 100644 --- a/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.cc +++ b/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.cc @@ -20,7 +20,6 @@ #include "modules/audio_processing/aec3/render_signal_analyzer.h" #include "modules/audio_processing/aec3/subtractor_output.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" namespace webrtc { @@ -31,13 +30,12 @@ constexpr int kPoorExcitationCounterInitial = 1000; } // namespace -int RefinedFilterUpdateGain::instance_count_ = 0; +std::atomic RefinedFilterUpdateGain::instance_count_(0); RefinedFilterUpdateGain::RefinedFilterUpdateGain( const EchoCanceller3Config::Filter::RefinedConfiguration& config, size_t config_change_duration_blocks) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), config_change_duration_blocks_( static_cast(config_change_duration_blocks)), poor_excitation_counter_(kPoorExcitationCounterInitial) { @@ -73,6 +71,7 @@ void RefinedFilterUpdateGain::Compute( rtc::ArrayView erl, size_t size_partitions, bool saturated_capture_signal, + bool disallow_leakage_diverged, FftData* gain_fft) { RTC_DCHECK(gain_fft); // Introducing shorter notation to improve readability. @@ -125,7 +124,7 @@ void RefinedFilterUpdateGain::Compute( // H_error = H_error + factor * erl. for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { - if (E2_coarse[k] >= E2_refined[k]) { + if (E2_refined[k] <= E2_coarse[k] || disallow_leakage_diverged) { H_error_[k] += current_config_.leakage_converged * erl[k]; } else { H_error_[k] += current_config_.leakage_diverged * erl[k]; diff --git a/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.h b/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.h index 5730979..1a68ebc 100644 --- a/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.h +++ b/webrtc/modules/audio_processing/aec3/refined_filter_update_gain.h @@ -14,6 +14,7 @@ #include #include +#include #include #include "api/array_view.h" @@ -51,6 +52,7 @@ class RefinedFilterUpdateGain { rtc::ArrayView erl, size_t size_partitions, bool saturated_capture_signal, + bool disallow_leakage_diverged, FftData* gain_fft); // Sets a new config. @@ -68,7 +70,7 @@ class RefinedFilterUpdateGain { } private: - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const int config_change_duration_blocks_; float one_by_config_change_duration_blocks_; diff --git a/webrtc/modules/audio_processing/aec3/render_buffer.cc b/webrtc/modules/audio_processing/aec3/render_buffer.cc index 60ea69c..aa511e2 100644 --- a/webrtc/modules/audio_processing/aec3/render_buffer.cc +++ b/webrtc/modules/audio_processing/aec3/render_buffer.cc @@ -42,8 +42,9 @@ void RenderBuffer::SpectralSum( int position = spectrum_buffer_->read; for (size_t j = 0; j < num_spectra; ++j) { for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) { - std::transform(X2->begin(), X2->end(), channel_spectrum.begin(), - X2->begin(), std::plus()); + for (size_t k = 0; k < X2->size(); ++k) { + (*X2)[k] += channel_spectrum[k]; + } } position = spectrum_buffer_->IncIndex(position); } @@ -60,18 +61,18 @@ void RenderBuffer::SpectralSums( size_t j = 0; for (; j < num_spectra_shorter; ++j) { for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) { - std::transform(X2_shorter->begin(), X2_shorter->end(), - channel_spectrum.begin(), X2_shorter->begin(), - std::plus()); + for (size_t k = 0; k < X2_shorter->size(); ++k) { + (*X2_shorter)[k] += channel_spectrum[k]; + } } position = spectrum_buffer_->IncIndex(position); } std::copy(X2_shorter->begin(), X2_shorter->end(), X2_longer->begin()); for (; j < num_spectra_longer; ++j) { for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) { - std::transform(X2_longer->begin(), X2_longer->end(), - channel_spectrum.begin(), X2_longer->begin(), - std::plus()); + for (size_t k = 0; k < X2_longer->size(); ++k) { + (*X2_longer)[k] += channel_spectrum[k]; + } } position = spectrum_buffer_->IncIndex(position); } diff --git a/webrtc/modules/audio_processing/aec3/render_buffer.h b/webrtc/modules/audio_processing/aec3/render_buffer.h index b8be6f5..8adc996 100644 --- a/webrtc/modules/audio_processing/aec3/render_buffer.h +++ b/webrtc/modules/audio_processing/aec3/render_buffer.h @@ -40,8 +40,7 @@ class RenderBuffer { ~RenderBuffer(); // Get a block. - const std::vector>>& Block( - int buffer_offset_blocks) const { + const Block& GetBlock(int buffer_offset_blocks) const { int position = block_buffer_->OffsetIndex(block_buffer_->read, buffer_offset_blocks); return block_buffer_->buffer[position]; diff --git a/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc b/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc index 7bebc6f..ca77a58 100644 --- a/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -32,7 +33,6 @@ #include "modules/audio_processing/aec3/render_buffer.h" #include "modules/audio_processing/aec3/spectrum_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" @@ -40,11 +40,6 @@ namespace webrtc { namespace { -bool UpdateCaptureCallCounterOnSkippedBlocks() { - return !field_trial::IsEnabled( - "WebRTC-Aec3RenderBufferCallCounterUpdateKillSwitch"); -} - class RenderDelayBufferImpl final : public RenderDelayBuffer { public: RenderDelayBufferImpl(const EchoCanceller3Config& config, @@ -54,8 +49,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { ~RenderDelayBufferImpl() override; void Reset() override; - BufferingEvent Insert( - const std::vector>>& block) override; + BufferingEvent Insert(const Block& block) override; BufferingEvent PrepareCaptureProcessing() override; void HandleSkippedCaptureProcessing() override; bool AlignFromDelay(size_t delay) override; @@ -75,11 +69,10 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { bool HasReceivedBufferDelay() override; private: - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const EchoCanceller3Config config_; - const bool update_capture_call_counter_on_skipped_blocks_; const float render_linear_amplitude_gain_; const rtc::LoggingSeverity delay_log_level_; size_t down_sampling_factor_; @@ -110,8 +103,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { int MapDelayToTotalDelay(size_t delay) const; int ComputeDelay() const; void ApplyTotalDelay(int delay); - void InsertBlock(const std::vector>>& block, - int previous_write); + void InsertBlock(const Block& block, int previous_write); bool DetectActiveRender(rtc::ArrayView x) const; bool DetectExcessRenderBlocks(); void IncrementWriteIndices(); @@ -121,17 +113,14 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { bool RenderUnderrun(); }; -int RenderDelayBufferImpl::instance_count_ = 0; +std::atomic RenderDelayBufferImpl::instance_count_ = 0; RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, int sample_rate_hz, size_t num_render_channels) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), optimization_(DetectOptimization()), config_(config), - update_capture_call_counter_on_skipped_blocks_( - UpdateCaptureCallCounterOnSkippedBlocks()), render_linear_amplitude_gain_( std::pow(10.0f, config_.render_levels.render_power_gain_db / 20.f)), delay_log_level_(config_.delay.log_warning_on_delay_changes @@ -145,8 +134,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, config.delay.num_filters, config.filter.refined.length_blocks), NumBandsForRate(sample_rate_hz), - num_render_channels, - kBlockSize), + num_render_channels), spectra_(blocks_.buffer.size(), num_render_channels), ffts_(blocks_.buffer.size(), num_render_channels), delay_(config_.delay.default_delay), @@ -161,7 +149,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size()); RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size()); for (size_t i = 0; i < blocks_.buffer.size(); ++i) { - RTC_DCHECK_EQ(blocks_.buffer[i][0].size(), ffts_.buffer[i].size()); + RTC_DCHECK_EQ(blocks_.buffer[i].NumChannels(), ffts_.buffer[i].size()); RTC_DCHECK_EQ(spectra_.buffer[i].size(), ffts_.buffer[i].size()); } @@ -211,7 +199,7 @@ void RenderDelayBufferImpl::Reset() { // Inserts a new block into the render buffers. RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( - const std::vector>>& block) { + const Block& block) { ++render_call_counter_; if (delay_) { if (!last_call_was_render_) { @@ -239,7 +227,8 @@ RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( // Detect and update render activity. if (!render_activity_) { - render_activity_counter_ += DetectActiveRender(block[0][0]) ? 1 : 0; + render_activity_counter_ += + DetectActiveRender(block.View(/*band=*/0, /*channel=*/0)) ? 1 : 0; render_activity_ = render_activity_counter_ >= 20; } @@ -254,9 +243,7 @@ RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( } void RenderDelayBufferImpl::HandleSkippedCaptureProcessing() { - if (update_capture_call_counter_on_skipped_blocks_) { - ++capture_call_counter_; - } + ++capture_call_counter_; } // Prepares the render buffers for processing another capture block. @@ -394,46 +381,45 @@ void RenderDelayBufferImpl::AlignFromExternalDelay() { } // Inserts a block into the render buffers. -void RenderDelayBufferImpl::InsertBlock( - const std::vector>>& block, - int previous_write) { +void RenderDelayBufferImpl::InsertBlock(const Block& block, + int previous_write) { auto& b = blocks_; auto& lr = low_rate_; auto& ds = render_ds_; auto& f = ffts_; auto& s = spectra_; - const size_t num_bands = b.buffer[b.write].size(); - const size_t num_render_channels = b.buffer[b.write][0].size(); - RTC_DCHECK_EQ(block.size(), b.buffer[b.write].size()); + const size_t num_bands = b.buffer[b.write].NumBands(); + const size_t num_render_channels = b.buffer[b.write].NumChannels(); + RTC_DCHECK_EQ(block.NumBands(), num_bands); + RTC_DCHECK_EQ(block.NumChannels(), num_render_channels); for (size_t band = 0; band < num_bands; ++band) { - RTC_DCHECK_EQ(block[band].size(), num_render_channels); - RTC_DCHECK_EQ(b.buffer[b.write][band].size(), num_render_channels); for (size_t ch = 0; ch < num_render_channels; ++ch) { - RTC_DCHECK_EQ(block[band][ch].size(), b.buffer[b.write][band][ch].size()); - std::copy(block[band][ch].begin(), block[band][ch].end(), - b.buffer[b.write][band][ch].begin()); + std::copy(block.begin(band, ch), block.end(band, ch), + b.buffer[b.write].begin(band, ch)); } } if (render_linear_amplitude_gain_ != 1.f) { for (size_t band = 0; band < num_bands; ++band) { for (size_t ch = 0; ch < num_render_channels; ++ch) { - for (size_t k = 0; k < 64; ++k) { - b.buffer[b.write][band][ch][k] *= render_linear_amplitude_gain_; + rtc::ArrayView b_view = + b.buffer[b.write].View(band, ch); + for (float& sample : b_view) { + sample *= render_linear_amplitude_gain_; } } } } std::array downmixed_render; - render_mixer_.ProduceOutput(b.buffer[b.write][0], downmixed_render); + render_mixer_.ProduceOutput(b.buffer[b.write], downmixed_render); render_decimator_.Decimate(downmixed_render, ds); data_dumper_->DumpWav("aec3_render_decimator_output", ds.size(), ds.data(), 16000 / down_sampling_factor_, 1); std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write); - for (size_t channel = 0; channel < b.buffer[b.write][0].size(); ++channel) { - fft_.PaddedFft(b.buffer[b.write][0][channel], - b.buffer[previous_write][0][channel], + for (int channel = 0; channel < b.buffer[b.write].NumChannels(); ++channel) { + fft_.PaddedFft(b.buffer[b.write].View(/*band=*/0, channel), + b.buffer[previous_write].View(/*band=*/0, channel), &f.buffer[f.write][channel]); f.buffer[f.write][channel].Spectrum(optimization_, s.buffer[s.write][channel]); diff --git a/webrtc/modules/audio_processing/aec3/render_delay_buffer.h b/webrtc/modules/audio_processing/aec3/render_delay_buffer.h index 79ffc4d..6dc1aef 100644 --- a/webrtc/modules/audio_processing/aec3/render_delay_buffer.h +++ b/webrtc/modules/audio_processing/aec3/render_delay_buffer.h @@ -16,6 +16,7 @@ #include #include "api/audio/echo_canceller3_config.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/downsampled_render_buffer.h" #include "modules/audio_processing/aec3/render_buffer.h" @@ -41,8 +42,7 @@ class RenderDelayBuffer { virtual void Reset() = 0; // Inserts a block into the buffer. - virtual BufferingEvent Insert( - const std::vector>>& block) = 0; + virtual BufferingEvent Insert(const Block& block) = 0; // Updates the buffers one step based on the specified buffer delay. Returns // an enum indicating whether there was a special event that occurred. diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller.cc b/webrtc/modules/audio_processing/aec3/render_delay_controller.cc index 3677085..465e77f 100644 --- a/webrtc/modules/audio_processing/aec3/render_delay_controller.cc +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include "absl/types/optional.h" @@ -23,7 +24,6 @@ #include "modules/audio_processing/aec3/echo_path_delay_estimator.h" #include "modules/audio_processing/aec3/render_delay_controller_metrics.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" namespace webrtc { @@ -47,14 +47,13 @@ class RenderDelayControllerImpl final : public RenderDelayController { absl::optional GetDelay( const DownsampledRenderBuffer& render_buffer, size_t render_delay_buffer_delay, - const std::vector>& capture) override; + const Block& capture) override; bool HasClockdrift() const override; private: - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const int hysteresis_limit_blocks_; - const int delay_headroom_samples_; absl::optional delay_; EchoPathDelayEstimator delay_estimator_; RenderDelayControllerMetrics metrics_; @@ -67,15 +66,9 @@ class RenderDelayControllerImpl final : public RenderDelayController { DelayEstimate ComputeBufferDelay( const absl::optional& current_delay, int hysteresis_limit_blocks, - int delay_headroom_samples, DelayEstimate estimated_delay) { - // Subtract delay headroom. - const int delay_with_headroom_samples = std::max( - static_cast(estimated_delay.delay) - delay_headroom_samples, 0); - // Compute the buffer delay increase required to achieve the desired latency. - size_t new_delay_blocks = delay_with_headroom_samples >> kBlockSizeLog2; - + size_t new_delay_blocks = estimated_delay.delay >> kBlockSizeLog2; // Add hysteresis. if (current_delay) { size_t current_delay_blocks = current_delay->delay; @@ -84,23 +77,20 @@ DelayEstimate ComputeBufferDelay( new_delay_blocks = current_delay_blocks; } } - DelayEstimate new_delay = estimated_delay; new_delay.delay = new_delay_blocks; return new_delay; } -int RenderDelayControllerImpl::instance_count_ = 0; +std::atomic RenderDelayControllerImpl::instance_count_(0); RenderDelayControllerImpl::RenderDelayControllerImpl( const EchoCanceller3Config& config, int sample_rate_hz, size_t num_capture_channels) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), hysteresis_limit_blocks_( static_cast(config.delay.hysteresis_limit_blocks)), - delay_headroom_samples_(config.delay.delay_headroom_samples), delay_estimator_(data_dumper_.get(), config, num_capture_channels), last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); @@ -124,8 +114,7 @@ void RenderDelayControllerImpl::LogRenderCall() {} absl::optional RenderDelayControllerImpl::GetDelay( const DownsampledRenderBuffer& render_buffer, size_t render_delay_buffer_delay, - const std::vector>& capture) { - RTC_DCHECK_EQ(kBlockSize, capture[0].size()); + const Block& capture) { ++capture_call_counter_; auto delay_samples = delay_estimator_.EstimateDelay(render_buffer, capture); @@ -161,15 +150,16 @@ absl::optional RenderDelayControllerImpl::GetDelay( const bool use_hysteresis = last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined && delay_samples_->quality == DelayEstimate::Quality::kRefined; - delay_ = ComputeBufferDelay(delay_, - use_hysteresis ? hysteresis_limit_blocks_ : 0, - delay_headroom_samples_, *delay_samples_); + delay_ = ComputeBufferDelay( + delay_, use_hysteresis ? hysteresis_limit_blocks_ : 0, *delay_samples_); last_delay_estimate_quality_ = delay_samples_->quality; } - metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) - : absl::nullopt, - delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift()); + metrics_.Update( + delay_samples_ ? absl::optional(delay_samples_->delay) + : absl::nullopt, + delay_ ? absl::optional(delay_->delay) : absl::nullopt, + delay_estimator_.Clockdrift()); data_dumper_->DumpRaw("aec3_render_delay_controller_delay", delay_samples ? delay_samples->delay : 0); diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller.h b/webrtc/modules/audio_processing/aec3/render_delay_controller.h index c45ab1f..4a18a11 100644 --- a/webrtc/modules/audio_processing/aec3/render_delay_controller.h +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller.h @@ -14,6 +14,7 @@ #include "absl/types/optional.h" #include "api/array_view.h" #include "api/audio/echo_canceller3_config.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/delay_estimate.h" #include "modules/audio_processing/aec3/downsampled_render_buffer.h" #include "modules/audio_processing/aec3/render_delay_buffer.h" @@ -40,7 +41,7 @@ class RenderDelayController { virtual absl::optional GetDelay( const DownsampledRenderBuffer& render_buffer, size_t render_delay_buffer_delay, - const std::vector>& capture) = 0; + const Block& capture) = 0; // Returns true if clockdrift has been detected. virtual bool HasClockdrift() const = 0; diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc b/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc index 582e033..1e0a0f4 100644 --- a/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc @@ -37,16 +37,13 @@ enum class DelayChangesCategory { kNumCategories }; -constexpr int kMaxSkewShiftCount = 20; - } // namespace RenderDelayControllerMetrics::RenderDelayControllerMetrics() = default; void RenderDelayControllerMetrics::Update( absl::optional delay_samples, - size_t buffer_delay_blocks, - absl::optional skew_shift_blocks, + absl::optional buffer_delay_blocks, ClockdriftDetector::Level clockdrift) { ++call_counter_; @@ -54,6 +51,8 @@ void RenderDelayControllerMetrics::Update( size_t delay_blocks; if (delay_samples) { ++reliable_delay_estimate_counter_; + // Add an offset by 1 (metric is halved before reporting) to reserve 0 for + // absent delay. delay_blocks = (*delay_samples) / kBlockSize + 2; } else { delay_blocks = 0; @@ -64,21 +63,21 @@ void RenderDelayControllerMetrics::Update( delay_blocks_ = delay_blocks; } - if (skew_shift_blocks) { - skew_shift_count_ = std::min(kMaxSkewShiftCount, skew_shift_count_); - } } else if (++initial_call_counter_ == 5 * kNumBlocksPerSecond) { initial_update = false; } if (call_counter_ == kMetricsReportingIntervalBlocks) { int value_to_report = static_cast(delay_blocks_); + // Divide by 2 to compress metric range. value_to_report = std::min(124, value_to_report >> 1); RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.EchoPathDelay", value_to_report, 0, 124, 125); - value_to_report = static_cast(buffer_delay_blocks + 2); - value_to_report = std::min(124, value_to_report >> 1); + // Divide by 2 to compress metric range. + // Offset by 1 to reserve 0 for absent delay. + value_to_report = buffer_delay_blocks ? (*buffer_delay_blocks + 2) >> 1 : 0; + value_to_report = std::min(124, value_to_report); RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.BufferDelay", value_to_report, 0, 124, 125); @@ -120,20 +119,8 @@ void RenderDelayControllerMetrics::Update( "WebRTC.Audio.EchoCanceller.Clockdrift", static_cast(clockdrift), static_cast(ClockdriftDetector::Level::kNumCategories)); - metrics_reported_ = true; call_counter_ = 0; ResetMetrics(); - } else { - metrics_reported_ = false; - } - - if (!initial_update && ++skew_report_timer_ == 60 * kNumBlocksPerSecond) { - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.MaxSkewShiftCount", - skew_shift_count_, 0, kMaxSkewShiftCount, - kMaxSkewShiftCount + 1); - - skew_shift_count_ = 0; - skew_report_timer_ = 0; } } diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h b/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h index 8c527a1..b81833b 100644 --- a/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h @@ -15,7 +15,6 @@ #include "absl/types/optional.h" #include "modules/audio_processing/aec3/clockdrift_detector.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -24,15 +23,15 @@ class RenderDelayControllerMetrics { public: RenderDelayControllerMetrics(); + RenderDelayControllerMetrics(const RenderDelayControllerMetrics&) = delete; + RenderDelayControllerMetrics& operator=(const RenderDelayControllerMetrics&) = + delete; + // Updates the metric with new data. void Update(absl::optional delay_samples, - size_t buffer_delay_blocks, - absl::optional skew_shift_blocks, + absl::optional buffer_delay_blocks, ClockdriftDetector::Level clockdrift); - // Returns true if the metrics have just been reported, otherwise false. - bool MetricsReported() { return metrics_reported_; } - private: // Resets the metrics. void ResetMetrics(); @@ -41,13 +40,8 @@ class RenderDelayControllerMetrics { int reliable_delay_estimate_counter_ = 0; int delay_change_counter_ = 0; int call_counter_ = 0; - int skew_report_timer_ = 0; int initial_call_counter_ = 0; - bool metrics_reported_ = false; bool initial_update = true; - int skew_shift_count_ = 0; - - RTC_DISALLOW_COPY_AND_ASSIGN(RenderDelayControllerMetrics); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc b/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc index f570aac..bfbeb0e 100644 --- a/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc +++ b/webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc @@ -66,10 +66,9 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer, *narrow_peak_band = absl::nullopt; } - const std::vector>>& x_latest = - render_buffer.Block(0); + const Block& x_latest = render_buffer.GetBlock(0); float max_peak_level = 0.f; - for (size_t channel = 0; channel < x_latest[0].size(); ++channel) { + for (int channel = 0; channel < x_latest.NumChannels(); ++channel) { rtc::ArrayView X2_latest = render_buffer.Spectrum(0)[channel]; @@ -90,13 +89,14 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer, } // Assess the render signal strength. - auto result0 = std::minmax_element(x_latest[0][channel].begin(), - x_latest[0][channel].end()); + auto result0 = std::minmax_element(x_latest.begin(/*band=*/0, channel), + x_latest.end(/*band=*/0, channel)); float max_abs = std::max(fabs(*result0.first), fabs(*result0.second)); - if (x_latest.size() > 1) { - const auto result1 = std::minmax_element(x_latest[1][channel].begin(), - x_latest[1][channel].end()); + if (x_latest.NumBands() > 1) { + const auto result1 = + std::minmax_element(x_latest.begin(/*band=*/1, channel), + x_latest.end(/*band=*/1, channel)); max_abs = std::max(max_abs, static_cast(std::max( fabs(*result1.first), fabs(*result1.second)))); diff --git a/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h b/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h index c7a3d8b..2e4aaa4 100644 --- a/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h +++ b/webrtc/modules/audio_processing/aec3/render_signal_analyzer.h @@ -20,7 +20,6 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/render_buffer.h" #include "rtc_base/checks.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -30,6 +29,9 @@ class RenderSignalAnalyzer { explicit RenderSignalAnalyzer(const EchoCanceller3Config& config); ~RenderSignalAnalyzer(); + RenderSignalAnalyzer(const RenderSignalAnalyzer&) = delete; + RenderSignalAnalyzer& operator=(const RenderSignalAnalyzer&) = delete; + // Updates the render signal analysis with the most recent render signal. void Update(const RenderBuffer& render_buffer, const absl::optional& delay_partitions); @@ -53,8 +55,6 @@ class RenderSignalAnalyzer { std::array narrow_band_counters_; absl::optional narrow_peak_band_; size_t narrow_peak_counter_; - - RTC_DISALLOW_COPY_AND_ASSIGN(RenderSignalAnalyzer); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc b/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc index 46db233..640a3e3 100644 --- a/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/residual_echo_estimator.cc @@ -23,15 +23,10 @@ namespace webrtc { namespace { -constexpr float kDefaultTransparentModeGain = 0.f; +constexpr float kDefaultTransparentModeGain = 0.01f; float GetTransparentModeGain() { - if (field_trial::IsEnabled( - "WebRTC-Aec3NoSuppressionInTransparentModeKillSwitch")) { - return 0.01f; - } else { - return kDefaultTransparentModeGain; - } + return kDefaultTransparentModeGain; } float GetEarlyReflectionsDefaultModeGain( @@ -50,6 +45,13 @@ float GetLateReflectionsDefaultModeGain( return config.default_gain; } +bool UseErleOnsetCompensationInDominantNearend( + const EchoCanceller3Config::EpStrength& config) { + return config.erle_onset_compensation_in_dominant_nearend || + field_trial::IsEnabled( + "WebRTC-Aec3UseErleOnsetCompensationInDominantNearend"); +} + // Computes the indexes that will be used for computing spectral power over // the blocks surrounding the delay. void GetRenderIndexesToAnalyze( @@ -89,22 +91,6 @@ void LinearEstimate( } } -// Estimates the residual echo power based on an uncertainty estimate of the -// echo return loss enhancement (ERLE) and the linear power estimate. -void LinearEstimate( - rtc::ArrayView> S2_linear, - float erle_uncertainty, - rtc::ArrayView> R2) { - RTC_DCHECK_EQ(S2_linear.size(), R2.size()); - - const size_t num_capture_channels = R2.size(); - for (size_t ch = 0; ch < num_capture_channels; ++ch) { - for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { - R2[ch][k] = S2_linear[ch][k] * erle_uncertainty; - } - } -} - // Estimates the residual echo power based on the estimate of the echo path // gain. void NonLinearEstimate( @@ -177,7 +163,9 @@ ResidualEchoEstimator::ResidualEchoEstimator(const EchoCanceller3Config& config, early_reflections_general_gain_( GetEarlyReflectionsDefaultModeGain(config_.ep_strength)), late_reflections_general_gain_( - GetLateReflectionsDefaultModeGain(config_.ep_strength)) { + GetLateReflectionsDefaultModeGain(config_.ep_strength)), + erle_onset_compensation_in_dominant_nearend_( + UseErleOnsetCompensationInDominantNearend(config_.ep_strength)) { Reset(); } @@ -188,7 +176,9 @@ void ResidualEchoEstimator::Estimate( const RenderBuffer& render_buffer, rtc::ArrayView> S2_linear, rtc::ArrayView> Y2, - rtc::ArrayView> R2) { + bool dominant_nearend, + rtc::ArrayView> R2, + rtc::ArrayView> R2_unbounded) { RTC_DCHECK_EQ(R2.size(), Y2.size()); RTC_DCHECK_EQ(R2.size(), S2_linear.size()); @@ -204,17 +194,19 @@ void ResidualEchoEstimator::Estimate( if (aec_state.SaturatedEcho()) { for (size_t ch = 0; ch < num_capture_channels; ++ch) { std::copy(Y2[ch].begin(), Y2[ch].end(), R2[ch].begin()); + std::copy(Y2[ch].begin(), Y2[ch].end(), R2_unbounded[ch].begin()); } } else { - absl::optional erle_uncertainty = aec_state.ErleUncertainty(); - if (erle_uncertainty) { - LinearEstimate(S2_linear, *erle_uncertainty, R2); - } else { - LinearEstimate(S2_linear, aec_state.Erle(), R2); - } + const bool onset_compensated = + erle_onset_compensation_in_dominant_nearend_ || !dominant_nearend; + LinearEstimate(S2_linear, aec_state.Erle(onset_compensated), R2); + LinearEstimate(S2_linear, aec_state.ErleUnbounded(), R2_unbounded); } - AddReverb(ReverbType::kLinear, aec_state, render_buffer, R2); + UpdateReverb(ReverbType::kLinear, aec_state, render_buffer, + dominant_nearend); + AddReverb(R2); + AddReverb(R2_unbounded); } else { const float echo_path_gain = GetEchoPathGain(aec_state, /*gain_for_early_reflections=*/true); @@ -224,6 +216,7 @@ void ResidualEchoEstimator::Estimate( if (aec_state.SaturatedEcho()) { for (size_t ch = 0; ch < num_capture_channels; ++ch) { std::copy(Y2[ch].begin(), Y2[ch].end(), R2[ch].begin()); + std::copy(Y2[ch].begin(), Y2[ch].end(), R2_unbounded[ch].begin()); } } else { // Estimate the echo generating signal power. @@ -243,11 +236,15 @@ void ResidualEchoEstimator::Estimate( } NonLinearEstimate(echo_path_gain, X2, R2); + NonLinearEstimate(echo_path_gain, X2, R2_unbounded); } if (config_.echo_model.model_reverb_in_nonlinear_mode && !aec_state.TransparentModeActive()) { - AddReverb(ReverbType::kNonLinear, aec_state, render_buffer, R2); + UpdateReverb(ReverbType::kNonLinear, aec_state, render_buffer, + dominant_nearend); + AddReverb(R2); + AddReverb(R2_unbounded); } } @@ -258,6 +255,7 @@ void ResidualEchoEstimator::Estimate( for (size_t ch = 0; ch < num_capture_channels; ++ch) { for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { R2[ch][k] *= residual_scaling[k]; + R2_unbounded[ch][k] *= residual_scaling[k]; } } } @@ -306,14 +304,11 @@ void ResidualEchoEstimator::UpdateRenderNoisePower( } } -// Adds the estimated power of the reverb to the residual echo power. -void ResidualEchoEstimator::AddReverb( - ReverbType reverb_type, - const AecState& aec_state, - const RenderBuffer& render_buffer, - rtc::ArrayView> R2) { - const size_t num_capture_channels = R2.size(); - +// Updates the reverb estimation. +void ResidualEchoEstimator::UpdateReverb(ReverbType reverb_type, + const AecState& aec_state, + const RenderBuffer& render_buffer, + bool dominant_nearend) { // Choose reverb partition based on what type of echo power model is used. const size_t first_reverb_partition = reverb_type == ReverbType::kLinear @@ -338,16 +333,21 @@ void ResidualEchoEstimator::AddReverb( } // Update the reverb estimate. + float reverb_decay = aec_state.ReverbDecay(/*mild=*/dominant_nearend); if (reverb_type == ReverbType::kLinear) { - echo_reverb_.UpdateReverb(render_power, - aec_state.GetReverbFrequencyResponse(), - aec_state.ReverbDecay()); + echo_reverb_.UpdateReverb( + render_power, aec_state.GetReverbFrequencyResponse(), reverb_decay); } else { const float echo_path_gain = GetEchoPathGain(aec_state, /*gain_for_early_reflections=*/false); echo_reverb_.UpdateReverbNoFreqShaping(render_power, echo_path_gain, - aec_state.ReverbDecay()); + reverb_decay); } +} +// Adds the estimated power of the reverb to the residual echo power. +void ResidualEchoEstimator::AddReverb( + rtc::ArrayView> R2) const { + const size_t num_capture_channels = R2.size(); // Add the reverb power. rtc::ArrayView reverb_power = diff --git a/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h b/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h index 8fe7a84..c468764 100644 --- a/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h +++ b/webrtc/modules/audio_processing/aec3/residual_echo_estimator.h @@ -39,7 +39,9 @@ class ResidualEchoEstimator { const RenderBuffer& render_buffer, rtc::ArrayView> S2_linear, rtc::ArrayView> Y2, - rtc::ArrayView> R2); + bool dominant_nearend, + rtc::ArrayView> R2, + rtc::ArrayView> R2_unbounded); private: enum class ReverbType { kLinear, kNonLinear }; @@ -51,12 +53,16 @@ class ResidualEchoEstimator { // render signal. void UpdateRenderNoisePower(const RenderBuffer& render_buffer); + // Updates the reverb estimation. + void UpdateReverb(ReverbType reverb_type, + const AecState& aec_state, + const RenderBuffer& render_buffer, + bool dominant_nearend); + // Adds the estimated unmodelled echo power to the residual echo power // estimate. - void AddReverb(ReverbType reverb_type, - const AecState& aec_state, - const RenderBuffer& render_buffer, - rtc::ArrayView> R2); + void AddReverb( + rtc::ArrayView> R2) const; // Gets the echo path gain to apply. float GetEchoPathGain(const AecState& aec_state, @@ -68,6 +74,7 @@ class ResidualEchoEstimator { const float late_reflections_transparent_mode_gain_; const float early_reflections_general_gain_; const float late_reflections_general_gain_; + const bool erle_onset_compensation_in_dominant_nearend_; std::array X2_noise_floor_; std::array X2_noise_floor_counter_; ReverbModel echo_reverb_; diff --git a/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.cc b/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.cc index f160b83..2daf376 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.cc @@ -93,7 +93,8 @@ ReverbDecayEstimator::ReverbDecayEstimator(const EchoCanceller3Config& config) late_reverb_start_(kEarlyReverbMinSizeBlocks), late_reverb_end_(kEarlyReverbMinSizeBlocks), previous_gains_(config.filter.refined.length_blocks, 0.f), - decay_(std::fabs(config.ep_strength.default_len)) { + decay_(std::fabs(config.ep_strength.default_len)), + mild_decay_(std::fabs(config.ep_strength.nearend_len)) { RTC_DCHECK_GT(config.filter.refined.length_blocks, static_cast(kEarlyReverbMinSizeBlocks)); } @@ -295,7 +296,7 @@ void ReverbDecayEstimator::LateReverbLinearRegressor::Accumulate(float z) { float ReverbDecayEstimator::LateReverbLinearRegressor::Estimate() { RTC_DCHECK(EstimateAvailable()); if (nn_ == 0.f) { - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return 0.f; } return nz_ / nn_; diff --git a/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.h b/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.h index 3bb9b2b..fee5421 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.h +++ b/webrtc/modules/audio_processing/aec3/reverb_decay_estimator.h @@ -34,8 +34,15 @@ class ReverbDecayEstimator { int filter_delay_blocks, bool usable_linear_filter, bool stationary_signal); - // Returns the decay for the exponential model. - float Decay() const { return decay_; } + // Returns the decay for the exponential model. The parameter `mild` indicates + // which exponential decay to return, the default one or a milder one. + float Decay(bool mild) const { + if (use_adaptive_echo_decay_) { + return decay_; + } else { + return mild ? mild_decay_ : decay_; + } + } // Dumps debug data. void Dump(ApmDataDumper* data_dumper) const; @@ -103,6 +110,7 @@ class ReverbDecayEstimator { bool estimation_region_identified_ = false; std::vector previous_gains_; float decay_; + float mild_decay_; float tail_gain_ = 0.f; float smoothing_constant_ = 0.f; }; diff --git a/webrtc/modules/audio_processing/aec3/reverb_frequency_response.cc b/webrtc/modules/audio_processing/aec3/reverb_frequency_response.cc index f4bd91f..6e7282a 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_frequency_response.cc +++ b/webrtc/modules/audio_processing/aec3/reverb_frequency_response.cc @@ -49,9 +49,13 @@ float AverageDecayWithinFilter( } // namespace -ReverbFrequencyResponse::ReverbFrequencyResponse() { - tail_response_.fill(0.f); +ReverbFrequencyResponse::ReverbFrequencyResponse( + bool use_conservative_tail_frequency_response) + : use_conservative_tail_frequency_response_( + use_conservative_tail_frequency_response) { + tail_response_.fill(0.0f); } + ReverbFrequencyResponse::~ReverbFrequencyResponse() = default; void ReverbFrequencyResponse::Update( @@ -88,6 +92,12 @@ void ReverbFrequencyResponse::Update( tail_response_[k] = freq_resp_direct_path[k] * average_decay_; } + if (use_conservative_tail_frequency_response_) { + for (size_t k = 0; k < kFftLengthBy2Plus1; ++k) { + tail_response_[k] = std::max(freq_resp_tail[k], tail_response_[k]); + } + } + for (size_t k = 1; k < kFftLengthBy2; ++k) { const float avg_neighbour = 0.5f * (tail_response_[k - 1] + tail_response_[k + 1]); diff --git a/webrtc/modules/audio_processing/aec3/reverb_frequency_response.h b/webrtc/modules/audio_processing/aec3/reverb_frequency_response.h index 84ea441..69b16b5 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_frequency_response.h +++ b/webrtc/modules/audio_processing/aec3/reverb_frequency_response.h @@ -12,7 +12,6 @@ #define MODULES_AUDIO_PROCESSING_AEC3_REVERB_FREQUENCY_RESPONSE_H_ #include -#include #include #include "absl/types/optional.h" @@ -24,7 +23,8 @@ namespace webrtc { // Class for updating the frequency response for the reverb. class ReverbFrequencyResponse { public: - ReverbFrequencyResponse(); + explicit ReverbFrequencyResponse( + bool use_conservative_tail_frequency_response); ~ReverbFrequencyResponse(); // Updates the frequency response estimate of the reverb. @@ -45,6 +45,7 @@ class ReverbFrequencyResponse { int filter_delay_blocks, float linear_filter_quality); + const bool use_conservative_tail_frequency_response_; float average_decay_ = 0.f; std::array tail_response_; }; diff --git a/webrtc/modules/audio_processing/aec3/reverb_model.h b/webrtc/modules/audio_processing/aec3/reverb_model.h index 5ba5485..47ed2f7 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_model.h +++ b/webrtc/modules/audio_processing/aec3/reverb_model.h @@ -49,7 +49,6 @@ class ReverbModel { float reverb_decay); private: - std::array reverb_; }; diff --git a/webrtc/modules/audio_processing/aec3/reverb_model_estimator.cc b/webrtc/modules/audio_processing/aec3/reverb_model_estimator.cc index 7174311..5cd7a78 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_model_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/reverb_model_estimator.cc @@ -15,7 +15,10 @@ namespace webrtc { ReverbModelEstimator::ReverbModelEstimator(const EchoCanceller3Config& config, size_t num_capture_channels) : reverb_decay_estimators_(num_capture_channels), - reverb_frequency_responses_(num_capture_channels) { + reverb_frequency_responses_( + num_capture_channels, + ReverbFrequencyResponse( + config.ep_strength.use_conservative_tail_frequency_response)) { for (size_t ch = 0; ch < reverb_decay_estimators_.size(); ++ch) { reverb_decay_estimators_[ch] = std::make_unique(config); diff --git a/webrtc/modules/audio_processing/aec3/reverb_model_estimator.h b/webrtc/modules/audio_processing/aec3/reverb_model_estimator.h index 3b9971a..63bade9 100644 --- a/webrtc/modules/audio_processing/aec3/reverb_model_estimator.h +++ b/webrtc/modules/audio_processing/aec3/reverb_model_estimator.h @@ -12,6 +12,7 @@ #define MODULES_AUDIO_PROCESSING_AEC3_REVERB_MODEL_ESTIMATOR_H_ #include +#include #include #include "absl/types/optional.h" @@ -42,9 +43,13 @@ class ReverbModelEstimator { const std::vector& usable_linear_estimates, bool stationary_block); - // Returns the exponential decay of the reverberant echo. + // Returns the exponential decay of the reverberant echo. The parameter `mild` + // indicates which exponential decay to return, the default one or a milder + // one. // TODO(peah): Correct to properly support multiple channels. - float ReverbDecay() const { return reverb_decay_estimators_[0]->Decay(); } + float ReverbDecay(bool mild) const { + return reverb_decay_estimators_[0]->Decay(mild); + } // Return the frequency response of the reverberant echo. // TODO(peah): Correct to properly support multiple channels. diff --git a/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc b/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc index 5a3ba6c..a5e7709 100644 --- a/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc @@ -131,7 +131,9 @@ SignalDependentErleEstimator::SignalDependentErleEstimator( section_boundaries_blocks_(SetSectionsBoundaries(delay_headroom_blocks_, num_blocks_, num_sections_)), + use_onset_detection_(config.erle.onset_detection), erle_(num_capture_channels), + erle_onset_compensated_(num_capture_channels), S2_section_accum_( num_capture_channels, std::vector>(num_sections_)), @@ -154,6 +156,7 @@ SignalDependentErleEstimator::~SignalDependentErleEstimator() = default; void SignalDependentErleEstimator::Reset() { for (size_t ch = 0; ch < erle_.size(); ++ch) { erle_[ch].fill(min_erle_); + erle_onset_compensated_[ch].fill(min_erle_); for (auto& erle_estimator : erle_estimators_[ch]) { erle_estimator.fill(min_erle_); } @@ -180,6 +183,8 @@ void SignalDependentErleEstimator::Update( rtc::ArrayView> Y2, rtc::ArrayView> E2, rtc::ArrayView> average_erle, + rtc::ArrayView> + average_erle_onset_compensated, const std::vector& converged_filters) { RTC_DCHECK_GT(num_sections_, 1); @@ -202,6 +207,11 @@ void SignalDependentErleEstimator::Update( [band_to_subband_[k]]; erle_[ch][k] = rtc::SafeClamp(average_erle[ch][k] * correction_factor, min_erle_, max_erle_[band_to_subband_[k]]); + if (use_onset_detection_) { + erle_onset_compensated_[ch][k] = rtc::SafeClamp( + average_erle_onset_compensated[ch][k] * correction_factor, + min_erle_, max_erle_[band_to_subband_[k]]); + } } } } diff --git a/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.h b/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.h index 498e922..6847c1a 100644 --- a/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.h +++ b/webrtc/modules/audio_processing/aec3/signal_dependent_erle_estimator.h @@ -37,8 +37,10 @@ class SignalDependentErleEstimator { void Reset(); // Returns the Erle per frequency subband. - rtc::ArrayView> Erle() const { - return erle_; + rtc::ArrayView> Erle( + bool onset_compensated) const { + return onset_compensated && use_onset_detection_ ? erle_onset_compensated_ + : erle_; } // Updates the Erle estimate. The Erle that is passed as an input is required @@ -51,6 +53,8 @@ class SignalDependentErleEstimator { rtc::ArrayView> Y2, rtc::ArrayView> E2, rtc::ArrayView> average_erle, + rtc::ArrayView> + average_erle_onset_compensated, const std::vector& converged_filters); void Dump(const std::unique_ptr& data_dumper) const; @@ -83,7 +87,9 @@ class SignalDependentErleEstimator { const std::array band_to_subband_; const std::array max_erle_; const std::vector section_boundaries_blocks_; + const bool use_onset_detection_; std::vector> erle_; + std::vector> erle_onset_compensated_; std::vector>> S2_section_accum_; std::vector>> erle_estimators_; diff --git a/webrtc/modules/audio_processing/aec3/stationarity_estimator.cc b/webrtc/modules/audio_processing/aec3/stationarity_estimator.cc index 01628f3..4d36404 100644 --- a/webrtc/modules/audio_processing/aec3/stationarity_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/stationarity_estimator.cc @@ -17,7 +17,6 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/spectrum_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" namespace webrtc { @@ -29,8 +28,7 @@ constexpr int kNBlocksInitialPhase = kNumBlocksPerSecond * 2.; } // namespace StationarityEstimator::StationarityEstimator() - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))) { + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)) { Reset(); } @@ -153,7 +151,7 @@ void StationarityEstimator::SmoothStationaryPerFreq() { stationarity_flags_ = all_ahead_stationary_smooth; } -int StationarityEstimator::instance_count_ = 0; +std::atomic StationarityEstimator::instance_count_(0); StationarityEstimator::NoiseSpectrum::NoiseSpectrum() { Reset(); diff --git a/webrtc/modules/audio_processing/aec3/stationarity_estimator.h b/webrtc/modules/audio_processing/aec3/stationarity_estimator.h index 6f7ad40..8bcd3b7 100644 --- a/webrtc/modules/audio_processing/aec3/stationarity_estimator.h +++ b/webrtc/modules/audio_processing/aec3/stationarity_estimator.h @@ -14,6 +14,7 @@ #include #include +#include #include #include "api/array_view.h" @@ -110,7 +111,7 @@ class StationarityEstimator { size_t block_counter_; }; - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; NoiseSpectrum noise_; std::array hangovers_; diff --git a/webrtc/modules/audio_processing/aec3/subband_erle_estimator.cc b/webrtc/modules/audio_processing/aec3/subband_erle_estimator.cc index 6c00091..dc7f92f 100644 --- a/webrtc/modules/audio_processing/aec3/subband_erle_estimator.cc +++ b/webrtc/modules/audio_processing/aec3/subband_erle_estimator.cc @@ -48,7 +48,9 @@ SubbandErleEstimator::SubbandErleEstimator(const EchoCanceller3Config& config, use_min_erle_during_onsets_(EnableMinErleDuringOnsets()), accum_spectra_(num_capture_channels), erle_(num_capture_channels), - erle_onsets_(num_capture_channels), + erle_onset_compensated_(num_capture_channels), + erle_unbounded_(num_capture_channels), + erle_during_onsets_(num_capture_channels), coming_onset_(num_capture_channels), hold_counters_(num_capture_channels) { Reset(); @@ -57,11 +59,12 @@ SubbandErleEstimator::SubbandErleEstimator(const EchoCanceller3Config& config, SubbandErleEstimator::~SubbandErleEstimator() = default; void SubbandErleEstimator::Reset() { - for (auto& erle : erle_) { - erle.fill(min_erle_); - } - for (size_t ch = 0; ch < erle_onsets_.size(); ++ch) { - erle_onsets_[ch].fill(min_erle_); + const size_t num_capture_channels = erle_.size(); + for (size_t ch = 0; ch < num_capture_channels; ++ch) { + erle_[ch].fill(min_erle_); + erle_onset_compensated_[ch].fill(min_erle_); + erle_unbounded_[ch].fill(min_erle_); + erle_during_onsets_[ch].fill(min_erle_); coming_onset_[ch].fill(true); hold_counters_[ch].fill(0); } @@ -80,15 +83,25 @@ void SubbandErleEstimator::Update( DecreaseErlePerBandForLowRenderSignals(); } - for (auto& erle : erle_) { + const size_t num_capture_channels = erle_.size(); + for (size_t ch = 0; ch < num_capture_channels; ++ch) { + auto& erle = erle_[ch]; erle[0] = erle[1]; erle[kFftLengthBy2] = erle[kFftLengthBy2 - 1]; + + auto& erle_oc = erle_onset_compensated_[ch]; + erle_oc[0] = erle_oc[1]; + erle_oc[kFftLengthBy2] = erle_oc[kFftLengthBy2 - 1]; + + auto& erle_u = erle_unbounded_[ch]; + erle_u[0] = erle_u[1]; + erle_u[kFftLengthBy2] = erle_u[kFftLengthBy2 - 1]; } } void SubbandErleEstimator::Dump( const std::unique_ptr& data_dumper) const { - data_dumper->DumpRaw("aec3_erle_onset", ErleOnsets()[0]); + data_dumper->DumpRaw("aec3_erle_onset", ErleDuringOnsets()[0]); } void SubbandErleEstimator::UpdateBands( @@ -102,13 +115,16 @@ void SubbandErleEstimator::UpdateBands( continue; } + if (accum_spectra_.num_points[ch] != kPointsToAccumulate) { + continue; + } + std::array new_erle; std::array is_erle_updated; is_erle_updated.fill(false); for (size_t k = 1; k < kFftLengthBy2; ++k) { - if (accum_spectra_.num_points[ch] == kPointsToAccumulate && - accum_spectra_.E2[ch][k] > 0.f) { + if (accum_spectra_.E2[ch][k] > 0.f) { new_erle[k] = accum_spectra_.Y2[ch][k] / accum_spectra_.E2[ch][k]; is_erle_updated[k] = true; } @@ -120,10 +136,11 @@ void SubbandErleEstimator::UpdateBands( if (coming_onset_[ch][k]) { coming_onset_[ch][k] = false; if (!use_min_erle_during_onsets_) { - float alpha = new_erle[k] < erle_onsets_[ch][k] ? 0.3f : 0.15f; - erle_onsets_[ch][k] = rtc::SafeClamp( - erle_onsets_[ch][k] + - alpha * (new_erle[k] - erle_onsets_[ch][k]), + float alpha = + new_erle[k] < erle_during_onsets_[ch][k] ? 0.3f : 0.15f; + erle_during_onsets_[ch][k] = rtc::SafeClamp( + erle_during_onsets_[ch][k] + + alpha * (new_erle[k] - erle_during_onsets_[ch][k]), min_erle_, max_erle_[k]); } } @@ -132,15 +149,31 @@ void SubbandErleEstimator::UpdateBands( } } + auto update_erle_band = [](float& erle, float new_erle, + bool low_render_energy, float min_erle, + float max_erle) { + float alpha = 0.05f; + if (new_erle < erle) { + alpha = low_render_energy ? 0.f : 0.1f; + } + erle = + rtc::SafeClamp(erle + alpha * (new_erle - erle), min_erle, max_erle); + }; + for (size_t k = 1; k < kFftLengthBy2; ++k) { if (is_erle_updated[k]) { - float alpha = 0.05f; - if (new_erle[k] < erle_[ch][k]) { - alpha = accum_spectra_.low_render_energy[ch][k] ? 0.f : 0.1f; + const bool low_render_energy = accum_spectra_.low_render_energy[ch][k]; + update_erle_band(erle_[ch][k], new_erle[k], low_render_energy, + min_erle_, max_erle_[k]); + if (use_onset_detection_) { + update_erle_band(erle_onset_compensated_[ch][k], new_erle[k], + low_render_energy, min_erle_, max_erle_[k]); } - erle_[ch][k] = - rtc::SafeClamp(erle_[ch][k] + alpha * (new_erle[k] - erle_[ch][k]), - min_erle_, max_erle_[k]); + + // Virtually unbounded ERLE. + constexpr float kUnboundedErleMax = 100000.0f; + update_erle_band(erle_unbounded_[ch][k], new_erle[k], low_render_energy, + min_erle_, kUnboundedErleMax); } } } @@ -153,9 +186,11 @@ void SubbandErleEstimator::DecreaseErlePerBandForLowRenderSignals() { --hold_counters_[ch][k]; if (hold_counters_[ch][k] <= (kBlocksForOnsetDetection - kBlocksToHoldErle)) { - if (erle_[ch][k] > erle_onsets_[ch][k]) { - erle_[ch][k] = std::max(erle_onsets_[ch][k], 0.97f * erle_[ch][k]); - RTC_DCHECK_LE(min_erle_, erle_[ch][k]); + if (erle_onset_compensated_[ch][k] > erle_during_onsets_[ch][k]) { + erle_onset_compensated_[ch][k] = + std::max(erle_during_onsets_[ch][k], + 0.97f * erle_onset_compensated_[ch][k]); + RTC_DCHECK_LE(min_erle_, erle_onset_compensated_[ch][k]); } if (hold_counters_[ch][k] <= 0) { coming_onset_[ch][k] = true; @@ -167,7 +202,7 @@ void SubbandErleEstimator::DecreaseErlePerBandForLowRenderSignals() { } void SubbandErleEstimator::ResetAccumulatedSpectra() { - for (size_t ch = 0; ch < erle_onsets_.size(); ++ch) { + for (size_t ch = 0; ch < erle_during_onsets_.size(); ++ch) { accum_spectra_.Y2[ch].fill(0.f); accum_spectra_.E2[ch].fill(0.f); accum_spectra_.num_points[ch] = 0; diff --git a/webrtc/modules/audio_processing/aec3/subband_erle_estimator.h b/webrtc/modules/audio_processing/aec3/subband_erle_estimator.h index 90363e0..8bf9c4d 100644 --- a/webrtc/modules/audio_processing/aec3/subband_erle_estimator.h +++ b/webrtc/modules/audio_processing/aec3/subband_erle_estimator.h @@ -41,14 +41,22 @@ class SubbandErleEstimator { const std::vector& converged_filters); // Returns the ERLE estimate. - rtc::ArrayView> Erle() const { - return erle_; + rtc::ArrayView> Erle( + bool onset_compensated) const { + return onset_compensated && use_onset_detection_ ? erle_onset_compensated_ + : erle_; + } + + // Returns the non-capped ERLE estimate. + rtc::ArrayView> ErleUnbounded() + const { + return erle_unbounded_; } // Returns the ERLE estimate at onsets (only used for testing). - rtc::ArrayView> ErleOnsets() + rtc::ArrayView> ErleDuringOnsets() const { - return erle_onsets_; + return erle_during_onsets_; } void Dump(const std::unique_ptr& data_dumper) const; @@ -82,8 +90,13 @@ class SubbandErleEstimator { const std::array max_erle_; const bool use_min_erle_during_onsets_; AccumulatedSpectra accum_spectra_; + // ERLE without special handling of render onsets. std::vector> erle_; - std::vector> erle_onsets_; + // ERLE lowered during render onsets. + std::vector> erle_onset_compensated_; + std::vector> erle_unbounded_; + // Estimation of ERLE during render onsets. + std::vector> erle_during_onsets_; std::vector> coming_onset_; std::vector> hold_counters_; }; diff --git a/webrtc/modules/audio_processing/aec3/subtractor.cc b/webrtc/modules/audio_processing/aec3/subtractor.cc index d152299..aa36bb2 100644 --- a/webrtc/modules/audio_processing/aec3/subtractor.cc +++ b/webrtc/modules/audio_processing/aec3/subtractor.cc @@ -19,11 +19,17 @@ #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" #include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { +bool UseCoarseFilterResetHangover() { + return !field_trial::IsEnabled( + "WebRTC-Aec3CoarseFilterResetHangoverKillSwitch"); +} + void PredictionError(const Aec3Fft& fft, const FftData& S, rtc::ArrayView y, @@ -66,12 +72,14 @@ Subtractor::Subtractor(const EchoCanceller3Config& config, optimization_(optimization), config_(config), num_capture_channels_(num_capture_channels), + use_coarse_filter_reset_hangover_(UseCoarseFilterResetHangover()), refined_filters_(num_capture_channels_), coarse_filter_(num_capture_channels_), refined_gains_(num_capture_channels_), coarse_gains_(num_capture_channels_), filter_misadjustment_estimators_(num_capture_channels_), poor_coarse_filter_counters_(num_capture_channels_, 0), + coarse_filter_reset_hangover_(num_capture_channels_, 0), refined_frequency_responses_( num_capture_channels_, std::vector>( @@ -83,7 +91,20 @@ Subtractor::Subtractor(const EchoCanceller3Config& config, std::vector(GetTimeDomainLength(std::max( config_.filter.refined_initial.length_blocks, config_.filter.refined.length_blocks)), - 0.f)) { + 0.f)), + coarse_impulse_responses_(0) { + // Set up the storing of coarse impulse responses if data dumping is + // available. + if (ApmDataDumper::IsAvailable()) { + coarse_impulse_responses_.resize(num_capture_channels_); + const size_t filter_size = GetTimeDomainLength( + std::max(config_.filter.coarse_initial.length_blocks, + config_.filter.coarse.length_blocks)); + for (std::vector& impulse_response : coarse_impulse_responses_) { + impulse_response.resize(filter_size, 0.f); + } + } + for (size_t ch = 0; ch < num_capture_channels_; ++ch) { refined_filters_[ch] = std::make_unique( config_.filter.refined.length_blocks, @@ -155,11 +176,11 @@ void Subtractor::ExitInitialState() { } void Subtractor::Process(const RenderBuffer& render_buffer, - const std::vector>& capture, + const Block& capture, const RenderSignalAnalyzer& render_signal_analyzer, const AecState& aec_state, rtc::ArrayView outputs) { - RTC_DCHECK_EQ(num_capture_channels_, capture.size()); + RTC_DCHECK_EQ(num_capture_channels_, capture.NumChannels()); // Compute the render powers. const bool same_filter_sizes = refined_filters_[0]->SizePartitions() == @@ -183,9 +204,8 @@ void Subtractor::Process(const RenderBuffer& render_buffer, // Process all capture channels for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - RTC_DCHECK_EQ(kBlockSize, capture[ch].size()); SubtractorOutput& output = outputs[ch]; - rtc::ArrayView y = capture[ch]; + rtc::ArrayView y = capture.View(/*band=*/0, ch); FftData& E_refined = output.E_refined; FftData E_coarse; std::array& e_refined = output.e_refined; @@ -228,11 +248,19 @@ void Subtractor::Process(const RenderBuffer& render_buffer, // Update the refined filter. if (!refined_filters_adjusted) { + // Do not allow the performance of the coarse filter to affect the + // adaptation speed of the refined filter just after the coarse filter has + // been reset. + const bool disallow_leakage_diverged = + coarse_filter_reset_hangover_[ch] > 0 && + use_coarse_filter_reset_hangover_; + std::array erl; ComputeErl(optimization_, refined_frequency_responses_[ch], erl); refined_gains_[ch]->Compute(X2_refined, render_signal_analyzer, output, erl, refined_filters_[ch]->SizePartitions(), - aec_state.SaturatedCapture(), &G); + aec_state.SaturatedCapture(), + disallow_leakage_diverged, &G); } else { G.re.fill(0.f); G.im.fill(0.f); @@ -256,6 +284,8 @@ void Subtractor::Process(const RenderBuffer& render_buffer, coarse_gains_[ch]->Compute(X2_coarse, render_signal_analyzer, E_coarse, coarse_filter_[ch]->SizePartitions(), aec_state.SaturatedCapture(), &G); + coarse_filter_reset_hangover_[ch] = + std::max(coarse_filter_reset_hangover_[ch] - 1, 0); } else { poor_coarse_filter_counters_[ch] = 0; coarse_filter_[ch]->SetFilter(refined_filters_[ch]->SizePartitions(), @@ -263,9 +293,18 @@ void Subtractor::Process(const RenderBuffer& render_buffer, coarse_gains_[ch]->Compute(X2_coarse, render_signal_analyzer, E_refined, coarse_filter_[ch]->SizePartitions(), aec_state.SaturatedCapture(), &G); + coarse_filter_reset_hangover_[ch] = + config_.filter.coarse_reset_hangover_blocks; + } + + if (ApmDataDumper::IsAvailable()) { + RTC_DCHECK_LT(ch, coarse_impulse_responses_.size()); + coarse_filter_[ch]->Adapt(render_buffer, G, + &coarse_impulse_responses_[ch]); + } else { + coarse_filter_[ch]->Adapt(render_buffer, G); } - coarse_filter_[ch]->Adapt(render_buffer, G); if (ch == 0) { data_dumper_->DumpRaw("aec3_subtractor_G_coarse", G.re); data_dumper_->DumpRaw("aec3_subtractor_G_coarse", G.im); diff --git a/webrtc/modules/audio_processing/aec3/subtractor.h b/webrtc/modules/audio_processing/aec3/subtractor.h index 42ca372..86159a3 100644 --- a/webrtc/modules/audio_processing/aec3/subtractor.h +++ b/webrtc/modules/audio_processing/aec3/subtractor.h @@ -23,6 +23,7 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/coarse_filter_update_gain.h" #include "modules/audio_processing/aec3/echo_path_variability.h" #include "modules/audio_processing/aec3/refined_filter_update_gain.h" @@ -48,7 +49,7 @@ class Subtractor { // Performs the echo subtraction. void Process(const RenderBuffer& render_buffer, - const std::vector>& capture, + const Block& capture, const RenderSignalAnalyzer& render_signal_analyzer, const AecState& aec_state, rtc::ArrayView outputs); @@ -78,6 +79,15 @@ class Subtractor { refined_impulse_responses_[0].data(), GetTimeDomainLength( refined_filters_[0]->max_filter_size_partitions()))); + if (ApmDataDumper::IsAvailable()) { + RTC_DCHECK_GT(coarse_impulse_responses_.size(), 0); + data_dumper_->DumpRaw( + "aec3_subtractor_h_coarse", + rtc::ArrayView( + coarse_impulse_responses_[0].data(), + GetTimeDomainLength( + coarse_filter_[0]->max_filter_size_partitions()))); + } refined_filters_[0]->DumpFilter("aec3_subtractor_H_refined"); coarse_filter_[0]->DumpFilter("aec3_subtractor_H_coarse"); @@ -120,6 +130,7 @@ class Subtractor { const Aec3Optimization optimization_; const EchoCanceller3Config config_; const size_t num_capture_channels_; + const bool use_coarse_filter_reset_hangover_; std::vector> refined_filters_; std::vector> coarse_filter_; @@ -127,9 +138,11 @@ class Subtractor { std::vector> coarse_gains_; std::vector filter_misadjustment_estimators_; std::vector poor_coarse_filter_counters_; + std::vector coarse_filter_reset_hangover_; std::vector>> refined_frequency_responses_; std::vector> refined_impulse_responses_; + std::vector> coarse_impulse_responses_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.cc b/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.cc index 8b22185..baf0600 100644 --- a/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.cc +++ b/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.cc @@ -22,12 +22,14 @@ SubtractorOutputAnalyzer::SubtractorOutputAnalyzer(size_t num_capture_channels) void SubtractorOutputAnalyzer::Update( rtc::ArrayView subtractor_output, bool* any_filter_converged, + bool* any_coarse_filter_converged, bool* all_filters_diverged) { RTC_DCHECK(any_filter_converged); RTC_DCHECK(all_filters_diverged); RTC_DCHECK_EQ(subtractor_output.size(), filters_converged_.size()); *any_filter_converged = false; + *any_coarse_filter_converged = false; *all_filters_diverged = true; for (size_t ch = 0; ch < subtractor_output.size(); ++ch) { @@ -36,16 +38,21 @@ void SubtractorOutputAnalyzer::Update( const float e2_coarse = subtractor_output[ch].e2_coarse; constexpr float kConvergenceThreshold = 50 * 50 * kBlockSize; + constexpr float kConvergenceThresholdLowLevel = 20 * 20 * kBlockSize; bool refined_filter_converged = e2_refined < 0.5f * y2 && y2 > kConvergenceThreshold; - bool coarse_filter_converged = + bool coarse_filter_converged_strict = e2_coarse < 0.05f * y2 && y2 > kConvergenceThreshold; + bool coarse_filter_converged_relaxed = + e2_coarse < 0.2f * y2 && y2 > kConvergenceThresholdLowLevel; float min_e2 = std::min(e2_refined, e2_coarse); bool filter_diverged = min_e2 > 1.5f * y2 && y2 > 30.f * 30.f * kBlockSize; filters_converged_[ch] = - refined_filter_converged || coarse_filter_converged; + refined_filter_converged || coarse_filter_converged_strict; *any_filter_converged = *any_filter_converged || filters_converged_[ch]; + *any_coarse_filter_converged = + *any_coarse_filter_converged || coarse_filter_converged_relaxed; *all_filters_diverged = *all_filters_diverged && filter_diverged; } } diff --git a/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.h b/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.h index 5328ae7..32707db 100644 --- a/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.h +++ b/webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.h @@ -26,6 +26,7 @@ class SubtractorOutputAnalyzer { // Analyses the subtractor output. void Update(rtc::ArrayView subtractor_output, bool* any_filter_converged, + bool* any_coarse_filter_converged, bool* all_filters_diverged); const std::vector& ConvergedFilters() const { diff --git a/webrtc/modules/audio_processing/aec3/suppression_filter.cc b/webrtc/modules/audio_processing/aec3/suppression_filter.cc index 8a813d9..83ded42 100644 --- a/webrtc/modules/audio_processing/aec3/suppression_filter.cc +++ b/webrtc/modules/audio_processing/aec3/suppression_filter.cc @@ -86,9 +86,9 @@ void SuppressionFilter::ApplyGain( const std::array& suppression_gain, float high_bands_gain, rtc::ArrayView E_lowest_band, - std::vector>>* e) { + Block* e) { RTC_DCHECK(e); - RTC_DCHECK_EQ(e->size(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(e->NumBands(), NumBandsForRate(sample_rate_hz_)); // Comfort noise gain is sqrt(1-g^2), where g is the suppression gain. std::array noise_gain; @@ -108,12 +108,12 @@ void SuppressionFilter::ApplyGain( for (size_t i = 0; i < kFftLengthBy2Plus1; ++i) { // Apply suppression gains. - E.re[i] *= suppression_gain[i]; - E.im[i] *= suppression_gain[i]; + float E_real = E.re[i] * suppression_gain[i]; + float E_imag = E.im[i] * suppression_gain[i]; // Scale and add the comfort noise. - E.re[i] += noise_gain[i] * comfort_noise[ch].re[i]; - E.im[i] += noise_gain[i] * comfort_noise[ch].im[i]; + E.re[i] = E_real + noise_gain[i] * comfort_noise[ch].re[i]; + E.im[i] = E_imag + noise_gain[i] * comfort_noise[ch].im[i]; } // Synthesis filterbank. @@ -121,36 +121,37 @@ void SuppressionFilter::ApplyGain( constexpr float kIfftNormalization = 2.f / kFftLength; fft_.Ifft(E, &e_extended); - auto& e0 = (*e)[0][ch]; - auto& e0_old = e_output_old_[0][ch]; + auto e0 = e->View(/*band=*/0, ch); + float* e0_old = e_output_old_[0][ch].data(); // Window and add the first half of e_extended with the second half of // e_extended from the previous block. for (size_t i = 0; i < kFftLengthBy2; ++i) { - e0[i] = e0_old[i] * kSqrtHanning[kFftLengthBy2 + i]; - e0[i] += e_extended[i] * kSqrtHanning[i]; - e0[i] *= kIfftNormalization; + float e0_i = e0_old[i] * kSqrtHanning[kFftLengthBy2 + i]; + e0_i += e_extended[i] * kSqrtHanning[i]; + e0[i] = e0_i * kIfftNormalization; } // The second half of e_extended is stored for the succeeding frame. std::copy(e_extended.begin() + kFftLengthBy2, - e_extended.begin() + kFftLength, std::begin(e0_old)); + e_extended.begin() + kFftLength, + std::begin(e_output_old_[0][ch])); // Apply suppression gain to upper bands. - for (size_t b = 1; b < e->size(); ++b) { - auto& e_band = (*e)[b][ch]; + for (int b = 1; b < e->NumBands(); ++b) { + auto e_band = e->View(b, ch); for (size_t i = 0; i < kFftLengthBy2; ++i) { e_band[i] *= high_bands_gain; } } // Add comfort noise to band 1. - if (e->size() > 1) { + if (e->NumBands() > 1) { E.Assign(comfort_noise_high_band[ch]); std::array time_domain_high_band_noise; fft_.Ifft(E, &time_domain_high_band_noise); - auto& e1 = (*e)[1][ch]; + auto e1 = e->View(/*band=*/1, ch); const float gain = high_bands_noise_scaling * kIfftNormalization; for (size_t i = 0; i < kFftLengthBy2; ++i) { e1[i] += time_domain_high_band_noise[i] * gain; @@ -158,17 +159,17 @@ void SuppressionFilter::ApplyGain( } // Delay upper bands to match the delay of the filter bank. - for (size_t b = 1; b < e->size(); ++b) { - auto& e_band = (*e)[b][ch]; - auto& e_band_old = e_output_old_[b][ch]; + for (int b = 1; b < e->NumBands(); ++b) { + auto e_band = e->View(b, ch); + float* e_band_old = e_output_old_[b][ch].data(); for (size_t i = 0; i < kFftLengthBy2; ++i) { std::swap(e_band[i], e_band_old[i]); } } // Clamp output of all bands. - for (size_t b = 0; b < e->size(); ++b) { - auto& e_band = (*e)[b][ch]; + for (int b = 0; b < e->NumBands(); ++b) { + auto e_band = e->View(b, ch); for (size_t i = 0; i < kFftLengthBy2; ++i) { e_band[i] = rtc::SafeClamp(e_band[i], -32768.f, 32767.f); } diff --git a/webrtc/modules/audio_processing/aec3/suppression_filter.h b/webrtc/modules/audio_processing/aec3/suppression_filter.h index dcf2292..c18b233 100644 --- a/webrtc/modules/audio_processing/aec3/suppression_filter.h +++ b/webrtc/modules/audio_processing/aec3/suppression_filter.h @@ -16,8 +16,8 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/block.h" #include "modules/audio_processing/aec3/fft_data.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -27,12 +27,16 @@ class SuppressionFilter { int sample_rate_hz, size_t num_capture_channels_); ~SuppressionFilter(); + + SuppressionFilter(const SuppressionFilter&) = delete; + SuppressionFilter& operator=(const SuppressionFilter&) = delete; + void ApplyGain(rtc::ArrayView comfort_noise, rtc::ArrayView comfort_noise_high_bands, const std::array& suppression_gain, float high_bands_gain, rtc::ArrayView E_lowest_band, - std::vector>>* e); + Block* e); private: const Aec3Optimization optimization_; @@ -40,7 +44,6 @@ class SuppressionFilter { const size_t num_capture_channels_; const Aec3Fft fft_; std::vector>> e_output_old_; - RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionFilter); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/suppression_gain.cc b/webrtc/modules/audio_processing/aec3/suppression_gain.cc index c1f12b7..037daba 100644 --- a/webrtc/modules/audio_processing/aec3/suppression_gain.cc +++ b/webrtc/modules/audio_processing/aec3/suppression_gain.cc @@ -21,45 +21,46 @@ #include "modules/audio_processing/aec3/subband_nearend_detector.h" #include "modules/audio_processing/aec3/vector_math.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { -void PostprocessGains(std::array* gain) { - // TODO(gustaf): Investigate if this can be relaxed to achieve higher - // transparency above 2 kHz. - +void LimitLowFrequencyGains(std::array* gain) { // Limit the low frequency gains to avoid the impact of the high-pass filter // on the lower-frequency gain influencing the overall achieved gain. (*gain)[0] = (*gain)[1] = std::min((*gain)[1], (*gain)[2]); +} - // Limit the high frequency gains to avoid the impact of the anti-aliasing - // filter on the upper-frequency gains influencing the overall achieved - // gain. TODO(peah): Update this when new anti-aliasing filters are - // implemented. - constexpr size_t kAntiAliasingImpactLimit = (64 * 2000) / 8000; - const float min_upper_gain = (*gain)[kAntiAliasingImpactLimit]; +void LimitHighFrequencyGains(bool conservative_hf_suppression, + std::array* gain) { + // Limit the high frequency gains to avoid echo leakage due to an imperfect + // filter. + constexpr size_t kFirstBandToLimit = (64 * 2000) / 8000; + const float min_upper_gain = (*gain)[kFirstBandToLimit]; std::for_each( - gain->begin() + kAntiAliasingImpactLimit, gain->end() - 1, + gain->begin() + kFirstBandToLimit + 1, gain->end(), [min_upper_gain](float& a) { a = std::min(a, min_upper_gain); }); (*gain)[kFftLengthBy2] = (*gain)[kFftLengthBy2Minus1]; - // Limits the gain in the frequencies for which the adaptive filter has not - // converged. - // TODO(peah): Make adaptive to take the actual filter error into account. - constexpr size_t kUpperAccurateBandPlus1 = 29; + if (conservative_hf_suppression) { + // Limits the gain in the frequencies for which the adaptive filter has not + // converged. + // TODO(peah): Make adaptive to take the actual filter error into account. + constexpr size_t kUpperAccurateBandPlus1 = 29; - constexpr float oneByBandsInSum = - 1 / static_cast(kUpperAccurateBandPlus1 - 20); - const float hf_gain_bound = - std::accumulate(gain->begin() + 20, - gain->begin() + kUpperAccurateBandPlus1, 0.f) * - oneByBandsInSum; + constexpr float oneByBandsInSum = + 1 / static_cast(kUpperAccurateBandPlus1 - 20); + const float hf_gain_bound = + std::accumulate(gain->begin() + 20, + gain->begin() + kUpperAccurateBandPlus1, 0.f) * + oneByBandsInSum; - std::for_each(gain->begin() + kUpperAccurateBandPlus1, gain->end(), - [hf_gain_bound](float& a) { a = std::min(a, hf_gain_bound); }); + std::for_each( + gain->begin() + kUpperAccurateBandPlus1, gain->end(), + [hf_gain_bound](float& a) { a = std::min(a, hf_gain_bound); }); + } } // Scales the echo according to assessed audibility at the other end. @@ -100,7 +101,7 @@ void WeightEchoForAudibility(const EchoCanceller3Config& config, } // namespace -int SuppressionGain::instance_count_ = 0; +std::atomic SuppressionGain::instance_count_(0); float SuppressionGain::UpperBandsGain( rtc::ArrayView> echo_spectrum, @@ -108,13 +109,13 @@ float SuppressionGain::UpperBandsGain( comfort_noise_spectrum, const absl::optional& narrow_peak_band, bool saturated_echo, - const std::vector>>& render, + const Block& render, const std::array& low_band_gain) const { - RTC_DCHECK_LT(0, render.size()); - if (render.size() == 1) { + RTC_DCHECK_LT(0, render.NumBands()); + if (render.NumBands() == 1) { return 1.f; } - const size_t num_render_channels = render[0].size(); + const int num_render_channels = render.NumChannels(); if (narrow_peak_band && (*narrow_peak_band > static_cast(kFftLengthBy2Plus1 - 10))) { @@ -133,16 +134,17 @@ float SuppressionGain::UpperBandsGain( // Compute the upper and lower band energies. const auto sum_of_squares = [](float a, float b) { return a + b * b; }; float low_band_energy = 0.f; - for (size_t ch = 0; ch < num_render_channels; ++ch) { - const float channel_energy = std::accumulate( - render[0][0].begin(), render[0][0].end(), 0.f, sum_of_squares); + for (int ch = 0; ch < num_render_channels; ++ch) { + const float channel_energy = + std::accumulate(render.begin(/*band=*/0, ch), + render.end(/*band=*/0, ch), 0.0f, sum_of_squares); low_band_energy = std::max(low_band_energy, channel_energy); } float high_band_energy = 0.f; - for (size_t k = 1; k < render.size(); ++k) { - for (size_t ch = 0; ch < num_render_channels; ++ch) { + for (int k = 1; k < render.NumBands(); ++k) { + for (int ch = 0; ch < num_render_channels; ++ch) { const float energy = std::accumulate( - render[k][ch].begin(), render[k][ch].end(), 0.f, sum_of_squares); + render.begin(k, ch), render.end(k, ch), 0.f, sum_of_squares); high_band_energy = std::max(high_band_energy, energy); } } @@ -229,16 +231,20 @@ void SuppressionGain::GetMinGain( min_gain[k] = std::min(min_gain[k], 1.f); } - const bool is_nearend_state = dominant_nearend_detector_->IsNearendState(); - for (size_t k = 0; k < 6; ++k) { - const auto& dec = is_nearend_state ? nearend_params_.max_dec_factor_lf - : normal_params_.max_dec_factor_lf; + if (!initial_state_ || + config_.suppressor.lf_smoothing_during_initial_phase) { + const float& dec = dominant_nearend_detector_->IsNearendState() + ? nearend_params_.max_dec_factor_lf + : normal_params_.max_dec_factor_lf; - // Make sure the gains of the low frequencies do not decrease too - // quickly after strong nearend. - if (last_nearend[k] > last_echo[k]) { - min_gain[k] = std::max(min_gain[k], last_gain_[k] * dec); - min_gain[k] = std::min(min_gain[k], 1.f); + for (int k = 0; k <= config_.suppressor.last_lf_smoothing_band; ++k) { + // Make sure the gains of the low frequencies do not decrease too + // quickly after strong nearend. + if (last_nearend[k] > last_echo[k] || + k <= config_.suppressor.last_permanent_lf_smoothing_band) { + min_gain[k] = std::max(min_gain[k], last_gain_[k] * dec); + min_gain[k] = std::min(min_gain[k], 1.f); + } } } } else { @@ -265,6 +271,7 @@ void SuppressionGain::LowerBandGain( suppressor_input, rtc::ArrayView> residual_echo, rtc::ArrayView> comfort_noise, + bool clock_drift, std::array* gain) { gain->fill(1.f); const bool saturated_echo = aec_state.SaturatedEcho(); @@ -298,8 +305,14 @@ void SuppressionGain::LowerBandGain( last_echo_[ch].begin()); } - // Limit high-frequency gains. - PostprocessGains(gain); + LimitLowFrequencyGains(gain); + // Use conservative high-frequency gains during clock-drift or when not in + // dominant nearend. + if (!dominant_nearend_detector_->IsNearendState() || clock_drift || + config_.suppressor.conservative_hf_suppression) { + LimitHighFrequencyGains(config_.suppressor.conservative_hf_suppression, + gain); + } // Store computed gains. std::copy(gain->begin(), gain->end(), last_gain_.begin()); @@ -312,8 +325,7 @@ SuppressionGain::SuppressionGain(const EchoCanceller3Config& config, Aec3Optimization optimization, int sample_rate_hz, size_t num_capture_channels) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), optimization_(optimization), config_(config), num_capture_channels_(num_capture_channels), @@ -325,8 +337,14 @@ SuppressionGain::SuppressionGain(const EchoCanceller3Config& config, num_capture_channels_, aec3::MovingAverage(kFftLengthBy2Plus1, config.suppressor.nearend_average_blocks)), - nearend_params_(config_.suppressor.nearend_tuning), - normal_params_(config_.suppressor.normal_tuning) { + nearend_params_(config_.suppressor.last_lf_band, + config_.suppressor.first_hf_band, + config_.suppressor.nearend_tuning), + normal_params_(config_.suppressor.last_lf_band, + config_.suppressor.first_hf_band, + config_.suppressor.normal_tuning), + use_unbounded_echo_spectrum_(config.suppressor.dominant_nearend_detection + .use_unbounded_echo_spectrum) { RTC_DCHECK_LT(0, state_change_duration_blocks_); last_gain_.fill(1.f); if (config_.suppressor.use_subband_nearend_detection) { @@ -347,24 +365,33 @@ void SuppressionGain::GetGain( rtc::ArrayView> echo_spectrum, rtc::ArrayView> residual_echo_spectrum, + rtc::ArrayView> + residual_echo_spectrum_unbounded, rtc::ArrayView> comfort_noise_spectrum, const RenderSignalAnalyzer& render_signal_analyzer, const AecState& aec_state, - const std::vector>>& render, + const Block& render, + bool clock_drift, float* high_bands_gain, std::array* low_band_gain) { RTC_DCHECK(high_bands_gain); RTC_DCHECK(low_band_gain); + // Choose residual echo spectrum for dominant nearend detection. + const auto echo = use_unbounded_echo_spectrum_ + ? residual_echo_spectrum_unbounded + : residual_echo_spectrum; + // Update the nearend state selection. - dominant_nearend_detector_->Update(nearend_spectrum, residual_echo_spectrum, + dominant_nearend_detector_->Update(nearend_spectrum, echo, comfort_noise_spectrum, initial_state_); // Compute gain for the lower band. bool low_noise_render = low_render_detector_.Detect(render); LowerBandGain(low_noise_render, aec_state, nearend_spectrum, - residual_echo_spectrum, comfort_noise_spectrum, low_band_gain); + residual_echo_spectrum, comfort_noise_spectrum, clock_drift, + low_band_gain); // Compute the gain for the upper bands. const absl::optional narrow_peak_band = @@ -373,6 +400,9 @@ void SuppressionGain::GetGain( *high_bands_gain = UpperBandsGain(echo_spectrum, comfort_noise_spectrum, narrow_peak_band, aec_state.SaturatedEcho(), render, *low_band_gain); + + data_dumper_->DumpRaw("aec3_dominant_nearend", + dominant_nearend_detector_->IsNearendState()); } void SuppressionGain::SetInitialState(bool state) { @@ -386,20 +416,17 @@ void SuppressionGain::SetInitialState(bool state) { // Detects when the render signal can be considered to have low power and // consist of stationary noise. -bool SuppressionGain::LowNoiseRenderDetector::Detect( - const std::vector>>& render) { +bool SuppressionGain::LowNoiseRenderDetector::Detect(const Block& render) { float x2_sum = 0.f; float x2_max = 0.f; - for (const auto& x_ch : render[0]) { - for (const auto& x_k : x_ch) { + for (int ch = 0; ch < render.NumChannels(); ++ch) { + for (float x_k : render.View(/*band=*/0, ch)) { const float x2 = x_k * x_k; x2_sum += x2; x2_max = std::max(x2_max, x2); } } - const size_t num_render_channels = render[0].size(); - x2_sum = x2_sum / num_render_channels; - ; + x2_sum = x2_sum / render.NumChannels(); constexpr float kThreshold = 50.f * 50.f * 64.f; const bool low_noise_render = @@ -409,23 +436,23 @@ bool SuppressionGain::LowNoiseRenderDetector::Detect( } SuppressionGain::GainParameters::GainParameters( + int last_lf_band, + int first_hf_band, const EchoCanceller3Config::Suppressor::Tuning& tuning) : max_inc_factor(tuning.max_inc_factor), max_dec_factor_lf(tuning.max_dec_factor_lf) { // Compute per-band masking thresholds. - constexpr size_t kLastLfBand = 5; - constexpr size_t kFirstHfBand = 8; - RTC_DCHECK_LT(kLastLfBand, kFirstHfBand); + RTC_DCHECK_LT(last_lf_band, first_hf_band); auto& lf = tuning.mask_lf; auto& hf = tuning.mask_hf; RTC_DCHECK_LT(lf.enr_transparent, lf.enr_suppress); RTC_DCHECK_LT(hf.enr_transparent, hf.enr_suppress); - for (size_t k = 0; k < kFftLengthBy2Plus1; k++) { + for (int k = 0; k < static_cast(kFftLengthBy2Plus1); k++) { float a; - if (k <= kLastLfBand) { + if (k <= last_lf_band) { a = 0.f; - } else if (k < kFirstHfBand) { - a = (k - kLastLfBand) / static_cast(kFirstHfBand - kLastLfBand); + } else if (k < first_hf_band) { + a = (k - last_lf_band) / static_cast(first_hf_band - last_lf_band); } else { a = 1.f; } diff --git a/webrtc/modules/audio_processing/aec3/suppression_gain.h b/webrtc/modules/audio_processing/aec3/suppression_gain.h index f46db0b..c19ddd7 100644 --- a/webrtc/modules/audio_processing/aec3/suppression_gain.h +++ b/webrtc/modules/audio_processing/aec3/suppression_gain.h @@ -12,6 +12,7 @@ #define MODULES_AUDIO_PROCESSING_AEC3_SUPPRESSION_GAIN_H_ #include +#include #include #include @@ -25,7 +26,6 @@ #include "modules/audio_processing/aec3/nearend_detector.h" #include "modules/audio_processing/aec3/render_signal_analyzer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -36,20 +36,31 @@ class SuppressionGain { int sample_rate_hz, size_t num_capture_channels); ~SuppressionGain(); + + SuppressionGain(const SuppressionGain&) = delete; + SuppressionGain& operator=(const SuppressionGain&) = delete; + void GetGain( rtc::ArrayView> nearend_spectrum, rtc::ArrayView> echo_spectrum, rtc::ArrayView> residual_echo_spectrum, + rtc::ArrayView> + residual_echo_spectrum_unbounded, rtc::ArrayView> comfort_noise_spectrum, const RenderSignalAnalyzer& render_signal_analyzer, const AecState& aec_state, - const std::vector>>& render, + const Block& render, + bool clock_drift, float* high_bands_gain, std::array* low_band_gain); + bool IsDominantNearend() { + return dominant_nearend_detector_->IsNearendState(); + } + // Toggles the usage of the initial state. void SetInitialState(bool state); @@ -61,7 +72,7 @@ class SuppressionGain { comfort_noise_spectrum, const absl::optional& narrow_peak_band, bool saturated_echo, - const std::vector>>& render, + const Block& render, const std::array& low_band_gain) const; void GainToNoAudibleEcho(const std::array& nearend, @@ -76,6 +87,7 @@ class SuppressionGain { suppressor_input, rtc::ArrayView> residual_echo, rtc::ArrayView> comfort_noise, + bool clock_drift, std::array* gain); void GetMinGain(rtc::ArrayView weighted_residual_echo, @@ -89,7 +101,7 @@ class SuppressionGain { class LowNoiseRenderDetector { public: - bool Detect(const std::vector>>& render); + bool Detect(const Block& render); private: float average_power_ = 32768.f * 32768.f; @@ -97,6 +109,8 @@ class SuppressionGain { struct GainParameters { explicit GainParameters( + int last_lf_band, + int first_hf_band, const EchoCanceller3Config::Suppressor::Tuning& tuning); const float max_inc_factor; const float max_dec_factor_lf; @@ -105,7 +119,7 @@ class SuppressionGain { std::array emr_transparent_; }; - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const EchoCanceller3Config config_; @@ -120,9 +134,10 @@ class SuppressionGain { std::vector nearend_smoothers_; const GainParameters nearend_params_; const GainParameters normal_params_; + // Determines if the dominant nearend detector uses the unbounded residual + // echo spectrum. + const bool use_unbounded_echo_spectrum_; std::unique_ptr dominant_nearend_detector_; - - RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionGain); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/transparent_mode.cc b/webrtc/modules/audio_processing/aec3/transparent_mode.cc index 1820e16..489f53f 100644 --- a/webrtc/modules/audio_processing/aec3/transparent_mode.cc +++ b/webrtc/modules/audio_processing/aec3/transparent_mode.cc @@ -11,6 +11,7 @@ #include "modules/audio_processing/aec3/transparent_mode.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" namespace webrtc { @@ -23,8 +24,8 @@ bool DeactivateTransparentMode() { return field_trial::IsEnabled("WebRTC-Aec3TransparentModeKillSwitch"); } -bool DeactivateTransparentModeHmm() { - return field_trial::IsEnabled("WebRTC-Aec3TransparentModeHmmKillSwitch"); +bool ActivateTransparentModeHmm() { + return field_trial::IsEnabled("WebRTC-Aec3TransparentModeHmm"); } } // namespace @@ -46,6 +47,7 @@ class TransparentModeImpl : public TransparentMode { void Update(int filter_delay_blocks, bool any_filter_consistent, bool any_filter_converged, + bool any_coarse_filter_converged, bool all_filters_diverged, bool active_render, bool saturated_capture) override { @@ -56,9 +58,9 @@ class TransparentModeImpl : public TransparentMode { // there is no echo present in the microphone signal. // The constants have been obtained by observing active_render and - // any_filter_converged under varying call scenarios. They have further been - // hand tuned to prefer normal state during uncertain regions (to avoid echo - // leaks). + // any_coarse_filter_converged under varying call scenarios. They + // have further been hand tuned to prefer normal state during uncertain + // regions (to avoid echo leaks). // The model is only updated during active render. if (!active_render) @@ -69,8 +71,8 @@ class TransparentModeImpl : public TransparentMode { // Probability of observing converged filters in states "normal" and // "transparent" during active render. - constexpr float kConvergedNormal = 0.03f; - constexpr float kConvergedTransparent = 0.005f; + constexpr float kConvergedNormal = 0.01f; + constexpr float kConvergedTransparent = 0.001f; // Probability of transitioning to transparent state from normal state and // transparent state respectively. @@ -92,7 +94,7 @@ class TransparentModeImpl : public TransparentMode { const float prob_transition_normal = 1.f - prob_transition_transparent; // Observed output. - const int out = any_filter_converged; + const int out = static_cast(any_coarse_filter_converged); // Joint probabilites of the observed output and respective states. const float prob_joint_normal = prob_transition_normal * kB[0][out]; @@ -142,6 +144,7 @@ class LegacyTransparentModeImpl : public TransparentMode { void Update(int filter_delay_blocks, bool any_filter_consistent, bool any_filter_converged, + bool any_coarse_filter_converged, bool all_filters_diverged, bool active_render, bool saturated_capture) override { @@ -226,12 +229,15 @@ class LegacyTransparentModeImpl : public TransparentMode { std::unique_ptr TransparentMode::Create( const EchoCanceller3Config& config) { if (config.ep_strength.bounded_erl || DeactivateTransparentMode()) { + RTC_LOG(LS_INFO) << "AEC3 Transparent Mode: Disabled"; return nullptr; } - if (DeactivateTransparentModeHmm()) { - return std::make_unique(config); + if (ActivateTransparentModeHmm()) { + RTC_LOG(LS_INFO) << "AEC3 Transparent Mode: HMM"; + return std::make_unique(); } - return std::make_unique(); + RTC_LOG(LS_INFO) << "AEC3 Transparent Mode: Legacy"; + return std::make_unique(config); } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/transparent_mode.h b/webrtc/modules/audio_processing/aec3/transparent_mode.h index b1be69b..bc5dd03 100644 --- a/webrtc/modules/audio_processing/aec3/transparent_mode.h +++ b/webrtc/modules/audio_processing/aec3/transparent_mode.h @@ -37,6 +37,7 @@ class TransparentMode { virtual void Update(int filter_delay_blocks, bool any_filter_consistent, bool any_filter_converged, + bool any_coarse_filter_converged, bool all_filters_diverged, bool active_render, bool saturated_capture) = 0; diff --git a/webrtc/modules/audio_processing/aec3/vector_math_avx2.cc b/webrtc/modules/audio_processing/aec3/vector_math_avx2.cc index 0b5f3c1..a9805da 100644 --- a/webrtc/modules/audio_processing/aec3/vector_math_avx2.cc +++ b/webrtc/modules/audio_processing/aec3/vector_math_avx2.cc @@ -8,12 +8,11 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/aec3/vector_math.h" - #include #include #include "api/array_view.h" +#include "modules/audio_processing/aec3/vector_math.h" #include "rtc_base/checks.h" namespace webrtc { diff --git a/webrtc/modules/audio_processing/aec_dump/BUILD.gn b/webrtc/modules/audio_processing/aec_dump/BUILD.gn index 9887f7d..38d8776 100644 --- a/webrtc/modules/audio_processing/aec_dump/BUILD.gn +++ b/webrtc/modules/audio_processing/aec_dump/BUILD.gn @@ -14,10 +14,10 @@ rtc_source_set("aec_dump") { deps = [ "..:aec_dump_interface", - "../../../rtc_base:rtc_base_approved", "../../../rtc_base/system:file_wrapper", "../../../rtc_base/system:rtc_export", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } if (rtc_include_tests) { @@ -46,7 +46,6 @@ if (rtc_include_tests) { "..:api", "..:audioproc_test_utils", "../", - "../../../rtc_base:rtc_base_approved", "//testing/gtest", ] } @@ -59,8 +58,6 @@ if (rtc_enable_protobuf) { "aec_dump_impl.h", "capture_stream_info.cc", "capture_stream_info.h", - "write_to_file_task.cc", - "write_to_file_task.h", ] deps = [ @@ -70,12 +67,16 @@ if (rtc_enable_protobuf) { "../../../api/task_queue", "../../../rtc_base:checks", "../../../rtc_base:ignore_wundef", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", "../../../rtc_base:protobuf_utils", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:race_checker", + "../../../rtc_base:rtc_event", "../../../rtc_base:rtc_task_queue", "../../../rtc_base/system:file_wrapper", "../../../system_wrappers", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] deps += [ "../:audioproc_debug_proto" ] } @@ -107,4 +108,5 @@ rtc_library("null_aec_dump_factory") { ":aec_dump", "..:aec_dump_interface", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } diff --git a/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h b/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h index 429a8a5..20718c3 100644 --- a/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h +++ b/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h @@ -12,8 +12,8 @@ #define MODULES_AUDIO_PROCESSING_AEC_DUMP_AEC_DUMP_FACTORY_H_ #include -#include +#include "absl/strings/string_view.h" #include "modules/audio_processing/include/aec_dump.h" #include "rtc_base/system/file_wrapper.h" #include "rtc_base/system/rtc_export.h" @@ -26,16 +26,16 @@ namespace webrtc { class RTC_EXPORT AecDumpFactory { public: - // The |worker_queue| may not be null and must outlive the created - // AecDump instance. |max_log_size_bytes == -1| means the log size - // will be unlimited. |handle| may not be null. The AecDump takes - // responsibility for |handle| and closes it in the destructor. A + // The `worker_queue` may not be null and must outlive the created + // AecDump instance. `max_log_size_bytes == -1` means the log size + // will be unlimited. `handle` may not be null. The AecDump takes + // responsibility for `handle` and closes it in the destructor. A // non-null return value indicates that the file has been // sucessfully opened. static std::unique_ptr Create(webrtc::FileWrapper file, int64_t max_log_size_bytes, rtc::TaskQueue* worker_queue); - static std::unique_ptr Create(std::string file_name, + static std::unique_ptr Create(absl::string_view file_name, int64_t max_log_size_bytes, rtc::TaskQueue* worker_queue); static std::unique_ptr Create(FILE* handle, diff --git a/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc b/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc index 126adeb..9bd9745 100644 --- a/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc +++ b/webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "absl/strings/string_view.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" #include "modules/audio_processing/include/aec_dump.h" @@ -19,7 +20,7 @@ std::unique_ptr AecDumpFactory::Create(webrtc::FileWrapper file, return nullptr; } -std::unique_ptr AecDumpFactory::Create(std::string file_name, +std::unique_ptr AecDumpFactory::Create(absl::string_view file_name, int64_t max_log_size_bytes, rtc::TaskQueue* worker_queue) { return nullptr; diff --git a/webrtc/modules/audio_processing/aecm/BUILD.gn b/webrtc/modules/audio_processing/aecm/BUILD.gn index 61e9aff..80f2901 100644 --- a/webrtc/modules/audio_processing/aecm/BUILD.gn +++ b/webrtc/modules/audio_processing/aecm/BUILD.gn @@ -19,7 +19,7 @@ rtc_library("aecm_core") { deps = [ "../../../common_audio:common_audio_c", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:safe_conversions", "../../../rtc_base:sanitizer", "../../../system_wrappers", "../utility:legacy_delay_estimator", diff --git a/webrtc/modules/audio_processing/aecm/aecm_core.cc b/webrtc/modules/audio_processing/aecm/aecm_core.cc index 78c0133..b463146 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core.cc +++ b/webrtc/modules/audio_processing/aecm/aecm_core.cc @@ -123,8 +123,7 @@ const int16_t WebRtcAecm_kSinTable[] = { -2667, -2531, -2395, -2258, -2120, -1981, -1842, -1703, -1563, -1422, -1281, -1140, -998, -856, -713, -571, -428, -285, -142}; - -// Moves the pointer to the next entry and inserts |far_spectrum| and +// Moves the pointer to the next entry and inserts `far_spectrum` and // corresponding Q-domain in its buffer. // // Inputs: @@ -574,7 +573,7 @@ int WebRtcAecm_ProcessFrame(AecmCore* aecm, // Obtain an output frame. WebRtc_ReadBuffer(aecm->outFrameBuf, (void**)&out_ptr, out, FRAME_LEN); if (out_ptr != out) { - // ReadBuffer() hasn't copied to |out| in this case. + // ReadBuffer() hasn't copied to `out` in this case. memcpy(out, out_ptr, FRAME_LEN * sizeof(int16_t)); } @@ -616,22 +615,22 @@ int16_t WebRtcAecm_AsymFilt(const int16_t filtOld, // ExtractFractionPart(a, zeros) // -// returns the fraction part of |a|, with |zeros| number of leading zeros, as an -// int16_t scaled to Q8. There is no sanity check of |a| in the sense that the +// returns the fraction part of `a`, with `zeros` number of leading zeros, as an +// int16_t scaled to Q8. There is no sanity check of `a` in the sense that the // number of zeros match. static int16_t ExtractFractionPart(uint32_t a, int zeros) { return (int16_t)(((a << zeros) & 0x7FFFFFFF) >> 23); } -// Calculates and returns the log of |energy| in Q8. The input |energy| is -// supposed to be in Q(|q_domain|). +// Calculates and returns the log of `energy` in Q8. The input `energy` is +// supposed to be in Q(`q_domain`). static int16_t LogOfEnergyInQ8(uint32_t energy, int q_domain) { static const int16_t kLogLowValue = PART_LEN_SHIFT << 7; int16_t log_energy_q8 = kLogLowValue; if (energy > 0) { int zeros = WebRtcSpl_NormU32(energy); int16_t frac = ExtractFractionPart(energy, zeros); - // log2 of |energy| in Q8. + // log2 of `energy` in Q8. log_energy_q8 += ((31 - zeros) << 8) + frac - (q_domain << 8); } return log_energy_q8; diff --git a/webrtc/modules/audio_processing/aecm/aecm_core.h b/webrtc/modules/audio_processing/aecm/aecm_core.h index aaa74e1..3de4931 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core.h +++ b/webrtc/modules/audio_processing/aecm/aecm_core.h @@ -58,7 +58,7 @@ typedef struct { void* delay_estimator; uint16_t currentDelay; // Far end history variables - // TODO(bjornv): Replace |far_history| with ring_buffer. + // TODO(bjornv): Replace `far_history` with ring_buffer. uint16_t far_history[PART_LEN1 * MAX_DELAY]; int far_history_pos; int far_q_domains[MAX_DELAY]; @@ -248,7 +248,7 @@ int WebRtcAecm_ProcessBlock(AecmCore* aecm, // void WebRtcAecm_BufferFarFrame(AecmCore* const aecm, const int16_t* const farend, - const int farLen); + int farLen); //////////////////////////////////////////////////////////////////////////////// // WebRtcAecm_FetchFarFrame() @@ -263,15 +263,15 @@ void WebRtcAecm_BufferFarFrame(AecmCore* const aecm, // void WebRtcAecm_FetchFarFrame(AecmCore* const aecm, int16_t* const farend, - const int farLen, - const int knownDelay); + int farLen, + int knownDelay); // All the functions below are intended to be private //////////////////////////////////////////////////////////////////////////////// // WebRtcAecm_UpdateFarHistory() // -// Moves the pointer to the next entry and inserts |far_spectrum| and +// Moves the pointer to the next entry and inserts `far_spectrum` and // corresponding Q-domain in its buffer. // // Inputs: @@ -339,8 +339,8 @@ int16_t WebRtcAecm_CalcSuppressionGain(AecmCore* const aecm); // void WebRtcAecm_CalcEnergies(AecmCore* aecm, const uint16_t* far_spectrum, - const int16_t far_q, - const uint32_t nearEner, + int16_t far_q, + uint32_t nearEner, int32_t* echoEst); /////////////////////////////////////////////////////////////////////////////// @@ -374,9 +374,9 @@ int16_t WebRtcAecm_CalcStepSize(AecmCore* const aecm); // void WebRtcAecm_UpdateChannel(AecmCore* aecm, const uint16_t* far_spectrum, - const int16_t far_q, + int16_t far_q, const uint16_t* const dfa, - const int16_t mu, + int16_t mu, int32_t* echoEst); extern const int16_t WebRtcAecm_kCosTable[]; diff --git a/webrtc/modules/audio_processing/aecm/aecm_core_c.cc b/webrtc/modules/audio_processing/aecm/aecm_core_c.cc index 7b6ca59..59e0296 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core_c.cc +++ b/webrtc/modules/audio_processing/aecm/aecm_core_c.cc @@ -98,7 +98,7 @@ static void ComfortNoise(AecmCore* aecm, // Track the minimum. if (aecm->noiseEst[i] < (1 << minTrackShift)) { // For small values, decrease noiseEst[i] every - // |kNoiseEstIncCount| block. The regular approach below can not + // `kNoiseEstIncCount` block. The regular approach below can not // go further down due to truncation. aecm->noiseEstTooHighCtr[i]++; if (aecm->noiseEstTooHighCtr[i] >= kNoiseEstIncCount) { @@ -125,7 +125,7 @@ static void ComfortNoise(AecmCore* aecm, aecm->noiseEst[i] >>= 11; } else { // Make incremental increases based on size every - // |kNoiseEstIncCount| block + // `kNoiseEstIncCount` block aecm->noiseEstTooLowCtr[i]++; if (aecm->noiseEstTooLowCtr[i] >= kNoiseEstIncCount) { aecm->noiseEst[i] += (aecm->noiseEst[i] >> 9) + 1; @@ -181,12 +181,13 @@ static void WindowAndFFT(AecmCore* aecm, // FFT of signal for (i = 0; i < PART_LEN; i++) { // Window time domain signal and insert into real part of - // transformation array |fft| + // transformation array `fft` int16_t scaled_time_signal = time_signal[i] * (1 << time_signal_scaling); fft[i] = (int16_t)((scaled_time_signal * WebRtcAecm_kSqrtHanning[i]) >> 14); scaled_time_signal = time_signal[i + PART_LEN] * (1 << time_signal_scaling); - fft[PART_LEN + i] = (int16_t)( - (scaled_time_signal * WebRtcAecm_kSqrtHanning[PART_LEN - i]) >> 14); + fft[PART_LEN + i] = (int16_t)((scaled_time_signal * + WebRtcAecm_kSqrtHanning[PART_LEN - i]) >> + 14); } // Do forward FFT, then take only the first PART_LEN complex samples, @@ -204,8 +205,8 @@ static void InverseFFTAndWindow(AecmCore* aecm, const int16_t* nearendClean) { int i, j, outCFFT; int32_t tmp32no1; - // Reuse |efw| for the inverse FFT output after transferring - // the contents to |fft|. + // Reuse `efw` for the inverse FFT output after transferring + // the contents to `fft`. int16_t* ifft_out = (int16_t*)efw; // Synthesis @@ -312,7 +313,7 @@ static int TimeToFrequencyDomain(AecmCore* aecm, } else { // Approximation for magnitude of complex fft output // magn = sqrt(real^2 + imag^2) - // magn ~= alpha * max(|imag|,|real|) + beta * min(|imag|,|real|) + // magn ~= alpha * max(`imag`,`real`) + beta * min(`imag`,`real`) // // The parameters alpha and beta are stored in Q15 @@ -541,7 +542,7 @@ int RTC_NO_SANITIZE("signed-integer-overflow") // bugs.webrtc.org/8200 } zeros16 = WebRtcSpl_NormW16(aecm->nearFilt[i]); - RTC_DCHECK_GE(zeros16, 0); // |zeros16| is a norm, hence non-negative. + RTC_DCHECK_GE(zeros16, 0); // `zeros16` is a norm, hence non-negative. dfa_clean_q_domain_diff = aecm->dfaCleanQDomain - aecm->dfaCleanQDomainOld; if (zeros16 < dfa_clean_q_domain_diff && aecm->nearFilt[i]) { tmp16no1 = aecm->nearFilt[i] * (1 << zeros16); @@ -644,18 +645,18 @@ int RTC_NO_SANITIZE("signed-integer-overflow") // bugs.webrtc.org/8200 } // multiply with Wiener coefficients - efw[i].real = (int16_t)( - WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, hnl[i], 14)); - efw[i].imag = (int16_t)( - WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, hnl[i], 14)); + efw[i].real = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, + hnl[i], 14)); + efw[i].imag = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, + hnl[i], 14)); } } else { // multiply with Wiener coefficients for (i = 0; i < PART_LEN1; i++) { - efw[i].real = (int16_t)( - WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, hnl[i], 14)); - efw[i].imag = (int16_t)( - WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, hnl[i], 14)); + efw[i].real = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, + hnl[i], 14)); + efw[i].imag = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, + hnl[i], 14)); } } diff --git a/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc b/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc index f2f43e1..16b03cf 100644 --- a/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc +++ b/webrtc/modules/audio_processing/aecm/aecm_core_mips.cc @@ -569,8 +569,8 @@ static void InverseFFTAndWindow(AecmCore* aecm, [paecm_buf] "+r"(paecm_buf), [i] "=&r"(i), [pp_kSqrtHanning] "+r"(pp_kSqrtHanning), [p_kSqrtHanning] "+r"(p_kSqrtHanning) - : [out_aecm] "r"(out_aecm), - [WebRtcAecm_kSqrtHanning] "r"(WebRtcAecm_kSqrtHanning) + : [out_aecm] "r"(out_aecm), [WebRtcAecm_kSqrtHanning] "r"( + WebRtcAecm_kSqrtHanning) : "hi", "lo", "memory"); // Copy the current block to the old position @@ -822,7 +822,7 @@ static int TimeToFrequencyDomain(AecmCore* aecm, } else { // Approximation for magnitude of complex fft output // magn = sqrt(real^2 + imag^2) - // magn ~= alpha * max(|imag|,|real|) + beta * min(|imag|,|real|) + // magn ~= alpha * max(`imag`,`real`) + beta * min(`imag`,`real`) // // The parameters alpha and beta are stored in Q15 tmp16no1 = WEBRTC_SPL_ABS_W16(freq_signal[i].real); @@ -1106,7 +1106,7 @@ int WebRtcAecm_ProcessBlock(AecmCore* aecm, } zeros16 = WebRtcSpl_NormW16(aecm->nearFilt[i]); - RTC_DCHECK_GE(zeros16, 0); // |zeros16| is a norm, hence non-negative. + RTC_DCHECK_GE(zeros16, 0); // `zeros16` is a norm, hence non-negative. dfa_clean_q_domain_diff = aecm->dfaCleanQDomain - aecm->dfaCleanQDomainOld; if (zeros16 < dfa_clean_q_domain_diff && aecm->nearFilt[i]) { tmp16no1 = aecm->nearFilt[i] << zeros16; @@ -1334,10 +1334,10 @@ int WebRtcAecm_ProcessBlock(AecmCore* aecm, } else { // multiply with Wiener coefficients for (i = 0; i < PART_LEN1; i++) { - efw[i].real = (int16_t)( - WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, hnl[i], 14)); - efw[i].imag = (int16_t)( - WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, hnl[i], 14)); + efw[i].real = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].real, + hnl[i], 14)); + efw[i].imag = (int16_t)(WEBRTC_SPL_MUL_16_16_RSFT_WITH_ROUND(dfw[i].imag, + hnl[i], 14)); } } @@ -1411,7 +1411,7 @@ static void ComfortNoise(AecmCore* aecm, // Track the minimum. if (tnoise < (1 << minTrackShift)) { // For small values, decrease noiseEst[i] every - // |kNoiseEstIncCount| block. The regular approach below can not + // `kNoiseEstIncCount` block. The regular approach below can not // go further down due to truncation. aecm->noiseEstTooHighCtr[i]++; if (aecm->noiseEstTooHighCtr[i] >= kNoiseEstIncCount) { @@ -1424,8 +1424,8 @@ static void ComfortNoise(AecmCore* aecm, "srav %[tmp32], %[tmp32], %[minTrackShift] \n\t" "subu %[tnoise], %[tnoise], %[tmp32] \n\t" : [tmp32] "=&r"(tmp32), [tnoise] "+r"(tnoise) - : - [outLShift32] "r"(outLShift32), [minTrackShift] "r"(minTrackShift)); + : [outLShift32] "r"(outLShift32), [minTrackShift] "r"( + minTrackShift)); } } else { // Reset "too high" counter @@ -1442,7 +1442,7 @@ static void ComfortNoise(AecmCore* aecm, : "hi", "lo"); } else { // Make incremental increases based on size every - // |kNoiseEstIncCount| block + // `kNoiseEstIncCount` block aecm->noiseEstTooLowCtr[i]++; if (aecm->noiseEstTooLowCtr[i] >= kNoiseEstIncCount) { __asm __volatile( @@ -1484,7 +1484,7 @@ static void ComfortNoise(AecmCore* aecm, // Track the minimum. if (tnoise1 < (1 << minTrackShift)) { // For small values, decrease noiseEst[i] every - // |kNoiseEstIncCount| block. The regular approach below can not + // `kNoiseEstIncCount` block. The regular approach below can not // go further down due to truncation. aecm->noiseEstTooHighCtr[i + 1]++; if (aecm->noiseEstTooHighCtr[i + 1] >= kNoiseEstIncCount) { @@ -1497,8 +1497,8 @@ static void ComfortNoise(AecmCore* aecm, "srav %[tmp32], %[tmp32], %[minTrackShift] \n\t" "subu %[tnoise1], %[tnoise1], %[tmp32] \n\t" : [tmp32] "=&r"(tmp32), [tnoise1] "+r"(tnoise1) - : - [outLShift32] "r"(outLShift32), [minTrackShift] "r"(minTrackShift)); + : [outLShift32] "r"(outLShift32), [minTrackShift] "r"( + minTrackShift)); } } else { // Reset "too high" counter @@ -1515,7 +1515,7 @@ static void ComfortNoise(AecmCore* aecm, : "hi", "lo"); } else { // Make incremental increases based on size every - // |kNoiseEstIncCount| block + // `kNoiseEstIncCount` block aecm->noiseEstTooLowCtr[i + 1]++; if (aecm->noiseEstTooLowCtr[i + 1] >= kNoiseEstIncCount) { __asm __volatile( diff --git a/webrtc/modules/audio_processing/agc/BUILD.gn b/webrtc/modules/audio_processing/agc/BUILD.gn index 8235456..508f901 100644 --- a/webrtc/modules/audio_processing/agc/BUILD.gn +++ b/webrtc/modules/audio_processing/agc/BUILD.gn @@ -20,20 +20,23 @@ rtc_library("agc") { configs += [ "..:apm_debug_dump" ] deps = [ ":gain_control_interface", - ":gain_map", ":level_estimation", + "..:api", "..:apm_logging", "..:audio_buffer", + "..:audio_frame_view", + "../../../api:array_view", "../../../common_audio", "../../../common_audio:common_audio_c", "../../../rtc_base:checks", "../../../rtc_base:gtest_prod", "../../../rtc_base:logging", - "../../../rtc_base:rtc_base_approved", "../../../rtc_base:safe_minmax", "../../../system_wrappers:field_trial", "../../../system_wrappers:metrics", - "../agc2:level_estimation_agc", + "../agc2:clipping_predictor", + "../agc2:gain_map", + "../agc2:input_volume_stats_reporter", "../vad", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] @@ -49,6 +52,7 @@ rtc_library("level_estimation") { "utility.h", ] deps = [ + "../../../api:array_view", "../../../rtc_base:checks", "../vad", ] @@ -75,7 +79,6 @@ rtc_library("legacy_agc") { "../../../common_audio:common_audio_c", "../../../common_audio/third_party/ooura:fft_size_256", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", "../../../system_wrappers", ] @@ -88,10 +91,6 @@ rtc_library("legacy_agc") { } } -rtc_source_set("gain_map") { - sources = [ "gain_map_internal.h" ] -} - if (rtc_include_tests) { rtc_library("agc_unittests") { testonly = true @@ -107,10 +106,21 @@ if (rtc_include_tests) { ":gain_control_interface", ":level_estimation", "..:mocks", + "../../../api:array_view", + "../../../rtc_base:checks", + "../../../rtc_base:random", + "../../../rtc_base:safe_conversions", + "../../../rtc_base:safe_minmax", + "../../../rtc_base:stringutils", + "../../../system_wrappers:metrics", "../../../test:field_trial", "../../../test:fileutils", "../../../test:test_support", "//testing/gtest", ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] } } diff --git a/webrtc/modules/audio_processing/agc/agc.cc b/webrtc/modules/audio_processing/agc/agc.cc index a89ae11..a018ff9 100644 --- a/webrtc/modules/audio_processing/agc/agc.cc +++ b/webrtc/modules/audio_processing/agc/agc.cc @@ -21,9 +21,11 @@ namespace webrtc { namespace { -const int kDefaultLevelDbfs = -18; -const int kNumAnalysisFrames = 100; -const double kActivityThreshold = 0.3; +constexpr int kDefaultLevelDbfs = -18; +constexpr int kNumAnalysisFrames = 100; +constexpr double kActivityThreshold = 0.3; +constexpr int kNum10msFramesInOneSecond = 100; +constexpr int kMaxSampleRateHz = 384000; } // namespace @@ -35,8 +37,10 @@ Agc::Agc() Agc::~Agc() = default; -void Agc::Process(const int16_t* audio, size_t length, int sample_rate_hz) { - vad_.ProcessChunk(audio, length, sample_rate_hz); +void Agc::Process(rtc::ArrayView audio) { + const int sample_rate_hz = audio.size() * kNum10msFramesInOneSecond; + RTC_DCHECK_LE(sample_rate_hz, kMaxSampleRateHz); + vad_.ProcessChunk(audio.data(), audio.size(), sample_rate_hz); const std::vector& rms = vad_.chunkwise_rms(); const std::vector& probabilities = vad_.chunkwise_voice_probabilities(); @@ -48,7 +52,7 @@ void Agc::Process(const int16_t* audio, size_t length, int sample_rate_hz) { bool Agc::GetRmsErrorDb(int* error) { if (!error) { - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return false; } diff --git a/webrtc/modules/audio_processing/agc/agc.h b/webrtc/modules/audio_processing/agc/agc.h index b9bd5ea..da42808 100644 --- a/webrtc/modules/audio_processing/agc/agc.h +++ b/webrtc/modules/audio_processing/agc/agc.h @@ -13,6 +13,7 @@ #include +#include "api/array_view.h" #include "modules/audio_processing/vad/voice_activity_detector.h" namespace webrtc { @@ -24,13 +25,13 @@ class Agc { Agc(); virtual ~Agc(); - // |audio| must be mono; in a multi-channel stream, provide the first (usually + // `audio` must be mono; in a multi-channel stream, provide the first (usually // left) channel. - virtual void Process(const int16_t* audio, size_t length, int sample_rate_hz); + virtual void Process(rtc::ArrayView audio); // Retrieves the difference between the target RMS level and the current // signal RMS level in dB. Returns true if an update is available and false - // otherwise, in which case |error| should be ignored and no action taken. + // otherwise, in which case `error` should be ignored and no action taken. virtual bool GetRmsErrorDb(int* error); virtual void Reset(); diff --git a/webrtc/modules/audio_processing/agc/agc_manager_direct.cc b/webrtc/modules/audio_processing/agc/agc_manager_direct.cc index 1428d2a..b8ad4a8 100644 --- a/webrtc/modules/audio_processing/agc/agc_manager_direct.cc +++ b/webrtc/modules/audio_processing/agc/agc_manager_direct.cc @@ -13,11 +13,12 @@ #include #include +#include "api/array_view.h" #include "common_audio/include/audio_util.h" #include "modules/audio_processing/agc/gain_control.h" -#include "modules/audio_processing/agc/gain_map_internal.h" -#include "modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.h" -#include "rtc_base/atomic_ops.h" +#include "modules/audio_processing/agc2/gain_map_internal.h" +#include "modules/audio_processing/agc2/input_volume_stats_reporter.h" +#include "modules/audio_processing/include/audio_frame_view.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_minmax.h" @@ -28,69 +29,65 @@ namespace webrtc { namespace { -// Amount the microphone level is lowered with every clipping event. -const int kClippedLevelStep = 15; -// Proportion of clipped samples required to declare a clipping event. -const float kClippedRatioThreshold = 0.1f; -// Time in frames to wait after a clipping event before checking again. -const int kClippedWaitFrames = 300; - // Amount of error we tolerate in the microphone level (presumably due to OS // quantization) before we assume the user has manually adjusted the microphone. -const int kLevelQuantizationSlack = 25; +constexpr int kLevelQuantizationSlack = 25; -const int kDefaultCompressionGain = 7; -const int kMaxCompressionGain = 12; -const int kMinCompressionGain = 2; +constexpr int kDefaultCompressionGain = 7; +constexpr int kMaxCompressionGain = 12; +constexpr int kMinCompressionGain = 2; // Controls the rate of compression changes towards the target. -const float kCompressionGainStep = 0.05f; +constexpr float kCompressionGainStep = 0.05f; -const int kMaxMicLevel = 255; +constexpr int kMaxMicLevel = 255; static_assert(kGainMapSize > kMaxMicLevel, "gain map too small"); -const int kMinMicLevel = 12; +constexpr int kMinMicLevel = 12; // Prevent very large microphone level changes. -const int kMaxResidualGainChange = 15; +constexpr int kMaxResidualGainChange = 15; // Maximum additional gain allowed to compensate for microphone level // restrictions from clipping events. -const int kSurplusCompressionGain = 6; +constexpr int kSurplusCompressionGain = 6; -// Returns whether a fall-back solution to choose the maximum level should be -// chosen. -bool UseMaxAnalogChannelLevel() { - return field_trial::IsEnabled("WebRTC-UseMaxAnalogAgcChannelLevel"); -} +// Target speech level (dBFs) and speech probability threshold used to compute +// the RMS error override in `GetSpeechLevelErrorDb()`. These are only used for +// computing the error override and they are not passed to `agc_`. +// TODO(webrtc:7494): Move these to a config and pass in the ctor. +constexpr float kOverrideTargetSpeechLevelDbfs = -18.0f; +constexpr float kOverrideSpeechProbabilitySilenceThreshold = 0.5f; +// The minimum number of frames between `UpdateGain()` calls. +// TODO(webrtc:7494): Move this to a config and pass in the ctor with +// kOverrideWaitFrames = 100. Default value zero needed for the unit tests. +constexpr int kOverrideWaitFrames = 0; -// Returns kMinMicLevel if no field trial exists or if it has been disabled. -// Returns a value between 0 and 255 depending on the field-trial string. -// Example: 'WebRTC-Audio-AgcMinMicLevelExperiment/Enabled-80' => returns 80. -int GetMinMicLevel() { - RTC_LOG(LS_INFO) << "[agc] GetMinMicLevel"; +using AnalogAgcConfig = + AudioProcessing::Config::GainController1::AnalogGainController; + +// If the "WebRTC-Audio-2ndAgcMinMicLevelExperiment" field trial is specified, +// parses it and returns a value between 0 and 255 depending on the field-trial +// string. Returns an unspecified value if the field trial is not specified, if +// disabled or if it cannot be parsed. Example: +// 'WebRTC-Audio-2ndAgcMinMicLevelExperiment/Enabled-80' => returns 80. +absl::optional GetMinMicLevelOverride() { constexpr char kMinMicLevelFieldTrial[] = - "WebRTC-Audio-AgcMinMicLevelExperiment"; + "WebRTC-Audio-2ndAgcMinMicLevelExperiment"; if (!webrtc::field_trial::IsEnabled(kMinMicLevelFieldTrial)) { - RTC_LOG(LS_INFO) << "[agc] Using default min mic level: " << kMinMicLevel; - return kMinMicLevel; + return absl::nullopt; } const auto field_trial_string = webrtc::field_trial::FindFullName(kMinMicLevelFieldTrial); int min_mic_level = -1; sscanf(field_trial_string.c_str(), "Enabled-%d", &min_mic_level); if (min_mic_level >= 0 && min_mic_level <= 255) { - RTC_LOG(LS_INFO) << "[agc] Experimental min mic level: " << min_mic_level; return min_mic_level; } else { RTC_LOG(LS_WARNING) << "[agc] Invalid parameter for " << kMinMicLevelFieldTrial << ", ignored."; - return kMinMicLevel; + return absl::nullopt; } } -int ClampLevel(int mic_level, int min_mic_level) { - return rtc::SafeClamp(mic_level, min_mic_level, kMaxMicLevel); -} - int LevelFromGainError(int gain_error, int level, int min_mic_level) { RTC_DCHECK_GE(level, 0); RTC_DCHECK_LE(level, kMaxMicLevel); @@ -124,7 +121,7 @@ float ComputeClippedRatio(const float* const* audio, int num_clipped_in_ch = 0; for (size_t i = 0; i < samples_per_channel; ++i) { RTC_DCHECK(audio[ch]); - if (audio[ch][i] >= 32767.f || audio[ch][i] <= -32768.f) { + if (audio[ch][i] >= 32767.0f || audio[ch][i] <= -32768.0f) { ++num_clipped_in_ch; } } @@ -133,29 +130,49 @@ float ComputeClippedRatio(const float* const* audio, return static_cast(num_clipped) / (samples_per_channel); } +void LogClippingMetrics(int clipping_rate) { + RTC_LOG(LS_INFO) << "Input clipping rate: " << clipping_rate << "%"; + RTC_HISTOGRAM_COUNTS_LINEAR(/*name=*/"WebRTC.Audio.Agc.InputClippingRate", + /*sample=*/clipping_rate, /*min=*/0, /*max=*/100, + /*bucket_count=*/50); +} + +// Computes the speech level error in dB. `speech_level_dbfs` is required to be +// in the range [-90.0f, 30.0f] and `speech_probability` in the range +// [0.0f, 1.0f]. +int GetSpeechLevelErrorDb(float speech_level_dbfs, float speech_probability) { + constexpr float kMinSpeechLevelDbfs = -90.0f; + constexpr float kMaxSpeechLevelDbfs = 30.0f; + RTC_DCHECK_GE(speech_level_dbfs, kMinSpeechLevelDbfs); + RTC_DCHECK_LE(speech_level_dbfs, kMaxSpeechLevelDbfs); + RTC_DCHECK_GE(speech_probability, 0.0f); + RTC_DCHECK_LE(speech_probability, 1.0f); + + if (speech_probability < kOverrideSpeechProbabilitySilenceThreshold) { + return 0; + } + + const float speech_level = rtc::SafeClamp( + speech_level_dbfs, kMinSpeechLevelDbfs, kMaxSpeechLevelDbfs); + + return std::round(kOverrideTargetSpeechLevelDbfs - speech_level); +} + } // namespace MonoAgc::MonoAgc(ApmDataDumper* data_dumper, - int startup_min_level, int clipped_level_min, - bool use_agc2_level_estimation, bool disable_digital_adaptive, int min_mic_level) : min_mic_level_(min_mic_level), disable_digital_adaptive_(disable_digital_adaptive), + agc_(std::make_unique()), max_level_(kMaxMicLevel), max_compression_gain_(kMaxCompressionGain), target_compression_(kDefaultCompressionGain), compression_(target_compression_), compression_accumulator_(compression_), - startup_min_level_(ClampLevel(startup_min_level, min_mic_level_)), - clipped_level_min_(clipped_level_min) { - if (use_agc2_level_estimation) { - agc_ = std::make_unique(data_dumper); - } else { - agc_ = std::make_unique(); - } -} + clipped_level_min_(clipped_level_min) {} MonoAgc::~MonoAgc() = default; @@ -165,13 +182,14 @@ void MonoAgc::Initialize() { target_compression_ = disable_digital_adaptive_ ? 0 : kDefaultCompressionGain; compression_ = disable_digital_adaptive_ ? 0 : target_compression_; compression_accumulator_ = compression_; - capture_muted_ = false; + capture_output_used_ = true; check_volume_on_next_process_ = true; + frames_since_update_gain_ = 0; + is_first_frame_ = true; } -void MonoAgc::Process(const int16_t* audio, - size_t samples_per_channel, - int sample_rate_hz) { +void MonoAgc::Process(rtc::ArrayView audio, + absl::optional rms_error_override) { new_compression_to_set_ = absl::nullopt; if (check_volume_on_next_process_) { @@ -181,34 +199,60 @@ void MonoAgc::Process(const int16_t* audio, CheckVolumeAndReset(); } - agc_->Process(audio, samples_per_channel, sample_rate_hz); + agc_->Process(audio); + + // Always check if `agc_` has a new error available. If yes, `agc_` gets + // reset. + // TODO(webrtc:7494) Replace the `agc_` call `GetRmsErrorDb()` with `Reset()` + // if an error override is used. + int rms_error = 0; + bool update_gain = agc_->GetRmsErrorDb(&rms_error); + if (rms_error_override.has_value()) { + if (is_first_frame_ || frames_since_update_gain_ < kOverrideWaitFrames) { + update_gain = false; + } else { + rms_error = *rms_error_override; + update_gain = true; + } + } + + if (update_gain) { + UpdateGain(rms_error); + } - UpdateGain(); if (!disable_digital_adaptive_) { UpdateCompressor(); } + + is_first_frame_ = false; + if (frames_since_update_gain_ < kOverrideWaitFrames) { + ++frames_since_update_gain_; + } } -void MonoAgc::HandleClipping() { +void MonoAgc::HandleClipping(int clipped_level_step) { + RTC_DCHECK_GT(clipped_level_step, 0); // Always decrease the maximum level, even if the current level is below // threshold. - SetMaxLevel(std::max(clipped_level_min_, max_level_ - kClippedLevelStep)); + SetMaxLevel(std::max(clipped_level_min_, max_level_ - clipped_level_step)); if (log_to_histograms_) { RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.AgcClippingAdjustmentAllowed", - level_ - kClippedLevelStep >= clipped_level_min_); + level_ - clipped_level_step >= clipped_level_min_); } if (level_ > clipped_level_min_) { // Don't try to adjust the level if we're already below the limit. As // a consequence, if the user has brought the level above the limit, we // will still not react until the postproc updates the level. - SetLevel(std::max(clipped_level_min_, level_ - kClippedLevelStep)); + SetLevel(std::max(clipped_level_min_, level_ - clipped_level_step)); // Reset the AGCs for all channels since the level has changed. agc_->Reset(); + frames_since_update_gain_ = 0; + is_first_frame_ = false; } } void MonoAgc::SetLevel(int new_level) { - int voe_level = stream_analog_level_; + int voe_level = recommended_input_volume_; if (voe_level == 0) { RTC_DLOG(LS_INFO) << "[agc] VolumeCallbacks returned level=0, taking no action."; @@ -220,6 +264,10 @@ void MonoAgc::SetLevel(int new_level) { return; } + // Detect manual input volume adjustments by checking if the current level + // `voe_level` is outside of the `[level_ - kLevelQuantizationSlack, level_ + + // kLevelQuantizationSlack]` range where `level_` is the last input volume + // known by this gain controller. if (voe_level > level_ + kLevelQuantizationSlack || voe_level < level_ - kLevelQuantizationSlack) { RTC_DLOG(LS_INFO) << "[agc] Mic volume was manually adjusted. Updating " @@ -234,7 +282,8 @@ void MonoAgc::SetLevel(int new_level) { // was manually adjusted. The compressor will still provide some of the // desired gain change. agc_->Reset(); - + frames_since_update_gain_ = 0; + is_first_frame_ = false; return; } @@ -243,7 +292,7 @@ void MonoAgc::SetLevel(int new_level) { return; } - stream_analog_level_ = new_level; + recommended_input_volume_ = new_level; RTC_DLOG(LS_INFO) << "[agc] voe_level=" << voe_level << ", level_=" << level_ << ", new_level=" << new_level; level_ = new_level; @@ -252,7 +301,7 @@ void MonoAgc::SetLevel(int new_level) { void MonoAgc::SetMaxLevel(int level) { RTC_DCHECK_GE(level, clipped_level_min_); max_level_ = level; - // Scale the |kSurplusCompressionGain| linearly across the restricted + // Scale the `kSurplusCompressionGain` linearly across the restricted // level range. max_compression_gain_ = kMaxCompressionGain + std::floor((1.f * kMaxMicLevel - max_level_) / @@ -263,23 +312,23 @@ void MonoAgc::SetMaxLevel(int level) { << ", max_compression_gain_=" << max_compression_gain_; } -void MonoAgc::SetCaptureMuted(bool muted) { - if (capture_muted_ == muted) { +void MonoAgc::HandleCaptureOutputUsedChange(bool capture_output_used) { + if (capture_output_used_ == capture_output_used) { return; } - capture_muted_ = muted; + capture_output_used_ = capture_output_used; - if (!muted) { - // When we unmute, we should reset things to be safe. + if (capture_output_used) { + // When we start using the output, we should reset things to be safe. check_volume_on_next_process_ = true; } } int MonoAgc::CheckVolumeAndReset() { - int level = stream_analog_level_; + int level = recommended_input_volume_; // Reasons for taking action at startup: // 1) A person starting a call is expected to be heard. - // 2) Independent of interpretation of |level| == 0 we should raise it so the + // 2) Independent of interpretation of `level` == 0 we should raise it so the // AGC can do its job properly. if (level == 0 && !startup_) { RTC_DLOG(LS_INFO) @@ -293,31 +342,33 @@ int MonoAgc::CheckVolumeAndReset() { } RTC_DLOG(LS_INFO) << "[agc] Initial GetMicVolume()=" << level; - int minLevel = startup_ ? startup_min_level_ : min_mic_level_; - if (level < minLevel) { - level = minLevel; + if (level < min_mic_level_) { + level = min_mic_level_; RTC_DLOG(LS_INFO) << "[agc] Initial volume too low, raising to " << level; - stream_analog_level_ = level; + recommended_input_volume_ = level; } agc_->Reset(); level_ = level; startup_ = false; + frames_since_update_gain_ = 0; + is_first_frame_ = true; return 0; } -// Requests the RMS error from AGC and distributes the required gain change -// between the digital compression stage and volume slider. We use the -// compressor first, providing a slack region around the current slider -// position to reduce movement. +// Distributes the required gain change between the digital compression stage +// and volume slider. We use the compressor first, providing a slack region +// around the current slider position to reduce movement. // // If the slider needs to be moved, we check first if the user has adjusted // it, in which case we take no action and cache the updated level. -void MonoAgc::UpdateGain() { - int rms_error = 0; - if (!agc_->GetRmsErrorDb(&rms_error)) { - // No error update ready. - return; - } +void MonoAgc::UpdateGain(int rms_error_db) { + int rms_error = rms_error_db; + + // Always reset the counter regardless of whether the gain is changed + // or not. This matches with the bahvior of `agc_` where the histogram is + // reset every time an RMS error is successfully read. + frames_since_update_gain_ = 0; + // The compressor will always add at least kMinCompressionGain. In effect, // this adjusts our target gain upward by the same amount and rms_error // needs to reflect that. @@ -357,22 +408,12 @@ void MonoAgc::UpdateGain() { int old_level = level_; SetLevel(LevelFromGainError(residual_gain, level_, min_mic_level_)); if (old_level != level_) { - // level_ was updated by SetLevel; log the new value. - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.AgcSetLevel", level_, 1, - kMaxMicLevel, 50); // Reset the AGC since the level has changed. agc_->Reset(); } } void MonoAgc::UpdateCompressor() { - calls_since_last_gain_log_++; - if (calls_since_last_gain_log_ == 100) { - calls_since_last_gain_log_ = 0; - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc.DigitalGainApplied", - compression_, 0, kMaxCompressionGain, - kMaxCompressionGain + 1); - } if (compression_ == target_compression_) { return; } @@ -397,57 +438,66 @@ void MonoAgc::UpdateCompressor() { // Set the new compression gain. if (new_compression != compression_) { - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc.DigitalGainUpdated", - new_compression, 0, kMaxCompressionGain, - kMaxCompressionGain + 1); compression_ = new_compression; compression_accumulator_ = new_compression; new_compression_to_set_ = compression_; } } -int AgcManagerDirect::instance_counter_ = 0; +std::atomic AgcManagerDirect::instance_counter_(0); -AgcManagerDirect::AgcManagerDirect(Agc* agc, - int startup_min_level, - int clipped_level_min, - int sample_rate_hz) - : AgcManagerDirect(/*num_capture_channels*/ 1, - startup_min_level, - clipped_level_min, - /*use_agc2_level_estimation*/ false, - /*disable_digital_adaptive*/ false, - sample_rate_hz) { +AgcManagerDirect::AgcManagerDirect( + const AudioProcessing::Config::GainController1::AnalogGainController& + analog_config, + Agc* agc) + : AgcManagerDirect(/*num_capture_channels=*/1, analog_config) { RTC_DCHECK(channel_agcs_[0]); RTC_DCHECK(agc); channel_agcs_[0]->set_agc(agc); } AgcManagerDirect::AgcManagerDirect(int num_capture_channels, - int startup_min_level, - int clipped_level_min, - bool use_agc2_level_estimation, - bool disable_digital_adaptive, - int sample_rate_hz) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_counter_))), - use_min_channel_level_(!UseMaxAnalogChannelLevel()), - sample_rate_hz_(sample_rate_hz), + const AnalogAgcConfig& analog_config) + : analog_controller_enabled_(analog_config.enabled), + min_mic_level_override_(GetMinMicLevelOverride()), + data_dumper_(new ApmDataDumper(instance_counter_.fetch_add(1) + 1)), num_capture_channels_(num_capture_channels), - disable_digital_adaptive_(disable_digital_adaptive), - frames_since_clipped_(kClippedWaitFrames), - capture_muted_(false), + disable_digital_adaptive_(!analog_config.enable_digital_adaptive), + frames_since_clipped_(analog_config.clipped_wait_frames), + capture_output_used_(true), + clipped_level_step_(analog_config.clipped_level_step), + clipped_ratio_threshold_(analog_config.clipped_ratio_threshold), + clipped_wait_frames_(analog_config.clipped_wait_frames), channel_agcs_(num_capture_channels), - new_compressions_to_set_(num_capture_channels) { - const int min_mic_level = GetMinMicLevel(); + new_compressions_to_set_(num_capture_channels), + clipping_predictor_( + CreateClippingPredictor(num_capture_channels, + analog_config.clipping_predictor)), + use_clipping_predictor_step_( + !!clipping_predictor_ && + analog_config.clipping_predictor.use_predicted_step), + clipping_rate_log_(0.0f), + clipping_rate_log_counter_(0) { + RTC_LOG(LS_INFO) << "[agc] analog controller enabled: " + << (analog_controller_enabled_ ? "yes" : "no"); + const int min_mic_level = min_mic_level_override_.value_or(kMinMicLevel); + RTC_LOG(LS_INFO) << "[agc] Min mic level: " << min_mic_level + << " (overridden: " + << (min_mic_level_override_.has_value() ? "yes" : "no") + << ")"; for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { ApmDataDumper* data_dumper_ch = ch == 0 ? data_dumper_.get() : nullptr; channel_agcs_[ch] = std::make_unique( - data_dumper_ch, startup_min_level, clipped_level_min, - use_agc2_level_estimation, disable_digital_adaptive_, min_mic_level); + data_dumper_ch, analog_config.clipped_level_min, + disable_digital_adaptive_, min_mic_level); } - RTC_DCHECK_LT(0, channel_agcs_.size()); + RTC_DCHECK(!channel_agcs_.empty()); + RTC_DCHECK_GT(clipped_level_step_, 0); + RTC_DCHECK_LE(clipped_level_step_, 255); + RTC_DCHECK_GT(clipped_ratio_threshold_, 0.0f); + RTC_DCHECK_LT(clipped_ratio_threshold_, 1.0f); + RTC_DCHECK_GT(clipped_wait_frames_, 0); channel_agcs_[0]->ActivateLogging(); } @@ -459,48 +509,47 @@ void AgcManagerDirect::Initialize() { for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { channel_agcs_[ch]->Initialize(); } - capture_muted_ = false; + capture_output_used_ = true; AggregateChannelLevels(); + clipping_rate_log_ = 0.0f; + clipping_rate_log_counter_ = 0; } void AgcManagerDirect::SetupDigitalGainControl( - GainControl* gain_control) const { - RTC_DCHECK(gain_control); - if (gain_control->set_mode(GainControl::kFixedDigital) != 0) { + GainControl& gain_control) const { + if (gain_control.set_mode(GainControl::kFixedDigital) != 0) { RTC_LOG(LS_ERROR) << "set_mode(GainControl::kFixedDigital) failed."; } const int target_level_dbfs = disable_digital_adaptive_ ? 0 : 2; - if (gain_control->set_target_level_dbfs(target_level_dbfs) != 0) { + if (gain_control.set_target_level_dbfs(target_level_dbfs) != 0) { RTC_LOG(LS_ERROR) << "set_target_level_dbfs() failed."; } const int compression_gain_db = disable_digital_adaptive_ ? 0 : kDefaultCompressionGain; - if (gain_control->set_compression_gain_db(compression_gain_db) != 0) { + if (gain_control.set_compression_gain_db(compression_gain_db) != 0) { RTC_LOG(LS_ERROR) << "set_compression_gain_db() failed."; } const bool enable_limiter = !disable_digital_adaptive_; - if (gain_control->enable_limiter(enable_limiter) != 0) { + if (gain_control.enable_limiter(enable_limiter) != 0) { RTC_LOG(LS_ERROR) << "enable_limiter() failed."; } } -void AgcManagerDirect::AnalyzePreProcess(const AudioBuffer* audio) { +void AgcManagerDirect::AnalyzePreProcess(const AudioBuffer& audio_buffer) { + const float* const* audio = audio_buffer.channels_const(); + size_t samples_per_channel = audio_buffer.num_frames(); RTC_DCHECK(audio); - AnalyzePreProcess(audio->channels_const(), audio->num_frames()); -} -void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, - size_t samples_per_channel) { - RTC_DCHECK(audio); AggregateChannelLevels(); - if (capture_muted_) { + if (!capture_output_used_) { return; } - if (frames_since_clipped_ < kClippedWaitFrames) { - ++frames_since_clipped_; - return; + if (!!clipping_predictor_) { + AudioFrameView frame = AudioFrameView( + audio, num_capture_channels_, static_cast(samples_per_channel)); + clipping_predictor_->Analyze(frame); } // Check for clipped samples, as the AGC has difficulty detecting pitch @@ -514,55 +563,108 @@ void AgcManagerDirect::AnalyzePreProcess(const float* const* audio, // gain is increased, through SetMaxLevel(). float clipped_ratio = ComputeClippedRatio(audio, num_capture_channels_, samples_per_channel); + clipping_rate_log_ = std::max(clipped_ratio, clipping_rate_log_); + clipping_rate_log_counter_++; + constexpr int kNumFramesIn30Seconds = 3000; + if (clipping_rate_log_counter_ == kNumFramesIn30Seconds) { + LogClippingMetrics(std::round(100.0f * clipping_rate_log_)); + clipping_rate_log_ = 0.0f; + clipping_rate_log_counter_ = 0; + } - if (clipped_ratio > kClippedRatioThreshold) { + if (frames_since_clipped_ < clipped_wait_frames_) { + ++frames_since_clipped_; + return; + } + + const bool clipping_detected = clipped_ratio > clipped_ratio_threshold_; + bool clipping_predicted = false; + int predicted_step = 0; + if (!!clipping_predictor_) { + for (int channel = 0; channel < num_capture_channels_; ++channel) { + const auto step = clipping_predictor_->EstimateClippedLevelStep( + channel, recommended_input_volume_, clipped_level_step_, + channel_agcs_[channel]->min_mic_level(), kMaxMicLevel); + if (step.has_value()) { + predicted_step = std::max(predicted_step, step.value()); + clipping_predicted = true; + } + } + } + if (clipping_detected) { RTC_DLOG(LS_INFO) << "[agc] Clipping detected. clipped_ratio=" << clipped_ratio; + } + int step = clipped_level_step_; + if (clipping_predicted) { + predicted_step = std::max(predicted_step, clipped_level_step_); + RTC_DLOG(LS_INFO) << "[agc] Clipping predicted. step=" << predicted_step; + if (use_clipping_predictor_step_) { + step = predicted_step; + } + } + if (clipping_detected || + (clipping_predicted && use_clipping_predictor_step_)) { for (auto& state_ch : channel_agcs_) { - state_ch->HandleClipping(); + state_ch->HandleClipping(step); } frames_since_clipped_ = 0; + if (!!clipping_predictor_) { + clipping_predictor_->Reset(); + } } AggregateChannelLevels(); } -void AgcManagerDirect::Process(const AudioBuffer* audio) { - AggregateChannelLevels(); +void AgcManagerDirect::Process(const AudioBuffer& audio_buffer) { + Process(audio_buffer, /*speech_probability=*/absl::nullopt, + /*speech_level_dbfs=*/absl::nullopt); +} - if (capture_muted_) { +void AgcManagerDirect::Process(const AudioBuffer& audio_buffer, + absl::optional speech_probability, + absl::optional speech_level_dbfs) { + AggregateChannelLevels(); + const int volume_after_clipping_handling = recommended_input_volume_; + + if (!capture_output_used_) { return; } + const size_t num_frames_per_band = audio_buffer.num_frames_per_band(); + absl::optional rms_error_override = absl::nullopt; + if (speech_probability.has_value() && speech_level_dbfs.has_value()) { + rms_error_override = + GetSpeechLevelErrorDb(*speech_level_dbfs, *speech_probability); + } for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { - int16_t* audio_use = nullptr; std::array audio_data; - int num_frames_per_band; - if (audio) { - FloatS16ToS16(audio->split_bands_const_f(ch)[0], - audio->num_frames_per_band(), audio_data.data()); - audio_use = audio_data.data(); - num_frames_per_band = audio->num_frames_per_band(); - } else { - // Only used for testing. - // TODO(peah): Change unittests to only allow on non-null audio input. - num_frames_per_band = 320; - } - channel_agcs_[ch]->Process(audio_use, num_frames_per_band, sample_rate_hz_); + int16_t* audio_use = audio_data.data(); + FloatS16ToS16(audio_buffer.split_bands_const_f(ch)[0], num_frames_per_band, + audio_use); + channel_agcs_[ch]->Process({audio_use, num_frames_per_band}, + rms_error_override); new_compressions_to_set_[ch] = channel_agcs_[ch]->new_compression(); } AggregateChannelLevels(); + if (volume_after_clipping_handling != recommended_input_volume_) { + // The recommended input volume was adjusted in order to match the target + // level. + UpdateHistogramOnRecommendedInputVolumeChangeToMatchTarget( + recommended_input_volume_); + } } absl::optional AgcManagerDirect::GetDigitalComressionGain() { return new_compressions_to_set_[channel_controlling_gain_]; } -void AgcManagerDirect::SetCaptureMuted(bool muted) { +void AgcManagerDirect::HandleCaptureOutputUsedChange(bool capture_output_used) { for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { - channel_agcs_[ch]->SetCaptureMuted(muted); + channel_agcs_[ch]->HandleCaptureOutputUsedChange(capture_output_used); } - capture_muted_ = muted; + capture_output_used_ = capture_output_used; } float AgcManagerDirect::voice_probability() const { @@ -575,6 +677,10 @@ float AgcManagerDirect::voice_probability() const { } void AgcManagerDirect::set_stream_analog_level(int level) { + if (!analog_controller_enabled_) { + recommended_input_volume_ = level; + } + for (size_t ch = 0; ch < channel_agcs_.size(); ++ch) { channel_agcs_[ch]->set_stream_analog_level(level); } @@ -583,25 +689,25 @@ void AgcManagerDirect::set_stream_analog_level(int level) { } void AgcManagerDirect::AggregateChannelLevels() { - stream_analog_level_ = channel_agcs_[0]->stream_analog_level(); + int new_recommended_input_volume = + channel_agcs_[0]->recommended_analog_level(); channel_controlling_gain_ = 0; - if (use_min_channel_level_) { - for (size_t ch = 1; ch < channel_agcs_.size(); ++ch) { - int level = channel_agcs_[ch]->stream_analog_level(); - if (level < stream_analog_level_) { - stream_analog_level_ = level; - channel_controlling_gain_ = static_cast(ch); - } - } - } else { - for (size_t ch = 1; ch < channel_agcs_.size(); ++ch) { - int level = channel_agcs_[ch]->stream_analog_level(); - if (level > stream_analog_level_) { - stream_analog_level_ = level; - channel_controlling_gain_ = static_cast(ch); - } + for (size_t ch = 1; ch < channel_agcs_.size(); ++ch) { + int level = channel_agcs_[ch]->recommended_analog_level(); + if (level < new_recommended_input_volume) { + new_recommended_input_volume = level; + channel_controlling_gain_ = static_cast(ch); } } + + if (min_mic_level_override_.has_value() && new_recommended_input_volume > 0) { + new_recommended_input_volume = + std::max(new_recommended_input_volume, *min_mic_level_override_); + } + + if (analog_controller_enabled_) { + recommended_input_volume_ = new_recommended_input_volume; + } } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc/agc_manager_direct.h b/webrtc/modules/audio_processing/agc/agc_manager_direct.h index d3663be..adb2f5a 100644 --- a/webrtc/modules/audio_processing/agc/agc_manager_direct.h +++ b/webrtc/modules/audio_processing/agc/agc_manager_direct.h @@ -11,11 +11,15 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC_AGC_MANAGER_DIRECT_H_ #define MODULES_AUDIO_PROCESSING_AGC_AGC_MANAGER_DIRECT_H_ +#include #include #include "absl/types/optional.h" +#include "api/array_view.h" #include "modules/audio_processing/agc/agc.h" +#include "modules/audio_processing/agc2/clipping_predictor.h" #include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/gtest_prod_util.h" @@ -24,89 +28,168 @@ namespace webrtc { class MonoAgc; class GainControl; -// Direct interface to use AGC to set volume and compression values. -// AudioProcessing uses this interface directly to integrate the callback-less -// AGC. -// +// Adaptive Gain Controller (AGC) that controls the input volume and a digital +// gain. The input volume controller recommends what volume to use, handles +// volume changes and clipping. In particular, it handles changes triggered by +// the user (e.g., volume set to zero by a HW mute button). The digital +// controller chooses and applies the digital compression gain. // This class is not thread-safe. +// TODO(bugs.webrtc.org/7494): Use applied/recommended input volume naming +// convention. class AgcManagerDirect final { public: - // AgcManagerDirect will configure GainControl internally. The user is - // responsible for processing the audio using it after the call to Process. - // The operating range of startup_min_level is [12, 255] and any input value - // outside that range will be clamped. - AgcManagerDirect(int num_capture_channels, - int startup_min_level, - int clipped_level_min, - bool use_agc2_level_estimation, - bool disable_digital_adaptive, - int sample_rate_hz); + // Ctor. `num_capture_channels` specifies the number of channels for the audio + // passed to `AnalyzePreProcess()` and `Process()`. Clamps + // `analog_config.startup_min_level` in the [12, 255] range. + AgcManagerDirect( + int num_capture_channels, + const AudioProcessing::Config::GainController1::AnalogGainController& + analog_config); ~AgcManagerDirect(); AgcManagerDirect(const AgcManagerDirect&) = delete; AgcManagerDirect& operator=(const AgcManagerDirect&) = delete; void Initialize(); - void SetupDigitalGainControl(GainControl* gain_control) const; - void AnalyzePreProcess(const AudioBuffer* audio); - void Process(const AudioBuffer* audio); + // Configures `gain_control` to work as a fixed digital controller so that the + // adaptive part is only handled by this gain controller. Must be called if + // `gain_control` is also used to avoid the side-effects of running two AGCs. + void SetupDigitalGainControl(GainControl& gain_control) const; + + // Sets the applied input volume. + void set_stream_analog_level(int level); + + // TODO(bugs.webrtc.org/7494): Add argument for the applied input volume and + // remove `set_stream_analog_level()`. + // Analyzes `audio` before `Process()` is called so that the analysis can be + // performed before external digital processing operations take place (e.g., + // echo cancellation). The analysis consists of input clipping detection and + // prediction (if enabled). Must be called after `set_stream_analog_level()`. + void AnalyzePreProcess(const AudioBuffer& audio_buffer); + + // Processes `audio_buffer`. Chooses a digital compression gain and the new + // input volume to recommend. Must be called after `AnalyzePreProcess()`. If + // `speech_probability` (range [0.0f, 1.0f]) and `speech_level_dbfs` (range + // [-90.f, 30.0f]) are given, uses them to override the estimated RMS error. + // TODO(webrtc:7494): This signature is needed for testing purposes, unify + // the signatures when the clean-up is done. + void Process(const AudioBuffer& audio_buffer, + absl::optional speech_probability, + absl::optional speech_level_dbfs); + + // Processes `audio_buffer`. Chooses a digital compression gain and the new + // input volume to recommend. Must be called after `AnalyzePreProcess()`. + void Process(const AudioBuffer& audio_buffer); + + // TODO(bugs.webrtc.org/7494): Return recommended input volume and remove + // `recommended_analog_level()`. + // Returns the recommended input volume. If the input volume contoller is + // disabled, returns the input volume set via the latest + // `set_stream_analog_level()` call. Must be called after + // `AnalyzePreProcess()` and `Process()`. + int recommended_analog_level() const { return recommended_input_volume_; } + + // Call when the capture stream output has been flagged to be used/not-used. + // If unused, the manager disregards all incoming audio. + void HandleCaptureOutputUsedChange(bool capture_output_used); - // Call when the capture stream has been muted/unmuted. This causes the - // manager to disregard all incoming audio; chances are good it's background - // noise to which we'd like to avoid adapting. - void SetCaptureMuted(bool muted); float voice_probability() const; - int stream_analog_level() const { return stream_analog_level_; } - void set_stream_analog_level(int level); int num_channels() const { return num_capture_channels_; } - int sample_rate_hz() const { return sample_rate_hz_; } - // If available, returns a new compression gain for the digital gain control. + // If available, returns the latest digital compression gain that has been + // chosen. absl::optional GetDigitalComressionGain(); + // Returns true if clipping prediction is enabled. + bool clipping_predictor_enabled() const { return !!clipping_predictor_; } + + // Returns true if clipping prediction is used to adjust the input volume. + bool use_clipping_predictor_step() const { + return use_clipping_predictor_step_; + } + private: - friend class AgcManagerDirectTest; + friend class AgcManagerDirectTestHelper; - FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, - DisableDigitalDisablesDigital); - FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectStandaloneTest, - AgcMinMicLevelExperiment); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, DisableDigitalDisablesDigital); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, + AgcMinMicLevelExperimentDefault); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, + AgcMinMicLevelExperimentDisabled); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, + AgcMinMicLevelExperimentOutOfRangeAbove); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, + AgcMinMicLevelExperimentOutOfRangeBelow); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, + AgcMinMicLevelExperimentEnabled50); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectTest, + AgcMinMicLevelExperimentEnabledAboveStartupLevel); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectParametrizedTest, + ClippingParametersVerified); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectParametrizedTest, + DisableClippingPredictorDoesNotLowerVolume); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectParametrizedTest, + UsedClippingPredictionsProduceLowerAnalogLevels); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectParametrizedTest, + UnusedClippingPredictionsProduceEqualAnalogLevels); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectParametrizedTest, + EmptyRmsErrorOverrideHasNoEffect); + FRIEND_TEST_ALL_PREFIXES(AgcManagerDirectParametrizedTest, + NonEmptyRmsErrorOverrideHasEffect); - // Dependency injection for testing. Don't delete |agc| as the memory is owned - // by the manager. - AgcManagerDirect(Agc* agc, - int startup_min_level, - int clipped_level_min, - int sample_rate_hz); - - void AnalyzePreProcess(const float* const* audio, size_t samples_per_channel); + // Ctor that creates a single channel AGC and by injecting `agc`. + // `agc` will be owned by this class; hence, do not delete it. + AgcManagerDirect( + const AudioProcessing::Config::GainController1::AnalogGainController& + analog_config, + Agc* agc); void AggregateChannelLevels(); + const bool analog_controller_enabled_; + + const absl::optional min_mic_level_override_; std::unique_ptr data_dumper_; - static int instance_counter_; - const bool use_min_channel_level_; - const int sample_rate_hz_; + static std::atomic instance_counter_; const int num_capture_channels_; const bool disable_digital_adaptive_; int frames_since_clipped_; - int stream_analog_level_ = 0; - bool capture_muted_; + + // TODO(bugs.webrtc.org/7494): Create a separate member for the applied input + // volume. + // TODO(bugs.webrtc.org/7494): Once + // `AudioProcessingImpl::recommended_stream_analog_level()` becomes a trivial + // getter, leave uninitialized. + // Recommended input volume. After `set_stream_analog_level()` is called it + // holds the observed input volume. Possibly updated by `AnalyzePreProcess()` + // and `Process()`; after these calls, holds the recommended input volume. + int recommended_input_volume_ = 0; + + bool capture_output_used_; int channel_controlling_gain_ = 0; + const int clipped_level_step_; + const float clipped_ratio_threshold_; + const int clipped_wait_frames_; + std::vector> channel_agcs_; std::vector> new_compressions_to_set_; + + const std::unique_ptr clipping_predictor_; + const bool use_clipping_predictor_step_; + float clipping_rate_log_; + int clipping_rate_log_counter_; }; +// TODO(bugs.webrtc.org/7494): Use applied/recommended input volume naming +// convention. class MonoAgc { public: MonoAgc(ApmDataDumper* data_dumper, - int startup_min_level, int clipped_level_min, - bool use_agc2_level_estimation, bool disable_digital_adaptive, int min_mic_level); ~MonoAgc(); @@ -114,16 +197,27 @@ class MonoAgc { MonoAgc& operator=(const MonoAgc&) = delete; void Initialize(); - void SetCaptureMuted(bool muted); + void HandleCaptureOutputUsedChange(bool capture_output_used); - void HandleClipping(); + // Sets the current input volume. + void set_stream_analog_level(int level) { recommended_input_volume_ = level; } - void Process(const int16_t* audio, - size_t samples_per_channel, - int sample_rate_hz); + // Lowers the recommended input volume in response to clipping based on the + // suggested reduction `clipped_level_step`. Must be called after + // `set_stream_analog_level()`. + void HandleClipping(int clipped_level_step); + + // Analyzes `audio`, requests the RMS error from AGC, updates the recommended + // input volume based on the estimated speech level and, if enabled, updates + // the (digital) compression gain to be applied by `agc_`. Must be called + // after `HandleClipping()`. If `rms_error_override` has a value, RMS error + // from AGC is overridden by it. + void Process(rtc::ArrayView audio, + absl::optional rms_error_override); + + // Returns the recommended input volume. Must be called after `Process()`. + int recommended_analog_level() const { return recommended_input_volume_; } - void set_stream_analog_level(int level) { stream_analog_level_ = level; } - int stream_analog_level() const { return stream_analog_level_; } float voice_probability() const { return agc_->voice_probability(); } void ActivateLogging() { log_to_histograms_ = true; } absl::optional new_compression() const { @@ -133,20 +227,19 @@ class MonoAgc { // Only used for testing. void set_agc(Agc* agc) { agc_.reset(agc); } int min_mic_level() const { return min_mic_level_; } - int startup_min_level() const { return startup_min_level_; } private: - // Sets a new microphone level, after first checking that it hasn't been - // updated by the user, in which case no action is taken. + // Sets a new input volume, after first checking that it hasn't been updated + // by the user, in which case no action is taken. void SetLevel(int new_level); - // Set the maximum level the AGC is allowed to apply. Also updates the - // maximum compression gain to compensate. The level must be at least - // |kClippedLevelMin|. + // Set the maximum input volume the AGC is allowed to apply. Also updates the + // maximum compression gain to compensate. The volume must be at least + // `kClippedLevelMin`. void SetMaxLevel(int level); int CheckVolumeAndReset(); - void UpdateGain(); + void UpdateGain(int rms_error_db); void UpdateCompressor(); const int min_mic_level_; @@ -158,15 +251,26 @@ class MonoAgc { int target_compression_; int compression_; float compression_accumulator_; - bool capture_muted_ = false; + bool capture_output_used_ = true; bool check_volume_on_next_process_ = true; bool startup_ = true; - int startup_min_level_; - int calls_since_last_gain_log_ = 0; - int stream_analog_level_ = 0; + + // TODO(bugs.webrtc.org/7494): Create a separate member for the applied + // input volume. + // Recommended input volume. After `set_stream_analog_level()` is + // called, it holds the observed applied input volume. Possibly updated by + // `HandleClipping()` and `Process()`; after these calls, holds the + // recommended input volume. + int recommended_input_volume_ = 0; + absl::optional new_compression_to_set_; bool log_to_histograms_ = false; const int clipped_level_min_; + + // Frames since the last `UpdateGain()` call. + int frames_since_update_gain_ = 0; + // Set to true for the first frame after startup and reset, otherwise false. + bool is_first_frame_ = true; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc/gain_control.h b/webrtc/modules/audio_processing/agc/gain_control.h index f8c706b..389b211 100644 --- a/webrtc/modules/audio_processing/agc/gain_control.h +++ b/webrtc/modules/audio_processing/agc/gain_control.h @@ -20,12 +20,12 @@ namespace webrtc { // Recommended to be enabled on the client-side. class GainControl { public: - // When an analog mode is set, this must be called prior to |ProcessStream()| + // When an analog mode is set, this must be called prior to `ProcessStream()` // to pass the current analog level from the audio HAL. Must be within the - // range provided to |set_analog_level_limits()|. + // range provided to `set_analog_level_limits()`. virtual int set_stream_analog_level(int level) = 0; - // When an analog mode is set, this should be called after |ProcessStream()| + // When an analog mode is set, this should be called after `ProcessStream()` // to obtain the recommended new analog level for the audio HAL. It is the // users responsibility to apply this level. virtual int stream_analog_level() const = 0; @@ -33,7 +33,7 @@ class GainControl { enum Mode { // Adaptive mode intended for use if an analog volume control is available // on the capture device. It will require the user to provide coupling - // between the OS mixer controls and AGC through the |stream_analog_level()| + // between the OS mixer controls and AGC through the `stream_analog_level()` // functions. // // It consists of an analog gain prescription for the audio device and a @@ -61,7 +61,7 @@ class GainControl { virtual int set_mode(Mode mode) = 0; virtual Mode mode() const = 0; - // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels + // Sets the target peak `level` (or envelope) of the AGC in dBFs (decibels // from digital full-scale). The convention is to use positive values. For // instance, passing in a value of 3 corresponds to -3 dBFs, or a target // level 3 dB below full-scale. Limited to [0, 31]. @@ -71,7 +71,7 @@ class GainControl { virtual int set_target_level_dbfs(int level) = 0; virtual int target_level_dbfs() const = 0; - // Sets the maximum |gain| the digital compression stage may apply, in dB. A + // Sets the maximum `gain` the digital compression stage may apply, in dB. A // higher number corresponds to greater compression, while a value of 0 will // leave the signal uncompressed. Limited to [0, 90]. virtual int set_compression_gain_db(int gain) = 0; @@ -83,7 +83,7 @@ class GainControl { virtual int enable_limiter(bool enable) = 0; virtual bool is_limiter_enabled() const = 0; - // Sets the |minimum| and |maximum| analog levels of the audio capture device. + // Sets the `minimum` and `maximum` analog levels of the audio capture device. // Must be set if and only if an analog mode is used. Limited to [0, 65535]. virtual int set_analog_level_limits(int minimum, int maximum) = 0; virtual int analog_level_minimum() const = 0; diff --git a/webrtc/modules/audio_processing/agc/legacy/analog_agc.cc b/webrtc/modules/audio_processing/agc/legacy/analog_agc.cc index b53e3f9..e40a3f1 100644 --- a/webrtc/modules/audio_processing/agc/legacy/analog_agc.cc +++ b/webrtc/modules/audio_processing/agc/legacy/analog_agc.cc @@ -160,7 +160,7 @@ int WebRtcAgc_AddMic(void* state, /* apply slowly varying digital gain */ if (stt->micVol > stt->maxAnalog) { - /* |maxLevel| is strictly >= |micVol|, so this condition should be + /* `maxLevel` is strictly >= `micVol`, so this condition should be * satisfied here, ensuring there is no divide-by-zero. */ RTC_DCHECK_GT(stt->maxLevel, stt->maxAnalog); diff --git a/webrtc/modules/audio_processing/agc/legacy/analog_agc.h b/webrtc/modules/audio_processing/agc/legacy/analog_agc.h index 22cd924..7a231c8 100644 --- a/webrtc/modules/audio_processing/agc/legacy/analog_agc.h +++ b/webrtc/modules/audio_processing/agc/legacy/analog_agc.h @@ -11,7 +11,6 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC_LEGACY_ANALOG_AGC_H_ #define MODULES_AUDIO_PROCESSING_AGC_LEGACY_ANALOG_AGC_H_ - #include "modules/audio_processing/agc/legacy/digital_agc.h" #include "modules/audio_processing/agc/legacy/gain_control.h" @@ -63,7 +62,7 @@ typedef struct { int32_t upperSecondaryLimit; // = kRxxBufferLen * 2677832; -17 dBfs int32_t lowerSecondaryLimit; // = kRxxBufferLen * 267783; -27 dBfs uint16_t targetIdx; // Table index for corresponding target level - int16_t analogTarget; // Digital reference level in ENV scale + int16_t analogTarget; // Digital reference level in ENV scale // Analog AGC specific variables int32_t filterState[8]; // For downsampling wb to nb @@ -74,8 +73,8 @@ typedef struct { int32_t Rxx160_LPw32; // Low pass filtered frame energies int32_t Rxx16_LPw32Max; // Keeps track of largest energy subframe int32_t Rxx16_vectorw32[kRxxBufferLen]; // Array with subframe energies - int32_t Rxx16w32_array[2][5]; // Energy values of microphone signal - int32_t env[2][10]; // Envelope values of subframes + int32_t Rxx16w32_array[2][5]; // Energy values of microphone signal + int32_t env[2][10]; // Envelope values of subframes int16_t Rxx16pos; // Current position in the Rxx16_vectorw32 int16_t envSum; // Filtered scaled envelope in subframes diff --git a/webrtc/modules/audio_processing/agc/legacy/digital_agc.cc b/webrtc/modules/audio_processing/agc/legacy/digital_agc.cc index 185e849..4cd86ac 100644 --- a/webrtc/modules/audio_processing/agc/legacy/digital_agc.cc +++ b/webrtc/modules/audio_processing/agc/legacy/digital_agc.cc @@ -79,10 +79,9 @@ int32_t WebRtcAgc_CalculateGainTable(int32_t* gainTable, // Q16 uint16_t constMaxGain; uint16_t tmpU16, intPart, fracPart; const int16_t kCompRatio = 3; - const int16_t kSoftLimiterLeft = 1; int16_t limiterOffset = 0; // Limiter offset int16_t limiterIdx, limiterLvlX; - int16_t constLinApprox, zeroGainLvl, maxGain, diffGain; + int16_t constLinApprox, maxGain, diffGain; int16_t i, tmp16, tmp16no1; int zeros, zerosScale; @@ -98,17 +97,11 @@ int32_t WebRtcAgc_CalculateGainTable(int32_t* gainTable, // Q16 WebRtcSpl_DivW32W16ResW16(tmp32no1 + (kCompRatio >> 1), kCompRatio); maxGain = WEBRTC_SPL_MAX(tmp16no1, (analogTarget - targetLevelDbfs)); tmp32no1 = maxGain * kCompRatio; - zeroGainLvl = digCompGaindB; - zeroGainLvl -= WebRtcSpl_DivW32W16ResW16(tmp32no1 + ((kCompRatio - 1) >> 1), - kCompRatio - 1); if ((digCompGaindB <= analogTarget) && (limiterEnable)) { - zeroGainLvl += (analogTarget - digCompGaindB + kSoftLimiterLeft); limiterOffset = 0; } - // Calculate the difference between maximum gain and gain at 0dB0v: - // diffGain = maxGain + (compRatio-1)*zeroGainLvl/compRatio - // = (compRatio-1)*digCompGaindB/compRatio + // Calculate the difference between maximum gain and gain at 0dB0v tmp32no1 = digCompGaindB * (kCompRatio - 1); diffGain = WebRtcSpl_DivW32W16ResW16(tmp32no1 + (kCompRatio >> 1), kCompRatio); @@ -191,9 +184,9 @@ int32_t WebRtcAgc_CalculateGainTable(int32_t* gainTable, // Q16 numFIX -= (int32_t)logApprox * diffGain; // Q14 // Calculate ratio - // Shift |numFIX| as much as possible. - // Ensure we avoid wrap-around in |den| as well. - if (numFIX > (den >> 8) || -numFIX > (den >> 8)) { // |den| is Q8. + // Shift `numFIX` as much as possible. + // Ensure we avoid wrap-around in `den` as well. + if (numFIX > (den >> 8) || -numFIX > (den >> 8)) { // `den` is Q8. zeros = WebRtcSpl_NormW32(numFIX); } else { zeros = WebRtcSpl_NormW32(den) + 8; @@ -294,15 +287,12 @@ int32_t WebRtcAgc_ComputeDigitalGains(DigitalAgc* stt, int16_t gate, gain_adj; int16_t k; size_t n, L; - int16_t L2; // samples/subframe // determine number of samples per ms if (FS == 8000) { L = 8; - L2 = 3; } else if (FS == 16000 || FS == 32000 || FS == 48000) { L = 16; - L2 = 4; } else { return -1; } diff --git a/webrtc/modules/audio_processing/agc/legacy/gain_control.h b/webrtc/modules/audio_processing/agc/legacy/gain_control.h index abb8e63..6010a98 100644 --- a/webrtc/modules/audio_processing/agc/legacy/gain_control.h +++ b/webrtc/modules/audio_processing/agc/legacy/gain_control.h @@ -11,6 +11,9 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC_LEGACY_GAIN_CONTROL_H_ #define MODULES_AUDIO_PROCESSING_AGC_LEGACY_GAIN_CONTROL_H_ +#include +#include + namespace webrtc { enum { diff --git a/webrtc/modules/audio_processing/agc/loudness_histogram.cc b/webrtc/modules/audio_processing/agc/loudness_histogram.cc index 4775ff7..b0a1f53 100644 --- a/webrtc/modules/audio_processing/agc/loudness_histogram.cc +++ b/webrtc/modules/audio_processing/agc/loudness_histogram.cc @@ -114,7 +114,7 @@ void LoudnessHistogram::RemoveOldestEntryAndUpdate() { void LoudnessHistogram::RemoveTransient() { // Don't expect to be here if high-activity region is longer than - // |kTransientWidthThreshold| or there has not been any transient. + // `kTransientWidthThreshold` or there has not been any transient. RTC_DCHECK_LE(len_high_activity_, kTransientWidthThreshold); int index = (buffer_index_ > 0) ? (buffer_index_ - 1) : len_circular_buffer_ - 1; diff --git a/webrtc/modules/audio_processing/agc/loudness_histogram.h b/webrtc/modules/audio_processing/agc/loudness_histogram.h index badd443..51b3871 100644 --- a/webrtc/modules/audio_processing/agc/loudness_histogram.h +++ b/webrtc/modules/audio_processing/agc/loudness_histogram.h @@ -25,7 +25,7 @@ class LoudnessHistogram { static LoudnessHistogram* Create(); // Create a sliding LoudnessHistogram, i.e. the histogram represents the last - // |window_size| samples. + // `window_size` samples. static LoudnessHistogram* Create(int window_size); ~LoudnessHistogram(); @@ -49,7 +49,7 @@ class LoudnessHistogram { LoudnessHistogram(); explicit LoudnessHistogram(int window); - // Find the histogram bin associated with the given |rms|. + // Find the histogram bin associated with the given `rms`. int GetBinIndex(double rms); void RemoveOldestEntryAndUpdate(); @@ -63,10 +63,10 @@ class LoudnessHistogram { // Number of times the histogram is updated int num_updates_; // Audio content, this should be equal to the sum of the components of - // |bin_count_q10_|. + // `bin_count_q10_`. int64_t audio_content_q10_; - // LoudnessHistogram of input RMS in Q10 with |kHistSize_| bins. In each + // LoudnessHistogram of input RMS in Q10 with `kHistSize_` bins. In each // 'Update(),' we increment the associated histogram-bin with the given // probability. The increment is implemented in Q10 to avoid rounding errors. int64_t bin_count_q10_[kHistSize]; diff --git a/webrtc/modules/audio_processing/agc/mock_agc.h b/webrtc/modules/audio_processing/agc/mock_agc.h index 0ef41c6..3080e15 100644 --- a/webrtc/modules/audio_processing/agc/mock_agc.h +++ b/webrtc/modules/audio_processing/agc/mock_agc.h @@ -11,6 +11,7 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC_MOCK_AGC_H_ #define MODULES_AUDIO_PROCESSING_AGC_MOCK_AGC_H_ +#include "api/array_view.h" #include "modules/audio_processing/agc/agc.h" #include "test/gmock.h" @@ -19,10 +20,7 @@ namespace webrtc { class MockAgc : public Agc { public: virtual ~MockAgc() {} - MOCK_METHOD(void, - Process, - (const int16_t* audio, size_t length, int sample_rate_hz), - (override)); + MOCK_METHOD(void, Process, (rtc::ArrayView audio), (override)); MOCK_METHOD(bool, GetRmsErrorDb, (int* error), (override)); MOCK_METHOD(void, Reset, (), (override)); MOCK_METHOD(int, set_target_level_dbfs, (int level), (override)); diff --git a/webrtc/modules/audio_processing/agc2/BUILD.gn b/webrtc/modules/audio_processing/agc2/BUILD.gn index bf09533..bd59ad3 100644 --- a/webrtc/modules/audio_processing/agc2/BUILD.gn +++ b/webrtc/modules/audio_processing/agc2/BUILD.gn @@ -8,48 +8,39 @@ import("../../../webrtc.gni") -group("agc2") { - deps = [ - ":adaptive_digital", - ":fixed_digital", - ] -} - -rtc_library("level_estimation_agc") { +rtc_library("speech_level_estimator") { sources = [ - "adaptive_mode_level_estimator_agc.cc", - "adaptive_mode_level_estimator_agc.h", + "speech_level_estimator.cc", + "speech_level_estimator.h", ] + + visibility = [ + "..:gain_controller2", + "./*", + ] + configs += [ "..:apm_debug_dump" ] + deps = [ - ":adaptive_digital", ":common", - ":gain_applier", - ":noise_level_estimator", - ":rnn_vad_with_level", "..:api", "..:apm_logging", - "..:audio_frame_view", "../../../api:array_view", - "../../../common_audio", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:logging", "../../../rtc_base:safe_minmax", - "../agc:level_estimation", - "../vad", ] } -rtc_library("adaptive_digital") { +rtc_library("adaptive_digital_gain_controller") { sources = [ - "adaptive_agc.cc", - "adaptive_agc.h", - "adaptive_digital_gain_applier.cc", - "adaptive_digital_gain_applier.h", - "adaptive_mode_level_estimator.cc", - "adaptive_mode_level_estimator.h", - "saturation_protector.cc", - "saturation_protector.h", + "adaptive_digital_gain_controller.cc", + "adaptive_digital_gain_controller.h", + ] + + visibility = [ + "..:gain_controller2", + "./*", ] configs += [ "..:apm_debug_dump" ] @@ -57,20 +48,39 @@ rtc_library("adaptive_digital") { deps = [ ":common", ":gain_applier", - ":noise_level_estimator", - ":rnn_vad_with_level", "..:api", "..:apm_logging", "..:audio_frame_view", - "../../../api:array_view", "../../../common_audio", "../../../rtc_base:checks", "../../../rtc_base:logging", - "../../../rtc_base:rtc_base_approved", - "../../../rtc_base:safe_compare", "../../../rtc_base:safe_minmax", "../../../system_wrappers:metrics", ] +} + +rtc_library("saturation_protector") { + sources = [ + "saturation_protector.cc", + "saturation_protector.h", + "saturation_protector_buffer.cc", + "saturation_protector_buffer.h", + ] + + visibility = [ + "..:gain_controller2", + "./*", + ] + + configs += [ "..:apm_debug_dump" ] + + deps = [ + ":common", + "..:apm_logging", + "../../../rtc_base:checks", + "../../../rtc_base:safe_compare", + "../../../rtc_base:safe_minmax", + ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } @@ -83,10 +93,36 @@ rtc_library("biquad_filter") { ] deps = [ "../../../api:array_view", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:macromagic", ] } +rtc_library("clipping_predictor") { + visibility = [ + "../agc:agc", + "./*", + ] + + sources = [ + "clipping_predictor.cc", + "clipping_predictor.h", + "clipping_predictor_level_buffer.cc", + "clipping_predictor_level_buffer.h", + ] + + deps = [ + ":gain_map", + "..:api", + "..:audio_frame_view", + "../../../common_audio", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../../rtc_base:safe_minmax", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_source_set("common") { sources = [ "agc2_common.h" ] } @@ -101,6 +137,12 @@ rtc_library("fixed_digital") { "limiter.h", ] + visibility = [ + "..:gain_controller2", + "../../audio_mixer:audio_mixer_impl", + "./*", + ] + configs += [ "..:apm_debug_dump" ] deps = [ @@ -111,10 +153,12 @@ rtc_library("fixed_digital") { "../../../common_audio", "../../../rtc_base:checks", "../../../rtc_base:gtest_prod", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:safe_conversions", "../../../rtc_base:safe_minmax", + "../../../rtc_base:stringutils", "../../../system_wrappers:metrics", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("gain_applier") { @@ -122,6 +166,12 @@ rtc_library("gain_applier") { "gain_applier.cc", "gain_applier.h", ] + + visibility = [ + "..:gain_controller2", + "./*", + ] + deps = [ ":common", "..:audio_frame_view", @@ -130,39 +180,94 @@ rtc_library("gain_applier") { ] } +rtc_source_set("gain_map") { + visibility = [ + "..:analog_mic_simulation", + "../agc:agc", + "./*", + ] + + sources = [ "gain_map_internal.h" ] +} + +rtc_library("input_volume_controller") { + sources = [ + "input_volume_controller.cc", + "input_volume_controller.h", + "speech_probability_buffer.cc", + "speech_probability_buffer.h", + ] + + visibility = [ + "..:gain_controller2", + "./*", + ] + + configs += [ "..:apm_debug_dump" ] + + deps = [ + ":clipping_predictor", + ":gain_map", + ":input_volume_stats_reporter", + "..:api", + "..:audio_buffer", + "..:audio_frame_view", + "../../../api:array_view", + "../../../rtc_base:checks", + "../../../rtc_base:checks", + "../../../rtc_base:gtest_prod", + "../../../rtc_base:gtest_prod", + "../../../rtc_base:logging", + "../../../rtc_base:safe_minmax", + "../../../system_wrappers:field_trial", + "../../../system_wrappers:metrics", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("noise_level_estimator") { sources = [ - "down_sampler.cc", - "down_sampler.h", "noise_level_estimator.cc", "noise_level_estimator.h", - "noise_spectrum_estimator.cc", - "noise_spectrum_estimator.h", - "signal_classifier.cc", - "signal_classifier.h", ] deps = [ ":biquad_filter", "..:apm_logging", "..:audio_frame_view", "../../../api:array_view", - "../../../common_audio", - "../../../common_audio/third_party/ooura:fft_size_128", "../../../rtc_base:checks", - "../../../rtc_base:macromagic", "../../../system_wrappers", ] + visibility = [ + "..:gain_controller2", + "./*", + ] + configs += [ "..:apm_debug_dump" ] } -rtc_library("rnn_vad_with_level") { +rtc_library("vad_wrapper") { sources = [ - "vad_with_level.cc", - "vad_with_level.h", + "vad_wrapper.cc", + "vad_wrapper.h", ] + + visibility = [ + "..:gain_controller2", + "./*", + ] + + defines = [] + if (rtc_build_with_neon && current_cpu != "arm64") { + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + deps = [ ":common", + ":cpu_features", "..:audio_frame_view", "../../../api:array_view", "../../../common_audio", @@ -172,28 +277,85 @@ rtc_library("rnn_vad_with_level") { ] } -rtc_library("adaptive_digital_unittests") { +rtc_library("cpu_features") { + sources = [ + "cpu_features.cc", + "cpu_features.h", + ] + + visibility = [ + "..:gain_controller2", + "./*", + ] + + deps = [ + "../../../rtc_base:stringutils", + "../../../rtc_base/system:arch", + "../../../system_wrappers", + ] +} + +rtc_library("speech_level_estimator_unittest") { + testonly = true + configs += [ "..:apm_debug_dump" ] + + sources = [ "speech_level_estimator_unittest.cc" ] + deps = [ + ":common", + ":speech_level_estimator", + "..:api", + "..:apm_logging", + "../../../rtc_base:gunit_helpers", + "../../../test:test_support", + ] +} + +rtc_library("adaptive_digital_gain_controller_unittest") { + testonly = true + configs += [ "..:apm_debug_dump" ] + + sources = [ "adaptive_digital_gain_controller_unittest.cc" ] + + deps = [ + ":adaptive_digital_gain_controller", + ":common", + ":test_utils", + "..:api", + "..:apm_logging", + "..:audio_frame_view", + "../../../common_audio", + "../../../rtc_base:gunit_helpers", + "../../../test:test_support", + ] +} + +rtc_library("gain_applier_unittest") { + testonly = true + configs += [ "..:apm_debug_dump" ] + + sources = [ "gain_applier_unittest.cc" ] + deps = [ + ":gain_applier", + ":test_utils", + "..:audio_frame_view", + "../../../rtc_base:gunit_helpers", + "../../../test:test_support", + ] +} + +rtc_library("saturation_protector_unittest") { testonly = true configs += [ "..:apm_debug_dump" ] sources = [ - "adaptive_digital_gain_applier_unittest.cc", - "adaptive_mode_level_estimator_unittest.cc", - "gain_applier_unittest.cc", + "saturation_protector_buffer_unittest.cc", "saturation_protector_unittest.cc", ] deps = [ - ":adaptive_digital", ":common", - ":gain_applier", - ":test_utils", + ":saturation_protector", "..:apm_logging", - "..:audio_frame_view", - "../../../api:array_view", - "../../../common_audio", - "../../../rtc_base:checks", "../../../rtc_base:gunit_helpers", - "../../../rtc_base:rtc_base_approved", "../../../test:test_support", ] } @@ -232,38 +394,67 @@ rtc_library("fixed_digital_unittests") { "../../../common_audio", "../../../rtc_base:checks", "../../../rtc_base:gunit_helpers", - "../../../rtc_base:rtc_base_approved", "../../../system_wrappers:metrics", ] } +rtc_library("input_volume_controller_unittests") { + testonly = true + sources = [ + "clipping_predictor_level_buffer_unittest.cc", + "clipping_predictor_unittest.cc", + "input_volume_controller_unittest.cc", + "speech_probability_buffer_unittest.cc", + ] + + configs += [ "..:apm_debug_dump" ] + + deps = [ + ":clipping_predictor", + ":gain_map", + ":input_volume_controller", + "..:api", + "../../../api:array_view", + "../../../rtc_base:checks", + "../../../rtc_base:random", + "../../../rtc_base:safe_conversions", + "../../../rtc_base:safe_minmax", + "../../../rtc_base:stringutils", + "../../../system_wrappers:metrics", + "../../../test:field_trial", + "../../../test:fileutils", + "../../../test:test_support", + "//testing/gtest", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("noise_estimator_unittests") { testonly = true configs += [ "..:apm_debug_dump" ] - sources = [ - "noise_level_estimator_unittest.cc", - "signal_classifier_unittest.cc", - ] + sources = [ "noise_level_estimator_unittest.cc" ] deps = [ ":noise_level_estimator", ":test_utils", "..:apm_logging", "..:audio_frame_view", "../../../api:array_view", + "../../../api:function_view", "../../../rtc_base:checks", "../../../rtc_base:gunit_helpers", - "../../../rtc_base:rtc_base_approved", ] } -rtc_library("rnn_vad_with_level_unittests") { +rtc_library("vad_wrapper_unittests") { testonly = true - sources = [ "vad_with_level_unittest.cc" ] + sources = [ "vad_wrapper_unittest.cc" ] deps = [ ":common", - ":rnn_vad_with_level", + ":vad_wrapper", "..:audio_frame_view", + "../../../rtc_base:checks", "../../../rtc_base:gunit_helpers", "../../../rtc_base:safe_compare", "../../../test:test_support", @@ -285,6 +476,36 @@ rtc_library("test_utils") { deps = [ "..:audio_frame_view", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", + "../../../rtc_base:random", ] } + +rtc_library("input_volume_stats_reporter") { + sources = [ + "input_volume_stats_reporter.cc", + "input_volume_stats_reporter.h", + ] + deps = [ + "../../../rtc_base:gtest_prod", + "../../../rtc_base:logging", + "../../../rtc_base:safe_minmax", + "../../../rtc_base:stringutils", + "../../../system_wrappers:metrics", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("input_volume_stats_reporter_unittests") { + testonly = true + sources = [ "input_volume_stats_reporter_unittest.cc" ] + deps = [ + ":input_volume_stats_reporter", + "../../../rtc_base:stringutils", + "../../../system_wrappers:metrics", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} diff --git a/webrtc/modules/audio_processing/agc2/adaptive_agc.cc b/webrtc/modules/audio_processing/agc2/adaptive_agc.cc deleted file mode 100644 index 0372ccf..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_agc.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/adaptive_agc.h" - -#include "common_audio/include/audio_util.h" -#include "modules/audio_processing/agc2/vad_with_level.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" - -namespace webrtc { -namespace { - -void DumpDebugData(const AdaptiveDigitalGainApplier::FrameInfo& info, - ApmDataDumper& dumper) { - dumper.DumpRaw("agc2_vad_probability", info.vad_result.speech_probability); - dumper.DumpRaw("agc2_vad_rms_dbfs", info.vad_result.rms_dbfs); - dumper.DumpRaw("agc2_vad_peak_dbfs", info.vad_result.peak_dbfs); - dumper.DumpRaw("agc2_noise_estimate_dbfs", info.input_noise_level_dbfs); - dumper.DumpRaw("agc2_last_limiter_audio_level", info.limiter_envelope_dbfs); -} - -constexpr int kGainApplierAdjacentSpeechFramesThreshold = 1; -constexpr float kMaxGainChangePerSecondDb = 3.f; -constexpr float kMaxOutputNoiseLevelDbfs = -50.f; - -} // namespace - -AdaptiveAgc::AdaptiveAgc(ApmDataDumper* apm_data_dumper) - : speech_level_estimator_(apm_data_dumper), - gain_applier_(apm_data_dumper, - kGainApplierAdjacentSpeechFramesThreshold, - kMaxGainChangePerSecondDb, - kMaxOutputNoiseLevelDbfs), - apm_data_dumper_(apm_data_dumper), - noise_level_estimator_(apm_data_dumper) { - RTC_DCHECK(apm_data_dumper); -} - -AdaptiveAgc::AdaptiveAgc(ApmDataDumper* apm_data_dumper, - const AudioProcessing::Config::GainController2& config) - : speech_level_estimator_( - apm_data_dumper, - config.adaptive_digital.level_estimator, - config.adaptive_digital - .level_estimator_adjacent_speech_frames_threshold, - config.adaptive_digital.initial_saturation_margin_db, - config.adaptive_digital.extra_saturation_margin_db), - vad_(config.adaptive_digital.vad_probability_attack), - gain_applier_( - apm_data_dumper, - config.adaptive_digital.gain_applier_adjacent_speech_frames_threshold, - config.adaptive_digital.max_gain_change_db_per_second, - config.adaptive_digital.max_output_noise_level_dbfs), - apm_data_dumper_(apm_data_dumper), - noise_level_estimator_(apm_data_dumper) { - RTC_DCHECK(apm_data_dumper); - if (!config.adaptive_digital.use_saturation_protector) { - RTC_LOG(LS_WARNING) << "The saturation protector cannot be disabled."; - } -} - -AdaptiveAgc::~AdaptiveAgc() = default; - -void AdaptiveAgc::Process(AudioFrameView frame, float limiter_envelope) { - AdaptiveDigitalGainApplier::FrameInfo info; - info.vad_result = vad_.AnalyzeFrame(frame); - speech_level_estimator_.Update(info.vad_result); - info.input_level_dbfs = speech_level_estimator_.level_dbfs(); - info.input_noise_level_dbfs = noise_level_estimator_.Analyze(frame); - info.limiter_envelope_dbfs = - limiter_envelope > 0 ? FloatS16ToDbfs(limiter_envelope) : -90.f; - info.estimate_is_confident = speech_level_estimator_.IsConfident(); - DumpDebugData(info, *apm_data_dumper_); - gain_applier_.Process(info, frame); -} - -void AdaptiveAgc::Reset() { - speech_level_estimator_.Reset(); -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/adaptive_agc.h b/webrtc/modules/audio_processing/agc2/adaptive_agc.h deleted file mode 100644 index f3c7854..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_agc.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_AGC_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_AGC_H_ - -#include "modules/audio_processing/agc2/adaptive_digital_gain_applier.h" -#include "modules/audio_processing/agc2/adaptive_mode_level_estimator.h" -#include "modules/audio_processing/agc2/noise_level_estimator.h" -#include "modules/audio_processing/agc2/vad_with_level.h" -#include "modules/audio_processing/include/audio_frame_view.h" -#include "modules/audio_processing/include/audio_processing.h" - -namespace webrtc { -class ApmDataDumper; - -// Adaptive digital gain controller. -// TODO(crbug.com/webrtc/7494): Unify with `AdaptiveDigitalGainApplier`. -class AdaptiveAgc { - public: - explicit AdaptiveAgc(ApmDataDumper* apm_data_dumper); - // TODO(crbug.com/webrtc/7494): Remove ctor above. - AdaptiveAgc(ApmDataDumper* apm_data_dumper, - const AudioProcessing::Config::GainController2& config); - ~AdaptiveAgc(); - - // Analyzes `frame` and applies a digital adaptive gain to it. Takes into - // account the envelope measured by the limiter. - // TODO(crbug.com/webrtc/7494): Make the class depend on the limiter. - void Process(AudioFrameView frame, float limiter_envelope); - void Reset(); - - private: - AdaptiveModeLevelEstimator speech_level_estimator_; - VadLevelAnalyzer vad_; - AdaptiveDigitalGainApplier gain_applier_; - ApmDataDumper* const apm_data_dumper_; - NoiseLevelEstimator noise_level_estimator_; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_AGC_H_ diff --git a/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc b/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc deleted file mode 100644 index 36ef9be..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/adaptive_digital_gain_applier.h" - -#include - -#include "common_audio/include/audio_util.h" -#include "modules/audio_processing/agc2/agc2_common.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/numerics/safe_minmax.h" -#include "system_wrappers/include/metrics.h" - -namespace webrtc { -namespace { - -// This function maps input level to desired applied gain. We want to -// boost the signal so that peaks are at -kHeadroomDbfs. We can't -// apply more than kMaxGainDb gain. -float ComputeGainDb(float input_level_dbfs) { - // If the level is very low, boost it as much as we can. - if (input_level_dbfs < -(kHeadroomDbfs + kMaxGainDb)) { - return kMaxGainDb; - } - - // We expect to end up here most of the time: the level is below - // -headroom, but we can boost it to -headroom. - if (input_level_dbfs < -kHeadroomDbfs) { - return -kHeadroomDbfs - input_level_dbfs; - } - - // Otherwise, the level is too high and we can't boost. The - // LevelEstimator is responsible for not reporting bogus gain - // values. - RTC_DCHECK_LE(input_level_dbfs, 0.f); - return 0.f; -} - -// Returns `target_gain` if the output noise level is below -// `max_output_noise_level_dbfs`; otherwise returns a capped gain so that the -// output noise level equals `max_output_noise_level_dbfs`. -float LimitGainByNoise(float target_gain, - float input_noise_level_dbfs, - float max_output_noise_level_dbfs, - ApmDataDumper& apm_data_dumper) { - const float noise_headroom_db = - max_output_noise_level_dbfs - input_noise_level_dbfs; - apm_data_dumper.DumpRaw("agc2_noise_headroom_db", noise_headroom_db); - return std::min(target_gain, std::max(noise_headroom_db, 0.f)); -} - -float LimitGainByLowConfidence(float target_gain, - float last_gain, - float limiter_audio_level_dbfs, - bool estimate_is_confident) { - if (estimate_is_confident || - limiter_audio_level_dbfs <= kLimiterThresholdForAgcGainDbfs) { - return target_gain; - } - const float limiter_level_before_gain = limiter_audio_level_dbfs - last_gain; - - // Compute a new gain so that limiter_level_before_gain + new_gain <= - // kLimiterThreshold. - const float new_target_gain = std::max( - kLimiterThresholdForAgcGainDbfs - limiter_level_before_gain, 0.f); - return std::min(new_target_gain, target_gain); -} - -// Computes how the gain should change during this frame. -// Return the gain difference in db to 'last_gain_db'. -float ComputeGainChangeThisFrameDb(float target_gain_db, - float last_gain_db, - bool gain_increase_allowed, - float max_gain_change_db) { - float target_gain_difference_db = target_gain_db - last_gain_db; - if (!gain_increase_allowed) { - target_gain_difference_db = std::min(target_gain_difference_db, 0.f); - } - return rtc::SafeClamp(target_gain_difference_db, -max_gain_change_db, - max_gain_change_db); -} - -} // namespace - -AdaptiveDigitalGainApplier::AdaptiveDigitalGainApplier( - ApmDataDumper* apm_data_dumper, - int adjacent_speech_frames_threshold, - float max_gain_change_db_per_second, - float max_output_noise_level_dbfs) - : apm_data_dumper_(apm_data_dumper), - gain_applier_( - /*hard_clip_samples=*/false, - /*initial_gain_factor=*/DbToRatio(kInitialAdaptiveDigitalGainDb)), - adjacent_speech_frames_threshold_(adjacent_speech_frames_threshold), - max_gain_change_db_per_10ms_(max_gain_change_db_per_second * - kFrameDurationMs / 1000.f), - max_output_noise_level_dbfs_(max_output_noise_level_dbfs), - calls_since_last_gain_log_(0), - frames_to_gain_increase_allowed_(adjacent_speech_frames_threshold_), - last_gain_db_(kInitialAdaptiveDigitalGainDb) { - RTC_DCHECK_GT(max_gain_change_db_per_second, 0.f); - RTC_DCHECK_GE(frames_to_gain_increase_allowed_, 1); - RTC_DCHECK_GE(max_output_noise_level_dbfs_, -90.f); - RTC_DCHECK_LE(max_output_noise_level_dbfs_, 0.f); -} - -void AdaptiveDigitalGainApplier::Process(const FrameInfo& info, - AudioFrameView frame) { - RTC_DCHECK_GE(info.input_level_dbfs, -150.f); - RTC_DCHECK_GE(frame.num_channels(), 1); - RTC_DCHECK( - frame.samples_per_channel() == 80 || frame.samples_per_channel() == 160 || - frame.samples_per_channel() == 320 || frame.samples_per_channel() == 480) - << "`frame` does not look like a 10 ms frame for an APM supported sample " - "rate"; - - const float target_gain_db = LimitGainByLowConfidence( - LimitGainByNoise(ComputeGainDb(std::min(info.input_level_dbfs, 0.f)), - info.input_noise_level_dbfs, - max_output_noise_level_dbfs_, *apm_data_dumper_), - last_gain_db_, info.limiter_envelope_dbfs, info.estimate_is_confident); - - // Forbid increasing the gain until enough adjacent speech frames are - // observed. - if (info.vad_result.speech_probability < kVadConfidenceThreshold) { - frames_to_gain_increase_allowed_ = adjacent_speech_frames_threshold_; - } else if (frames_to_gain_increase_allowed_ > 0) { - frames_to_gain_increase_allowed_--; - } - - const float gain_change_this_frame_db = ComputeGainChangeThisFrameDb( - target_gain_db, last_gain_db_, - /*gain_increase_allowed=*/frames_to_gain_increase_allowed_ == 0, - max_gain_change_db_per_10ms_); - - apm_data_dumper_->DumpRaw("agc2_want_to_change_by_db", - target_gain_db - last_gain_db_); - apm_data_dumper_->DumpRaw("agc2_will_change_by_db", - gain_change_this_frame_db); - - // Optimization: avoid calling math functions if gain does not - // change. - if (gain_change_this_frame_db != 0.f) { - gain_applier_.SetGainFactor( - DbToRatio(last_gain_db_ + gain_change_this_frame_db)); - } - gain_applier_.ApplyGain(frame); - - // Remember that the gain has changed for the next iteration. - last_gain_db_ = last_gain_db_ + gain_change_this_frame_db; - apm_data_dumper_->DumpRaw("agc2_applied_gain_db", last_gain_db_); - - // Log every 10 seconds. - calls_since_last_gain_log_++; - if (calls_since_last_gain_log_ == 1000) { - calls_since_last_gain_log_ = 0; - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc2.DigitalGainApplied", - last_gain_db_, 0, kMaxGainDb, kMaxGainDb + 1); - RTC_HISTOGRAM_COUNTS_LINEAR( - "WebRTC.Audio.Agc2.EstimatedSpeechPlusNoiseLevel", - -info.input_level_dbfs, 0, 100, 101); - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc2.EstimatedNoiseLevel", - -info.input_noise_level_dbfs, 0, 100, 101); - RTC_LOG(LS_INFO) << "AGC2 adaptive digital" - << " | speech_plus_noise_dbfs: " << info.input_level_dbfs - << " | noise_dbfs: " << info.input_noise_level_dbfs - << " | gain_db: " << last_gain_db_; - } -} -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.h b/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.h deleted file mode 100644 index a65379f..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_applier.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_APPLIER_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_APPLIER_H_ - -#include "modules/audio_processing/agc2/gain_applier.h" -#include "modules/audio_processing/agc2/vad_with_level.h" -#include "modules/audio_processing/include/audio_frame_view.h" - -namespace webrtc { - -class ApmDataDumper; - -// Part of the adaptive digital controller that applies a digital adaptive gain. -// The gain is updated towards a target. The logic decides when gain updates are -// allowed, it controls the adaptation speed and caps the target based on the -// estimated noise level and the speech level estimate confidence. -class AdaptiveDigitalGainApplier { - public: - // Information about a frame to process. - struct FrameInfo { - float input_level_dbfs; // Estimated speech plus noise level. - float input_noise_level_dbfs; // Estimated noise level. - VadLevelAnalyzer::Result vad_result; - float limiter_envelope_dbfs; // Envelope level from the limiter. - bool estimate_is_confident; - }; - - // Ctor. - // `adjacent_speech_frames_threshold` indicates how many speech frames are - // required before a gain increase is allowed. `max_gain_change_db_per_second` - // limits the adaptation speed (uniformly operated across frames). - // `max_output_noise_level_dbfs` limits the output noise level. - AdaptiveDigitalGainApplier(ApmDataDumper* apm_data_dumper, - int adjacent_speech_frames_threshold, - float max_gain_change_db_per_second, - float max_output_noise_level_dbfs); - AdaptiveDigitalGainApplier(const AdaptiveDigitalGainApplier&) = delete; - AdaptiveDigitalGainApplier& operator=(const AdaptiveDigitalGainApplier&) = - delete; - - // Analyzes `info`, updates the digital gain and applies it to a 10 ms - // `frame`. Supports any sample rate supported by APM. - void Process(const FrameInfo& info, AudioFrameView frame); - - private: - ApmDataDumper* const apm_data_dumper_; - GainApplier gain_applier_; - - const int adjacent_speech_frames_threshold_; - const float max_gain_change_db_per_10ms_; - const float max_output_noise_level_dbfs_; - - int calls_since_last_gain_log_; - int frames_to_gain_increase_allowed_; - float last_gain_db_; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_APPLIER_H_ diff --git a/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.cc b/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.cc new file mode 100644 index 0000000..e8edab6 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.cc @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/adaptive_digital_gain_controller.h" + +#include + +#include "common_audio/include/audio_util.h" +#include "modules/audio_processing/agc2/agc2_common.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { + +using AdaptiveDigitalConfig = + AudioProcessing::Config::GainController2::AdaptiveDigital; + +constexpr int kHeadroomHistogramMin = 0; +constexpr int kHeadroomHistogramMax = 50; +constexpr int kGainDbHistogramMax = 30; + +// Computes the gain for `input_level_dbfs` to reach `-config.headroom_db`. +// Clamps the gain in [0, `config.max_gain_db`]. `config.headroom_db` is a +// safety margin to allow transient peaks to exceed the target peak level +// without clipping. +float ComputeGainDb(float input_level_dbfs, + const AdaptiveDigitalConfig& config) { + // If the level is very low, apply the maximum gain. + if (input_level_dbfs < -(config.headroom_db + config.max_gain_db)) { + return config.max_gain_db; + } + // We expect to end up here most of the time: the level is below + // -headroom, but we can boost it to -headroom. + if (input_level_dbfs < -config.headroom_db) { + return -config.headroom_db - input_level_dbfs; + } + // The level is too high and we can't boost. + RTC_DCHECK_GE(input_level_dbfs, -config.headroom_db); + return 0.0f; +} + +// Returns `target_gain_db` if applying such a gain to `input_noise_level_dbfs` +// does not exceed `max_output_noise_level_dbfs`. Otherwise lowers and returns +// `target_gain_db` so that the output noise level equals +// `max_output_noise_level_dbfs`. +float LimitGainByNoise(float target_gain_db, + float input_noise_level_dbfs, + float max_output_noise_level_dbfs, + ApmDataDumper& apm_data_dumper) { + const float max_allowed_gain_db = + max_output_noise_level_dbfs - input_noise_level_dbfs; + apm_data_dumper.DumpRaw("agc2_adaptive_gain_applier_max_allowed_gain_db", + max_allowed_gain_db); + return std::min(target_gain_db, std::max(max_allowed_gain_db, 0.0f)); +} + +float LimitGainByLowConfidence(float target_gain_db, + float last_gain_db, + float limiter_audio_level_dbfs, + bool estimate_is_confident) { + if (estimate_is_confident || + limiter_audio_level_dbfs <= kLimiterThresholdForAgcGainDbfs) { + return target_gain_db; + } + const float limiter_level_dbfs_before_gain = + limiter_audio_level_dbfs - last_gain_db; + + // Compute a new gain so that `limiter_level_dbfs_before_gain` + + // `new_target_gain_db` is not great than `kLimiterThresholdForAgcGainDbfs`. + const float new_target_gain_db = std::max( + kLimiterThresholdForAgcGainDbfs - limiter_level_dbfs_before_gain, 0.0f); + return std::min(new_target_gain_db, target_gain_db); +} + +// Computes how the gain should change during this frame. +// Return the gain difference in db to 'last_gain_db'. +float ComputeGainChangeThisFrameDb(float target_gain_db, + float last_gain_db, + bool gain_increase_allowed, + float max_gain_decrease_db, + float max_gain_increase_db) { + RTC_DCHECK_GT(max_gain_decrease_db, 0); + RTC_DCHECK_GT(max_gain_increase_db, 0); + float target_gain_difference_db = target_gain_db - last_gain_db; + if (!gain_increase_allowed) { + target_gain_difference_db = std::min(target_gain_difference_db, 0.0f); + } + return rtc::SafeClamp(target_gain_difference_db, -max_gain_decrease_db, + max_gain_increase_db); +} + +} // namespace + +AdaptiveDigitalGainController::AdaptiveDigitalGainController( + ApmDataDumper* apm_data_dumper, + const AudioProcessing::Config::GainController2::AdaptiveDigital& config, + int adjacent_speech_frames_threshold) + : apm_data_dumper_(apm_data_dumper), + gain_applier_( + /*hard_clip_samples=*/false, + /*initial_gain_factor=*/DbToRatio(config.initial_gain_db)), + config_(config), + adjacent_speech_frames_threshold_(adjacent_speech_frames_threshold), + max_gain_change_db_per_10ms_(config_.max_gain_change_db_per_second * + kFrameDurationMs / 1000.0f), + calls_since_last_gain_log_(0), + frames_to_gain_increase_allowed_(adjacent_speech_frames_threshold), + last_gain_db_(config_.initial_gain_db) { + RTC_DCHECK_GT(max_gain_change_db_per_10ms_, 0.0f); + RTC_DCHECK_GE(frames_to_gain_increase_allowed_, 1); + RTC_DCHECK_GE(config_.max_output_noise_level_dbfs, -90.0f); + RTC_DCHECK_LE(config_.max_output_noise_level_dbfs, 0.0f); +} + +void AdaptiveDigitalGainController::Process(const FrameInfo& info, + AudioFrameView frame) { + RTC_DCHECK_GE(info.speech_level_dbfs, -150.0f); + RTC_DCHECK_GE(frame.num_channels(), 1); + RTC_DCHECK( + frame.samples_per_channel() == 80 || frame.samples_per_channel() == 160 || + frame.samples_per_channel() == 320 || frame.samples_per_channel() == 480) + << "`frame` does not look like a 10 ms frame for an APM supported sample " + "rate"; + + // Compute the input level used to select the desired gain. + RTC_DCHECK_GT(info.headroom_db, 0.0f); + const float input_level_dbfs = info.speech_level_dbfs + info.headroom_db; + + const float target_gain_db = LimitGainByLowConfidence( + LimitGainByNoise(ComputeGainDb(input_level_dbfs, config_), + info.noise_rms_dbfs, config_.max_output_noise_level_dbfs, + *apm_data_dumper_), + last_gain_db_, info.limiter_envelope_dbfs, info.speech_level_reliable); + + // Forbid increasing the gain until enough adjacent speech frames are + // observed. + bool first_confident_speech_frame = false; + if (info.speech_probability < kVadConfidenceThreshold) { + frames_to_gain_increase_allowed_ = adjacent_speech_frames_threshold_; + } else if (frames_to_gain_increase_allowed_ > 0) { + frames_to_gain_increase_allowed_--; + first_confident_speech_frame = frames_to_gain_increase_allowed_ == 0; + } + apm_data_dumper_->DumpRaw( + "agc2_adaptive_gain_applier_frames_to_gain_increase_allowed", + frames_to_gain_increase_allowed_); + + const bool gain_increase_allowed = frames_to_gain_increase_allowed_ == 0; + + float max_gain_increase_db = max_gain_change_db_per_10ms_; + if (first_confident_speech_frame) { + // No gain increase happened while waiting for a long enough speech + // sequence. Therefore, temporarily allow a faster gain increase. + RTC_DCHECK(gain_increase_allowed); + max_gain_increase_db *= adjacent_speech_frames_threshold_; + } + + const float gain_change_this_frame_db = ComputeGainChangeThisFrameDb( + target_gain_db, last_gain_db_, gain_increase_allowed, + /*max_gain_decrease_db=*/max_gain_change_db_per_10ms_, + max_gain_increase_db); + + apm_data_dumper_->DumpRaw("agc2_adaptive_gain_applier_want_to_change_by_db", + target_gain_db - last_gain_db_); + apm_data_dumper_->DumpRaw("agc2_adaptive_gain_applier_will_change_by_db", + gain_change_this_frame_db); + + // Optimization: avoid calling math functions if gain does not + // change. + if (gain_change_this_frame_db != 0.f) { + gain_applier_.SetGainFactor( + DbToRatio(last_gain_db_ + gain_change_this_frame_db)); + } + + gain_applier_.ApplyGain(frame); + + // Remember that the gain has changed for the next iteration. + last_gain_db_ = last_gain_db_ + gain_change_this_frame_db; + apm_data_dumper_->DumpRaw("agc2_adaptive_gain_applier_applied_gain_db", + last_gain_db_); + + // Log every 10 seconds. + calls_since_last_gain_log_++; + if (calls_since_last_gain_log_ == 1000) { + calls_since_last_gain_log_ = 0; + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc2.EstimatedSpeechLevel", + -info.speech_level_dbfs, 0, 100, 101); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc2.EstimatedNoiseLevel", + -info.noise_rms_dbfs, 0, 100, 101); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.Agc2.Headroom", info.headroom_db, kHeadroomHistogramMin, + kHeadroomHistogramMax, + kHeadroomHistogramMax - kHeadroomHistogramMin + 1); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.Agc2.DigitalGainApplied", + last_gain_db_, 0, kGainDbHistogramMax, + kGainDbHistogramMax + 1); + RTC_LOG(LS_INFO) << "AGC2 adaptive digital" + << " | speech_dbfs: " << info.speech_level_dbfs + << " | noise_dbfs: " << info.noise_rms_dbfs + << " | headroom_db: " << info.headroom_db + << " | gain_db: " << last_gain_db_; + } +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.h b/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.h new file mode 100644 index 0000000..01335e7 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_CONTROLLER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_CONTROLLER_H_ + +#include + +#include "modules/audio_processing/agc2/gain_applier.h" +#include "modules/audio_processing/include/audio_frame_view.h" +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { + +class ApmDataDumper; + +// Selects the target digital gain, decides when and how quickly to adapt to the +// target and applies the current gain to 10 ms frames. +class AdaptiveDigitalGainController { + public: + // Information about a frame to process. + struct FrameInfo { + float speech_probability; // Probability of speech in the [0, 1] range. + float speech_level_dbfs; // Estimated speech level (dBFS). + bool speech_level_reliable; // True with reliable speech level estimation. + float noise_rms_dbfs; // Estimated noise RMS level (dBFS). + float headroom_db; // Headroom (dB). + // TODO(bugs.webrtc.org/7494): Remove `limiter_envelope_dbfs`. + float limiter_envelope_dbfs; // Envelope level from the limiter (dBFS). + }; + + AdaptiveDigitalGainController( + ApmDataDumper* apm_data_dumper, + const AudioProcessing::Config::GainController2::AdaptiveDigital& config, + int adjacent_speech_frames_threshold); + AdaptiveDigitalGainController(const AdaptiveDigitalGainController&) = delete; + AdaptiveDigitalGainController& operator=( + const AdaptiveDigitalGainController&) = delete; + + // Analyzes `info`, updates the digital gain and applies it to a 10 ms + // `frame`. Supports any sample rate supported by APM. + void Process(const FrameInfo& info, AudioFrameView frame); + + private: + ApmDataDumper* const apm_data_dumper_; + GainApplier gain_applier_; + + const AudioProcessing::Config::GainController2::AdaptiveDigital config_; + const int adjacent_speech_frames_threshold_; + const float max_gain_change_db_per_10ms_; + + int calls_since_last_gain_log_; + int frames_to_gain_increase_allowed_; + float last_gain_db_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_CONTROLLER_H_ diff --git a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc b/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc deleted file mode 100644 index f09f63b..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/adaptive_mode_level_estimator.h" - -#include "modules/audio_processing/agc2/agc2_common.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/numerics/safe_minmax.h" - -namespace webrtc { -namespace { - -using LevelEstimatorType = - AudioProcessing::Config::GainController2::LevelEstimator; - -// Combines a level estimation with the saturation protector margins. -float ComputeLevelEstimateDbfs(float level_estimate_dbfs, - float saturation_margin_db, - float extra_saturation_margin_db) { - return rtc::SafeClamp( - level_estimate_dbfs + saturation_margin_db + extra_saturation_margin_db, - -90.f, 30.f); -} - -// Returns the level of given type from `vad_level`. -float GetLevel(const VadLevelAnalyzer::Result& vad_level, - LevelEstimatorType type) { - switch (type) { - case LevelEstimatorType::kRms: - return vad_level.rms_dbfs; - break; - case LevelEstimatorType::kPeak: - return vad_level.peak_dbfs; - break; - } -} - -} // namespace - -bool AdaptiveModeLevelEstimator::LevelEstimatorState::operator==( - const AdaptiveModeLevelEstimator::LevelEstimatorState& b) const { - return time_to_full_buffer_ms == b.time_to_full_buffer_ms && - level_dbfs.numerator == b.level_dbfs.numerator && - level_dbfs.denominator == b.level_dbfs.denominator && - saturation_protector == b.saturation_protector; -} - -float AdaptiveModeLevelEstimator::LevelEstimatorState::Ratio::GetRatio() const { - RTC_DCHECK_NE(denominator, 0.f); - return numerator / denominator; -} - -AdaptiveModeLevelEstimator::AdaptiveModeLevelEstimator( - ApmDataDumper* apm_data_dumper) - : AdaptiveModeLevelEstimator( - apm_data_dumper, - AudioProcessing::Config::GainController2::LevelEstimator::kRms, - kDefaultLevelEstimatorAdjacentSpeechFramesThreshold, - kDefaultInitialSaturationMarginDb, - kDefaultExtraSaturationMarginDb) {} - -AdaptiveModeLevelEstimator::AdaptiveModeLevelEstimator( - ApmDataDumper* apm_data_dumper, - AudioProcessing::Config::GainController2::LevelEstimator level_estimator, - int adjacent_speech_frames_threshold, - float initial_saturation_margin_db, - float extra_saturation_margin_db) - : apm_data_dumper_(apm_data_dumper), - level_estimator_type_(level_estimator), - adjacent_speech_frames_threshold_(adjacent_speech_frames_threshold), - initial_saturation_margin_db_(initial_saturation_margin_db), - extra_saturation_margin_db_(extra_saturation_margin_db), - level_dbfs_(ComputeLevelEstimateDbfs(kInitialSpeechLevelEstimateDbfs, - initial_saturation_margin_db_, - extra_saturation_margin_db_)) { - RTC_DCHECK(apm_data_dumper_); - RTC_DCHECK_GE(adjacent_speech_frames_threshold_, 1); - Reset(); -} - -void AdaptiveModeLevelEstimator::Update( - const VadLevelAnalyzer::Result& vad_level) { - RTC_DCHECK_GT(vad_level.rms_dbfs, -150.f); - RTC_DCHECK_LT(vad_level.rms_dbfs, 50.f); - RTC_DCHECK_GT(vad_level.peak_dbfs, -150.f); - RTC_DCHECK_LT(vad_level.peak_dbfs, 50.f); - RTC_DCHECK_GE(vad_level.speech_probability, 0.f); - RTC_DCHECK_LE(vad_level.speech_probability, 1.f); - DumpDebugData(); - - if (vad_level.speech_probability < kVadConfidenceThreshold) { - // Not a speech frame. - if (adjacent_speech_frames_threshold_ > 1) { - // When two or more adjacent speech frames are required in order to update - // the state, we need to decide whether to discard or confirm the updates - // based on the speech sequence length. - if (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_) { - // First non-speech frame after a long enough sequence of speech frames. - // Update the reliable state. - reliable_state_ = preliminary_state_; - } else if (num_adjacent_speech_frames_ > 0) { - // First non-speech frame after a too short sequence of speech frames. - // Reset to the last reliable state. - preliminary_state_ = reliable_state_; - } - } - num_adjacent_speech_frames_ = 0; - return; - } - - // Speech frame observed. - num_adjacent_speech_frames_++; - - // Update preliminary level estimate. - RTC_DCHECK_GE(preliminary_state_.time_to_full_buffer_ms, 0); - const bool buffer_is_full = preliminary_state_.time_to_full_buffer_ms == 0; - if (!buffer_is_full) { - preliminary_state_.time_to_full_buffer_ms -= kFrameDurationMs; - } - // Weighted average of levels with speech probability as weight. - RTC_DCHECK_GT(vad_level.speech_probability, 0.f); - const float leak_factor = buffer_is_full ? kFullBufferLeakFactor : 1.f; - preliminary_state_.level_dbfs.numerator = - preliminary_state_.level_dbfs.numerator * leak_factor + - GetLevel(vad_level, level_estimator_type_) * vad_level.speech_probability; - preliminary_state_.level_dbfs.denominator = - preliminary_state_.level_dbfs.denominator * leak_factor + - vad_level.speech_probability; - - const float level_dbfs = preliminary_state_.level_dbfs.GetRatio(); - - UpdateSaturationProtectorState(vad_level.peak_dbfs, level_dbfs, - preliminary_state_.saturation_protector); - - if (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_) { - // `preliminary_state_` is now reliable. Update the last level estimation. - level_dbfs_ = ComputeLevelEstimateDbfs( - level_dbfs, preliminary_state_.saturation_protector.margin_db, - extra_saturation_margin_db_); - } -} - -bool AdaptiveModeLevelEstimator::IsConfident() const { - if (adjacent_speech_frames_threshold_ == 1) { - // Ignore `reliable_state_` when a single frame is enough to update the - // level estimate (because it is not used). - return preliminary_state_.time_to_full_buffer_ms == 0; - } - // Once confident, it remains confident. - RTC_DCHECK(reliable_state_.time_to_full_buffer_ms != 0 || - preliminary_state_.time_to_full_buffer_ms == 0); - // During the first long enough speech sequence, `reliable_state_` must be - // ignored since `preliminary_state_` is used. - return reliable_state_.time_to_full_buffer_ms == 0 || - (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_ && - preliminary_state_.time_to_full_buffer_ms == 0); -} - -void AdaptiveModeLevelEstimator::Reset() { - ResetLevelEstimatorState(preliminary_state_); - ResetLevelEstimatorState(reliable_state_); - level_dbfs_ = ComputeLevelEstimateDbfs(kInitialSpeechLevelEstimateDbfs, - initial_saturation_margin_db_, - extra_saturation_margin_db_); - num_adjacent_speech_frames_ = 0; -} - -void AdaptiveModeLevelEstimator::ResetLevelEstimatorState( - LevelEstimatorState& state) const { - state.time_to_full_buffer_ms = kFullBufferSizeMs; - state.level_dbfs.numerator = 0.f; - state.level_dbfs.denominator = 0.f; - ResetSaturationProtectorState(initial_saturation_margin_db_, - state.saturation_protector); -} - -void AdaptiveModeLevelEstimator::DumpDebugData() const { - apm_data_dumper_->DumpRaw("agc2_adaptive_level_estimate_dbfs", level_dbfs_); - apm_data_dumper_->DumpRaw("agc2_adaptive_num_adjacent_speech_frames_", - num_adjacent_speech_frames_); - apm_data_dumper_->DumpRaw("agc2_adaptive_preliminary_level_estimate_num", - preliminary_state_.level_dbfs.numerator); - apm_data_dumper_->DumpRaw("agc2_adaptive_preliminary_level_estimate_den", - preliminary_state_.level_dbfs.denominator); - apm_data_dumper_->DumpRaw("agc2_adaptive_preliminary_saturation_margin_db", - preliminary_state_.saturation_protector.margin_db); -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.cc b/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.cc deleted file mode 100644 index 5ceeb7d..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.cc +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.h" - -#include -#include - -#include "modules/audio_processing/agc2/agc2_common.h" -#include "modules/audio_processing/include/audio_frame_view.h" - -namespace webrtc { - -AdaptiveModeLevelEstimatorAgc::AdaptiveModeLevelEstimatorAgc( - ApmDataDumper* apm_data_dumper) - : level_estimator_(apm_data_dumper) { - set_target_level_dbfs(kDefaultAgc2LevelHeadroomDbfs); -} - -// |audio| must be mono; in a multi-channel stream, provide the first (usually -// left) channel. -void AdaptiveModeLevelEstimatorAgc::Process(const int16_t* audio, - size_t length, - int sample_rate_hz) { - std::vector float_audio_frame(audio, audio + length); - const float* const first_channel = &float_audio_frame[0]; - AudioFrameView frame_view(&first_channel, 1 /* num channels */, - length); - const auto vad_prob = agc2_vad_.AnalyzeFrame(frame_view); - latest_voice_probability_ = vad_prob.speech_probability; - if (latest_voice_probability_ > kVadConfidenceThreshold) { - time_in_ms_since_last_estimate_ += kFrameDurationMs; - } - level_estimator_.Update(vad_prob); -} - -// Retrieves the difference between the target RMS level and the current -// signal RMS level in dB. Returns true if an update is available and false -// otherwise, in which case |error| should be ignored and no action taken. -bool AdaptiveModeLevelEstimatorAgc::GetRmsErrorDb(int* error) { - if (time_in_ms_since_last_estimate_ <= kTimeUntilConfidentMs) { - return false; - } - *error = - std::floor(target_level_dbfs() - level_estimator_.level_dbfs() + 0.5f); - time_in_ms_since_last_estimate_ = 0; - return true; -} - -void AdaptiveModeLevelEstimatorAgc::Reset() { - level_estimator_.Reset(); -} - -float AdaptiveModeLevelEstimatorAgc::voice_probability() const { - return latest_voice_probability_; -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.h b/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.h deleted file mode 100644 index bc6fa84..0000000 --- a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator_agc.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_AGC_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_AGC_H_ - -#include -#include - -#include "modules/audio_processing/agc/agc.h" -#include "modules/audio_processing/agc2/adaptive_mode_level_estimator.h" -#include "modules/audio_processing/agc2/saturation_protector.h" -#include "modules/audio_processing/agc2/vad_with_level.h" - -namespace webrtc { -class AdaptiveModeLevelEstimatorAgc : public Agc { - public: - explicit AdaptiveModeLevelEstimatorAgc(ApmDataDumper* apm_data_dumper); - - // |audio| must be mono; in a multi-channel stream, provide the first (usually - // left) channel. - void Process(const int16_t* audio, - size_t length, - int sample_rate_hz) override; - - // Retrieves the difference between the target RMS level and the current - // signal RMS level in dB. Returns true if an update is available and false - // otherwise, in which case |error| should be ignored and no action taken. - bool GetRmsErrorDb(int* error) override; - void Reset() override; - - float voice_probability() const override; - - private: - static constexpr int kTimeUntilConfidentMs = 700; - static constexpr int kDefaultAgc2LevelHeadroomDbfs = -1; - int32_t time_in_ms_since_last_estimate_ = 0; - AdaptiveModeLevelEstimator level_estimator_; - VadLevelAnalyzer agc2_vad_; - float latest_voice_probability_ = 0.f; -}; -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_AGC_H_ diff --git a/webrtc/modules/audio_processing/agc2/agc2_common.h b/webrtc/modules/audio_processing/agc2/agc2_common.h index 5d01100..4597bcd 100644 --- a/webrtc/modules/audio_processing/agc2/agc2_common.h +++ b/webrtc/modules/audio_processing/agc2/agc2_common.h @@ -11,74 +11,50 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_AGC2_COMMON_H_ #define MODULES_AUDIO_PROCESSING_AGC2_AGC2_COMMON_H_ -#include - namespace webrtc { -constexpr float kMinFloatS16Value = -32768.f; -constexpr float kMaxFloatS16Value = 32767.f; +constexpr float kMinFloatS16Value = -32768.0f; +constexpr float kMaxFloatS16Value = 32767.0f; constexpr float kMaxAbsFloatS16Value = 32768.0f; -constexpr size_t kFrameDurationMs = 10; -constexpr size_t kSubFramesInFrame = 20; -constexpr size_t kMaximalNumberOfSamplesPerChannel = 480; +// Minimum audio level in dBFS scale for S16 samples. +constexpr float kMinLevelDbfs = -90.31f; -constexpr float kAttackFilterConstant = 0.f; +constexpr int kFrameDurationMs = 10; +constexpr int kSubFramesInFrame = 20; +constexpr int kMaximalNumberOfSamplesPerChannel = 480; + +// Adaptive digital gain applier settings. -// Adaptive digital gain applier settings below. -constexpr float kHeadroomDbfs = 1.f; -constexpr float kMaxGainDb = 30.f; -constexpr float kInitialAdaptiveDigitalGainDb = 8.f; // At what limiter levels should we start decreasing the adaptive digital gain. -constexpr float kLimiterThresholdForAgcGainDbfs = -kHeadroomDbfs; +constexpr float kLimiterThresholdForAgcGainDbfs = -1.0f; -// This is the threshold for speech. Speech frames are used for updating the -// speech level, measuring the amount of speech, and decide when to allow target -// gain reduction. -constexpr float kVadConfidenceThreshold = 0.9f; +// Number of milliseconds to wait to periodically reset the VAD. +constexpr int kVadResetPeriodMs = 1500; -// The amount of 'memory' of the Level Estimator. Decides leak factors. -constexpr size_t kFullBufferSizeMs = 1200; -constexpr float kFullBufferLeakFactor = 1.f - 1.f / kFullBufferSizeMs; +// Speech probability threshold to detect speech activity. +constexpr float kVadConfidenceThreshold = 0.95f; -constexpr float kInitialSpeechLevelEstimateDbfs = -30.f; +// Minimum number of adjacent speech frames having a sufficiently high speech +// probability to reliably detect speech activity. +constexpr int kAdjacentSpeechFramesThreshold = 12; -// Robust VAD probability and speech decisions. -constexpr float kDefaultSmoothedVadProbabilityAttack = 1.f; -constexpr int kDefaultLevelEstimatorAdjacentSpeechFramesThreshold = 1; +// Number of milliseconds of speech frames to observe to make the estimator +// confident. +constexpr float kLevelEstimatorTimeToConfidenceMs = 400; +constexpr float kLevelEstimatorLeakFactor = + 1.0f - 1.0f / kLevelEstimatorTimeToConfidenceMs; // Saturation Protector settings. -constexpr float kDefaultInitialSaturationMarginDb = 20.f; -constexpr float kDefaultExtraSaturationMarginDb = 2.f; - -constexpr size_t kPeakEnveloperSuperFrameLengthMs = 400; -static_assert(kFullBufferSizeMs % kPeakEnveloperSuperFrameLengthMs == 0, - "Full buffer size should be a multiple of super frame length for " - "optimal Saturation Protector performance."); - -constexpr size_t kPeakEnveloperBufferSize = - kFullBufferSizeMs / kPeakEnveloperSuperFrameLengthMs + 1; - -// This value is 10 ** (-1/20 * frame_size_ms / satproc_attack_ms), -// where satproc_attack_ms is 5000. -constexpr float kSaturationProtectorAttackConstant = 0.9988493699365052f; - -// This value is 10 ** (-1/20 * frame_size_ms / satproc_decay_ms), -// where satproc_decay_ms is 1000. -constexpr float kSaturationProtectorDecayConstant = 0.9997697679981565f; - -// This is computed from kDecayMs by -// 10 ** (-1/20 * subframe_duration / kDecayMs). -// |subframe_duration| is |kFrameDurationMs / kSubFramesInFrame|. -// kDecayMs is defined in agc2_testing_common.h -constexpr float kDecayFilterConstant = 0.9998848773724686f; +constexpr float kSaturationProtectorInitialHeadroomDb = 20.0f; +constexpr int kSaturationProtectorBufferSize = 4; // Number of interpolation points for each region of the limiter. // These values have been tuned to limit the interpolated gain curve error given // the limiter parameters and allowing a maximum error of +/- 32768^-1. -constexpr size_t kInterpolatedGainCurveKneePoints = 22; -constexpr size_t kInterpolatedGainCurveBeyondKneePoints = 10; -constexpr size_t kInterpolatedGainCurveTotalPoints = +constexpr int kInterpolatedGainCurveKneePoints = 22; +constexpr int kInterpolatedGainCurveBeyondKneePoints = 10; +constexpr int kInterpolatedGainCurveTotalPoints = kInterpolatedGainCurveKneePoints + kInterpolatedGainCurveBeyondKneePoints; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/agc2_testing_common.cc b/webrtc/modules/audio_processing/agc2/agc2_testing_common.cc index 6c22492..8f3e9db 100644 --- a/webrtc/modules/audio_processing/agc2/agc2_testing_common.cc +++ b/webrtc/modules/audio_processing/agc2/agc2_testing_common.cc @@ -10,24 +10,85 @@ #include "modules/audio_processing/agc2/agc2_testing_common.h" +#include + #include "rtc_base/checks.h" namespace webrtc { - namespace test { -std::vector LinSpace(const double l, - const double r, - size_t num_points) { - RTC_CHECK(num_points >= 2); +std::vector LinSpace(double l, double r, int num_points) { + RTC_CHECK_GE(num_points, 2); std::vector points(num_points); const double step = (r - l) / (num_points - 1.0); points[0] = l; - for (size_t i = 1; i < num_points - 1; i++) { + for (int i = 1; i < num_points - 1; i++) { points[i] = static_cast(l) + i * step; } points[num_points - 1] = r; return points; } + +WhiteNoiseGenerator::WhiteNoiseGenerator(int min_amplitude, int max_amplitude) + : rand_gen_(42), + min_amplitude_(min_amplitude), + max_amplitude_(max_amplitude) { + RTC_DCHECK_LT(min_amplitude_, max_amplitude_); + RTC_DCHECK_LE(kMinS16, min_amplitude_); + RTC_DCHECK_LE(min_amplitude_, kMaxS16); + RTC_DCHECK_LE(kMinS16, max_amplitude_); + RTC_DCHECK_LE(max_amplitude_, kMaxS16); +} + +float WhiteNoiseGenerator::operator()() { + return static_cast(rand_gen_.Rand(min_amplitude_, max_amplitude_)); +} + +SineGenerator::SineGenerator(float amplitude, + float frequency_hz, + int sample_rate_hz) + : amplitude_(amplitude), + frequency_hz_(frequency_hz), + sample_rate_hz_(sample_rate_hz), + x_radians_(0.0f) { + RTC_DCHECK_GT(amplitude_, 0); + RTC_DCHECK_LE(amplitude_, kMaxS16); +} + +float SineGenerator::operator()() { + constexpr float kPi = 3.1415926536f; + x_radians_ += frequency_hz_ / sample_rate_hz_ * 2 * kPi; + if (x_radians_ >= 2 * kPi) { + x_radians_ -= 2 * kPi; + } + // Use sinf instead of std::sinf for libstdc++ compatibility. + return amplitude_ * sinf(x_radians_); +} + +PulseGenerator::PulseGenerator(float pulse_amplitude, + float no_pulse_amplitude, + float frequency_hz, + int sample_rate_hz) + : pulse_amplitude_(pulse_amplitude), + no_pulse_amplitude_(no_pulse_amplitude), + samples_period_( + static_cast(static_cast(sample_rate_hz) / frequency_hz)), + sample_counter_(0) { + RTC_DCHECK_GE(pulse_amplitude_, kMinS16); + RTC_DCHECK_LE(pulse_amplitude_, kMaxS16); + RTC_DCHECK_GT(no_pulse_amplitude_, kMinS16); + RTC_DCHECK_LE(no_pulse_amplitude_, kMaxS16); + RTC_DCHECK_GT(sample_rate_hz, frequency_hz); +} + +float PulseGenerator::operator()() { + sample_counter_++; + if (sample_counter_ >= samples_period_) { + sample_counter_ -= samples_period_; + } + return static_cast(sample_counter_ == 0 ? pulse_amplitude_ + : no_pulse_amplitude_); +} + } // namespace test } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/agc2_testing_common.h b/webrtc/modules/audio_processing/agc2/agc2_testing_common.h index 7bfadbb..afed97e 100644 --- a/webrtc/modules/audio_processing/agc2/agc2_testing_common.h +++ b/webrtc/modules/audio_processing/agc2/agc2_testing_common.h @@ -11,65 +11,69 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_AGC2_TESTING_COMMON_H_ #define MODULES_AUDIO_PROCESSING_AGC2_AGC2_TESTING_COMMON_H_ -#include - #include #include -#include "rtc_base/checks.h" +#include "rtc_base/random.h" namespace webrtc { - namespace test { +constexpr float kMinS16 = + static_cast(std::numeric_limits::min()); +constexpr float kMaxS16 = + static_cast(std::numeric_limits::max()); + // Level Estimator test parameters. -constexpr float kDecayMs = 500.f; +constexpr float kDecayMs = 20.0f; // Limiter parameters. constexpr float kLimiterMaxInputLevelDbFs = 1.f; constexpr float kLimiterKneeSmoothnessDb = 1.f; constexpr float kLimiterCompressionRatio = 5.f; -constexpr float kPi = 3.1415926536f; -std::vector LinSpace(const double l, const double r, size_t num_points); +// Returns evenly spaced `num_points` numbers over a specified interval [l, r]. +std::vector LinSpace(double l, double r, int num_points); -class SineGenerator { +// Generates white noise. +class WhiteNoiseGenerator { public: - SineGenerator(float frequency, int rate) - : frequency_(frequency), rate_(rate) {} - float operator()() { - x_radians_ += frequency_ / rate_ * 2 * kPi; - if (x_radians_ > 2 * kPi) { - x_radians_ -= 2 * kPi; - } - return 1000.f * sinf(x_radians_); - } + WhiteNoiseGenerator(int min_amplitude, int max_amplitude); + float operator()(); private: - float frequency_; - int rate_; - float x_radians_ = 0.f; + Random rand_gen_; + const int min_amplitude_; + const int max_amplitude_; }; -class PulseGenerator { +// Generates a sine function. +class SineGenerator { public: - PulseGenerator(float frequency, int rate) - : samples_period_( - static_cast(static_cast(rate) / frequency)) { - RTC_DCHECK_GT(rate, frequency); - } - float operator()() { - sample_counter_++; - if (sample_counter_ >= samples_period_) { - sample_counter_ -= samples_period_; - } - return static_cast( - sample_counter_ == 0 ? std::numeric_limits::max() : 10.f); - } + SineGenerator(float amplitude, float frequency_hz, int sample_rate_hz); + float operator()(); private: - int samples_period_; - int sample_counter_ = 0; + const float amplitude_; + const float frequency_hz_; + const int sample_rate_hz_; + float x_radians_; +}; + +// Generates periodic pulses. +class PulseGenerator { + public: + PulseGenerator(float pulse_amplitude, + float no_pulse_amplitude, + float frequency_hz, + int sample_rate_hz); + float operator()(); + + private: + const float pulse_amplitude_; + const float no_pulse_amplitude_; + const int samples_period_; + int sample_counter_; }; } // namespace test diff --git a/webrtc/modules/audio_processing/agc2/biquad_filter.cc b/webrtc/modules/audio_processing/agc2/biquad_filter.cc index da8557c..c1b80d7 100644 --- a/webrtc/modules/audio_processing/agc2/biquad_filter.cc +++ b/webrtc/modules/audio_processing/agc2/biquad_filter.cc @@ -10,27 +10,51 @@ #include "modules/audio_processing/agc2/biquad_filter.h" -#include +#include "rtc_base/arraysize.h" namespace webrtc { -// Transposed direct form I implementation of a bi-quad filter applied to an -// input signal |x| to produce an output signal |y|. +BiQuadFilter::BiQuadFilter(const Config& config) + : config_(config), state_({}) {} + +BiQuadFilter::~BiQuadFilter() = default; + +void BiQuadFilter::SetConfig(const Config& config) { + config_ = config; + state_ = {}; +} + +void BiQuadFilter::Reset() { + state_ = {}; +} + void BiQuadFilter::Process(rtc::ArrayView x, rtc::ArrayView y) { - for (size_t k = 0; k < x.size(); ++k) { - // Use temporary variable for x[k] to allow in-place function call - // (that x and y refer to the same array). + RTC_DCHECK_EQ(x.size(), y.size()); + const float config_a0 = config_.a[0]; + const float config_a1 = config_.a[1]; + const float config_b0 = config_.b[0]; + const float config_b1 = config_.b[1]; + const float config_b2 = config_.b[2]; + float state_a0 = state_.a[0]; + float state_a1 = state_.a[1]; + float state_b0 = state_.b[0]; + float state_b1 = state_.b[1]; + for (size_t k = 0, x_size = x.size(); k < x_size; ++k) { + // Use a temporary variable for `x[k]` to allow in-place processing. const float tmp = x[k]; - y[k] = coefficients_.b[0] * tmp + coefficients_.b[1] * biquad_state_.b[0] + - coefficients_.b[2] * biquad_state_.b[1] - - coefficients_.a[0] * biquad_state_.a[0] - - coefficients_.a[1] * biquad_state_.a[1]; - biquad_state_.b[1] = biquad_state_.b[0]; - biquad_state_.b[0] = tmp; - biquad_state_.a[1] = biquad_state_.a[0]; - biquad_state_.a[0] = y[k]; + float y_k = config_b0 * tmp + config_b1 * state_b0 + config_b2 * state_b1 - + config_a0 * state_a0 - config_a1 * state_a1; + state_b1 = state_b0; + state_b0 = tmp; + state_a1 = state_a0; + state_a0 = y_k; + y[k] = y_k; } + state_.a[0] = state_a0; + state_.a[1] = state_a1; + state_.b[0] = state_b0; + state_.b[1] = state_b1; } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/biquad_filter.h b/webrtc/modules/audio_processing/agc2/biquad_filter.h index 7bf3301..5273ff9 100644 --- a/webrtc/modules/audio_processing/agc2/biquad_filter.h +++ b/webrtc/modules/audio_processing/agc2/biquad_filter.h @@ -11,54 +11,44 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_BIQUAD_FILTER_H_ #define MODULES_AUDIO_PROCESSING_AGC2_BIQUAD_FILTER_H_ -#include - #include "api/array_view.h" -#include "rtc_base/arraysize.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { +// Transposed direct form I implementation of a bi-quad filter. +// b[0] + b[1] • z^(-1) + b[2] • z^(-2) +// H(z) = ------------------------------------ +// 1 + a[1] • z^(-1) + a[2] • z^(-2) class BiQuadFilter { public: // Normalized filter coefficients. - // b_0 + b_1 • z^(-1) + b_2 • z^(-2) - // H(z) = --------------------------------- - // 1 + a_1 • z^(-1) + a_2 • z^(-2) - struct BiQuadCoefficients { - float b[3]; - float a[2]; + // Computed as `[b, a] = scipy.signal.butter(N=2, Wn, btype)`. + struct Config { + float b[3]; // b[0], b[1], b[2]. + float a[2]; // a[1], a[2]. }; - BiQuadFilter() = default; + explicit BiQuadFilter(const Config& config); + BiQuadFilter(const BiQuadFilter&) = delete; + BiQuadFilter& operator=(const BiQuadFilter&) = delete; + ~BiQuadFilter(); - void Initialize(const BiQuadCoefficients& coefficients) { - coefficients_ = coefficients; - } + // Sets the filter configuration and resets the internal state. + void SetConfig(const Config& config); - void Reset() { biquad_state_.Reset(); } + // Zeroes the filter state. + void Reset(); - // Produces a filtered output y of the input x. Both x and y need to - // have the same length. In-place modification is allowed. + // Filters `x` and writes the output in `y`, which must have the same length + // of `x`. In-place processing is supported. void Process(rtc::ArrayView x, rtc::ArrayView y); private: - struct BiQuadState { - BiQuadState() { Reset(); } - - void Reset() { - std::fill(b, b + arraysize(b), 0.f); - std::fill(a, a + arraysize(a), 0.f); - } - + Config config_; + struct State { float b[2]; float a[2]; - }; - - BiQuadState biquad_state_; - BiQuadCoefficients coefficients_; - - RTC_DISALLOW_COPY_AND_ASSIGN(BiQuadFilter); + } state_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/clipping_predictor.cc b/webrtc/modules/audio_processing/agc2/clipping_predictor.cc new file mode 100644 index 0000000..fd759c6 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/clipping_predictor.cc @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/clipping_predictor.h" + +#include +#include + +#include "common_audio/include/audio_util.h" +#include "modules/audio_processing/agc2/clipping_predictor_level_buffer.h" +#include "modules/audio_processing/agc2/gain_map_internal.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { +namespace { + +constexpr int kClippingPredictorMaxGainChange = 15; + +// Returns an input volume in the [`min_input_volume`, `max_input_volume`] range +// that reduces `gain_error_db`, which is a gain error estimated when +// `input_volume` was applied, according to a fixed gain map. +int ComputeVolumeUpdate(int gain_error_db, + int input_volume, + int min_input_volume, + int max_input_volume) { + RTC_DCHECK_GE(input_volume, 0); + RTC_DCHECK_LE(input_volume, max_input_volume); + if (gain_error_db == 0) { + return input_volume; + } + int new_volume = input_volume; + if (gain_error_db > 0) { + while (kGainMap[new_volume] - kGainMap[input_volume] < gain_error_db && + new_volume < max_input_volume) { + ++new_volume; + } + } else { + while (kGainMap[new_volume] - kGainMap[input_volume] > gain_error_db && + new_volume > min_input_volume) { + --new_volume; + } + } + return new_volume; +} + +float ComputeCrestFactor(const ClippingPredictorLevelBuffer::Level& level) { + const float crest_factor = + FloatS16ToDbfs(level.max) - FloatS16ToDbfs(std::sqrt(level.average)); + return crest_factor; +} + +// Crest factor-based clipping prediction and clipped level step estimation. +class ClippingEventPredictor : public ClippingPredictor { + public: + // ClippingEventPredictor with `num_channels` channels (limited to values + // higher than zero); window size `window_length` and reference window size + // `reference_window_length` (both referring to the number of frames in the + // respective sliding windows and limited to values higher than zero); + // reference window delay `reference_window_delay` (delay in frames, limited + // to values zero and higher with an additional requirement of + // `window_length` < `reference_window_length` + reference_window_delay`); + // and an estimation peak threshold `clipping_threshold` and a crest factor + // drop threshold `crest_factor_margin` (both in dB). + ClippingEventPredictor(int num_channels, + int window_length, + int reference_window_length, + int reference_window_delay, + float clipping_threshold, + float crest_factor_margin) + : window_length_(window_length), + reference_window_length_(reference_window_length), + reference_window_delay_(reference_window_delay), + clipping_threshold_(clipping_threshold), + crest_factor_margin_(crest_factor_margin) { + RTC_DCHECK_GT(num_channels, 0); + RTC_DCHECK_GT(window_length, 0); + RTC_DCHECK_GT(reference_window_length, 0); + RTC_DCHECK_GE(reference_window_delay, 0); + RTC_DCHECK_GT(reference_window_length + reference_window_delay, + window_length); + const int buffer_length = GetMinFramesProcessed(); + RTC_DCHECK_GT(buffer_length, 0); + for (int i = 0; i < num_channels; ++i) { + ch_buffers_.push_back( + std::make_unique(buffer_length)); + } + } + + ClippingEventPredictor(const ClippingEventPredictor&) = delete; + ClippingEventPredictor& operator=(const ClippingEventPredictor&) = delete; + ~ClippingEventPredictor() {} + + void Reset() { + const int num_channels = ch_buffers_.size(); + for (int i = 0; i < num_channels; ++i) { + ch_buffers_[i]->Reset(); + } + } + + // Analyzes a frame of audio and stores the framewise metrics in + // `ch_buffers_`. + void Analyze(const AudioFrameView& frame) { + const int num_channels = frame.num_channels(); + RTC_DCHECK_EQ(num_channels, ch_buffers_.size()); + const int samples_per_channel = frame.samples_per_channel(); + RTC_DCHECK_GT(samples_per_channel, 0); + for (int channel = 0; channel < num_channels; ++channel) { + float sum_squares = 0.0f; + float peak = 0.0f; + for (const auto& sample : frame.channel(channel)) { + sum_squares += sample * sample; + peak = std::max(std::fabs(sample), peak); + } + ch_buffers_[channel]->Push( + {sum_squares / static_cast(samples_per_channel), peak}); + } + } + + // Estimates the analog gain adjustment for channel `channel` using a + // sliding window over the frame-wise metrics in `ch_buffers_`. Returns an + // estimate for the clipped level step equal to `default_clipped_level_step_` + // if at least `GetMinFramesProcessed()` frames have been processed since the + // last reset and a clipping event is predicted. `level`, `min_mic_level`, and + // `max_mic_level` are limited to [0, 255] and `default_step` to [1, 255]. + absl::optional EstimateClippedLevelStep(int channel, + int level, + int default_step, + int min_mic_level, + int max_mic_level) const { + RTC_CHECK_GE(channel, 0); + RTC_CHECK_LT(channel, ch_buffers_.size()); + RTC_DCHECK_GE(level, 0); + RTC_DCHECK_LE(level, 255); + RTC_DCHECK_GT(default_step, 0); + RTC_DCHECK_LE(default_step, 255); + RTC_DCHECK_GE(min_mic_level, 0); + RTC_DCHECK_LE(min_mic_level, 255); + RTC_DCHECK_GE(max_mic_level, 0); + RTC_DCHECK_LE(max_mic_level, 255); + if (level <= min_mic_level) { + return absl::nullopt; + } + if (PredictClippingEvent(channel)) { + const int new_level = + rtc::SafeClamp(level - default_step, min_mic_level, max_mic_level); + const int step = level - new_level; + if (step > 0) { + return step; + } + } + return absl::nullopt; + } + + private: + int GetMinFramesProcessed() const { + return reference_window_delay_ + reference_window_length_; + } + + // Predicts clipping events based on the processed audio frames. Returns + // true if a clipping event is likely. + bool PredictClippingEvent(int channel) const { + const auto metrics = + ch_buffers_[channel]->ComputePartialMetrics(0, window_length_); + if (!metrics.has_value() || + !(FloatS16ToDbfs(metrics.value().max) > clipping_threshold_)) { + return false; + } + const auto reference_metrics = ch_buffers_[channel]->ComputePartialMetrics( + reference_window_delay_, reference_window_length_); + if (!reference_metrics.has_value()) { + return false; + } + const float crest_factor = ComputeCrestFactor(metrics.value()); + const float reference_crest_factor = + ComputeCrestFactor(reference_metrics.value()); + if (crest_factor < reference_crest_factor - crest_factor_margin_) { + return true; + } + return false; + } + + std::vector> ch_buffers_; + const int window_length_; + const int reference_window_length_; + const int reference_window_delay_; + const float clipping_threshold_; + const float crest_factor_margin_; +}; + +// Performs crest factor-based clipping peak prediction. +class ClippingPeakPredictor : public ClippingPredictor { + public: + // Ctor. ClippingPeakPredictor with `num_channels` channels (limited to values + // higher than zero); window size `window_length` and reference window size + // `reference_window_length` (both referring to the number of frames in the + // respective sliding windows and limited to values higher than zero); + // reference window delay `reference_window_delay` (delay in frames, limited + // to values zero and higher with an additional requirement of + // `window_length` < `reference_window_length` + reference_window_delay`); + // and a clipping prediction threshold `clipping_threshold` (in dB). Adaptive + // clipped level step estimation is used if `adaptive_step_estimation` is + // true. + explicit ClippingPeakPredictor(int num_channels, + int window_length, + int reference_window_length, + int reference_window_delay, + int clipping_threshold, + bool adaptive_step_estimation) + : window_length_(window_length), + reference_window_length_(reference_window_length), + reference_window_delay_(reference_window_delay), + clipping_threshold_(clipping_threshold), + adaptive_step_estimation_(adaptive_step_estimation) { + RTC_DCHECK_GT(num_channels, 0); + RTC_DCHECK_GT(window_length, 0); + RTC_DCHECK_GT(reference_window_length, 0); + RTC_DCHECK_GE(reference_window_delay, 0); + RTC_DCHECK_GT(reference_window_length + reference_window_delay, + window_length); + const int buffer_length = GetMinFramesProcessed(); + RTC_DCHECK_GT(buffer_length, 0); + for (int i = 0; i < num_channels; ++i) { + ch_buffers_.push_back( + std::make_unique(buffer_length)); + } + } + + ClippingPeakPredictor(const ClippingPeakPredictor&) = delete; + ClippingPeakPredictor& operator=(const ClippingPeakPredictor&) = delete; + ~ClippingPeakPredictor() {} + + void Reset() { + const int num_channels = ch_buffers_.size(); + for (int i = 0; i < num_channels; ++i) { + ch_buffers_[i]->Reset(); + } + } + + // Analyzes a frame of audio and stores the framewise metrics in + // `ch_buffers_`. + void Analyze(const AudioFrameView& frame) { + const int num_channels = frame.num_channels(); + RTC_DCHECK_EQ(num_channels, ch_buffers_.size()); + const int samples_per_channel = frame.samples_per_channel(); + RTC_DCHECK_GT(samples_per_channel, 0); + for (int channel = 0; channel < num_channels; ++channel) { + float sum_squares = 0.0f; + float peak = 0.0f; + for (const auto& sample : frame.channel(channel)) { + sum_squares += sample * sample; + peak = std::max(std::fabs(sample), peak); + } + ch_buffers_[channel]->Push( + {sum_squares / static_cast(samples_per_channel), peak}); + } + } + + // Estimates the analog gain adjustment for channel `channel` using a + // sliding window over the frame-wise metrics in `ch_buffers_`. Returns an + // estimate for the clipped level step (equal to + // `default_clipped_level_step_` if `adaptive_estimation_` is false) if at + // least `GetMinFramesProcessed()` frames have been processed since the last + // reset and a clipping event is predicted. `level`, `min_mic_level`, and + // `max_mic_level` are limited to [0, 255] and `default_step` to [1, 255]. + absl::optional EstimateClippedLevelStep(int channel, + int level, + int default_step, + int min_mic_level, + int max_mic_level) const { + RTC_DCHECK_GE(channel, 0); + RTC_DCHECK_LT(channel, ch_buffers_.size()); + RTC_DCHECK_GE(level, 0); + RTC_DCHECK_LE(level, 255); + RTC_DCHECK_GT(default_step, 0); + RTC_DCHECK_LE(default_step, 255); + RTC_DCHECK_GE(min_mic_level, 0); + RTC_DCHECK_LE(min_mic_level, 255); + RTC_DCHECK_GE(max_mic_level, 0); + RTC_DCHECK_LE(max_mic_level, 255); + if (level <= min_mic_level) { + return absl::nullopt; + } + absl::optional estimate_db = EstimatePeakValue(channel); + if (estimate_db.has_value() && estimate_db.value() > clipping_threshold_) { + int step = 0; + if (!adaptive_step_estimation_) { + step = default_step; + } else { + const int estimated_gain_change = + rtc::SafeClamp(-static_cast(std::ceil(estimate_db.value())), + -kClippingPredictorMaxGainChange, 0); + step = + std::max(level - ComputeVolumeUpdate(estimated_gain_change, level, + min_mic_level, max_mic_level), + default_step); + } + const int new_level = + rtc::SafeClamp(level - step, min_mic_level, max_mic_level); + if (level > new_level) { + return level - new_level; + } + } + return absl::nullopt; + } + + private: + int GetMinFramesProcessed() { + return reference_window_delay_ + reference_window_length_; + } + + // Predicts clipping sample peaks based on the processed audio frames. + // Returns the estimated peak value if clipping is predicted. Otherwise + // returns absl::nullopt. + absl::optional EstimatePeakValue(int channel) const { + const auto reference_metrics = ch_buffers_[channel]->ComputePartialMetrics( + reference_window_delay_, reference_window_length_); + if (!reference_metrics.has_value()) { + return absl::nullopt; + } + const auto metrics = + ch_buffers_[channel]->ComputePartialMetrics(0, window_length_); + if (!metrics.has_value() || + !(FloatS16ToDbfs(metrics.value().max) > clipping_threshold_)) { + return absl::nullopt; + } + const float reference_crest_factor = + ComputeCrestFactor(reference_metrics.value()); + const float& mean_squares = metrics.value().average; + const float projected_peak = + reference_crest_factor + FloatS16ToDbfs(std::sqrt(mean_squares)); + return projected_peak; + } + + std::vector> ch_buffers_; + const int window_length_; + const int reference_window_length_; + const int reference_window_delay_; + const int clipping_threshold_; + const bool adaptive_step_estimation_; +}; + +} // namespace + +std::unique_ptr CreateClippingPredictor( + int num_channels, + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& config) { + if (!config.enabled) { + RTC_LOG(LS_INFO) << "[AGC2] Clipping prediction disabled."; + return nullptr; + } + RTC_LOG(LS_INFO) << "[AGC2] Clipping prediction enabled."; + using ClippingPredictorMode = AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor::Mode; + switch (config.mode) { + case ClippingPredictorMode::kClippingEventPrediction: + return std::make_unique( + num_channels, config.window_length, config.reference_window_length, + config.reference_window_delay, config.clipping_threshold, + config.crest_factor_margin); + case ClippingPredictorMode::kAdaptiveStepClippingPeakPrediction: + return std::make_unique( + num_channels, config.window_length, config.reference_window_length, + config.reference_window_delay, config.clipping_threshold, + /*adaptive_step_estimation=*/true); + case ClippingPredictorMode::kFixedStepClippingPeakPrediction: + return std::make_unique( + num_channels, config.window_length, config.reference_window_length, + config.reference_window_delay, config.clipping_threshold, + /*adaptive_step_estimation=*/false); + } + RTC_DCHECK_NOTREACHED(); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/clipping_predictor.h b/webrtc/modules/audio_processing/agc2/clipping_predictor.h new file mode 100644 index 0000000..1461250 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/clipping_predictor.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "modules/audio_processing/include/audio_frame_view.h" +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { + +// Frame-wise clipping prediction and clipped level step estimation. Analyzes +// 10 ms multi-channel frames and estimates an analog mic level decrease step +// to possibly avoid clipping when predicted. `Analyze()` and +// `EstimateClippedLevelStep()` can be called in any order. +class ClippingPredictor { + public: + virtual ~ClippingPredictor() = default; + + virtual void Reset() = 0; + + // Analyzes a 10 ms multi-channel audio frame. + virtual void Analyze(const AudioFrameView& frame) = 0; + + // Predicts if clipping is going to occur for the specified `channel` in the + // near-future and, if so, it returns a recommended analog mic level decrease + // step. Returns absl::nullopt if clipping is not predicted. + // `level` is the current analog mic level, `default_step` is the amount the + // mic level is lowered by the analog controller with every clipping event and + // `min_mic_level` and `max_mic_level` is the range of allowed analog mic + // levels. + virtual absl::optional EstimateClippedLevelStep( + int channel, + int level, + int default_step, + int min_mic_level, + int max_mic_level) const = 0; +}; + +// Creates a ClippingPredictor based on the provided `config`. When enabled, +// the following must hold for `config`: +// `window_length < reference_window_length + reference_window_delay`. +// Returns `nullptr` if `config.enabled` is false. +std::unique_ptr CreateClippingPredictor( + int num_channels, + const AudioProcessing::Config::GainController1::AnalogGainController:: + ClippingPredictor& config); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_H_ diff --git a/webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.cc b/webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.cc new file mode 100644 index 0000000..fe4cf2a --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/clipping_predictor_level_buffer.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +bool ClippingPredictorLevelBuffer::Level::operator==(const Level& level) const { + constexpr float kEpsilon = 1e-6f; + return std::fabs(average - level.average) < kEpsilon && + std::fabs(max - level.max) < kEpsilon; +} + +ClippingPredictorLevelBuffer::ClippingPredictorLevelBuffer(int capacity) + : tail_(-1), size_(0), data_(std::max(1, capacity)) { + if (capacity > kMaxCapacity) { + RTC_LOG(LS_WARNING) << "[agc]: ClippingPredictorLevelBuffer exceeds the " + << "maximum allowed capacity. Capacity: " << capacity; + } + RTC_DCHECK(!data_.empty()); +} + +void ClippingPredictorLevelBuffer::Reset() { + tail_ = -1; + size_ = 0; +} + +void ClippingPredictorLevelBuffer::Push(Level level) { + ++tail_; + if (tail_ == Capacity()) { + tail_ = 0; + } + if (size_ < Capacity()) { + size_++; + } + data_[tail_] = level; +} + +// TODO(bugs.webrtc.org/12774): Optimize partial computation for long buffers. +absl::optional +ClippingPredictorLevelBuffer::ComputePartialMetrics(int delay, + int num_items) const { + RTC_DCHECK_GE(delay, 0); + RTC_DCHECK_LT(delay, Capacity()); + RTC_DCHECK_GT(num_items, 0); + RTC_DCHECK_LE(num_items, Capacity()); + RTC_DCHECK_LE(delay + num_items, Capacity()); + if (delay + num_items > Size()) { + return absl::nullopt; + } + float sum = 0.0f; + float max = 0.0f; + for (int i = 0; i < num_items && i < Size(); ++i) { + int idx = tail_ - delay - i; + if (idx < 0) { + idx += Capacity(); + } + sum += data_[idx].average; + max = std::fmax(data_[idx].max, max); + } + return absl::optional({sum / static_cast(num_items), max}); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.h b/webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.h new file mode 100644 index 0000000..c903277 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_LEVEL_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_LEVEL_BUFFER_H_ + +#include +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +// A circular buffer to store frame-wise `Level` items for clipping prediction. +// The current implementation is not optimized for large buffer lengths. +class ClippingPredictorLevelBuffer { + public: + struct Level { + float average; + float max; + bool operator==(const Level& level) const; + }; + + // Recommended maximum capacity. It is possible to create a buffer with a + // larger capacity, but the implementation is not optimized for large values. + static constexpr int kMaxCapacity = 100; + + // Ctor. Sets the buffer capacity to max(1, `capacity`) and logs a warning + // message if the capacity is greater than `kMaxCapacity`. + explicit ClippingPredictorLevelBuffer(int capacity); + ~ClippingPredictorLevelBuffer() {} + ClippingPredictorLevelBuffer(const ClippingPredictorLevelBuffer&) = delete; + ClippingPredictorLevelBuffer& operator=(const ClippingPredictorLevelBuffer&) = + delete; + + void Reset(); + + // Returns the current number of items stored in the buffer. + int Size() const { return size_; } + + // Returns the capacity of the buffer. + int Capacity() const { return data_.size(); } + + // Adds a `level` item into the circular buffer `data_`. Stores at most + // `Capacity()` items. If more items are pushed, the new item replaces the + // least recently pushed item. + void Push(Level level); + + // If at least `num_items` + `delay` items have been pushed, returns the + // average and maximum value for the `num_items` most recently pushed items + // from `delay` to `delay` - `num_items` (a delay equal to zero corresponds + // to the most recently pushed item). The value of `delay` is limited to + // [0, N] and `num_items` to [1, M] where N + M is the capacity of the buffer. + absl::optional ComputePartialMetrics(int delay, int num_items) const; + + private: + int tail_; + int size_; + std::vector data_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_LEVEL_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.cc b/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.cc index bc92613..221b499 100644 --- a/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.cc +++ b/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.cc @@ -105,7 +105,7 @@ std::vector SampleLimiterRegion(const LimiterDbGainCurve* limiter) { const auto interval = q.top(); q.pop(); - // Split |interval| and enqueue. + // Split `interval` and enqueue. double x_split = (interval.x0 + interval.x1) / 2.0; q.emplace(interval.x0, x_split, LimiterUnderApproximationNegativeError(limiter, interval.x0, @@ -135,7 +135,7 @@ std::vector SampleLimiterRegion(const LimiterDbGainCurve* limiter) { void PrecomputeKneeApproxParams(const LimiterDbGainCurve* limiter, test::InterpolatedParameters* parameters) { static_assert(kInterpolatedGainCurveKneePoints > 2, ""); - // Get |kInterpolatedGainCurveKneePoints| - 1 equally spaced points. + // Get `kInterpolatedGainCurveKneePoints` - 1 equally spaced points. const std::vector points = test::LinSpace( limiter->knee_start_linear(), limiter->limiter_start_linear(), kInterpolatedGainCurveKneePoints - 1); diff --git a/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.h b/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.h index 5f52441..08b676f 100644 --- a/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.h +++ b/webrtc/modules/audio_processing/agc2/compute_interpolated_gain_curve.h @@ -29,8 +29,8 @@ namespace test { // Knee and beyond-knee regions approximation parameters. // The gain curve is approximated as a piece-wise linear function. -// |approx_params_x_| are the boundaries between adjacent linear pieces, -// |approx_params_m_| and |approx_params_q_| are the slope and the y-intercept +// `approx_params_x_` are the boundaries between adjacent linear pieces, +// `approx_params_m_` and `approx_params_q_` are the slope and the y-intercept // values of each piece. struct InterpolatedParameters { std::array diff --git a/webrtc/modules/audio_processing/agc2/cpu_features.cc b/webrtc/modules/audio_processing/agc2/cpu_features.cc new file mode 100644 index 0000000..cced761 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/cpu_features.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/cpu_features.h" + +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/system/arch.h" +#include "system_wrappers/include/cpu_features_wrapper.h" + +namespace webrtc { + +std::string AvailableCpuFeatures::ToString() const { + char buf[64]; + rtc::SimpleStringBuilder builder(buf); + bool first = true; + if (sse2) { + builder << (first ? "SSE2" : "_SSE2"); + first = false; + } + if (avx2) { + builder << (first ? "AVX2" : "_AVX2"); + first = false; + } + if (neon) { + builder << (first ? "NEON" : "_NEON"); + first = false; + } + if (first) { + return "none"; + } + return builder.str(); +} + +// Detects available CPU features. +AvailableCpuFeatures GetAvailableCpuFeatures() { +#if defined(WEBRTC_ARCH_X86_FAMILY) + return {/*sse2=*/GetCPUInfo(kSSE2) != 0, + /*avx2=*/GetCPUInfo(kAVX2) != 0, + /*neon=*/false}; +#elif defined(WEBRTC_HAS_NEON) + return {/*sse2=*/false, + /*avx2=*/false, + /*neon=*/true}; +#else + return {/*sse2=*/false, + /*avx2=*/false, + /*neon=*/false}; +#endif +} + +AvailableCpuFeatures NoAvailableCpuFeatures() { + return {/*sse2=*/false, /*avx2=*/false, /*neon=*/false}; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/cpu_features.h b/webrtc/modules/audio_processing/agc2/cpu_features.h new file mode 100644 index 0000000..54ddfb3 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/cpu_features.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_CPU_FEATURES_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_CPU_FEATURES_H_ + +#include + +namespace webrtc { + +// Collection of flags indicating which CPU features are available on the +// current platform. True means available. +struct AvailableCpuFeatures { + AvailableCpuFeatures(bool sse2, bool avx2, bool neon) + : sse2(sse2), avx2(avx2), neon(neon) {} + // Intel. + bool sse2; + bool avx2; + // ARM. + bool neon; + std::string ToString() const; +}; + +// Detects what CPU features are available. +AvailableCpuFeatures GetAvailableCpuFeatures(); + +// Returns the CPU feature flags all set to false. +AvailableCpuFeatures NoAvailableCpuFeatures(); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_CPU_FEATURES_H_ diff --git a/webrtc/modules/audio_processing/agc2/down_sampler.cc b/webrtc/modules/audio_processing/agc2/down_sampler.cc deleted file mode 100644 index 654ed4b..0000000 --- a/webrtc/modules/audio_processing/agc2/down_sampler.cc +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/down_sampler.h" - -#include - -#include - -#include "modules/audio_processing/agc2/biquad_filter.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/checks.h" - -namespace webrtc { -namespace { - -constexpr int kChunkSizeMs = 10; -constexpr int kSampleRate8kHz = 8000; -constexpr int kSampleRate16kHz = 16000; -constexpr int kSampleRate32kHz = 32000; -constexpr int kSampleRate48kHz = 48000; - -// Bandlimiter coefficients computed based on that only -// the first 40 bins of the spectrum for the downsampled -// signal are used. -// [B,A] = butter(2,(41/64*4000)/8000) -const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_16kHz = { - {0.1455f, 0.2911f, 0.1455f}, - {-0.6698f, 0.2520f}}; - -// [B,A] = butter(2,(41/64*4000)/16000) -const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_32kHz = { - {0.0462f, 0.0924f, 0.0462f}, - {-1.3066f, 0.4915f}}; - -// [B,A] = butter(2,(41/64*4000)/24000) -const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_48kHz = { - {0.0226f, 0.0452f, 0.0226f}, - {-1.5320f, 0.6224f}}; - -} // namespace - -DownSampler::DownSampler(ApmDataDumper* data_dumper) - : data_dumper_(data_dumper) { - Initialize(48000); -} -void DownSampler::Initialize(int sample_rate_hz) { - RTC_DCHECK( - sample_rate_hz == kSampleRate8kHz || sample_rate_hz == kSampleRate16kHz || - sample_rate_hz == kSampleRate32kHz || sample_rate_hz == kSampleRate48kHz); - - sample_rate_hz_ = sample_rate_hz; - down_sampling_factor_ = rtc::CheckedDivExact(sample_rate_hz_, 8000); - - /// Note that the down sampling filter is not used if the sample rate is 8 - /// kHz. - if (sample_rate_hz_ == kSampleRate16kHz) { - low_pass_filter_.Initialize(kLowPassFilterCoefficients_16kHz); - } else if (sample_rate_hz_ == kSampleRate32kHz) { - low_pass_filter_.Initialize(kLowPassFilterCoefficients_32kHz); - } else if (sample_rate_hz_ == kSampleRate48kHz) { - low_pass_filter_.Initialize(kLowPassFilterCoefficients_48kHz); - } -} - -void DownSampler::DownSample(rtc::ArrayView in, - rtc::ArrayView out) { - data_dumper_->DumpWav("lc_down_sampler_input", in, sample_rate_hz_, 1); - RTC_DCHECK_EQ(sample_rate_hz_ * kChunkSizeMs / 1000, in.size()); - RTC_DCHECK_EQ(kSampleRate8kHz * kChunkSizeMs / 1000, out.size()); - const size_t kMaxNumFrames = kSampleRate48kHz * kChunkSizeMs / 1000; - float x[kMaxNumFrames]; - - // Band-limit the signal to 4 kHz. - if (sample_rate_hz_ != kSampleRate8kHz) { - low_pass_filter_.Process(in, rtc::ArrayView(x, in.size())); - - // Downsample the signal. - size_t k = 0; - for (size_t j = 0; j < out.size(); ++j) { - RTC_DCHECK_GT(kMaxNumFrames, k); - out[j] = x[k]; - k += down_sampling_factor_; - } - } else { - std::copy(in.data(), in.data() + in.size(), out.data()); - } - - data_dumper_->DumpWav("lc_down_sampler_output", out, kSampleRate8kHz, 1); -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/down_sampler.h b/webrtc/modules/audio_processing/agc2/down_sampler.h deleted file mode 100644 index be7cbb3..0000000 --- a/webrtc/modules/audio_processing/agc2/down_sampler.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_DOWN_SAMPLER_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_DOWN_SAMPLER_H_ - -#include "api/array_view.h" -#include "modules/audio_processing/agc2/biquad_filter.h" - -namespace webrtc { - -class ApmDataDumper; - -class DownSampler { - public: - explicit DownSampler(ApmDataDumper* data_dumper); - - DownSampler() = delete; - DownSampler(const DownSampler&) = delete; - DownSampler& operator=(const DownSampler&) = delete; - - void Initialize(int sample_rate_hz); - - void DownSample(rtc::ArrayView in, rtc::ArrayView out); - - private: - ApmDataDumper* data_dumper_; - int sample_rate_hz_; - int down_sampling_factor_; - BiQuadFilter low_pass_filter_; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_DOWN_SAMPLER_H_ diff --git a/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.cc b/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.cc index 971f4f6..1995b24 100644 --- a/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.cc +++ b/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.cc @@ -20,12 +20,21 @@ namespace webrtc { namespace { -constexpr float kInitialFilterStateLevel = 0.f; +constexpr float kInitialFilterStateLevel = 0.0f; + +// Instant attack. +constexpr float kAttackFilterConstant = 0.0f; + +// Limiter decay constant. +// Computed as `10 ** (-1/20 * subframe_duration / kDecayMs)` where: +// - `subframe_duration` is `kFrameDurationMs / kSubFramesInFrame`; +// - `kDecayMs` is defined in agc2_testing_common.h. +constexpr float kDecayFilterConstant = 0.9971259f; } // namespace FixedDigitalLevelEstimator::FixedDigitalLevelEstimator( - size_t sample_rate_hz, + int sample_rate_hz, ApmDataDumper* apm_data_dumper) : apm_data_dumper_(apm_data_dumper), filter_state_level_(kInitialFilterStateLevel) { @@ -49,11 +58,11 @@ std::array FixedDigitalLevelEstimator::ComputeLevel( // Compute max envelope without smoothing. std::array envelope{}; - for (size_t channel_idx = 0; channel_idx < float_frame.num_channels(); + for (int channel_idx = 0; channel_idx < float_frame.num_channels(); ++channel_idx) { const auto channel = float_frame.channel(channel_idx); - for (size_t sub_frame = 0; sub_frame < kSubFramesInFrame; ++sub_frame) { - for (size_t sample_in_sub_frame = 0; + for (int sub_frame = 0; sub_frame < kSubFramesInFrame; ++sub_frame) { + for (int sample_in_sub_frame = 0; sample_in_sub_frame < samples_in_sub_frame_; ++sample_in_sub_frame) { envelope[sub_frame] = std::max(envelope[sub_frame], @@ -66,14 +75,14 @@ std::array FixedDigitalLevelEstimator::ComputeLevel( // Make sure envelope increases happen one step earlier so that the // corresponding *gain decrease* doesn't miss a sudden signal // increase due to interpolation. - for (size_t sub_frame = 0; sub_frame < kSubFramesInFrame - 1; ++sub_frame) { + for (int sub_frame = 0; sub_frame < kSubFramesInFrame - 1; ++sub_frame) { if (envelope[sub_frame] < envelope[sub_frame + 1]) { envelope[sub_frame] = envelope[sub_frame + 1]; } } // Add attack / decay smoothing. - for (size_t sub_frame = 0; sub_frame < kSubFramesInFrame; ++sub_frame) { + for (int sub_frame = 0; sub_frame < kSubFramesInFrame; ++sub_frame) { const float envelope_value = envelope[sub_frame]; if (envelope_value > filter_state_level_) { envelope[sub_frame] = envelope_value * (1 - kAttackFilterConstant) + @@ -97,9 +106,9 @@ std::array FixedDigitalLevelEstimator::ComputeLevel( return envelope; } -void FixedDigitalLevelEstimator::SetSampleRate(size_t sample_rate_hz) { - samples_in_frame_ = rtc::CheckedDivExact(sample_rate_hz * kFrameDurationMs, - static_cast(1000)); +void FixedDigitalLevelEstimator::SetSampleRate(int sample_rate_hz) { + samples_in_frame_ = + rtc::CheckedDivExact(sample_rate_hz * kFrameDurationMs, 1000); samples_in_sub_frame_ = rtc::CheckedDivExact(samples_in_frame_, kSubFramesInFrame); CheckParameterCombination(); diff --git a/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.h b/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.h index aa84a2e..d26b559 100644 --- a/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.h +++ b/webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.h @@ -16,7 +16,6 @@ #include "modules/audio_processing/agc2/agc2_common.h" #include "modules/audio_processing/include/audio_frame_view.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -31,9 +30,13 @@ class FixedDigitalLevelEstimator { // kSubFramesInSample. For kFrameDurationMs=10 and // kSubFramesInSample=20, this means that sample_rate_hz has to be // divisible by 2000. - FixedDigitalLevelEstimator(size_t sample_rate_hz, + FixedDigitalLevelEstimator(int sample_rate_hz, ApmDataDumper* apm_data_dumper); + FixedDigitalLevelEstimator(const FixedDigitalLevelEstimator&) = delete; + FixedDigitalLevelEstimator& operator=(const FixedDigitalLevelEstimator&) = + delete; + // The input is assumed to be in FloatS16 format. Scaled input will // produce similarly scaled output. A frame of with kFrameDurationMs // ms of audio produces a level estimates in the same scale. The @@ -43,7 +46,7 @@ class FixedDigitalLevelEstimator { // Rate may be changed at any time (but not concurrently) from the // value passed to the constructor. The class is not thread safe. - void SetSampleRate(size_t sample_rate_hz); + void SetSampleRate(int sample_rate_hz); // Resets the level estimator internal state. void Reset(); @@ -55,10 +58,8 @@ class FixedDigitalLevelEstimator { ApmDataDumper* const apm_data_dumper_ = nullptr; float filter_state_level_; - size_t samples_in_frame_; - size_t samples_in_sub_frame_; - - RTC_DISALLOW_COPY_AND_ASSIGN(FixedDigitalLevelEstimator); + int samples_in_frame_; + int samples_in_sub_frame_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/fixed_gain_controller.cc b/webrtc/modules/audio_processing/agc2/fixed_gain_controller.cc deleted file mode 100644 index ef908dc..0000000 --- a/webrtc/modules/audio_processing/agc2/fixed_gain_controller.cc +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/fixed_gain_controller.h" - -#include "api/array_view.h" -#include "common_audio/include/audio_util.h" -#include "modules/audio_processing/agc2/agc2_common.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/numerics/safe_minmax.h" - -namespace webrtc { -namespace { - -// Returns true when the gain factor is so close to 1 that it would -// not affect int16 samples. -bool CloseToOne(float gain_factor) { - return 1.f - 1.f / kMaxFloatS16Value <= gain_factor && - gain_factor <= 1.f + 1.f / kMaxFloatS16Value; -} -} // namespace - -FixedGainController::FixedGainController(ApmDataDumper* apm_data_dumper) - : FixedGainController(apm_data_dumper, "Agc2") {} - -FixedGainController::FixedGainController(ApmDataDumper* apm_data_dumper, - std::string histogram_name_prefix) - : apm_data_dumper_(apm_data_dumper), - limiter_(48000, apm_data_dumper_, histogram_name_prefix) { - // Do update histograms.xml when adding name prefixes. - RTC_DCHECK(histogram_name_prefix == "" || histogram_name_prefix == "Test" || - histogram_name_prefix == "AudioMixer" || - histogram_name_prefix == "Agc2"); -} - -void FixedGainController::SetGain(float gain_to_apply_db) { - // Changes in gain_to_apply_ cause discontinuities. We assume - // gain_to_apply_ is set in the beginning of the call. If it is - // frequently changed, we should add interpolation between the - // values. - // The gain - RTC_DCHECK_LE(-50.f, gain_to_apply_db); - RTC_DCHECK_LE(gain_to_apply_db, 50.f); - const float previous_applied_gained = gain_to_apply_; - gain_to_apply_ = DbToRatio(gain_to_apply_db); - RTC_DCHECK_LT(0.f, gain_to_apply_); - RTC_DLOG(LS_INFO) << "Gain to apply: " << gain_to_apply_db << " db."; - // Reset the gain curve applier to quickly react on abrupt level changes - // caused by large changes of the applied gain. - if (previous_applied_gained != gain_to_apply_) { - limiter_.Reset(); - } -} - -void FixedGainController::SetSampleRate(size_t sample_rate_hz) { - limiter_.SetSampleRate(sample_rate_hz); -} - -void FixedGainController::Process(AudioFrameView signal) { - // Apply fixed digital gain. One of the - // planned usages of the FGC is to only use the limiter. In that - // case, the gain would be 1.0. Not doing the multiplications speeds - // it up considerably. Hence the check. - if (!CloseToOne(gain_to_apply_)) { - for (size_t k = 0; k < signal.num_channels(); ++k) { - rtc::ArrayView channel_view = signal.channel(k); - for (auto& sample : channel_view) { - sample *= gain_to_apply_; - } - } - } - - // Use the limiter. - limiter_.Process(signal); - - // Dump data for debug. - const auto channel_view = signal.channel(0); - apm_data_dumper_->DumpRaw("agc2_fixed_digital_gain_curve_applier", - channel_view.size(), channel_view.data()); - // Hard-clipping. - for (size_t k = 0; k < signal.num_channels(); ++k) { - rtc::ArrayView channel_view = signal.channel(k); - for (auto& sample : channel_view) { - sample = rtc::SafeClamp(sample, kMinFloatS16Value, kMaxFloatS16Value); - } - } -} - -float FixedGainController::LastAudioLevel() const { - return limiter_.LastAudioLevel(); -} -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/gain_applier.cc b/webrtc/modules/audio_processing/agc2/gain_applier.cc index 8c43717..f9e276d 100644 --- a/webrtc/modules/audio_processing/agc2/gain_applier.cc +++ b/webrtc/modules/audio_processing/agc2/gain_applier.cc @@ -25,7 +25,7 @@ bool GainCloseToOne(float gain_factor) { } void ClipSignal(AudioFrameView signal) { - for (size_t k = 0; k < signal.num_channels(); ++k) { + for (int k = 0; k < signal.num_channels(); ++k) { rtc::ArrayView channel_view = signal.channel(k); for (auto& sample : channel_view) { sample = rtc::SafeClamp(sample, kMinFloatS16Value, kMaxFloatS16Value); @@ -45,7 +45,7 @@ void ApplyGainWithRamping(float last_gain_linear, // Gain is constant and different from 1. if (last_gain_linear == gain_at_end_of_frame_linear) { - for (size_t k = 0; k < float_frame.num_channels(); ++k) { + for (int k = 0; k < float_frame.num_channels(); ++k) { rtc::ArrayView channel_view = float_frame.channel(k); for (auto& sample : channel_view) { sample *= gain_at_end_of_frame_linear; @@ -58,8 +58,8 @@ void ApplyGainWithRamping(float last_gain_linear, const float increment = (gain_at_end_of_frame_linear - last_gain_linear) * inverse_samples_per_channel; float gain = last_gain_linear; - for (size_t i = 0; i < float_frame.samples_per_channel(); ++i) { - for (size_t ch = 0; ch < float_frame.num_channels(); ++ch) { + for (int i = 0; i < float_frame.samples_per_channel(); ++i) { + for (int ch = 0; ch < float_frame.num_channels(); ++ch) { float_frame.channel(ch)[i] *= gain; } gain += increment; @@ -88,12 +88,13 @@ void GainApplier::ApplyGain(AudioFrameView signal) { } } +// TODO(bugs.webrtc.org/7494): Remove once switched to gains in dB. void GainApplier::SetGainFactor(float gain_factor) { RTC_DCHECK_GT(gain_factor, 0.f); current_gain_factor_ = gain_factor; } -void GainApplier::Initialize(size_t samples_per_channel) { +void GainApplier::Initialize(int samples_per_channel) { RTC_DCHECK_GT(samples_per_channel, 0); samples_per_channel_ = static_cast(samples_per_channel); inverse_samples_per_channel_ = 1.f / samples_per_channel_; diff --git a/webrtc/modules/audio_processing/agc2/gain_applier.h b/webrtc/modules/audio_processing/agc2/gain_applier.h index d9aa19d..ba8a4a4 100644 --- a/webrtc/modules/audio_processing/agc2/gain_applier.h +++ b/webrtc/modules/audio_processing/agc2/gain_applier.h @@ -25,7 +25,7 @@ class GainApplier { float GetGainFactor() const { return current_gain_factor_; } private: - void Initialize(size_t samples_per_channel); + void Initialize(int samples_per_channel); // Whether to clip samples after gain is applied. If 'true', result // will fit in FloatS16 range. diff --git a/webrtc/modules/audio_processing/agc/gain_map_internal.h b/webrtc/modules/audio_processing/agc2/gain_map_internal.h similarity index 74% rename from webrtc/modules/audio_processing/agc/gain_map_internal.h rename to webrtc/modules/audio_processing/agc2/gain_map_internal.h index 547f0f3..7c669fc 100644 --- a/webrtc/modules/audio_processing/agc/gain_map_internal.h +++ b/webrtc/modules/audio_processing/agc2/gain_map_internal.h @@ -8,13 +8,19 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef MODULES_AUDIO_PROCESSING_AGC_GAIN_MAP_INTERNAL_H_ -#define MODULES_AUDIO_PROCESSING_AGC_GAIN_MAP_INTERNAL_H_ +#ifndef MODULES_AUDIO_PROCESSING_AGC2_GAIN_MAP_INTERNAL_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_GAIN_MAP_INTERNAL_H_ namespace webrtc { -static const int kGainMapSize = 256; -// Uses parameters: si = 2, sf = 0.25, D = 8/256 +static constexpr int kGainMapSize = 256; +// Maps input volumes, which are values in the [0, 255] range, to gains in dB. +// The values below are generated with numpy as follows: +// SI = 2 # Initial slope. +// SF = 0.25 # Final slope. +// D = 8/256 # Quantization factor. +// x = np.linspace(0, 255, 256) # Input volumes. +// y = (SF * x + (SI - SF) * (1 - np.exp(-D*x)) / D - 56).round() static const int kGainMap[kGainMapSize] = { -56, -54, -52, -50, -48, -47, -45, -43, -42, -40, -38, -37, -35, -34, -33, -31, -30, -29, -27, -26, -25, -24, -23, -22, -20, -19, -18, -17, -16, -15, @@ -37,4 +43,4 @@ static const int kGainMap[kGainMapSize] = { } // namespace webrtc -#endif // MODULES_AUDIO_PROCESSING_AGC_GAIN_MAP_INTERNAL_H_ +#endif // MODULES_AUDIO_PROCESSING_AGC2_GAIN_MAP_INTERNAL_H_ diff --git a/webrtc/modules/audio_processing/agc2/input_volume_controller.cc b/webrtc/modules/audio_processing/agc2/input_volume_controller.cc new file mode 100644 index 0000000..bcc650f --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/input_volume_controller.cc @@ -0,0 +1,580 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/input_volume_controller.h" + +#include +#include + +#include "api/array_view.h" +#include "modules/audio_processing/agc2/gain_map_internal.h" +#include "modules/audio_processing/agc2/input_volume_stats_reporter.h" +#include "modules/audio_processing/include/audio_frame_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +// Amount of error we tolerate in the microphone input volume (presumably due to +// OS quantization) before we assume the user has manually adjusted the volume. +constexpr int kVolumeQuantizationSlack = 25; + +constexpr int kMaxInputVolume = 255; +static_assert(kGainMapSize > kMaxInputVolume, "gain map too small"); + +// Maximum absolute RMS error. +constexpr int KMaxAbsRmsErrorDbfs = 15; +static_assert(KMaxAbsRmsErrorDbfs > 0, ""); + +using Agc1ClippingPredictorConfig = AudioProcessing::Config::GainController1:: + AnalogGainController::ClippingPredictor; + +// TODO(webrtc:7494): Hardcode clipping predictor parameters and remove this +// function after no longer needed in the ctor. +Agc1ClippingPredictorConfig CreateClippingPredictorConfig(bool enabled) { + Agc1ClippingPredictorConfig config; + config.enabled = enabled; + + return config; +} + +// Returns an input volume in the [`min_input_volume`, `kMaxInputVolume`] range +// that reduces `gain_error_db`, which is a gain error estimated when +// `input_volume` was applied, according to a fixed gain map. +int ComputeVolumeUpdate(int gain_error_db, + int input_volume, + int min_input_volume) { + RTC_DCHECK_GE(input_volume, 0); + RTC_DCHECK_LE(input_volume, kMaxInputVolume); + if (gain_error_db == 0) { + return input_volume; + } + + int new_volume = input_volume; + if (gain_error_db > 0) { + while (kGainMap[new_volume] - kGainMap[input_volume] < gain_error_db && + new_volume < kMaxInputVolume) { + ++new_volume; + } + } else { + while (kGainMap[new_volume] - kGainMap[input_volume] > gain_error_db && + new_volume > min_input_volume) { + --new_volume; + } + } + return new_volume; +} + +// Returns the proportion of samples in the buffer which are at full-scale +// (and presumably clipped). +float ComputeClippedRatio(const float* const* audio, + size_t num_channels, + size_t samples_per_channel) { + RTC_DCHECK_GT(samples_per_channel, 0); + int num_clipped = 0; + for (size_t ch = 0; ch < num_channels; ++ch) { + int num_clipped_in_ch = 0; + for (size_t i = 0; i < samples_per_channel; ++i) { + RTC_DCHECK(audio[ch]); + if (audio[ch][i] >= 32767.0f || audio[ch][i] <= -32768.0f) { + ++num_clipped_in_ch; + } + } + num_clipped = std::max(num_clipped, num_clipped_in_ch); + } + return static_cast(num_clipped) / (samples_per_channel); +} + +void LogClippingMetrics(int clipping_rate) { + RTC_LOG(LS_INFO) << "[AGC2] Input clipping rate: " << clipping_rate << "%"; + RTC_HISTOGRAM_COUNTS_LINEAR(/*name=*/"WebRTC.Audio.Agc.InputClippingRate", + /*sample=*/clipping_rate, /*min=*/0, /*max=*/100, + /*bucket_count=*/50); +} + +// Compares `speech_level_dbfs` to the [`target_range_min_dbfs`, +// `target_range_max_dbfs`] range and returns the error to be compensated via +// input volume adjustment. Returns a positive value when the level is below +// the range, a negative value when the level is above the range, zero +// otherwise. +int GetSpeechLevelRmsErrorDb(float speech_level_dbfs, + int target_range_min_dbfs, + int target_range_max_dbfs) { + constexpr float kMinSpeechLevelDbfs = -90.0f; + constexpr float kMaxSpeechLevelDbfs = 30.0f; + RTC_DCHECK_GE(speech_level_dbfs, kMinSpeechLevelDbfs); + RTC_DCHECK_LE(speech_level_dbfs, kMaxSpeechLevelDbfs); + speech_level_dbfs = rtc::SafeClamp( + speech_level_dbfs, kMinSpeechLevelDbfs, kMaxSpeechLevelDbfs); + + int rms_error_db = 0; + if (speech_level_dbfs > target_range_max_dbfs) { + rms_error_db = std::round(target_range_max_dbfs - speech_level_dbfs); + } else if (speech_level_dbfs < target_range_min_dbfs) { + rms_error_db = std::round(target_range_min_dbfs - speech_level_dbfs); + } + + return rms_error_db; +} + +} // namespace + +MonoInputVolumeController::MonoInputVolumeController( + int min_input_volume_after_clipping, + int min_input_volume, + int update_input_volume_wait_frames, + float speech_probability_threshold, + float speech_ratio_threshold) + : min_input_volume_(min_input_volume), + min_input_volume_after_clipping_(min_input_volume_after_clipping), + max_input_volume_(kMaxInputVolume), + update_input_volume_wait_frames_( + std::max(update_input_volume_wait_frames, 1)), + speech_probability_threshold_(speech_probability_threshold), + speech_ratio_threshold_(speech_ratio_threshold) { + RTC_DCHECK_GE(min_input_volume_, 0); + RTC_DCHECK_LE(min_input_volume_, 255); + RTC_DCHECK_GE(min_input_volume_after_clipping_, 0); + RTC_DCHECK_LE(min_input_volume_after_clipping_, 255); + RTC_DCHECK_GE(max_input_volume_, 0); + RTC_DCHECK_LE(max_input_volume_, 255); + RTC_DCHECK_GE(update_input_volume_wait_frames_, 0); + RTC_DCHECK_GE(speech_probability_threshold_, 0.0f); + RTC_DCHECK_LE(speech_probability_threshold_, 1.0f); + RTC_DCHECK_GE(speech_ratio_threshold_, 0.0f); + RTC_DCHECK_LE(speech_ratio_threshold_, 1.0f); +} + +MonoInputVolumeController::~MonoInputVolumeController() = default; + +void MonoInputVolumeController::Initialize() { + max_input_volume_ = kMaxInputVolume; + capture_output_used_ = true; + check_volume_on_next_process_ = true; + frames_since_update_input_volume_ = 0; + speech_frames_since_update_input_volume_ = 0; + is_first_frame_ = true; +} + +// A speeh segment is considered active if at least +// `update_input_volume_wait_frames_` new frames have been processed since the +// previous update and the ratio of non-silence frames (i.e., frames with a +// `speech_probability` higher than `speech_probability_threshold_`) is at least +// `speech_ratio_threshold_`. +void MonoInputVolumeController::Process(absl::optional rms_error_db, + float speech_probability) { + if (check_volume_on_next_process_) { + check_volume_on_next_process_ = false; + // We have to wait until the first process call to check the volume, + // because Chromium doesn't guarantee it to be valid any earlier. + CheckVolumeAndReset(); + } + + // Count frames with a high speech probability as speech. + if (speech_probability >= speech_probability_threshold_) { + ++speech_frames_since_update_input_volume_; + } + + // Reset the counters and maybe update the input volume. + if (++frames_since_update_input_volume_ >= update_input_volume_wait_frames_) { + const float speech_ratio = + static_cast(speech_frames_since_update_input_volume_) / + static_cast(update_input_volume_wait_frames_); + + // Always reset the counters regardless of whether the volume changes or + // not. + frames_since_update_input_volume_ = 0; + speech_frames_since_update_input_volume_ = 0; + + // Update the input volume if allowed. + if (!is_first_frame_ && speech_ratio >= speech_ratio_threshold_ && + rms_error_db.has_value()) { + UpdateInputVolume(*rms_error_db); + } + } + + is_first_frame_ = false; +} + +void MonoInputVolumeController::HandleClipping(int clipped_level_step) { + RTC_DCHECK_GT(clipped_level_step, 0); + // Always decrease the maximum input volume, even if the current input volume + // is below threshold. + SetMaxLevel(std::max(min_input_volume_after_clipping_, + max_input_volume_ - clipped_level_step)); + if (log_to_histograms_) { + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.AgcClippingAdjustmentAllowed", + last_recommended_input_volume_ - clipped_level_step >= + min_input_volume_after_clipping_); + } + if (last_recommended_input_volume_ > min_input_volume_after_clipping_) { + // Don't try to adjust the input volume if we're already below the limit. As + // a consequence, if the user has brought the input volume above the limit, + // we will still not react until the postproc updates the input volume. + SetInputVolume( + std::max(min_input_volume_after_clipping_, + last_recommended_input_volume_ - clipped_level_step)); + frames_since_update_input_volume_ = 0; + speech_frames_since_update_input_volume_ = 0; + is_first_frame_ = false; + } +} + +void MonoInputVolumeController::SetInputVolume(int new_volume) { + int applied_input_volume = recommended_input_volume_; + if (applied_input_volume == 0) { + RTC_DLOG(LS_INFO) + << "[AGC2] The applied input volume is zero, taking no action."; + return; + } + if (applied_input_volume < 0 || applied_input_volume > kMaxInputVolume) { + RTC_LOG(LS_ERROR) << "[AGC2] Invalid value for the applied input volume: " + << applied_input_volume; + return; + } + + // Detect manual input volume adjustments by checking if the + // `applied_input_volume` is outside of the `[last_recommended_input_volume_ - + // kVolumeQuantizationSlack, last_recommended_input_volume_ + + // kVolumeQuantizationSlack]` range. + if (applied_input_volume > + last_recommended_input_volume_ + kVolumeQuantizationSlack || + applied_input_volume < + last_recommended_input_volume_ - kVolumeQuantizationSlack) { + RTC_DLOG(LS_INFO) + << "[AGC2] The input volume was manually adjusted. Updating " + "stored input volume from " + << last_recommended_input_volume_ << " to " << applied_input_volume; + last_recommended_input_volume_ = applied_input_volume; + // Always allow the user to increase the volume. + if (last_recommended_input_volume_ > max_input_volume_) { + SetMaxLevel(last_recommended_input_volume_); + } + // Take no action in this case, since we can't be sure when the volume + // was manually adjusted. + frames_since_update_input_volume_ = 0; + speech_frames_since_update_input_volume_ = 0; + is_first_frame_ = false; + return; + } + + new_volume = std::min(new_volume, max_input_volume_); + if (new_volume == last_recommended_input_volume_) { + return; + } + + recommended_input_volume_ = new_volume; + RTC_DLOG(LS_INFO) << "[AGC2] Applied input volume: " << applied_input_volume + << " | last recommended input volume: " + << last_recommended_input_volume_ + << " | newly recommended input volume: " << new_volume; + last_recommended_input_volume_ = new_volume; +} + +void MonoInputVolumeController::SetMaxLevel(int input_volume) { + RTC_DCHECK_GE(input_volume, min_input_volume_after_clipping_); + max_input_volume_ = input_volume; + RTC_DLOG(LS_INFO) << "[AGC2] Maximum input volume updated: " + << max_input_volume_; +} + +void MonoInputVolumeController::HandleCaptureOutputUsedChange( + bool capture_output_used) { + if (capture_output_used_ == capture_output_used) { + return; + } + capture_output_used_ = capture_output_used; + + if (capture_output_used) { + // When we start using the output, we should reset things to be safe. + check_volume_on_next_process_ = true; + } +} + +int MonoInputVolumeController::CheckVolumeAndReset() { + int input_volume = recommended_input_volume_; + // Reasons for taking action at startup: + // 1) A person starting a call is expected to be heard. + // 2) Independent of interpretation of `input_volume` == 0 we should raise it + // so the AGC can do its job properly. + if (input_volume == 0 && !startup_) { + RTC_DLOG(LS_INFO) + << "[AGC2] The applied input volume is zero, taking no action."; + return 0; + } + if (input_volume < 0 || input_volume > kMaxInputVolume) { + RTC_LOG(LS_ERROR) << "[AGC2] Invalid value for the applied input volume: " + << input_volume; + return -1; + } + RTC_DLOG(LS_INFO) << "[AGC2] Initial input volume: " << input_volume; + + if (input_volume < min_input_volume_) { + input_volume = min_input_volume_; + RTC_DLOG(LS_INFO) + << "[AGC2] The initial input volume is too low, raising to " + << input_volume; + recommended_input_volume_ = input_volume; + } + + last_recommended_input_volume_ = input_volume; + startup_ = false; + frames_since_update_input_volume_ = 0; + speech_frames_since_update_input_volume_ = 0; + is_first_frame_ = true; + + return 0; +} + +void MonoInputVolumeController::UpdateInputVolume(int rms_error_db) { + RTC_DLOG(LS_INFO) << "[AGC2] RMS error: " << rms_error_db << " dB"; + // Prevent too large microphone input volume changes by clamping the RMS + // error. + rms_error_db = + rtc::SafeClamp(rms_error_db, -KMaxAbsRmsErrorDbfs, KMaxAbsRmsErrorDbfs); + if (rms_error_db == 0) { + return; + } + SetInputVolume(ComputeVolumeUpdate( + rms_error_db, last_recommended_input_volume_, min_input_volume_)); +} + +InputVolumeController::InputVolumeController(int num_capture_channels, + const Config& config) + : num_capture_channels_(num_capture_channels), + min_input_volume_(config.min_input_volume), + capture_output_used_(true), + clipped_level_step_(config.clipped_level_step), + clipped_ratio_threshold_(config.clipped_ratio_threshold), + clipped_wait_frames_(config.clipped_wait_frames), + clipping_predictor_(CreateClippingPredictor( + num_capture_channels, + CreateClippingPredictorConfig(config.enable_clipping_predictor))), + use_clipping_predictor_step_( + !!clipping_predictor_ && + CreateClippingPredictorConfig(config.enable_clipping_predictor) + .use_predicted_step), + frames_since_clipped_(config.clipped_wait_frames), + clipping_rate_log_counter_(0), + clipping_rate_log_(0.0f), + target_range_max_dbfs_(config.target_range_max_dbfs), + target_range_min_dbfs_(config.target_range_min_dbfs), + channel_controllers_(num_capture_channels) { + RTC_LOG(LS_INFO) + << "[AGC2] Input volume controller enabled. Minimum input volume: " + << min_input_volume_; + + for (auto& controller : channel_controllers_) { + controller = std::make_unique( + config.clipped_level_min, min_input_volume_, + config.update_input_volume_wait_frames, + config.speech_probability_threshold, config.speech_ratio_threshold); + } + + RTC_DCHECK(!channel_controllers_.empty()); + RTC_DCHECK_GT(clipped_level_step_, 0); + RTC_DCHECK_LE(clipped_level_step_, 255); + RTC_DCHECK_GT(clipped_ratio_threshold_, 0.0f); + RTC_DCHECK_LT(clipped_ratio_threshold_, 1.0f); + RTC_DCHECK_GT(clipped_wait_frames_, 0); + channel_controllers_[0]->ActivateLogging(); +} + +InputVolumeController::~InputVolumeController() {} + +void InputVolumeController::Initialize() { + for (auto& controller : channel_controllers_) { + controller->Initialize(); + } + capture_output_used_ = true; + + AggregateChannelLevels(); + clipping_rate_log_ = 0.0f; + clipping_rate_log_counter_ = 0; + + applied_input_volume_ = absl::nullopt; +} + +void InputVolumeController::AnalyzeInputAudio(int applied_input_volume, + const AudioBuffer& audio_buffer) { + RTC_DCHECK_GE(applied_input_volume, 0); + RTC_DCHECK_LE(applied_input_volume, 255); + + SetAppliedInputVolume(applied_input_volume); + + RTC_DCHECK_EQ(audio_buffer.num_channels(), channel_controllers_.size()); + const float* const* audio = audio_buffer.channels_const(); + size_t samples_per_channel = audio_buffer.num_frames(); + RTC_DCHECK(audio); + + AggregateChannelLevels(); + if (!capture_output_used_) { + return; + } + + if (!!clipping_predictor_) { + AudioFrameView frame = AudioFrameView( + audio, num_capture_channels_, static_cast(samples_per_channel)); + clipping_predictor_->Analyze(frame); + } + + // Check for clipped samples. We do this in the preprocessing phase in order + // to catch clipped echo as well. + // + // If we find a sufficiently clipped frame, drop the current microphone + // input volume and enforce a new maximum input volume, dropped the same + // amount from the current maximum. This harsh treatment is an effort to avoid + // repeated clipped echo events. + float clipped_ratio = + ComputeClippedRatio(audio, num_capture_channels_, samples_per_channel); + clipping_rate_log_ = std::max(clipped_ratio, clipping_rate_log_); + clipping_rate_log_counter_++; + constexpr int kNumFramesIn30Seconds = 3000; + if (clipping_rate_log_counter_ == kNumFramesIn30Seconds) { + LogClippingMetrics(std::round(100.0f * clipping_rate_log_)); + clipping_rate_log_ = 0.0f; + clipping_rate_log_counter_ = 0; + } + + if (frames_since_clipped_ < clipped_wait_frames_) { + ++frames_since_clipped_; + return; + } + + const bool clipping_detected = clipped_ratio > clipped_ratio_threshold_; + bool clipping_predicted = false; + int predicted_step = 0; + if (!!clipping_predictor_) { + for (int channel = 0; channel < num_capture_channels_; ++channel) { + const auto step = clipping_predictor_->EstimateClippedLevelStep( + channel, recommended_input_volume_, clipped_level_step_, + channel_controllers_[channel]->min_input_volume_after_clipping(), + kMaxInputVolume); + if (step.has_value()) { + predicted_step = std::max(predicted_step, step.value()); + clipping_predicted = true; + } + } + } + + if (clipping_detected) { + RTC_DLOG(LS_INFO) << "[AGC2] Clipping detected (ratio: " << clipped_ratio + << ")"; + } + + int step = clipped_level_step_; + if (clipping_predicted) { + predicted_step = std::max(predicted_step, clipped_level_step_); + RTC_DLOG(LS_INFO) << "[AGC2] Clipping predicted (volume down step: " + << predicted_step << ")"; + if (use_clipping_predictor_step_) { + step = predicted_step; + } + } + + if (clipping_detected || + (clipping_predicted && use_clipping_predictor_step_)) { + for (auto& state_ch : channel_controllers_) { + state_ch->HandleClipping(step); + } + frames_since_clipped_ = 0; + if (!!clipping_predictor_) { + clipping_predictor_->Reset(); + } + } + + AggregateChannelLevels(); +} + +absl::optional InputVolumeController::RecommendInputVolume( + float speech_probability, + absl::optional speech_level_dbfs) { + // Only process if applied input volume is set. + if (!applied_input_volume_.has_value()) { + RTC_LOG(LS_ERROR) << "[AGC2] Applied input volume not set."; + return absl::nullopt; + } + + AggregateChannelLevels(); + const int volume_after_clipping_handling = recommended_input_volume_; + + if (!capture_output_used_) { + return applied_input_volume_; + } + + absl::optional rms_error_db; + if (speech_level_dbfs.has_value()) { + // Compute the error for all frames (both speech and non-speech frames). + rms_error_db = GetSpeechLevelRmsErrorDb( + *speech_level_dbfs, target_range_min_dbfs_, target_range_max_dbfs_); + } + + for (auto& controller : channel_controllers_) { + controller->Process(rms_error_db, speech_probability); + } + + AggregateChannelLevels(); + if (volume_after_clipping_handling != recommended_input_volume_) { + // The recommended input volume was adjusted in order to match the target + // level. + UpdateHistogramOnRecommendedInputVolumeChangeToMatchTarget( + recommended_input_volume_); + } + + applied_input_volume_ = absl::nullopt; + return recommended_input_volume(); +} + +void InputVolumeController::HandleCaptureOutputUsedChange( + bool capture_output_used) { + for (auto& controller : channel_controllers_) { + controller->HandleCaptureOutputUsedChange(capture_output_used); + } + + capture_output_used_ = capture_output_used; +} + +void InputVolumeController::SetAppliedInputVolume(int input_volume) { + applied_input_volume_ = input_volume; + + for (auto& controller : channel_controllers_) { + controller->set_stream_analog_level(input_volume); + } + + AggregateChannelLevels(); +} + +void InputVolumeController::AggregateChannelLevels() { + int new_recommended_input_volume = + channel_controllers_[0]->recommended_analog_level(); + channel_controlling_gain_ = 0; + for (size_t ch = 1; ch < channel_controllers_.size(); ++ch) { + int input_volume = channel_controllers_[ch]->recommended_analog_level(); + if (input_volume < new_recommended_input_volume) { + new_recommended_input_volume = input_volume; + channel_controlling_gain_ = static_cast(ch); + } + } + + // Enforce the minimum input volume when a recommendation is made. + if (applied_input_volume_.has_value() && *applied_input_volume_ > 0) { + new_recommended_input_volume = + std::max(new_recommended_input_volume, min_input_volume_); + } + + recommended_input_volume_ = new_recommended_input_volume; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/input_volume_controller.h b/webrtc/modules/audio_processing/agc2/input_volume_controller.h new file mode 100644 index 0000000..2140554 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/input_volume_controller.h @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_CONTROLLER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_CONTROLLER_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "modules/audio_processing/agc2/clipping_predictor.h" +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/gtest_prod_util.h" + +namespace webrtc { + +class MonoInputVolumeController; + +// The input volume controller recommends what volume to use, handles volume +// changes and clipping detection and prediction. In particular, it handles +// changes triggered by the user (e.g., volume set to zero by a HW mute button). +// This class is not thread-safe. +// TODO(bugs.webrtc.org/7494): Use applied/recommended input volume naming +// convention. +class InputVolumeController final { + public: + // Config for the constructor. + struct Config { + // Minimum input volume that can be recommended. Not enforced when the + // applied input volume is zero outside startup. + int min_input_volume = 20; + // Lowest input volume level that will be applied in response to clipping. + int clipped_level_min = 70; + // Amount input volume level is lowered with every clipping event. Limited + // to (0, 255]. + int clipped_level_step = 15; + // Proportion of clipped samples required to declare a clipping event. + // Limited to (0.0f, 1.0f). + float clipped_ratio_threshold = 0.1f; + // Time in frames to wait after a clipping event before checking again. + // Limited to values higher than 0. + int clipped_wait_frames = 300; + // Enables clipping prediction functionality. + bool enable_clipping_predictor = false; + // Speech level target range (dBFS). If the speech level is in the range + // [`target_range_min_dbfs`, `target_range_max_dbfs`], no input volume + // adjustments are done based on the speech level. For speech levels below + // and above the range, the targets `target_range_min_dbfs` and + // `target_range_max_dbfs` are used, respectively. + int target_range_max_dbfs = -30; + int target_range_min_dbfs = -50; + // Number of wait frames between the recommended input volume updates. + int update_input_volume_wait_frames = 100; + // Speech probability threshold: speech probabilities below the threshold + // are considered silence. Limited to [0.0f, 1.0f]. + float speech_probability_threshold = 0.7f; + // Minimum speech frame ratio for volume updates to be allowed. Limited to + // [0.0f, 1.0f]. + float speech_ratio_threshold = 0.6f; + }; + + // Ctor. `num_capture_channels` specifies the number of channels for the audio + // passed to `AnalyzePreProcess()` and `Process()`. Clamps + // `config.startup_min_level` in the [12, 255] range. + InputVolumeController(int num_capture_channels, const Config& config); + + ~InputVolumeController(); + InputVolumeController(const InputVolumeController&) = delete; + InputVolumeController& operator=(const InputVolumeController&) = delete; + + // TODO(webrtc:7494): Integrate initialization into ctor and remove. + void Initialize(); + + // Analyzes `audio_buffer` before `RecommendInputVolume()` is called so tha + // the analysis can be performed before digital processing operations take + // place (e.g., echo cancellation). The analysis consists of input clipping + // detection and prediction (if enabled). + void AnalyzeInputAudio(int applied_input_volume, + const AudioBuffer& audio_buffer); + + // Adjusts the recommended input volume upwards/downwards based on the result + // of `AnalyzeInputAudio()` and on `speech_level_dbfs` (if specified). Must + // be called after `AnalyzeInputAudio()`. The value of `speech_probability` + // is expected to be in the range [0, 1] and `speech_level_dbfs` in the range + // [-90, 30] and both should be estimated after echo cancellation and noise + // suppression are applied. Returns a non-empty input volume recommendation if + // available. If `capture_output_used_` is true, returns the applied input + // volume. + absl::optional RecommendInputVolume( + float speech_probability, + absl::optional speech_level_dbfs); + + // Stores whether the capture output will be used or not. Call when the + // capture stream output has been flagged to be used/not-used. If unused, the + // controller disregards all incoming audio. + void HandleCaptureOutputUsedChange(bool capture_output_used); + + // Returns true if clipping prediction is enabled. + // TODO(bugs.webrtc.org/7494): Deprecate this method. + bool clipping_predictor_enabled() const { return !!clipping_predictor_; } + + // Returns true if clipping prediction is used to adjust the input volume. + // TODO(bugs.webrtc.org/7494): Deprecate this method. + bool use_clipping_predictor_step() const { + return use_clipping_predictor_step_; + } + + // Only use for testing: Use `RecommendInputVolume()` elsewhere. + // Returns the value of a member variable, needed for testing + // `AnalyzeInputAudio()`. + int recommended_input_volume() const { return recommended_input_volume_; } + + // Only use for testing. + bool capture_output_used() const { return capture_output_used_; } + + private: + friend class InputVolumeControllerTestHelper; + + FRIEND_TEST_ALL_PREFIXES(InputVolumeControllerTest, MinInputVolumeDefault); + FRIEND_TEST_ALL_PREFIXES(InputVolumeControllerTest, MinInputVolumeDisabled); + FRIEND_TEST_ALL_PREFIXES(InputVolumeControllerTest, + MinInputVolumeOutOfRangeAbove); + FRIEND_TEST_ALL_PREFIXES(InputVolumeControllerTest, + MinInputVolumeOutOfRangeBelow); + FRIEND_TEST_ALL_PREFIXES(InputVolumeControllerTest, MinInputVolumeEnabled50); + FRIEND_TEST_ALL_PREFIXES(InputVolumeControllerParametrizedTest, + ClippingParametersVerified); + + // Sets the applied input volume and resets the recommended input volume. + void SetAppliedInputVolume(int level); + + void AggregateChannelLevels(); + + const int num_capture_channels_; + + // Minimum input volume that can be recommended. + const int min_input_volume_; + + // TODO(bugs.webrtc.org/7494): Once + // `AudioProcessingImpl::recommended_stream_analog_level()` becomes a trivial + // getter, leave uninitialized. + // Recommended input volume. After `SetAppliedInputVolume()` is called it + // holds holds the observed input volume. Possibly updated by + // `AnalyzePreProcess()` and `Process()`; after these calls, holds the + // recommended input volume. + int recommended_input_volume_ = 0; + // Applied input volume. After `SetAppliedInputVolume()` is called it holds + // the current applied volume. + absl::optional applied_input_volume_; + + bool capture_output_used_; + + // Clipping detection and prediction. + const int clipped_level_step_; + const float clipped_ratio_threshold_; + const int clipped_wait_frames_; + const std::unique_ptr clipping_predictor_; + const bool use_clipping_predictor_step_; + int frames_since_clipped_; + int clipping_rate_log_counter_; + float clipping_rate_log_; + + // Target range minimum and maximum. If the seech level is in the range + // [`target_range_min_dbfs`, `target_range_max_dbfs`], no volume adjustments + // take place. Instead, the digital gain controller is assumed to adapt to + // compensate for the speech level RMS error. + const int target_range_max_dbfs_; + const int target_range_min_dbfs_; + + // Channel controllers updating the gain upwards/downwards. + std::vector> channel_controllers_; + int channel_controlling_gain_ = 0; +}; + +// TODO(bugs.webrtc.org/7494): Use applied/recommended input volume naming +// convention. +class MonoInputVolumeController { + public: + MonoInputVolumeController(int min_input_volume_after_clipping, + int min_input_volume, + int update_input_volume_wait_frames, + float speech_probability_threshold, + float speech_ratio_threshold); + ~MonoInputVolumeController(); + MonoInputVolumeController(const MonoInputVolumeController&) = delete; + MonoInputVolumeController& operator=(const MonoInputVolumeController&) = + delete; + + void Initialize(); + void HandleCaptureOutputUsedChange(bool capture_output_used); + + // Sets the current input volume. + void set_stream_analog_level(int input_volume) { + recommended_input_volume_ = input_volume; + } + + // Lowers the recommended input volume in response to clipping based on the + // suggested reduction `clipped_level_step`. Must be called after + // `set_stream_analog_level()`. + void HandleClipping(int clipped_level_step); + + // TODO(bugs.webrtc.org/7494): Rename, audio not passed to the method anymore. + // Adjusts the recommended input volume upwards/downwards depending on the + // result of `HandleClipping()` and on `rms_error_dbfs`. Updates are only + // allowed for active speech segments and when `rms_error_dbfs` is not empty. + // Must be called after `HandleClipping()`. + void Process(absl::optional rms_error_dbfs, float speech_probability); + + // Returns the recommended input volume. Must be called after `Process()`. + int recommended_analog_level() const { return recommended_input_volume_; } + + void ActivateLogging() { log_to_histograms_ = true; } + + int min_input_volume_after_clipping() const { + return min_input_volume_after_clipping_; + } + + // Only used for testing. + int min_input_volume() const { return min_input_volume_; } + + private: + // Sets a new input volume, after first checking that it hasn't been updated + // by the user, in which case no action is taken. + void SetInputVolume(int new_volume); + + // Sets the maximum input volume that the input volume controller is allowed + // to apply. The volume must be at least `kClippedLevelMin`. + void SetMaxLevel(int level); + + int CheckVolumeAndReset(); + + // Updates the recommended input volume. If the volume slider needs to be + // moved, we check first if the user has adjusted it, in which case we take no + // action and cache the updated level. + void UpdateInputVolume(int rms_error_dbfs); + + const int min_input_volume_; + const int min_input_volume_after_clipping_; + int max_input_volume_; + + int last_recommended_input_volume_ = 0; + + bool capture_output_used_ = true; + bool check_volume_on_next_process_ = true; + bool startup_ = true; + + // TODO(bugs.webrtc.org/7494): Create a separate member for the applied + // input volume. + // Recommended input volume. After `set_stream_analog_level()` is + // called, it holds the observed applied input volume. Possibly updated by + // `HandleClipping()` and `Process()`; after these calls, holds the + // recommended input volume. + int recommended_input_volume_ = 0; + + bool log_to_histograms_ = false; + + // Counters for frames and speech frames since the last update in the + // recommended input volume. + const int update_input_volume_wait_frames_; + int frames_since_update_input_volume_ = 0; + int speech_frames_since_update_input_volume_ = 0; + bool is_first_frame_ = true; + + // Speech probability threshold for a frame to be considered speech (instead + // of silence). Limited to [0.0f, 1.0f]. + const float speech_probability_threshold_; + // Minimum ratio of speech frames. Limited to [0.0f, 1.0f]. + const float speech_ratio_threshold_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_CONTROLLER_H_ diff --git a/webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.cc b/webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.cc new file mode 100644 index 0000000..05624b1 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/input_volume_stats_reporter.h" + +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { + +using InputVolumeType = InputVolumeStatsReporter::InputVolumeType; + +constexpr int kFramesIn60Seconds = 6000; +constexpr int kMinInputVolume = 0; +constexpr int kMaxInputVolume = 255; +constexpr int kMaxUpdate = kMaxInputVolume - kMinInputVolume; + +int ComputeAverageUpdate(int sum_updates, int num_updates) { + RTC_DCHECK_GE(sum_updates, 0); + RTC_DCHECK_LE(sum_updates, kMaxUpdate * kFramesIn60Seconds); + RTC_DCHECK_GE(num_updates, 0); + RTC_DCHECK_LE(num_updates, kFramesIn60Seconds); + if (num_updates == 0) { + return 0; + } + return std::round(static_cast(sum_updates) / + static_cast(num_updates)); +} + +constexpr absl::string_view MetricNamePrefix( + InputVolumeType input_volume_type) { + switch (input_volume_type) { + case InputVolumeType::kApplied: + return "WebRTC.Audio.Apm.AppliedInputVolume."; + case InputVolumeType::kRecommended: + return "WebRTC.Audio.Apm.RecommendedInputVolume."; + } +} + +metrics::Histogram* CreateVolumeHistogram(InputVolumeType input_volume_type) { + char buffer[64]; + rtc::SimpleStringBuilder builder(buffer); + builder << MetricNamePrefix(input_volume_type) << "OnChange"; + return metrics::HistogramFactoryGetCountsLinear(/*name=*/builder.str(), + /*min=*/1, + /*max=*/kMaxInputVolume, + /*bucket_count=*/50); +} + +metrics::Histogram* CreateRateHistogram(InputVolumeType input_volume_type, + absl::string_view name) { + char buffer[64]; + rtc::SimpleStringBuilder builder(buffer); + builder << MetricNamePrefix(input_volume_type) << name; + return metrics::HistogramFactoryGetCountsLinear(/*name=*/builder.str(), + /*min=*/1, + /*max=*/kFramesIn60Seconds, + /*bucket_count=*/50); +} + +metrics::Histogram* CreateAverageHistogram(InputVolumeType input_volume_type, + absl::string_view name) { + char buffer[64]; + rtc::SimpleStringBuilder builder(buffer); + builder << MetricNamePrefix(input_volume_type) << name; + return metrics::HistogramFactoryGetCountsLinear(/*name=*/builder.str(), + /*min=*/1, + /*max=*/kMaxUpdate, + /*bucket_count=*/50); +} + +} // namespace + +InputVolumeStatsReporter::InputVolumeStatsReporter(InputVolumeType type) + : histograms_( + {.on_volume_change = CreateVolumeHistogram(type), + .decrease_rate = CreateRateHistogram(type, "DecreaseRate"), + .decrease_average = CreateAverageHistogram(type, "DecreaseAverage"), + .increase_rate = CreateRateHistogram(type, "IncreaseRate"), + .increase_average = CreateAverageHistogram(type, "IncreaseAverage"), + .update_rate = CreateRateHistogram(type, "UpdateRate"), + .update_average = CreateAverageHistogram(type, "UpdateAverage")}), + cannot_log_stats_(!histograms_.AllPointersSet()) { + if (cannot_log_stats_) { + RTC_LOG(LS_WARNING) << "Will not log any `" << MetricNamePrefix(type) + << "*` histogram stats."; + } +} + +InputVolumeStatsReporter::~InputVolumeStatsReporter() = default; + +void InputVolumeStatsReporter::UpdateStatistics(int input_volume) { + if (cannot_log_stats_) { + // Since the stats cannot be logged, do not bother updating them. + return; + } + + RTC_DCHECK_GE(input_volume, kMinInputVolume); + RTC_DCHECK_LE(input_volume, kMaxInputVolume); + if (previous_input_volume_.has_value() && + input_volume != previous_input_volume_.value()) { + // Update stats when the input volume changes. + metrics::HistogramAdd(histograms_.on_volume_change, input_volume); + // Update stats that are periodically logged. + const int volume_change = input_volume - previous_input_volume_.value(); + if (volume_change < 0) { + ++volume_update_stats_.num_decreases; + volume_update_stats_.sum_decreases -= volume_change; + } else { + ++volume_update_stats_.num_increases; + volume_update_stats_.sum_increases += volume_change; + } + } + // Periodically log input volume change metrics. + if (++log_volume_update_stats_counter_ >= kFramesIn60Seconds) { + LogVolumeUpdateStats(); + volume_update_stats_ = {}; + log_volume_update_stats_counter_ = 0; + } + previous_input_volume_ = input_volume; +} + +void InputVolumeStatsReporter::LogVolumeUpdateStats() const { + // Decrease rate and average. + metrics::HistogramAdd(histograms_.decrease_rate, + volume_update_stats_.num_decreases); + if (volume_update_stats_.num_decreases > 0) { + int average_decrease = ComputeAverageUpdate( + volume_update_stats_.sum_decreases, volume_update_stats_.num_decreases); + metrics::HistogramAdd(histograms_.decrease_average, average_decrease); + } + // Increase rate and average. + metrics::HistogramAdd(histograms_.increase_rate, + volume_update_stats_.num_increases); + if (volume_update_stats_.num_increases > 0) { + int average_increase = ComputeAverageUpdate( + volume_update_stats_.sum_increases, volume_update_stats_.num_increases); + metrics::HistogramAdd(histograms_.increase_average, average_increase); + } + // Update rate and average. + int num_updates = + volume_update_stats_.num_decreases + volume_update_stats_.num_increases; + metrics::HistogramAdd(histograms_.update_rate, num_updates); + if (num_updates > 0) { + int average_update = ComputeAverageUpdate( + volume_update_stats_.sum_decreases + volume_update_stats_.sum_increases, + num_updates); + metrics::HistogramAdd(histograms_.update_average, average_update); + } +} + +void UpdateHistogramOnRecommendedInputVolumeChangeToMatchTarget(int volume) { + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.Apm.RecommendedInputVolume.OnChangeToMatchTarget", volume, + 1, kMaxInputVolume, 50); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.h b/webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.h new file mode 100644 index 0000000..31b1100 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_STATS_REPORTER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_STATS_REPORTER_H_ + +#include "absl/types/optional.h" +#include "rtc_base/gtest_prod_util.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +// Input volume statistics calculator. Computes aggregate stats based on the +// framewise input volume observed by `UpdateStatistics()`. Periodically logs +// the statistics into a histogram. +class InputVolumeStatsReporter { + public: + enum class InputVolumeType { + kApplied = 0, + kRecommended = 1, + }; + + explicit InputVolumeStatsReporter(InputVolumeType input_volume_type); + InputVolumeStatsReporter(const InputVolumeStatsReporter&) = delete; + InputVolumeStatsReporter operator=(const InputVolumeStatsReporter&) = delete; + ~InputVolumeStatsReporter(); + + // Updates the stats based on `input_volume`. Periodically logs the stats into + // a histogram. + void UpdateStatistics(int input_volume); + + private: + FRIEND_TEST_ALL_PREFIXES(InputVolumeStatsReporterTest, + CheckVolumeUpdateStatsForEmptyStats); + FRIEND_TEST_ALL_PREFIXES(InputVolumeStatsReporterTest, + CheckVolumeUpdateStatsAfterNoVolumeChange); + FRIEND_TEST_ALL_PREFIXES(InputVolumeStatsReporterTest, + CheckVolumeUpdateStatsAfterVolumeIncrease); + FRIEND_TEST_ALL_PREFIXES(InputVolumeStatsReporterTest, + CheckVolumeUpdateStatsAfterVolumeDecrease); + FRIEND_TEST_ALL_PREFIXES(InputVolumeStatsReporterTest, + CheckVolumeUpdateStatsAfterReset); + + // Stores input volume update stats to enable calculation of update rate and + // average update separately for volume increases and decreases. + struct VolumeUpdateStats { + int num_decreases = 0; + int num_increases = 0; + int sum_decreases = 0; + int sum_increases = 0; + } volume_update_stats_; + + // Returns a copy of the stored statistics. Use only for testing. + VolumeUpdateStats volume_update_stats() const { return volume_update_stats_; } + + // Computes aggregate stat and logs them into a histogram. + void LogVolumeUpdateStats() const; + + // Histograms. + struct Histograms { + metrics::Histogram* const on_volume_change; + metrics::Histogram* const decrease_rate; + metrics::Histogram* const decrease_average; + metrics::Histogram* const increase_rate; + metrics::Histogram* const increase_average; + metrics::Histogram* const update_rate; + metrics::Histogram* const update_average; + bool AllPointersSet() const { + return !!on_volume_change && !!decrease_rate && !!decrease_average && + !!increase_rate && !!increase_average && !!update_rate && + !!update_average; + } + } histograms_; + + // True if the stats cannot be logged. + const bool cannot_log_stats_; + + int log_volume_update_stats_counter_ = 0; + absl::optional previous_input_volume_ = absl::nullopt; +}; + +// Updates the histogram that keeps track of recommended input volume changes +// required in order to match the target level in the input volume adaptation +// process. +void UpdateHistogramOnRecommendedInputVolumeChangeToMatchTarget(int volume); + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_STATS_REPORTER_H_ diff --git a/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.cc b/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.cc index 502e702..bb6e038 100644 --- a/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.cc +++ b/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.cc @@ -13,9 +13,11 @@ #include #include +#include "absl/strings/string_view.h" #include "modules/audio_processing/agc2/agc2_common.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" namespace webrtc { @@ -28,16 +30,23 @@ constexpr std::array constexpr std::array InterpolatedGainCurve::approximation_params_q_; -InterpolatedGainCurve::InterpolatedGainCurve(ApmDataDumper* apm_data_dumper, - std::string histogram_name_prefix) - : region_logger_("WebRTC.Audio." + histogram_name_prefix + - ".FixedDigitalGainCurveRegion.Identity", - "WebRTC.Audio." + histogram_name_prefix + - ".FixedDigitalGainCurveRegion.Knee", - "WebRTC.Audio." + histogram_name_prefix + - ".FixedDigitalGainCurveRegion.Limiter", - "WebRTC.Audio." + histogram_name_prefix + - ".FixedDigitalGainCurveRegion.Saturation"), +InterpolatedGainCurve::InterpolatedGainCurve( + ApmDataDumper* apm_data_dumper, + absl::string_view histogram_name_prefix) + : region_logger_( + (rtc::StringBuilder("WebRTC.Audio.") + << histogram_name_prefix << ".FixedDigitalGainCurveRegion.Identity") + .str(), + (rtc::StringBuilder("WebRTC.Audio.") + << histogram_name_prefix << ".FixedDigitalGainCurveRegion.Knee") + .str(), + (rtc::StringBuilder("WebRTC.Audio.") + << histogram_name_prefix << ".FixedDigitalGainCurveRegion.Limiter") + .str(), + (rtc::StringBuilder("WebRTC.Audio.") + << histogram_name_prefix + << ".FixedDigitalGainCurveRegion.Saturation") + .str()), apm_data_dumper_(apm_data_dumper) {} InterpolatedGainCurve::~InterpolatedGainCurve() { @@ -56,10 +65,10 @@ InterpolatedGainCurve::~InterpolatedGainCurve() { } InterpolatedGainCurve::RegionLogger::RegionLogger( - std::string identity_histogram_name, - std::string knee_histogram_name, - std::string limiter_histogram_name, - std::string saturation_histogram_name) + absl::string_view identity_histogram_name, + absl::string_view knee_histogram_name, + absl::string_view limiter_histogram_name, + absl::string_view saturation_histogram_name) : identity_histogram( metrics::HistogramFactoryGetCounts(identity_histogram_name, 1, @@ -114,7 +123,7 @@ void InterpolatedGainCurve::RegionLogger::LogRegionStats( break; } default: { - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } } } @@ -150,11 +159,11 @@ void InterpolatedGainCurve::UpdateStats(float input_level) const { } // Looks up a gain to apply given a non-negative input level. -// The cost of this operation depends on the region in which |input_level| +// The cost of this operation depends on the region in which `input_level` // falls. // For the identity and the saturation regions the cost is O(1). // For the other regions, namely knee and limiter, the cost is -// O(2 + log2(|LightkInterpolatedGainCurveTotalPoints|), plus O(1) for the +// O(2 + log2(`LightkInterpolatedGainCurveTotalPoints`), plus O(1) for the // linear interpolation (one product and one sum). float InterpolatedGainCurve::LookUpGainToApply(float input_level) const { UpdateStats(input_level); diff --git a/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.h b/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.h index ef1c027..8dd3e48 100644 --- a/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.h +++ b/webrtc/modules/audio_processing/agc2/interpolated_gain_curve.h @@ -12,10 +12,9 @@ #define MODULES_AUDIO_PROCESSING_AGC2_INTERPOLATED_GAIN_CURVE_H_ #include -#include +#include "absl/strings/string_view.h" #include "modules/audio_processing/agc2/agc2_common.h" -#include "rtc_base/constructor_magic.h" #include "rtc_base/gtest_prod_util.h" #include "system_wrappers/include/metrics.h" @@ -61,9 +60,12 @@ class InterpolatedGainCurve { }; InterpolatedGainCurve(ApmDataDumper* apm_data_dumper, - std::string histogram_name_prefix); + absl::string_view histogram_name_prefix); ~InterpolatedGainCurve(); + InterpolatedGainCurve(const InterpolatedGainCurve&) = delete; + InterpolatedGainCurve& operator=(const InterpolatedGainCurve&) = delete; + Stats get_stats() const { return stats_; } // Given a non-negative input level (linear scale), a scalar factor to apply @@ -75,7 +77,7 @@ class InterpolatedGainCurve { private: // For comparing 'approximation_params_*_' with ones computed by // ComputeInterpolatedGainCurve. - FRIEND_TEST_ALL_PREFIXES(AutomaticGainController2InterpolatedGainCurve, + FRIEND_TEST_ALL_PREFIXES(GainController2InterpolatedGainCurve, CheckApproximationParams); struct RegionLogger { @@ -84,10 +86,10 @@ class InterpolatedGainCurve { metrics::Histogram* limiter_histogram; metrics::Histogram* saturation_histogram; - RegionLogger(std::string identity_histogram_name, - std::string knee_histogram_name, - std::string limiter_histogram_name, - std::string saturation_histogram_name); + RegionLogger(absl::string_view identity_histogram_name, + absl::string_view knee_histogram_name, + absl::string_view limiter_histogram_name, + absl::string_view saturation_histogram_name); ~RegionLogger(); @@ -143,8 +145,6 @@ class InterpolatedGainCurve { // Stats. mutable Stats stats_; - - RTC_DISALLOW_COPY_AND_ASSIGN(InterpolatedGainCurve); }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/limiter.cc b/webrtc/modules/audio_processing/agc2/limiter.cc index 1589f07..7a1e220 100644 --- a/webrtc/modules/audio_processing/agc2/limiter.cc +++ b/webrtc/modules/audio_processing/agc2/limiter.cc @@ -14,10 +14,12 @@ #include #include +#include "absl/strings/string_view.h" #include "api/array_view.h" #include "modules/audio_processing/agc2/agc2_common.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/numerics/safe_minmax.h" namespace webrtc { @@ -29,14 +31,14 @@ namespace { // sub-frame, linear interpolation is replaced with a power function which // reduces the chances of over-shooting (and hence saturation), however reducing // the fixed gain effectiveness. -constexpr float kAttackFirstSubframeInterpolationPower = 8.f; +constexpr float kAttackFirstSubframeInterpolationPower = 8.0f; void InterpolateFirstSubframe(float last_factor, float current_factor, rtc::ArrayView subframe) { - const auto n = subframe.size(); - constexpr auto p = kAttackFirstSubframeInterpolationPower; - for (size_t i = 0; i < n; ++i) { + const int n = rtc::dchecked_cast(subframe.size()); + constexpr float p = kAttackFirstSubframeInterpolationPower; + for (int i = 0; i < n; ++i) { subframe[i] = std::pow(1.f - i / n, p) * (last_factor - current_factor) + current_factor; } @@ -44,10 +46,10 @@ void InterpolateFirstSubframe(float last_factor, void ComputePerSampleSubframeFactors( const std::array& scaling_factors, - size_t samples_per_channel, + int samples_per_channel, rtc::ArrayView per_sample_scaling_factors) { - const size_t num_subframes = scaling_factors.size() - 1; - const size_t subframe_size = + const int num_subframes = scaling_factors.size() - 1; + const int subframe_size = rtc::CheckedDivExact(samples_per_channel, num_subframes); // Handle first sub-frame differently in case of attack. @@ -59,12 +61,12 @@ void ComputePerSampleSubframeFactors( per_sample_scaling_factors.subview(0, subframe_size))); } - for (size_t i = is_attack ? 1 : 0; i < num_subframes; ++i) { - const size_t subframe_start = i * subframe_size; + for (int i = is_attack ? 1 : 0; i < num_subframes; ++i) { + const int subframe_start = i * subframe_size; const float scaling_start = scaling_factors[i]; const float scaling_end = scaling_factors[i + 1]; const float scaling_diff = (scaling_end - scaling_start) / subframe_size; - for (size_t j = 0; j < subframe_size; ++j) { + for (int j = 0; j < subframe_size; ++j) { per_sample_scaling_factors[subframe_start + j] = scaling_start + scaling_diff * j; } @@ -73,18 +75,18 @@ void ComputePerSampleSubframeFactors( void ScaleSamples(rtc::ArrayView per_sample_scaling_factors, AudioFrameView signal) { - const size_t samples_per_channel = signal.samples_per_channel(); + const int samples_per_channel = signal.samples_per_channel(); RTC_DCHECK_EQ(samples_per_channel, per_sample_scaling_factors.size()); - for (size_t i = 0; i < signal.num_channels(); ++i) { - auto channel = signal.channel(i); - for (size_t j = 0; j < samples_per_channel; ++j) { + for (int i = 0; i < signal.num_channels(); ++i) { + rtc::ArrayView channel = signal.channel(i); + for (int j = 0; j < samples_per_channel; ++j) { channel[j] = rtc::SafeClamp(channel[j] * per_sample_scaling_factors[j], kMinFloatS16Value, kMaxFloatS16Value); } } } -void CheckLimiterSampleRate(size_t sample_rate_hz) { +void CheckLimiterSampleRate(int sample_rate_hz) { // Check that per_sample_scaling_factors_ is large enough. RTC_DCHECK_LE(sample_rate_hz, kMaximalNumberOfSamplesPerChannel * 1000 / kFrameDurationMs); @@ -92,9 +94,9 @@ void CheckLimiterSampleRate(size_t sample_rate_hz) { } // namespace -Limiter::Limiter(size_t sample_rate_hz, +Limiter::Limiter(int sample_rate_hz, ApmDataDumper* apm_data_dumper, - std::string histogram_name) + absl::string_view histogram_name) : interp_gain_curve_(apm_data_dumper, histogram_name), level_estimator_(sample_rate_hz, apm_data_dumper), apm_data_dumper_(apm_data_dumper) { @@ -104,7 +106,8 @@ Limiter::Limiter(size_t sample_rate_hz, Limiter::~Limiter() = default; void Limiter::Process(AudioFrameView signal) { - const auto level_estimate = level_estimator_.ComputeLevel(signal); + const std::array level_estimate = + level_estimator_.ComputeLevel(signal); RTC_DCHECK_EQ(level_estimate.size() + 1, scaling_factors_.size()); scaling_factors_[0] = last_scaling_factor_; @@ -113,7 +116,7 @@ void Limiter::Process(AudioFrameView signal) { return interp_gain_curve_.LookUpGainToApply(x); }); - const size_t samples_per_channel = signal.samples_per_channel(); + const int samples_per_channel = signal.samples_per_channel(); RTC_DCHECK_LE(samples_per_channel, kMaximalNumberOfSamplesPerChannel); auto per_sample_scaling_factors = rtc::ArrayView( @@ -125,16 +128,18 @@ void Limiter::Process(AudioFrameView signal) { last_scaling_factor_ = scaling_factors_.back(); // Dump data for debug. - apm_data_dumper_->DumpRaw("agc2_gain_curve_applier_scaling_factors", - samples_per_channel, - per_sample_scaling_factors_.data()); + apm_data_dumper_->DumpRaw("agc2_limiter_last_scaling_factor", + last_scaling_factor_); + apm_data_dumper_->DumpRaw( + "agc2_limiter_region", + static_cast(interp_gain_curve_.get_stats().region)); } InterpolatedGainCurve::Stats Limiter::GetGainCurveStats() const { return interp_gain_curve_.get_stats(); } -void Limiter::SetSampleRate(size_t sample_rate_hz) { +void Limiter::SetSampleRate(int sample_rate_hz) { CheckLimiterSampleRate(sample_rate_hz); level_estimator_.SetSampleRate(sample_rate_hz); } diff --git a/webrtc/modules/audio_processing/agc2/limiter.h b/webrtc/modules/audio_processing/agc2/limiter.h index 599fd0f..d4d5563 100644 --- a/webrtc/modules/audio_processing/agc2/limiter.h +++ b/webrtc/modules/audio_processing/agc2/limiter.h @@ -11,27 +11,26 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_LIMITER_H_ #define MODULES_AUDIO_PROCESSING_AGC2_LIMITER_H_ -#include #include +#include "absl/strings/string_view.h" #include "modules/audio_processing/agc2/fixed_digital_level_estimator.h" #include "modules/audio_processing/agc2/interpolated_gain_curve.h" #include "modules/audio_processing/include/audio_frame_view.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { class ApmDataDumper; class Limiter { public: - Limiter(size_t sample_rate_hz, + Limiter(int sample_rate_hz, ApmDataDumper* apm_data_dumper, - std::string histogram_name_prefix); + absl::string_view histogram_name_prefix); Limiter(const Limiter& limiter) = delete; Limiter& operator=(const Limiter& limiter) = delete; ~Limiter(); - // Applies limiter and hard-clipping to |signal|. + // Applies limiter and hard-clipping to `signal`. void Process(AudioFrameView signal); InterpolatedGainCurve::Stats GetGainCurveStats() const; @@ -40,7 +39,7 @@ class Limiter { // * below kMaximalNumberOfSamplesPerChannel*1000/kFrameDurationMs // so that samples_per_channel fit in the // per_sample_scaling_factors_ array. - void SetSampleRate(size_t sample_rate_hz); + void SetSampleRate(int sample_rate_hz); // Resets the internal state. void Reset(); diff --git a/webrtc/modules/audio_processing/agc2/limiter_db_gain_curve.cc b/webrtc/modules/audio_processing/agc2/limiter_db_gain_curve.cc index d55ed5d..d47c0b2 100644 --- a/webrtc/modules/audio_processing/agc2/limiter_db_gain_curve.cc +++ b/webrtc/modules/audio_processing/agc2/limiter_db_gain_curve.cc @@ -105,7 +105,7 @@ double LimiterDbGainCurve::GetGainLinear(double input_level_linear) const { input_level_linear; } -// Computes the first derivative of GetGainLinear() in |x|. +// Computes the first derivative of GetGainLinear() in `x`. double LimiterDbGainCurve::GetGainFirstDerivativeLinear(double x) const { // Beyond-knee region only. RTC_CHECK_GE(x, limiter_start_linear_ - 1e-7 * kMaxAbsFloatS16Value); diff --git a/webrtc/modules/audio_processing/agc2/noise_level_estimator.cc b/webrtc/modules/audio_processing/agc2/noise_level_estimator.cc index 2ca5034..691513b 100644 --- a/webrtc/modules/audio_processing/agc2/noise_level_estimator.cc +++ b/webrtc/modules/audio_processing/agc2/noise_level_estimator.cc @@ -17,98 +17,156 @@ #include #include "api/array_view.h" -#include "common_audio/include/audio_util.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" namespace webrtc { - namespace { + constexpr int kFramesPerSecond = 100; float FrameEnergy(const AudioFrameView& audio) { - float energy = 0.f; - for (size_t k = 0; k < audio.num_channels(); ++k) { + float energy = 0.0f; + for (int k = 0; k < audio.num_channels(); ++k) { float channel_energy = - std::accumulate(audio.channel(k).begin(), audio.channel(k).end(), 0.f, + std::accumulate(audio.channel(k).begin(), audio.channel(k).end(), 0.0f, [](float a, float b) -> float { return a + b * b; }); energy = std::max(channel_energy, energy); } return energy; } -float EnergyToDbfs(float signal_energy, size_t num_samples) { - const float rms = std::sqrt(signal_energy / num_samples); - return FloatS16ToDbfs(rms); +float EnergyToDbfs(float signal_energy, int num_samples) { + RTC_DCHECK_GE(signal_energy, 0.0f); + const float rms_square = signal_energy / num_samples; + constexpr float kMinDbfs = -90.30899869919436f; + if (rms_square <= 1.0f) { + return kMinDbfs; + } + return 10.0f * std::log10(rms_square) + kMinDbfs; } + +// Updates the noise floor with instant decay and slow attack. This tuning is +// specific for AGC2, so that (i) it can promptly increase the gain if the noise +// floor drops (instant decay) and (ii) in case of music or fast speech, due to +// which the noise floor can be overestimated, the gain reduction is slowed +// down. +float SmoothNoiseFloorEstimate(float current_estimate, float new_estimate) { + constexpr float kAttack = 0.5f; + if (current_estimate < new_estimate) { + // Attack phase. + return kAttack * new_estimate + (1.0f - kAttack) * current_estimate; + } + // Instant attack. + return new_estimate; +} + +class NoiseFloorEstimator : public NoiseLevelEstimator { + public: + // Update the noise floor every 5 seconds. + static constexpr int kUpdatePeriodNumFrames = 500; + static_assert(kUpdatePeriodNumFrames >= 200, + "A too small value may cause noise level overestimation."); + static_assert(kUpdatePeriodNumFrames <= 1500, + "A too large value may make AGC2 slow at reacting to increased " + "noise levels."); + + NoiseFloorEstimator(ApmDataDumper* data_dumper) : data_dumper_(data_dumper) { + RTC_DCHECK(data_dumper_); + // Initially assume that 48 kHz will be used. `Analyze()` will detect the + // used sample rate and call `Initialize()` again if needed. + Initialize(/*sample_rate_hz=*/48000); + } + NoiseFloorEstimator(const NoiseFloorEstimator&) = delete; + NoiseFloorEstimator& operator=(const NoiseFloorEstimator&) = delete; + ~NoiseFloorEstimator() = default; + + float Analyze(const AudioFrameView& frame) override { + // Detect sample rate changes. + const int sample_rate_hz = + static_cast(frame.samples_per_channel() * kFramesPerSecond); + if (sample_rate_hz != sample_rate_hz_) { + Initialize(sample_rate_hz); + } + + const float frame_energy = FrameEnergy(frame); + if (frame_energy <= min_noise_energy_) { + // Ignore frames when muted or below the minimum measurable energy. + if (data_dumper_) + data_dumper_->DumpRaw("agc2_noise_floor_estimator_preliminary_level", + noise_energy_); + return EnergyToDbfs(noise_energy_, + static_cast(frame.samples_per_channel())); + } + + if (preliminary_noise_energy_set_) { + preliminary_noise_energy_ = + std::min(preliminary_noise_energy_, frame_energy); + } else { + preliminary_noise_energy_ = frame_energy; + preliminary_noise_energy_set_ = true; + } + if (data_dumper_) + data_dumper_->DumpRaw("agc2_noise_floor_estimator_preliminary_level", + preliminary_noise_energy_); + + if (counter_ == 0) { + // Full period observed. + first_period_ = false; + // Update the estimated noise floor energy with the preliminary + // estimation. + noise_energy_ = SmoothNoiseFloorEstimate( + /*current_estimate=*/noise_energy_, + /*new_estimate=*/preliminary_noise_energy_); + // Reset for a new observation period. + counter_ = kUpdatePeriodNumFrames; + preliminary_noise_energy_set_ = false; + } else if (first_period_) { + // While analyzing the signal during the initial period, continuously + // update the estimated noise energy, which is monotonic. + noise_energy_ = preliminary_noise_energy_; + counter_--; + } else { + // During the observation period it's only allowed to lower the energy. + noise_energy_ = std::min(noise_energy_, preliminary_noise_energy_); + counter_--; + } + + float noise_rms_dbfs = EnergyToDbfs( + noise_energy_, static_cast(frame.samples_per_channel())); + if (data_dumper_) + data_dumper_->DumpRaw("agc2_noise_rms_dbfs", noise_rms_dbfs); + + return noise_rms_dbfs; + } + + private: + void Initialize(int sample_rate_hz) { + sample_rate_hz_ = sample_rate_hz; + first_period_ = true; + preliminary_noise_energy_set_ = false; + // Initialize the minimum noise energy to -84 dBFS. + min_noise_energy_ = sample_rate_hz * 2.0f * 2.0f / kFramesPerSecond; + preliminary_noise_energy_ = min_noise_energy_; + noise_energy_ = min_noise_energy_; + counter_ = kUpdatePeriodNumFrames; + } + + ApmDataDumper* const data_dumper_; + int sample_rate_hz_; + float min_noise_energy_; + bool first_period_; + bool preliminary_noise_energy_set_; + float preliminary_noise_energy_; + float noise_energy_; + int counter_; +}; + } // namespace -NoiseLevelEstimator::NoiseLevelEstimator(ApmDataDumper* data_dumper) - : signal_classifier_(data_dumper) { - Initialize(48000); -} - -NoiseLevelEstimator::~NoiseLevelEstimator() {} - -void NoiseLevelEstimator::Initialize(int sample_rate_hz) { - sample_rate_hz_ = sample_rate_hz; - noise_energy_ = 1.f; - first_update_ = true; - min_noise_energy_ = sample_rate_hz * 2.f * 2.f / kFramesPerSecond; - noise_energy_hold_counter_ = 0; - signal_classifier_.Initialize(sample_rate_hz); -} - -float NoiseLevelEstimator::Analyze(const AudioFrameView& frame) { - const int rate = - static_cast(frame.samples_per_channel() * kFramesPerSecond); - if (rate != sample_rate_hz_) { - Initialize(rate); - } - const float frame_energy = FrameEnergy(frame); - if (frame_energy <= 0.f) { - RTC_DCHECK_GE(frame_energy, 0.f); - return EnergyToDbfs(noise_energy_, frame.samples_per_channel()); - } - - if (first_update_) { - // Initialize the noise energy to the frame energy. - first_update_ = false; - return EnergyToDbfs( - noise_energy_ = std::max(frame_energy, min_noise_energy_), - frame.samples_per_channel()); - } - - const SignalClassifier::SignalType signal_type = - signal_classifier_.Analyze(frame.channel(0)); - - // Update the noise estimate in a minimum statistics-type manner. - if (signal_type == SignalClassifier::SignalType::kStationary) { - if (frame_energy > noise_energy_) { - // Leak the estimate upwards towards the frame energy if no recent - // downward update. - noise_energy_hold_counter_ = std::max(noise_energy_hold_counter_ - 1, 0); - - if (noise_energy_hold_counter_ == 0) { - noise_energy_ = std::min(noise_energy_ * 1.01f, frame_energy); - } - } else { - // Update smoothly downwards with a limited maximum update magnitude. - noise_energy_ = - std::max(noise_energy_ * 0.9f, - noise_energy_ + 0.05f * (frame_energy - noise_energy_)); - noise_energy_hold_counter_ = 1000; - } - } else { - // For a non-stationary signal, leak the estimate downwards in order to - // avoid estimate locking due to incorrect signal classification. - noise_energy_ = noise_energy_ * 0.99f; - } - - // Ensure a minimum of the estimate. - return EnergyToDbfs( - noise_energy_ = std::max(noise_energy_, min_noise_energy_), - frame.samples_per_channel()); +std::unique_ptr CreateNoiseFloorEstimator( + ApmDataDumper* data_dumper) { + return std::make_unique(data_dumper); } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/noise_level_estimator.h b/webrtc/modules/audio_processing/agc2/noise_level_estimator.h index ca2f9f2..9f3b957 100644 --- a/webrtc/modules/audio_processing/agc2/noise_level_estimator.h +++ b/webrtc/modules/audio_processing/agc2/noise_level_estimator.h @@ -11,33 +11,26 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_NOISE_LEVEL_ESTIMATOR_H_ #define MODULES_AUDIO_PROCESSING_AGC2_NOISE_LEVEL_ESTIMATOR_H_ -#include "modules/audio_processing/agc2/signal_classifier.h" +#include + #include "modules/audio_processing/include/audio_frame_view.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { class ApmDataDumper; +// Noise level estimator interface. class NoiseLevelEstimator { public: - NoiseLevelEstimator(ApmDataDumper* data_dumper); - ~NoiseLevelEstimator(); - // Returns the estimated noise level in dBFS. - float Analyze(const AudioFrameView& frame); - - private: - void Initialize(int sample_rate_hz); - - int sample_rate_hz_; - float min_noise_energy_; - bool first_update_; - float noise_energy_; - int noise_energy_hold_counter_; - SignalClassifier signal_classifier_; - - RTC_DISALLOW_COPY_AND_ASSIGN(NoiseLevelEstimator); + virtual ~NoiseLevelEstimator() = default; + // Analyzes a 10 ms `frame`, updates the noise level estimation and returns + // the value for the latter in dBFS. + virtual float Analyze(const AudioFrameView& frame) = 0; }; +// Creates a noise level estimator based on noise floor detection. +std::unique_ptr CreateNoiseFloorEstimator( + ApmDataDumper* data_dumper); + } // namespace webrtc #endif // MODULES_AUDIO_PROCESSING_AGC2_NOISE_LEVEL_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.cc b/webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.cc deleted file mode 100644 index 31438b1..0000000 --- a/webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.cc +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/noise_spectrum_estimator.h" - -#include - -#include - -#include "api/array_view.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/arraysize.h" -#include "rtc_base/checks.h" - -namespace webrtc { -namespace { -constexpr float kMinNoisePower = 100.f; -} // namespace - -NoiseSpectrumEstimator::NoiseSpectrumEstimator(ApmDataDumper* data_dumper) - : data_dumper_(data_dumper) { - Initialize(); -} - -void NoiseSpectrumEstimator::Initialize() { - std::fill(noise_spectrum_, noise_spectrum_ + arraysize(noise_spectrum_), - kMinNoisePower); -} - -void NoiseSpectrumEstimator::Update(rtc::ArrayView spectrum, - bool first_update) { - RTC_DCHECK_EQ(65, spectrum.size()); - - if (first_update) { - // Initialize the noise spectral estimate with the signal spectrum. - std::copy(spectrum.data(), spectrum.data() + spectrum.size(), - noise_spectrum_); - } else { - // Smoothly update the noise spectral estimate towards the signal spectrum - // such that the magnitude of the updates are limited. - for (size_t k = 0; k < spectrum.size(); ++k) { - if (noise_spectrum_[k] < spectrum[k]) { - noise_spectrum_[k] = std::min( - 1.01f * noise_spectrum_[k], - noise_spectrum_[k] + 0.05f * (spectrum[k] - noise_spectrum_[k])); - } else { - noise_spectrum_[k] = std::max( - 0.99f * noise_spectrum_[k], - noise_spectrum_[k] + 0.05f * (spectrum[k] - noise_spectrum_[k])); - } - } - } - - // Ensure that the noise spectal estimate does not become too low. - for (auto& v : noise_spectrum_) { - v = std::max(v, kMinNoisePower); - } - - data_dumper_->DumpRaw("lc_noise_spectrum", 65, noise_spectrum_); - data_dumper_->DumpRaw("lc_signal_spectrum", spectrum); -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.h b/webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.h deleted file mode 100644 index e9895f0..0000000 --- a/webrtc/modules/audio_processing/agc2/noise_spectrum_estimator.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_NOISE_SPECTRUM_ESTIMATOR_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_NOISE_SPECTRUM_ESTIMATOR_H_ - -#include "api/array_view.h" - -namespace webrtc { - -class ApmDataDumper; - -class NoiseSpectrumEstimator { - public: - explicit NoiseSpectrumEstimator(ApmDataDumper* data_dumper); - - NoiseSpectrumEstimator() = delete; - NoiseSpectrumEstimator(const NoiseSpectrumEstimator&) = delete; - NoiseSpectrumEstimator& operator=(const NoiseSpectrumEstimator&) = delete; - - void Initialize(); - void Update(rtc::ArrayView spectrum, bool first_update); - - rtc::ArrayView GetNoiseSpectrum() const { - return rtc::ArrayView(noise_spectrum_); - } - - private: - ApmDataDumper* data_dumper_; - float noise_spectrum_[65]; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_NOISE_SPECTRUM_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn b/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn index 0cc62a9..9093a68 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn @@ -17,6 +17,7 @@ rtc_library("rnn_vad") { "rnn.h", ] + defines = [] if (rtc_build_with_neon && current_cpu != "arm64") { suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] cflags = [ "-mfpu=neon" ] @@ -24,16 +25,17 @@ rtc_library("rnn_vad") { deps = [ ":rnn_vad_common", + ":rnn_vad_layers", ":rnn_vad_lp_residual", ":rnn_vad_pitch", ":rnn_vad_sequence_buffer", ":rnn_vad_spectral_features", "..:biquad_filter", + "..:cpu_features", "../../../../api:array_view", - "../../../../api:function_view", "../../../../rtc_base:checks", - "../../../../rtc_base:logging", - "../../../../rtc_base/system:arch", + "../../../../rtc_base:safe_compare", + "../../../../rtc_base:safe_conversions", "//third_party/rnnoise:rnn_vad", ] } @@ -51,16 +53,13 @@ rtc_library("rnn_vad_auto_correlation") { ] } -rtc_library("rnn_vad_common") { +rtc_source_set("rnn_vad_common") { # TODO(alessiob): Make this target visibility private. visibility = [ ":*", - "..:rnn_vad_with_level", - ] - sources = [ - "common.cc", - "common.h", + "..:vad_wrapper", ] + sources = [ "common.h" ] deps = [ "../../../../rtc_base/system:arch", "../../../../system_wrappers", @@ -75,23 +74,100 @@ rtc_library("rnn_vad_lp_residual") { deps = [ "../../../../api:array_view", "../../../../rtc_base:checks", + "../../../../rtc_base:safe_compare", ] } +rtc_source_set("rnn_vad_layers") { + sources = [ + "rnn_fc.cc", + "rnn_fc.h", + "rnn_gru.cc", + "rnn_gru.h", + ] + + defines = [] + if (rtc_build_with_neon && current_cpu != "arm64") { + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + + deps = [ + ":rnn_vad_common", + ":vector_math", + "..:cpu_features", + "../../../../api:array_view", + "../../../../api:function_view", + "../../../../rtc_base:checks", + "../../../../rtc_base:safe_conversions", + "//third_party/rnnoise:rnn_vad", + ] + if (current_cpu == "x86" || current_cpu == "x64") { + deps += [ ":vector_math_avx2" ] + } + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_source_set("vector_math") { + sources = [ "vector_math.h" ] + deps = [ + "..:cpu_features", + "../../../../api:array_view", + "../../../../rtc_base:checks", + "../../../../rtc_base:safe_conversions", + "../../../../rtc_base/system:arch", + ] +} + +if (current_cpu == "x86" || current_cpu == "x64") { + rtc_library("vector_math_avx2") { + sources = [ "vector_math_avx2.cc" ] + if (is_win) { + cflags = [ "/arch:AVX2" ] + } else { + cflags = [ + "-mavx2", + "-mfma", + ] + } + deps = [ + ":vector_math", + "../../../../api:array_view", + "../../../../rtc_base:checks", + "../../../../rtc_base:safe_conversions", + ] + } +} + rtc_library("rnn_vad_pitch") { sources = [ - "pitch_info.h", "pitch_search.cc", "pitch_search.h", "pitch_search_internal.cc", "pitch_search_internal.h", ] + + defines = [] + if (rtc_build_with_neon && current_cpu != "arm64") { + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + deps = [ ":rnn_vad_auto_correlation", ":rnn_vad_common", + ":vector_math", + "..:cpu_features", "../../../../api:array_view", "../../../../rtc_base:checks", + "../../../../rtc_base:gtest_prod", + "../../../../rtc_base:safe_compare", + "../../../../rtc_base:safe_conversions", + "../../../../rtc_base/system:arch", ] + if (current_cpu == "x86" || current_cpu == "x64") { + deps += [ ":vector_math_avx2" ] + } } rtc_source_set("rnn_vad_ring_buffer") { @@ -123,6 +199,7 @@ rtc_library("rnn_vad_spectral_features") { ":rnn_vad_symmetric_matrix_buffer", "../../../../api:array_view", "../../../../rtc_base:checks", + "../../../../rtc_base:safe_compare", "../../utility:pffft_wrapper", ] } @@ -132,6 +209,7 @@ rtc_source_set("rnn_vad_symmetric_matrix_buffer") { deps = [ "../../../../api:array_view", "../../../../rtc_base:checks", + "../../../../rtc_base:safe_compare", ] } @@ -148,11 +226,11 @@ if (rtc_include_tests) { "../../../../api:array_view", "../../../../api:scoped_refptr", "../../../../rtc_base:checks", - "../../../../rtc_base/system:arch", - "../../../../system_wrappers", + "../../../../rtc_base:safe_compare", "../../../../test:fileutils", "../../../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } unittest_resources = [ @@ -181,17 +259,28 @@ if (rtc_include_tests) { "pitch_search_internal_unittest.cc", "pitch_search_unittest.cc", "ring_buffer_unittest.cc", + "rnn_fc_unittest.cc", + "rnn_gru_unittest.cc", "rnn_unittest.cc", "rnn_vad_unittest.cc", "sequence_buffer_unittest.cc", "spectral_features_internal_unittest.cc", "spectral_features_unittest.cc", "symmetric_matrix_buffer_unittest.cc", + "vector_math_unittest.cc", ] + + defines = [] + if (rtc_build_with_neon && current_cpu != "arm64") { + suppressed_configs += [ "//build/config/compiler:compiler_arm_fpu" ] + cflags = [ "-mfpu=neon" ] + } + deps = [ ":rnn_vad", ":rnn_vad_auto_correlation", ":rnn_vad_common", + ":rnn_vad_layers", ":rnn_vad_lp_residual", ":rnn_vad_pitch", ":rnn_vad_ring_buffer", @@ -199,20 +288,47 @@ if (rtc_include_tests) { ":rnn_vad_spectral_features", ":rnn_vad_symmetric_matrix_buffer", ":test_utils", + ":vector_math", + "..:cpu_features", "../..:audioproc_test_utils", "../../../../api:array_view", "../../../../common_audio/", "../../../../rtc_base:checks", "../../../../rtc_base:logging", + "../../../../rtc_base:safe_compare", + "../../../../rtc_base:safe_conversions", + "../../../../rtc_base:stringutils", "../../../../rtc_base/system:arch", "../../../../test:test_support", "../../utility:pffft_wrapper", "//third_party/rnnoise:rnn_vad", ] + if (current_cpu == "x86" || current_cpu == "x64") { + deps += [ ":vector_math_avx2" ] + } absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] data = unittest_resources if (is_ios) { deps += [ ":unittests_bundle_data" ] } } + + if (!build_with_chromium) { + rtc_executable("rnn_vad_tool") { + testonly = true + sources = [ "rnn_vad_tool.cc" ] + deps = [ + ":rnn_vad", + ":rnn_vad_common", + "..:cpu_features", + "../../../../api:array_view", + "../../../../common_audio", + "../../../../rtc_base:logging", + "../../../../rtc_base:safe_compare", + "../../../../test:test_support", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + ] + } + } } diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.cc index d932c78..3ddeec8 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.cc @@ -20,7 +20,7 @@ namespace { constexpr int kAutoCorrelationFftOrder = 9; // Length-512 FFT. static_assert(1 << kAutoCorrelationFftOrder > - kNumInvertedLags12kHz + kBufSize12kHz - kMaxPitch12kHz, + kNumLags12kHz + kBufSize12kHz - kMaxPitch12kHz, ""); } // namespace @@ -40,20 +40,20 @@ AutoCorrelationCalculator::~AutoCorrelationCalculator() = default; // [ y_{m-1} ] // x and y are sub-array of equal length; x is never moved, whereas y slides. // The cross-correlation between y_0 and x corresponds to the auto-correlation -// for the maximum pitch period. Hence, the first value in |auto_corr| has an +// for the maximum pitch period. Hence, the first value in `auto_corr` has an // inverted lag equal to 0 that corresponds to a lag equal to the maximum // pitch period. void AutoCorrelationCalculator::ComputeOnPitchBuffer( rtc::ArrayView pitch_buf, - rtc::ArrayView auto_corr) { + rtc::ArrayView auto_corr) { RTC_DCHECK_LT(auto_corr.size(), kMaxPitch12kHz); RTC_DCHECK_GT(pitch_buf.size(), kMaxPitch12kHz); - constexpr size_t kFftFrameSize = 1 << kAutoCorrelationFftOrder; - constexpr size_t kConvolutionLength = kBufSize12kHz - kMaxPitch12kHz; + constexpr int kFftFrameSize = 1 << kAutoCorrelationFftOrder; + constexpr int kConvolutionLength = kBufSize12kHz - kMaxPitch12kHz; static_assert(kConvolutionLength == kFrameSize20ms12kHz, "Mismatch between pitch buffer size, frame size and maximum " "pitch period."); - static_assert(kFftFrameSize > kNumInvertedLags12kHz + kConvolutionLength, + static_assert(kFftFrameSize > kNumLags12kHz + kConvolutionLength, "The FFT length is not sufficiently big to avoid cyclic " "convolution errors."); auto tmp = tmp_->GetView(); @@ -67,13 +67,12 @@ void AutoCorrelationCalculator::ComputeOnPitchBuffer( // Compute the FFT for the sliding frames chunk. The sliding frames are // defined as pitch_buf[i:i+kConvolutionLength] where i in - // [0, kNumInvertedLags12kHz). The chunk includes all of them, hence it is - // defined as pitch_buf[:kNumInvertedLags12kHz+kConvolutionLength]. + // [0, kNumLags12kHz). The chunk includes all of them, hence it is + // defined as pitch_buf[:kNumLags12kHz+kConvolutionLength]. std::copy(pitch_buf.begin(), - pitch_buf.begin() + kConvolutionLength + kNumInvertedLags12kHz, + pitch_buf.begin() + kConvolutionLength + kNumLags12kHz, tmp.begin()); - std::fill(tmp.begin() + kNumInvertedLags12kHz + kConvolutionLength, tmp.end(), - 0.f); + std::fill(tmp.begin() + kNumLags12kHz + kConvolutionLength, tmp.end(), 0.f); fft_.ForwardTransform(*tmp_, X_.get(), /*ordered=*/false); // Convolve in the frequency domain. @@ -84,7 +83,7 @@ void AutoCorrelationCalculator::ComputeOnPitchBuffer( // Extract the auto-correlation coefficients. std::copy(tmp.begin() + kConvolutionLength - 1, - tmp.begin() + kConvolutionLength + kNumInvertedLags12kHz - 1, + tmp.begin() + kConvolutionLength + kNumLags12kHz - 1, auto_corr.begin()); } diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.h b/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.h index de7f453..1ae5054 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/auto_correlation.h @@ -31,10 +31,10 @@ class AutoCorrelationCalculator { ~AutoCorrelationCalculator(); // Computes the auto-correlation coefficients for a target pitch interval. - // |auto_corr| indexes are inverted lags. + // `auto_corr` indexes are inverted lags. void ComputeOnPitchBuffer( rtc::ArrayView pitch_buf, - rtc::ArrayView auto_corr); + rtc::ArrayView auto_corr); private: Pffft fft_; diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/common.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/common.cc deleted file mode 100644 index 5d76b52..0000000 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/common.cc +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/rnn_vad/common.h" - -#include "rtc_base/system/arch.h" -#include "system_wrappers/include/cpu_features_wrapper.h" - -namespace webrtc { -namespace rnn_vad { - -Optimization DetectOptimization() { -#if defined(WEBRTC_ARCH_X86_FAMILY) - if (GetCPUInfo(kSSE2) != 0) { - return Optimization::kSse2; - } -#endif - -#if defined(WEBRTC_HAS_NEON) - return Optimization::kNeon; -#endif - - return Optimization::kNone; -} - -} // namespace rnn_vad -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/common.h b/webrtc/modules/audio_processing/agc2/rnn_vad/common.h index c2e8df6..c099373 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/common.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/common.h @@ -18,57 +18,58 @@ namespace rnn_vad { constexpr double kPi = 3.14159265358979323846; -constexpr size_t kSampleRate24kHz = 24000; -constexpr size_t kFrameSize10ms24kHz = kSampleRate24kHz / 100; -constexpr size_t kFrameSize20ms24kHz = kFrameSize10ms24kHz * 2; +constexpr int kSampleRate24kHz = 24000; +constexpr int kFrameSize10ms24kHz = kSampleRate24kHz / 100; +constexpr int kFrameSize20ms24kHz = kFrameSize10ms24kHz * 2; // Pitch buffer. -constexpr size_t kMinPitch24kHz = kSampleRate24kHz / 800; // 0.00125 s. -constexpr size_t kMaxPitch24kHz = kSampleRate24kHz / 62.5; // 0.016 s. -constexpr size_t kBufSize24kHz = kMaxPitch24kHz + kFrameSize20ms24kHz; +constexpr int kMinPitch24kHz = kSampleRate24kHz / 800; // 0.00125 s. +constexpr int kMaxPitch24kHz = kSampleRate24kHz / 62.5; // 0.016 s. +constexpr int kBufSize24kHz = kMaxPitch24kHz + kFrameSize20ms24kHz; static_assert((kBufSize24kHz & 1) == 0, "The buffer size must be even."); // 24 kHz analysis. // Define a higher minimum pitch period for the initial search. This is used to // avoid searching for very short periods, for which a refinement step is // responsible. -constexpr size_t kInitialMinPitch24kHz = 3 * kMinPitch24kHz; +constexpr int kInitialMinPitch24kHz = 3 * kMinPitch24kHz; static_assert(kMinPitch24kHz < kInitialMinPitch24kHz, ""); static_assert(kInitialMinPitch24kHz < kMaxPitch24kHz, ""); static_assert(kMaxPitch24kHz > kInitialMinPitch24kHz, ""); -constexpr size_t kNumInvertedLags24kHz = kMaxPitch24kHz - kInitialMinPitch24kHz; +// Number of (inverted) lags during the initial pitch search phase at 24 kHz. +constexpr int kInitialNumLags24kHz = kMaxPitch24kHz - kInitialMinPitch24kHz; +// Number of (inverted) lags during the pitch search refinement phase at 24 kHz. +constexpr int kRefineNumLags24kHz = kMaxPitch24kHz + 1; +static_assert( + kRefineNumLags24kHz > kInitialNumLags24kHz, + "The refinement step must search the pitch in an extended pitch range."); // 12 kHz analysis. -constexpr size_t kSampleRate12kHz = 12000; -constexpr size_t kFrameSize10ms12kHz = kSampleRate12kHz / 100; -constexpr size_t kFrameSize20ms12kHz = kFrameSize10ms12kHz * 2; -constexpr size_t kBufSize12kHz = kBufSize24kHz / 2; -constexpr size_t kInitialMinPitch12kHz = kInitialMinPitch24kHz / 2; -constexpr size_t kMaxPitch12kHz = kMaxPitch24kHz / 2; +constexpr int kSampleRate12kHz = 12000; +constexpr int kFrameSize10ms12kHz = kSampleRate12kHz / 100; +constexpr int kFrameSize20ms12kHz = kFrameSize10ms12kHz * 2; +constexpr int kBufSize12kHz = kBufSize24kHz / 2; +constexpr int kInitialMinPitch12kHz = kInitialMinPitch24kHz / 2; +constexpr int kMaxPitch12kHz = kMaxPitch24kHz / 2; static_assert(kMaxPitch12kHz > kInitialMinPitch12kHz, ""); -// The inverted lags for the pitch interval [|kInitialMinPitch12kHz|, -// |kMaxPitch12kHz|] are in the range [0, |kNumInvertedLags12kHz|]. -constexpr size_t kNumInvertedLags12kHz = kMaxPitch12kHz - kInitialMinPitch12kHz; +// The inverted lags for the pitch interval [`kInitialMinPitch12kHz`, +// `kMaxPitch12kHz`] are in the range [0, `kNumLags12kHz`]. +constexpr int kNumLags12kHz = kMaxPitch12kHz - kInitialMinPitch12kHz; // 48 kHz constants. -constexpr size_t kMinPitch48kHz = kMinPitch24kHz * 2; -constexpr size_t kMaxPitch48kHz = kMaxPitch24kHz * 2; +constexpr int kMinPitch48kHz = kMinPitch24kHz * 2; +constexpr int kMaxPitch48kHz = kMaxPitch24kHz * 2; // Spectral features. -constexpr size_t kNumBands = 22; -constexpr size_t kNumLowerBands = 6; +constexpr int kNumBands = 22; +constexpr int kNumLowerBands = 6; static_assert((0 < kNumLowerBands) && (kNumLowerBands < kNumBands), ""); -constexpr size_t kCepstralCoeffsHistorySize = 8; +constexpr int kCepstralCoeffsHistorySize = 8; static_assert(kCepstralCoeffsHistorySize > 2, "The history size must at least be 3 to compute first and second " "derivatives."); -constexpr size_t kFeatureVectorSize = 42; - -enum class Optimization { kNone, kSse2, kNeon }; - -// Detects what kind of optimizations to use for the code. -Optimization DetectOptimization(); +constexpr int kFeatureVectorSize = 42; } // namespace rnn_vad } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.cc index e935179..5020234 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.cc @@ -19,23 +19,23 @@ namespace webrtc { namespace rnn_vad { namespace { -// Generated via "B, A = scipy.signal.butter(2, 30/12000, btype='highpass')" -const BiQuadFilter::BiQuadCoefficients kHpfConfig24k = { +// Computed as `scipy.signal.butter(N=2, Wn=60/24000, btype='highpass')`. +constexpr BiQuadFilter::Config kHpfConfig24k{ {0.99446179f, -1.98892358f, 0.99446179f}, {-1.98889291f, 0.98895425f}}; } // namespace -FeaturesExtractor::FeaturesExtractor() +FeaturesExtractor::FeaturesExtractor(const AvailableCpuFeatures& cpu_features) : use_high_pass_filter_(false), + hpf_(kHpfConfig24k), pitch_buf_24kHz_(), pitch_buf_24kHz_view_(pitch_buf_24kHz_.GetBufferView()), lp_residual_(kBufSize24kHz), lp_residual_view_(lp_residual_.data(), kBufSize24kHz), - pitch_estimator_(), + pitch_estimator_(cpu_features), reference_frame_view_(pitch_buf_24kHz_.GetMostRecentValuesView()) { RTC_DCHECK_EQ(kBufSize24kHz, lp_residual_.size()); - hpf_.Initialize(kHpfConfig24k); Reset(); } @@ -44,8 +44,9 @@ FeaturesExtractor::~FeaturesExtractor() = default; void FeaturesExtractor::Reset() { pitch_buf_24kHz_.Reset(); spectral_features_extractor_.Reset(); - if (use_high_pass_filter_) + if (use_high_pass_filter_) { hpf_.Reset(); + } } bool FeaturesExtractor::CheckSilenceComputeFeatures( @@ -55,10 +56,10 @@ bool FeaturesExtractor::CheckSilenceComputeFeatures( if (use_high_pass_filter_) { std::array samples_filtered; hpf_.Process(samples, samples_filtered); - // Feed buffer with the pre-processed version of |samples|. + // Feed buffer with the pre-processed version of `samples`. pitch_buf_24kHz_.Push(samples_filtered); } else { - // Feed buffer with |samples|. + // Feed buffer with `samples`. pitch_buf_24kHz_.Push(samples); } // Extract the LP residual. @@ -67,13 +68,12 @@ bool FeaturesExtractor::CheckSilenceComputeFeatures( ComputeLpResidual(lpc_coeffs, pitch_buf_24kHz_view_, lp_residual_view_); // Estimate pitch on the LP-residual and write the normalized pitch period // into the output vector (normalization based on training data stats). - pitch_info_48kHz_ = pitch_estimator_.Estimate(lp_residual_view_); - feature_vector[kFeatureVectorSize - 2] = - 0.01f * (static_cast(pitch_info_48kHz_.period) - 300); + pitch_period_48kHz_ = pitch_estimator_.Estimate(lp_residual_view_); + feature_vector[kFeatureVectorSize - 2] = 0.01f * (pitch_period_48kHz_ - 300); // Extract lagged frames (according to the estimated pitch period). - RTC_DCHECK_LE(pitch_info_48kHz_.period / 2, kMaxPitch24kHz); + RTC_DCHECK_LE(pitch_period_48kHz_ / 2, kMaxPitch24kHz); auto lagged_frame = pitch_buf_24kHz_view_.subview( - kMaxPitch24kHz - pitch_info_48kHz_.period / 2, kFrameSize20ms24kHz); + kMaxPitch24kHz - pitch_period_48kHz_ / 2, kFrameSize20ms24kHz); // Analyze reference and lagged frames checking if silence has been detected // and write the feature vector. return spectral_features_extractor_.CheckSilenceComputeFeatures( diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.h b/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.h index ce5cce1..d47a85b 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/features_extraction.h @@ -16,7 +16,6 @@ #include "api/array_view.h" #include "modules/audio_processing/agc2/biquad_filter.h" #include "modules/audio_processing/agc2/rnn_vad/common.h" -#include "modules/audio_processing/agc2/rnn_vad/pitch_info.h" #include "modules/audio_processing/agc2/rnn_vad/pitch_search.h" #include "modules/audio_processing/agc2/rnn_vad/sequence_buffer.h" #include "modules/audio_processing/agc2/rnn_vad/spectral_features.h" @@ -27,14 +26,14 @@ namespace rnn_vad { // Feature extractor to feed the VAD RNN. class FeaturesExtractor { public: - FeaturesExtractor(); + explicit FeaturesExtractor(const AvailableCpuFeatures& cpu_features); FeaturesExtractor(const FeaturesExtractor&) = delete; FeaturesExtractor& operator=(const FeaturesExtractor&) = delete; ~FeaturesExtractor(); void Reset(); // Analyzes the samples, computes the feature vector and returns true if // silence is detected (false if not). When silence is detected, - // |feature_vector| is partially written and therefore must not be used to + // `feature_vector` is partially written and therefore must not be used to // feed the VAD RNN. bool CheckSilenceComputeFeatures( rtc::ArrayView samples, @@ -53,7 +52,7 @@ class FeaturesExtractor { PitchEstimator pitch_estimator_; rtc::ArrayView reference_frame_view_; SpectralFeaturesExtractor spectral_features_extractor_; - PitchInfo pitch_info_48kHz_; + int pitch_period_48kHz_; }; } // namespace rnn_vad diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.cc index 1a124a3..484bfba 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.cc @@ -16,27 +16,23 @@ #include #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" namespace webrtc { namespace rnn_vad { namespace { -// Computes cross-correlation coefficients between |x| and |y| and writes them -// in |x_corr|. The lag values are in {0, ..., max_lag - 1}, where max_lag -// equals the size of |x_corr|. -// The |x| and |y| sub-arrays used to compute a cross-correlation coefficients -// for a lag l have both size "size of |x| - l" - i.e., the longest sub-array is -// used. |x| and |y| must have the same size. -void ComputeCrossCorrelation( +// Computes auto-correlation coefficients for `x` and writes them in +// `auto_corr`. The lag values are in {0, ..., max_lag - 1}, where max_lag +// equals the size of `auto_corr`. +void ComputeAutoCorrelation( rtc::ArrayView x, - rtc::ArrayView y, - rtc::ArrayView x_corr) { - constexpr size_t max_lag = x_corr.size(); - RTC_DCHECK_EQ(x.size(), y.size()); + rtc::ArrayView auto_corr) { + constexpr int max_lag = auto_corr.size(); RTC_DCHECK_LT(max_lag, x.size()); - for (size_t lag = 0; lag < max_lag; ++lag) { - x_corr[lag] = - std::inner_product(x.begin(), x.end() - lag, y.begin() + lag, 0.f); + for (int lag = 0; lag < max_lag; ++lag) { + auto_corr[lag] = + std::inner_product(x.begin(), x.end() - lag, x.begin() + lag, 0.f); } } @@ -45,9 +41,13 @@ void DenoiseAutoCorrelation( rtc::ArrayView auto_corr) { // Assume -40 dB white noise floor. auto_corr[0] *= 1.0001f; - for (size_t i = 1; i < kNumLpcCoefficients; ++i) { - auto_corr[i] -= auto_corr[i] * (0.008f * i) * (0.008f * i); - } + // Hard-coded values obtained as + // [np.float32((0.008*0.008*i*i)) for i in range(1,5)]. + auto_corr[1] -= auto_corr[1] * 0.000064f; + auto_corr[2] -= auto_corr[2] * 0.000256f; + auto_corr[3] -= auto_corr[3] * 0.000576f; + auto_corr[4] -= auto_corr[4] * 0.001024f; + static_assert(kNumLpcCoefficients == 5, "Update `auto_corr`."); } // Computes the initial inverse filter coefficients given the auto-correlation @@ -56,9 +56,9 @@ void ComputeInitialInverseFilterCoefficients( rtc::ArrayView auto_corr, rtc::ArrayView lpc_coeffs) { float error = auto_corr[0]; - for (size_t i = 0; i < kNumLpcCoefficients - 1; ++i) { + for (int i = 0; i < kNumLpcCoefficients - 1; ++i) { float reflection_coeff = 0.f; - for (size_t j = 0; j < i; ++j) { + for (int j = 0; j < i; ++j) { reflection_coeff += lpc_coeffs[j] * auto_corr[i - j]; } reflection_coeff += auto_corr[i + 1]; @@ -72,7 +72,7 @@ void ComputeInitialInverseFilterCoefficients( reflection_coeff /= -error; // Update LPC coefficients and total error. lpc_coeffs[i] = reflection_coeff; - for (size_t j = 0; j<(i + 1)>> 1; ++j) { + for (int j = 0; j < ((i + 1) >> 1); ++j) { const float tmp1 = lpc_coeffs[j]; const float tmp2 = lpc_coeffs[i - 1 - j]; lpc_coeffs[j] = tmp1 + reflection_coeff * tmp2; @@ -91,46 +91,49 @@ void ComputeAndPostProcessLpcCoefficients( rtc::ArrayView x, rtc::ArrayView lpc_coeffs) { std::array auto_corr; - ComputeCrossCorrelation(x, x, {auto_corr.data(), auto_corr.size()}); + ComputeAutoCorrelation(x, auto_corr); if (auto_corr[0] == 0.f) { // Empty frame. std::fill(lpc_coeffs.begin(), lpc_coeffs.end(), 0); return; } - DenoiseAutoCorrelation({auto_corr.data(), auto_corr.size()}); + DenoiseAutoCorrelation(auto_corr); std::array lpc_coeffs_pre{}; ComputeInitialInverseFilterCoefficients(auto_corr, lpc_coeffs_pre); // LPC coefficients post-processing. // TODO(bugs.webrtc.org/9076): Consider removing these steps. - float c1 = 1.f; - for (size_t i = 0; i < kNumLpcCoefficients - 1; ++i) { - c1 *= 0.9f; - lpc_coeffs_pre[i] *= c1; - } - const float c2 = 0.8f; - lpc_coeffs[0] = lpc_coeffs_pre[0] + c2; - lpc_coeffs[1] = lpc_coeffs_pre[1] + c2 * lpc_coeffs_pre[0]; - lpc_coeffs[2] = lpc_coeffs_pre[2] + c2 * lpc_coeffs_pre[1]; - lpc_coeffs[3] = lpc_coeffs_pre[3] + c2 * lpc_coeffs_pre[2]; - lpc_coeffs[4] = c2 * lpc_coeffs_pre[3]; + lpc_coeffs_pre[0] *= 0.9f; + lpc_coeffs_pre[1] *= 0.9f * 0.9f; + lpc_coeffs_pre[2] *= 0.9f * 0.9f * 0.9f; + lpc_coeffs_pre[3] *= 0.9f * 0.9f * 0.9f * 0.9f; + constexpr float kC = 0.8f; + lpc_coeffs[0] = lpc_coeffs_pre[0] + kC; + lpc_coeffs[1] = lpc_coeffs_pre[1] + kC * lpc_coeffs_pre[0]; + lpc_coeffs[2] = lpc_coeffs_pre[2] + kC * lpc_coeffs_pre[1]; + lpc_coeffs[3] = lpc_coeffs_pre[3] + kC * lpc_coeffs_pre[2]; + lpc_coeffs[4] = kC * lpc_coeffs_pre[3]; + static_assert(kNumLpcCoefficients == 5, "Update `lpc_coeffs(_pre)`."); } void ComputeLpResidual( rtc::ArrayView lpc_coeffs, rtc::ArrayView x, rtc::ArrayView y) { - RTC_DCHECK_LT(kNumLpcCoefficients, x.size()); + RTC_DCHECK_GT(x.size(), kNumLpcCoefficients); RTC_DCHECK_EQ(x.size(), y.size()); - std::array input_chunk; - input_chunk.fill(0.f); - for (size_t i = 0; i < y.size(); ++i) { - const float sum = std::inner_product(input_chunk.begin(), input_chunk.end(), - lpc_coeffs.begin(), x[i]); - // Circular shift and add a new sample. - for (size_t j = kNumLpcCoefficients - 1; j > 0; --j) - input_chunk[j] = input_chunk[j - 1]; - input_chunk[0] = x[i]; - // Copy result. - y[i] = sum; + // The code below implements the following operation: + // y[i] = x[i] + dot_product({x[i], ..., x[i - kNumLpcCoefficients + 1]}, + // lpc_coeffs) + // Edge case: i < kNumLpcCoefficients. + y[0] = x[0]; + for (int i = 1; i < kNumLpcCoefficients; ++i) { + y[i] = + std::inner_product(x.crend() - i, x.crend(), lpc_coeffs.cbegin(), x[i]); + } + // Regular case. + auto last = x.crend(); + for (int i = kNumLpcCoefficients; rtc::SafeLt(i, y.size()); ++i, --last) { + y[i] = std::inner_product(last - kNumLpcCoefficients, last, + lpc_coeffs.cbegin(), x[i]); } } diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.h b/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.h index cddedca..d04c536 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/lp_residual.h @@ -18,17 +18,17 @@ namespace webrtc { namespace rnn_vad { -// LPC inverse filter length. -constexpr size_t kNumLpcCoefficients = 5; +// Linear predictive coding (LPC) inverse filter length. +constexpr int kNumLpcCoefficients = 5; -// Given a frame |x|, computes a post-processed version of LPC coefficients +// Given a frame `x`, computes a post-processed version of LPC coefficients // tailored for pitch estimation. void ComputeAndPostProcessLpcCoefficients( rtc::ArrayView x, rtc::ArrayView lpc_coeffs); -// Computes the LP residual for the input frame |x| and the LPC coefficients -// |lpc_coeffs|. |y| and |x| can point to the same array for in-place +// Computes the LP residual for the input frame `x` and the LPC coefficients +// `lpc_coeffs`. `y` and `x` can point to the same array for in-place // computation. void ComputeLpResidual( rtc::ArrayView lpc_coeffs, diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_info.h b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_info.h deleted file mode 100644 index c9fdd18..0000000 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_info.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_INFO_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_INFO_H_ - -namespace webrtc { -namespace rnn_vad { - -// Stores pitch period and gain information. The pitch gain measures the -// strength of the pitch (the higher, the stronger). -struct PitchInfo { - PitchInfo() : period(0), gain(0.f) {} - PitchInfo(int p, float g) : period(p), gain(g) {} - int period; - float gain; -}; - -} // namespace rnn_vad -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_INFO_H_ diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.cc index 1b3b459..419620f 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.cc @@ -18,38 +18,52 @@ namespace webrtc { namespace rnn_vad { -PitchEstimator::PitchEstimator() - : pitch_buf_decimated_(kBufSize12kHz), - pitch_buf_decimated_view_(pitch_buf_decimated_.data(), kBufSize12kHz), - auto_corr_(kNumInvertedLags12kHz), - auto_corr_view_(auto_corr_.data(), kNumInvertedLags12kHz) { - RTC_DCHECK_EQ(kBufSize12kHz, pitch_buf_decimated_.size()); - RTC_DCHECK_EQ(kNumInvertedLags12kHz, auto_corr_view_.size()); -} +PitchEstimator::PitchEstimator(const AvailableCpuFeatures& cpu_features) + : cpu_features_(cpu_features), + y_energy_24kHz_(kRefineNumLags24kHz, 0.f), + pitch_buffer_12kHz_(kBufSize12kHz), + auto_correlation_12kHz_(kNumLags12kHz) {} PitchEstimator::~PitchEstimator() = default; -PitchInfo PitchEstimator::Estimate( - rtc::ArrayView pitch_buf) { +int PitchEstimator::Estimate( + rtc::ArrayView pitch_buffer) { + rtc::ArrayView pitch_buffer_12kHz_view( + pitch_buffer_12kHz_.data(), kBufSize12kHz); + RTC_DCHECK_EQ(pitch_buffer_12kHz_.size(), pitch_buffer_12kHz_view.size()); + rtc::ArrayView auto_correlation_12kHz_view( + auto_correlation_12kHz_.data(), kNumLags12kHz); + RTC_DCHECK_EQ(auto_correlation_12kHz_.size(), + auto_correlation_12kHz_view.size()); + + // TODO(bugs.chromium.org/10480): Use `cpu_features_` to estimate pitch. // Perform the initial pitch search at 12 kHz. - Decimate2x(pitch_buf, pitch_buf_decimated_view_); - auto_corr_calculator_.ComputeOnPitchBuffer(pitch_buf_decimated_view_, - auto_corr_view_); - std::array pitch_candidates_inv_lags = FindBestPitchPeriods( - auto_corr_view_, pitch_buf_decimated_view_, kMaxPitch12kHz); - // Refine the pitch period estimation. + Decimate2x(pitch_buffer, pitch_buffer_12kHz_view); + auto_corr_calculator_.ComputeOnPitchBuffer(pitch_buffer_12kHz_view, + auto_correlation_12kHz_view); + CandidatePitchPeriods pitch_periods = ComputePitchPeriod12kHz( + pitch_buffer_12kHz_view, auto_correlation_12kHz_view, cpu_features_); // The refinement is done using the pitch buffer that contains 24 kHz samples. - // Therefore, adapt the inverted lags in |pitch_candidates_inv_lags| from 12 + // Therefore, adapt the inverted lags in `pitch_candidates_inv_lags` from 12 // to 24 kHz. - pitch_candidates_inv_lags[0] *= 2; - pitch_candidates_inv_lags[1] *= 2; - size_t pitch_inv_lag_48kHz = - RefinePitchPeriod48kHz(pitch_buf, pitch_candidates_inv_lags); - // Look for stronger harmonics to find the final pitch period and its gain. - RTC_DCHECK_LT(pitch_inv_lag_48kHz, kMaxPitch48kHz); - last_pitch_48kHz_ = CheckLowerPitchPeriodsAndComputePitchGain( - pitch_buf, kMaxPitch48kHz - pitch_inv_lag_48kHz, last_pitch_48kHz_); - return last_pitch_48kHz_; + pitch_periods.best *= 2; + pitch_periods.second_best *= 2; + + // Refine the initial pitch period estimation from 12 kHz to 48 kHz. + // Pre-compute frame energies at 24 kHz. + rtc::ArrayView y_energy_24kHz_view( + y_energy_24kHz_.data(), kRefineNumLags24kHz); + RTC_DCHECK_EQ(y_energy_24kHz_.size(), y_energy_24kHz_view.size()); + ComputeSlidingFrameSquareEnergies24kHz(pitch_buffer, y_energy_24kHz_view, + cpu_features_); + // Estimation at 48 kHz. + const int pitch_lag_48kHz = ComputePitchPeriod48kHz( + pitch_buffer, y_energy_24kHz_view, pitch_periods, cpu_features_); + last_pitch_48kHz_ = ComputeExtendedPitchPeriod48kHz( + pitch_buffer, y_energy_24kHz_view, + /*initial_pitch_period_48kHz=*/kMaxPitch48kHz - pitch_lag_48kHz, + last_pitch_48kHz_, cpu_features_); + return last_pitch_48kHz_.period; } } // namespace rnn_vad diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.h b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.h index 74133d0..42c448e 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search.h @@ -15,10 +15,11 @@ #include #include "api/array_view.h" +#include "modules/audio_processing/agc2/cpu_features.h" #include "modules/audio_processing/agc2/rnn_vad/auto_correlation.h" #include "modules/audio_processing/agc2/rnn_vad/common.h" -#include "modules/audio_processing/agc2/rnn_vad/pitch_info.h" #include "modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h" +#include "rtc_base/gtest_prod_util.h" namespace webrtc { namespace rnn_vad { @@ -26,21 +27,25 @@ namespace rnn_vad { // Pitch estimator. class PitchEstimator { public: - PitchEstimator(); + explicit PitchEstimator(const AvailableCpuFeatures& cpu_features); PitchEstimator(const PitchEstimator&) = delete; PitchEstimator& operator=(const PitchEstimator&) = delete; ~PitchEstimator(); - // Estimates the pitch period and gain. Returns the pitch estimation data for - // 48 kHz. - PitchInfo Estimate(rtc::ArrayView pitch_buf); + // Returns the estimated pitch period at 48 kHz. + int Estimate(rtc::ArrayView pitch_buffer); private: - PitchInfo last_pitch_48kHz_; + FRIEND_TEST_ALL_PREFIXES(RnnVadTest, PitchSearchWithinTolerance); + float GetLastPitchStrengthForTesting() const { + return last_pitch_48kHz_.strength; + } + + const AvailableCpuFeatures cpu_features_; + PitchInfo last_pitch_48kHz_{}; AutoCorrelationCalculator auto_corr_calculator_; - std::vector pitch_buf_decimated_; - rtc::ArrayView pitch_buf_decimated_view_; - std::vector auto_corr_; - rtc::ArrayView auto_corr_view_; + std::vector y_energy_24kHz_; + std::vector pitch_buffer_12kHz_; + std::vector auto_correlation_12kHz_; }; } // namespace rnn_vad diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc index f24a76f..e8c9125 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc @@ -18,103 +18,81 @@ #include #include "modules/audio_processing/agc2/rnn_vad/common.h" +#include "modules/audio_processing/agc2/rnn_vad/vector_math.h" #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/system/arch.h" namespace webrtc { namespace rnn_vad { namespace { -// Converts a lag to an inverted lag (only for 24kHz). -size_t GetInvertedLag(size_t lag) { - RTC_DCHECK_LE(lag, kMaxPitch24kHz); - return kMaxPitch24kHz - lag; +float ComputeAutoCorrelation( + int inverted_lag, + rtc::ArrayView pitch_buffer, + const VectorMath& vector_math) { + RTC_DCHECK_LT(inverted_lag, kBufSize24kHz); + RTC_DCHECK_LT(inverted_lag, kRefineNumLags24kHz); + static_assert(kMaxPitch24kHz < kBufSize24kHz, ""); + return vector_math.DotProduct( + pitch_buffer.subview(/*offset=*/kMaxPitch24kHz), + pitch_buffer.subview(inverted_lag, kFrameSize20ms24kHz)); } -float ComputeAutoCorrelationCoeff(rtc::ArrayView pitch_buf, - size_t inv_lag, - size_t max_pitch_period) { - RTC_DCHECK_LT(inv_lag, pitch_buf.size()); - RTC_DCHECK_LT(max_pitch_period, pitch_buf.size()); - RTC_DCHECK_LE(inv_lag, max_pitch_period); - // TODO(bugs.webrtc.org/9076): Maybe optimize using vectorization. - return std::inner_product(pitch_buf.begin() + max_pitch_period, - pitch_buf.end(), pitch_buf.begin() + inv_lag, 0.f); -} - -// Computes a pseudo-interpolation offset for an estimated pitch period |lag| by -// looking at the auto-correlation coefficients in the neighborhood of |lag|. -// (namely, |prev_auto_corr|, |lag_auto_corr| and |next_auto_corr|). The output -// is a lag in {-1, 0, +1}. -// TODO(bugs.webrtc.org/9076): Consider removing pseudo-i since it -// is relevant only if the spectral analysis works at a sample rate that is -// twice as that of the pitch buffer (not so important instead for the estimated -// pitch period feature fed into the RNN). -int GetPitchPseudoInterpolationOffset(size_t lag, - float prev_auto_corr, - float lag_auto_corr, - float next_auto_corr) { - const float& a = prev_auto_corr; - const float& b = lag_auto_corr; - const float& c = next_auto_corr; - - int offset = 0; - if ((c - a) > 0.7f * (b - a)) { - offset = 1; // |c| is the largest auto-correlation coefficient. - } else if ((a - c) > 0.7f * (b - c)) { - offset = -1; // |a| is the largest auto-correlation coefficient. +// Given an auto-correlation coefficient `curr_auto_correlation` and its +// neighboring values `prev_auto_correlation` and `next_auto_correlation` +// computes a pseudo-interpolation offset to be applied to the pitch period +// associated to `curr`. The output is a lag in {-1, 0, +1}. +// TODO(bugs.webrtc.org/9076): Consider removing this method. +// `GetPitchPseudoInterpolationOffset()` it is relevant only if the spectral +// analysis works at a sample rate that is twice as that of the pitch buffer; +// In particular, it is not relevant for the estimated pitch period feature fed +// into the RNN. +int GetPitchPseudoInterpolationOffset(float prev_auto_correlation, + float curr_auto_correlation, + float next_auto_correlation) { + if ((next_auto_correlation - prev_auto_correlation) > + 0.7f * (curr_auto_correlation - prev_auto_correlation)) { + return 1; // `next_auto_correlation` is the largest auto-correlation + // coefficient. + } else if ((prev_auto_correlation - next_auto_correlation) > + 0.7f * (curr_auto_correlation - next_auto_correlation)) { + return -1; // `prev_auto_correlation` is the largest auto-correlation + // coefficient. } - return offset; + return 0; } -// Refines a pitch period |lag| encoded as lag with pseudo-interpolation. The -// output sample rate is twice as that of |lag|. -size_t PitchPseudoInterpolationLagPitchBuf( - size_t lag, - rtc::ArrayView pitch_buf) { +// Refines a pitch period `lag` encoded as lag with pseudo-interpolation. The +// output sample rate is twice as that of `lag`. +int PitchPseudoInterpolationLagPitchBuf( + int lag, + rtc::ArrayView pitch_buffer, + const VectorMath& vector_math) { int offset = 0; // Cannot apply pseudo-interpolation at the boundaries. if (lag > 0 && lag < kMaxPitch24kHz) { + const int inverted_lag = kMaxPitch24kHz - lag; offset = GetPitchPseudoInterpolationOffset( - lag, - ComputeAutoCorrelationCoeff(pitch_buf, GetInvertedLag(lag - 1), - kMaxPitch24kHz), - ComputeAutoCorrelationCoeff(pitch_buf, GetInvertedLag(lag), - kMaxPitch24kHz), - ComputeAutoCorrelationCoeff(pitch_buf, GetInvertedLag(lag + 1), - kMaxPitch24kHz)); + ComputeAutoCorrelation(inverted_lag + 1, pitch_buffer, vector_math), + ComputeAutoCorrelation(inverted_lag, pitch_buffer, vector_math), + ComputeAutoCorrelation(inverted_lag - 1, pitch_buffer, vector_math)); } return 2 * lag + offset; } -// Refines a pitch period |inv_lag| encoded as inverted lag with -// pseudo-interpolation. The output sample rate is twice as that of -// |inv_lag|. -size_t PitchPseudoInterpolationInvLagAutoCorr( - size_t inv_lag, - rtc::ArrayView auto_corr) { - int offset = 0; - // Cannot apply pseudo-interpolation at the boundaries. - if (inv_lag > 0 && inv_lag < auto_corr.size() - 1) { - offset = GetPitchPseudoInterpolationOffset(inv_lag, auto_corr[inv_lag + 1], - auto_corr[inv_lag], - auto_corr[inv_lag - 1]); - } - // TODO(bugs.webrtc.org/9076): When retraining, check if |offset| below should - // be subtracted since |inv_lag| is an inverted lag but offset is a lag. - return 2 * inv_lag + offset; -} - -// Integer multipliers used in CheckLowerPitchPeriodsAndComputePitchGain() when +// Integer multipliers used in ComputeExtendedPitchPeriod48kHz() when // looking for sub-harmonics. // The values have been chosen to serve the following algorithm. Given the // initial pitch period T, we examine whether one of its harmonics is the true // fundamental frequency. We consider T/k with k in {2, ..., 15}. For each of -// these harmonics, in addition to the pitch gain of itself, we choose one +// these harmonics, in addition to the pitch strength of itself, we choose one // multiple of its pitch period, n*T/k, to validate it (by averaging their pitch -// gains). The multiplier n is chosen so that n*T/k is used only one time over -// all k. When for example k = 4, we should also expect a peak at 3*T/4. When -// k = 8 instead we don't want to look at 2*T/8, since we have already checked -// T/4 before. Instead, we look at T*3/8. +// strengths). The multiplier n is chosen so that n*T/k is used only one time +// over all k. When for example k = 4, we should also expect a peak at 3*T/4. +// When k = 8 instead we don't want to look at 2*T/8, since we have already +// checked T/4 before. Instead, we look at T*3/8. // The array can be generate in Python as follows: // from fractions import Fraction // # Smallest positive integer not in X. @@ -131,96 +109,220 @@ size_t PitchPseudoInterpolationInvLagAutoCorr( constexpr std::array kSubHarmonicMultipliers = { {3, 2, 3, 2, 5, 2, 3, 2, 3, 2, 5, 2, 3, 2}}; -// Initial pitch period candidate thresholds for ComputePitchGainThreshold() for -// a sample rate of 24 kHz. Computed as [5*k*k for k in range(16)]. -constexpr std::array kInitialPitchPeriodThresholds = { - {20, 45, 80, 125, 180, 245, 320, 405, 500, 605, 720, 845, 980, 1125}}; +struct Range { + int min; + int max; +}; + +// Number of analyzed pitches to the left(right) of a pitch candidate. +constexpr int kPitchNeighborhoodRadius = 2; + +// Creates a pitch period interval centered in `inverted_lag` with hard-coded +// radius. Clipping is applied so that the interval is always valid for a 24 kHz +// pitch buffer. +Range CreateInvertedLagRange(int inverted_lag) { + return {std::max(inverted_lag - kPitchNeighborhoodRadius, 0), + std::min(inverted_lag + kPitchNeighborhoodRadius, + kInitialNumLags24kHz - 1)}; +} + +constexpr int kNumPitchCandidates = 2; // Best and second best. +// Maximum number of analyzed pitch periods. +constexpr int kMaxPitchPeriods24kHz = + kNumPitchCandidates * (2 * kPitchNeighborhoodRadius + 1); + +// Collection of inverted lags. +class InvertedLagsIndex { + public: + InvertedLagsIndex() : num_entries_(0) {} + // Adds an inverted lag to the index. Cannot add more than + // `kMaxPitchPeriods24kHz` values. + void Append(int inverted_lag) { + RTC_DCHECK_LT(num_entries_, kMaxPitchPeriods24kHz); + inverted_lags_[num_entries_++] = inverted_lag; + } + const int* data() const { return inverted_lags_.data(); } + int size() const { return num_entries_; } + + private: + std::array inverted_lags_; + int num_entries_; +}; + +// Computes the auto correlation coefficients for the inverted lags in the +// closed interval `inverted_lags`. Updates `inverted_lags_index` by appending +// the inverted lags for the computed auto correlation values. +void ComputeAutoCorrelation( + Range inverted_lags, + rtc::ArrayView pitch_buffer, + rtc::ArrayView auto_correlation, + InvertedLagsIndex& inverted_lags_index, + const VectorMath& vector_math) { + // Check valid range. + RTC_DCHECK_LE(inverted_lags.min, inverted_lags.max); + // Trick to avoid zero initialization of `auto_correlation`. + // Needed by the pseudo-interpolation. + if (inverted_lags.min > 0) { + auto_correlation[inverted_lags.min - 1] = 0.f; + } + if (inverted_lags.max < kInitialNumLags24kHz - 1) { + auto_correlation[inverted_lags.max + 1] = 0.f; + } + // Check valid `inverted_lag` indexes. + RTC_DCHECK_GE(inverted_lags.min, 0); + RTC_DCHECK_LT(inverted_lags.max, kInitialNumLags24kHz); + for (int inverted_lag = inverted_lags.min; inverted_lag <= inverted_lags.max; + ++inverted_lag) { + auto_correlation[inverted_lag] = + ComputeAutoCorrelation(inverted_lag, pitch_buffer, vector_math); + inverted_lags_index.Append(inverted_lag); + } +} + +// Searches the strongest pitch period at 24 kHz and returns its inverted lag at +// 48 kHz. +int ComputePitchPeriod48kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView inverted_lags, + rtc::ArrayView auto_correlation, + rtc::ArrayView y_energy, + const VectorMath& vector_math) { + static_assert(kMaxPitch24kHz > kInitialNumLags24kHz, ""); + static_assert(kMaxPitch24kHz < kBufSize24kHz, ""); + int best_inverted_lag = 0; // Pitch period. + float best_numerator = -1.f; // Pitch strength numerator. + float best_denominator = 0.f; // Pitch strength denominator. + for (int inverted_lag : inverted_lags) { + // A pitch candidate must have positive correlation. + if (auto_correlation[inverted_lag] > 0.f) { + // Auto-correlation energy normalized by frame energy. + const float numerator = + auto_correlation[inverted_lag] * auto_correlation[inverted_lag]; + const float denominator = y_energy[inverted_lag]; + // Compare numerator/denominator ratios without using divisions. + if (numerator * best_denominator > best_numerator * denominator) { + best_inverted_lag = inverted_lag; + best_numerator = numerator; + best_denominator = denominator; + } + } + } + // Pseudo-interpolation to transform `best_inverted_lag` (24 kHz pitch) to a + // 48 kHz pitch period. + if (best_inverted_lag == 0 || best_inverted_lag >= kInitialNumLags24kHz - 1) { + // Cannot apply pseudo-interpolation at the boundaries. + return best_inverted_lag * 2; + } + int offset = GetPitchPseudoInterpolationOffset( + auto_correlation[best_inverted_lag + 1], + auto_correlation[best_inverted_lag], + auto_correlation[best_inverted_lag - 1]); + // TODO(bugs.webrtc.org/9076): When retraining, check if `offset` below should + // be subtracted since `inverted_lag` is an inverted lag but offset is a lag. + return 2 * best_inverted_lag + offset; +} + +// Returns an alternative pitch period for `pitch_period` given a `multiplier` +// and a `divisor` of the period. +constexpr int GetAlternativePitchPeriod(int pitch_period, + int multiplier, + int divisor) { + RTC_DCHECK_GT(divisor, 0); + // Same as `round(multiplier * pitch_period / divisor)`. + return (2 * multiplier * pitch_period + divisor) / (2 * divisor); +} + +// Returns true if the alternative pitch period is stronger than the initial one +// given the last estimated pitch and the value of `period_divisor` used to +// compute the alternative pitch period via `GetAlternativePitchPeriod()`. +bool IsAlternativePitchStrongerThanInitial(PitchInfo last, + PitchInfo initial, + PitchInfo alternative, + int period_divisor) { + // Initial pitch period candidate thresholds for a sample rate of 24 kHz. + // Computed as [5*k*k for k in range(16)]. + constexpr std::array kInitialPitchPeriodThresholds = { + {20, 45, 80, 125, 180, 245, 320, 405, 500, 605, 720, 845, 980, 1125}}; + static_assert( + kInitialPitchPeriodThresholds.size() == kSubHarmonicMultipliers.size(), + ""); + RTC_DCHECK_GE(last.period, 0); + RTC_DCHECK_GE(initial.period, 0); + RTC_DCHECK_GE(alternative.period, 0); + RTC_DCHECK_GE(period_divisor, 2); + // Compute a term that lowers the threshold when `alternative.period` is close + // to the last estimated period `last.period` - i.e., pitch tracking. + float lower_threshold_term = 0.f; + if (std::abs(alternative.period - last.period) <= 1) { + // The candidate pitch period is within 1 sample from the last one. + // Make the candidate at `alternative.period` very easy to be accepted. + lower_threshold_term = last.strength; + } else if (std::abs(alternative.period - last.period) == 2 && + initial.period > + kInitialPitchPeriodThresholds[period_divisor - 2]) { + // The candidate pitch period is 2 samples far from the last one and the + // period `initial.period` (from which `alternative.period` has been + // derived) is greater than a threshold. Make `alternative.period` easy to + // be accepted. + lower_threshold_term = 0.5f * last.strength; + } + // Set the threshold based on the strength of the initial estimate + // `initial.period`. Also reduce the chance of false positives caused by a + // bias towards high frequencies (originating from short-term correlations). + float threshold = + std::max(0.3f, 0.7f * initial.strength - lower_threshold_term); + if (alternative.period < 3 * kMinPitch24kHz) { + // High frequency. + threshold = std::max(0.4f, 0.85f * initial.strength - lower_threshold_term); + } else if (alternative.period < 2 * kMinPitch24kHz) { + // Even higher frequency. + threshold = std::max(0.5f, 0.9f * initial.strength - lower_threshold_term); + } + return alternative.strength > threshold; +} } // namespace void Decimate2x(rtc::ArrayView src, rtc::ArrayView dst) { // TODO(bugs.webrtc.org/9076): Consider adding anti-aliasing filter. - static_assert(2 * dst.size() == src.size(), ""); - for (size_t i = 0; i < dst.size(); ++i) { + static_assert(2 * kBufSize12kHz == kBufSize24kHz, ""); + for (int i = 0; i < kBufSize12kHz; ++i) { dst[i] = src[2 * i]; } } -float ComputePitchGainThreshold(int candidate_pitch_period, - int pitch_period_ratio, - int initial_pitch_period, - float initial_pitch_gain, - int prev_pitch_period, - float prev_pitch_gain) { - // Map arguments to more compact aliases. - const int& t1 = candidate_pitch_period; - const int& k = pitch_period_ratio; - const int& t0 = initial_pitch_period; - const float& g0 = initial_pitch_gain; - const int& t_prev = prev_pitch_period; - const float& g_prev = prev_pitch_gain; - - // Validate input. - RTC_DCHECK_GE(t1, 0); - RTC_DCHECK_GE(k, 2); - RTC_DCHECK_GE(t0, 0); - RTC_DCHECK_GE(t_prev, 0); - - // Compute a term that lowers the threshold when |t1| is close to the last - // estimated period |t_prev| - i.e., pitch tracking. - float lower_threshold_term = 0; - if (abs(t1 - t_prev) <= 1) { - // The candidate pitch period is within 1 sample from the previous one. - // Make the candidate at |t1| very easy to be accepted. - lower_threshold_term = g_prev; - } else if (abs(t1 - t_prev) == 2 && - t0 > kInitialPitchPeriodThresholds[k - 2]) { - // The candidate pitch period is 2 samples far from the previous one and the - // period |t0| (from which |t1| has been derived) is greater than a - // threshold. Make |t1| easy to be accepted. - lower_threshold_term = 0.5f * g_prev; - } - // Set the threshold based on the gain of the initial estimate |t0|. Also - // reduce the chance of false positives caused by a bias towards high - // frequencies (originating from short-term correlations). - float threshold = std::max(0.3f, 0.7f * g0 - lower_threshold_term); - if (static_cast(t1) < 3 * kMinPitch24kHz) { - // High frequency. - threshold = std::max(0.4f, 0.85f * g0 - lower_threshold_term); - } else if (static_cast(t1) < 2 * kMinPitch24kHz) { - // Even higher frequency. - threshold = std::max(0.5f, 0.9f * g0 - lower_threshold_term); - } - return threshold; -} - -void ComputeSlidingFrameSquareEnergies( - rtc::ArrayView pitch_buf, - rtc::ArrayView yy_values) { - float yy = - ComputeAutoCorrelationCoeff(pitch_buf, kMaxPitch24kHz, kMaxPitch24kHz); - yy_values[0] = yy; - for (size_t i = 1; i < yy_values.size(); ++i) { - RTC_DCHECK_LE(i, kMaxPitch24kHz + kFrameSize20ms24kHz); - RTC_DCHECK_LE(i, kMaxPitch24kHz); - const float old_coeff = pitch_buf[kMaxPitch24kHz + kFrameSize20ms24kHz - i]; - const float new_coeff = pitch_buf[kMaxPitch24kHz - i]; - yy -= old_coeff * old_coeff; - yy += new_coeff * new_coeff; - yy = std::max(0.f, yy); - yy_values[i] = yy; +void ComputeSlidingFrameSquareEnergies24kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView y_energy, + AvailableCpuFeatures cpu_features) { + VectorMath vector_math(cpu_features); + static_assert(kFrameSize20ms24kHz < kBufSize24kHz, ""); + const auto frame_20ms_view = pitch_buffer.subview(0, kFrameSize20ms24kHz); + float yy = vector_math.DotProduct(frame_20ms_view, frame_20ms_view); + y_energy[0] = yy; + static_assert(kMaxPitch24kHz - 1 + kFrameSize20ms24kHz < kBufSize24kHz, ""); + static_assert(kMaxPitch24kHz < kRefineNumLags24kHz, ""); + for (int inverted_lag = 0; inverted_lag < kMaxPitch24kHz; ++inverted_lag) { + yy -= pitch_buffer[inverted_lag] * pitch_buffer[inverted_lag]; + yy += pitch_buffer[inverted_lag + kFrameSize20ms24kHz] * + pitch_buffer[inverted_lag + kFrameSize20ms24kHz]; + yy = std::max(1.f, yy); + y_energy[inverted_lag + 1] = yy; } } -std::array FindBestPitchPeriods( - rtc::ArrayView auto_corr, - rtc::ArrayView pitch_buf, - size_t max_pitch_period) { +CandidatePitchPeriods ComputePitchPeriod12kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView auto_correlation, + AvailableCpuFeatures cpu_features) { + static_assert(kMaxPitch12kHz > kNumLags12kHz, ""); + static_assert(kMaxPitch12kHz < kBufSize12kHz, ""); + // Stores a pitch candidate period and strength information. struct PitchCandidate { // Pitch period encoded as inverted lag. - size_t period_inverted_lag = 0; + int period_inverted_lag = 0; // Pitch strength encoded as a ratio. float strength_numerator = -1.f; float strength_denominator = 0.f; @@ -232,25 +334,22 @@ std::array FindBestPitchPeriods( } }; - RTC_DCHECK_GT(max_pitch_period, auto_corr.size()); - RTC_DCHECK_LT(max_pitch_period, pitch_buf.size()); - const size_t frame_size = pitch_buf.size() - max_pitch_period; - // TODO(bugs.webrtc.org/9076): Maybe optimize using vectorization. - float yy = - std::inner_product(pitch_buf.begin(), pitch_buf.begin() + frame_size + 1, - pitch_buf.begin(), 1.f); + VectorMath vector_math(cpu_features); + static_assert(kFrameSize20ms12kHz + 1 < kBufSize12kHz, ""); + const auto frame_view = pitch_buffer.subview(0, kFrameSize20ms12kHz + 1); + float denominator = 1.f + vector_math.DotProduct(frame_view, frame_view); // Search best and second best pitches by looking at the scaled // auto-correlation. - PitchCandidate candidate; PitchCandidate best; PitchCandidate second_best; second_best.period_inverted_lag = 1; - for (size_t inv_lag = 0; inv_lag < auto_corr.size(); ++inv_lag) { + for (int inverted_lag = 0; inverted_lag < kNumLags12kHz; ++inverted_lag) { // A pitch candidate must have positive correlation. - if (auto_corr[inv_lag] > 0) { - candidate.period_inverted_lag = inv_lag; - candidate.strength_numerator = auto_corr[inv_lag] * auto_corr[inv_lag]; - candidate.strength_denominator = yy; + if (auto_correlation[inverted_lag] > 0.f) { + PitchCandidate candidate{ + inverted_lag, + auto_correlation[inverted_lag] * auto_correlation[inverted_lag], + denominator}; if (candidate.HasStrongerPitchThan(second_best)) { if (candidate.HasStrongerPitchThan(best)) { second_best = best; @@ -260,143 +359,154 @@ std::array FindBestPitchPeriods( } } } - // Update |squared_energy_y| for the next inverted lag. - const float old_coeff = pitch_buf[inv_lag]; - const float new_coeff = pitch_buf[inv_lag + frame_size]; - yy -= old_coeff * old_coeff; - yy += new_coeff * new_coeff; - yy = std::max(0.f, yy); + // Update `squared_energy_y` for the next inverted lag. + const float y_old = pitch_buffer[inverted_lag]; + const float y_new = pitch_buffer[inverted_lag + kFrameSize20ms12kHz]; + denominator -= y_old * y_old; + denominator += y_new * y_new; + denominator = std::max(0.f, denominator); } - return {{best.period_inverted_lag, second_best.period_inverted_lag}}; + return {best.period_inverted_lag, second_best.period_inverted_lag}; } -size_t RefinePitchPeriod48kHz( - rtc::ArrayView pitch_buf, - rtc::ArrayView inv_lags) { - // Compute the auto-correlation terms only for neighbors of the given pitch - // candidates (similar to what is done in ComputePitchAutoCorrelation(), but - // for a few lag values). - std::array auto_corr; - auto_corr.fill(0.f); // Zeros become ignored lags in FindBestPitchPeriods(). - auto is_neighbor = [](size_t i, size_t j) { - return ((i > j) ? (i - j) : (j - i)) <= 2; - }; - for (size_t inv_lag = 0; inv_lag < auto_corr.size(); ++inv_lag) { - if (is_neighbor(inv_lag, inv_lags[0]) || is_neighbor(inv_lag, inv_lags[1])) - auto_corr[inv_lag] = - ComputeAutoCorrelationCoeff(pitch_buf, inv_lag, kMaxPitch24kHz); +int ComputePitchPeriod48kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView y_energy, + CandidatePitchPeriods pitch_candidates, + AvailableCpuFeatures cpu_features) { + // Compute the auto-correlation terms only for neighbors of the two pitch + // candidates (best and second best). + std::array auto_correlation; + InvertedLagsIndex inverted_lags_index; + // Create two inverted lag ranges so that `r1` precedes `r2`. + const bool swap_candidates = + pitch_candidates.best > pitch_candidates.second_best; + const Range r1 = CreateInvertedLagRange( + swap_candidates ? pitch_candidates.second_best : pitch_candidates.best); + const Range r2 = CreateInvertedLagRange( + swap_candidates ? pitch_candidates.best : pitch_candidates.second_best); + // Check valid ranges. + RTC_DCHECK_LE(r1.min, r1.max); + RTC_DCHECK_LE(r2.min, r2.max); + // Check `r1` precedes `r2`. + RTC_DCHECK_LE(r1.min, r2.min); + RTC_DCHECK_LE(r1.max, r2.max); + VectorMath vector_math(cpu_features); + if (r1.max + 1 >= r2.min) { + // Overlapping or adjacent ranges. + ComputeAutoCorrelation({r1.min, r2.max}, pitch_buffer, auto_correlation, + inverted_lags_index, vector_math); + } else { + // Disjoint ranges. + ComputeAutoCorrelation(r1, pitch_buffer, auto_correlation, + inverted_lags_index, vector_math); + ComputeAutoCorrelation(r2, pitch_buffer, auto_correlation, + inverted_lags_index, vector_math); } - // Find best pitch at 24 kHz. - const auto pitch_candidates_inv_lags = FindBestPitchPeriods( - {auto_corr.data(), auto_corr.size()}, - {pitch_buf.data(), pitch_buf.size()}, kMaxPitch24kHz); - const auto inv_lag = pitch_candidates_inv_lags[0]; // Refine the best. - // Pseudo-interpolation. - return PitchPseudoInterpolationInvLagAutoCorr(inv_lag, auto_corr); + return ComputePitchPeriod48kHz(pitch_buffer, inverted_lags_index, + auto_correlation, y_energy, vector_math); } -PitchInfo CheckLowerPitchPeriodsAndComputePitchGain( - rtc::ArrayView pitch_buf, +PitchInfo ComputeExtendedPitchPeriod48kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView y_energy, int initial_pitch_period_48kHz, - PitchInfo prev_pitch_48kHz) { + PitchInfo last_pitch_48kHz, + AvailableCpuFeatures cpu_features) { RTC_DCHECK_LE(kMinPitch48kHz, initial_pitch_period_48kHz); RTC_DCHECK_LE(initial_pitch_period_48kHz, kMaxPitch48kHz); + // Stores information for a refined pitch candidate. struct RefinedPitchCandidate { - RefinedPitchCandidate() {} - RefinedPitchCandidate(int period_24kHz, float gain, float xy, float yy) - : period_24kHz(period_24kHz), gain(gain), xy(xy), yy(yy) {} - int period_24kHz; - // Pitch strength information. - float gain; - // Additional pitch strength information used for the final estimation of - // pitch gain. - float xy; // Cross-correlation. - float yy; // Auto-correlation. + int period; + float strength; + // Additional strength data used for the final pitch estimation. + float xy; // Auto-correlation. + float y_energy; // Energy of the sliding frame `y`. }; - // Initialize. - std::array yy_values; - ComputeSlidingFrameSquareEnergies(pitch_buf, - {yy_values.data(), yy_values.size()}); - const float xx = yy_values[0]; - // Helper lambdas. - const auto pitch_gain = [](float xy, float yy, float xx) { - RTC_DCHECK_LE(0.f, xx * yy); - return xy / std::sqrt(1.f + xx * yy); + const float x_energy = y_energy[kMaxPitch24kHz]; + const auto pitch_strength = [x_energy](float xy, float y_energy) { + RTC_DCHECK_GE(x_energy * y_energy, 0.f); + return xy / std::sqrt(1.f + x_energy * y_energy); }; - // Initial pitch candidate gain. + VectorMath vector_math(cpu_features); + + // Initialize the best pitch candidate with `initial_pitch_period_48kHz`. RefinedPitchCandidate best_pitch; - best_pitch.period_24kHz = std::min(initial_pitch_period_48kHz / 2, - static_cast(kMaxPitch24kHz - 1)); - best_pitch.xy = ComputeAutoCorrelationCoeff( - pitch_buf, GetInvertedLag(best_pitch.period_24kHz), kMaxPitch24kHz); - best_pitch.yy = yy_values[best_pitch.period_24kHz]; - best_pitch.gain = pitch_gain(best_pitch.xy, best_pitch.yy, xx); + best_pitch.period = + std::min(initial_pitch_period_48kHz / 2, kMaxPitch24kHz - 1); + best_pitch.xy = ComputeAutoCorrelation(kMaxPitch24kHz - best_pitch.period, + pitch_buffer, vector_math); + best_pitch.y_energy = y_energy[kMaxPitch24kHz - best_pitch.period]; + best_pitch.strength = pitch_strength(best_pitch.xy, best_pitch.y_energy); + // Keep a copy of the initial pitch candidate. + const PitchInfo initial_pitch{best_pitch.period, best_pitch.strength}; + // 24 kHz version of the last estimated pitch. + const PitchInfo last_pitch{last_pitch_48kHz.period / 2, + last_pitch_48kHz.strength}; - // Store the initial pitch period information. - const size_t initial_pitch_period = best_pitch.period_24kHz; - const float initial_pitch_gain = best_pitch.gain; - - // Given the initial pitch estimation, check lower periods (i.e., harmonics). - const auto alternative_period = [](int period, int k, int n) -> int { - RTC_DCHECK_GT(k, 0); - return (2 * n * period + k) / (2 * k); // Same as round(n*period/k). - }; - for (int k = 2; k < static_cast(kSubHarmonicMultipliers.size() + 2); - ++k) { - int candidate_pitch_period = alternative_period(initial_pitch_period, k, 1); - if (static_cast(candidate_pitch_period) < kMinPitch24kHz) { - break; + // Find `max_period_divisor` such that the result of + // `GetAlternativePitchPeriod(initial_pitch_period, 1, max_period_divisor)` + // equals `kMinPitch24kHz`. + const int max_period_divisor = + (2 * initial_pitch.period) / (2 * kMinPitch24kHz - 1); + for (int period_divisor = 2; period_divisor <= max_period_divisor; + ++period_divisor) { + PitchInfo alternative_pitch; + alternative_pitch.period = GetAlternativePitchPeriod( + initial_pitch.period, /*multiplier=*/1, period_divisor); + RTC_DCHECK_GE(alternative_pitch.period, kMinPitch24kHz); + // When looking at `alternative_pitch.period`, we also look at one of its + // sub-harmonics. `kSubHarmonicMultipliers` is used to know where to look. + // `period_divisor` == 2 is a special case since `dual_alternative_period` + // might be greater than the maximum pitch period. + int dual_alternative_period = GetAlternativePitchPeriod( + initial_pitch.period, kSubHarmonicMultipliers[period_divisor - 2], + period_divisor); + RTC_DCHECK_GT(dual_alternative_period, 0); + if (period_divisor == 2 && dual_alternative_period > kMaxPitch24kHz) { + dual_alternative_period = initial_pitch.period; } - // When looking at |candidate_pitch_period|, we also look at one of its - // sub-harmonics. |kSubHarmonicMultipliers| is used to know where to look. - // |k| == 2 is a special case since |candidate_pitch_secondary_period| might - // be greater than the maximum pitch period. - int candidate_pitch_secondary_period = alternative_period( - initial_pitch_period, k, kSubHarmonicMultipliers[k - 2]); - RTC_DCHECK_GT(candidate_pitch_secondary_period, 0); - if (k == 2 && - candidate_pitch_secondary_period > static_cast(kMaxPitch24kHz)) { - candidate_pitch_secondary_period = initial_pitch_period; - } - RTC_DCHECK_NE(candidate_pitch_period, candidate_pitch_secondary_period) + RTC_DCHECK_NE(alternative_pitch.period, dual_alternative_period) << "The lower pitch period and the additional sub-harmonic must not " "coincide."; // Compute an auto-correlation score for the primary pitch candidate - // |candidate_pitch_period| by also looking at its possible sub-harmonic - // |candidate_pitch_secondary_period|. - float xy_primary_period = ComputeAutoCorrelationCoeff( - pitch_buf, GetInvertedLag(candidate_pitch_period), kMaxPitch24kHz); - float xy_secondary_period = ComputeAutoCorrelationCoeff( - pitch_buf, GetInvertedLag(candidate_pitch_secondary_period), - kMaxPitch24kHz); - float xy = 0.5f * (xy_primary_period + xy_secondary_period); - float yy = 0.5f * (yy_values[candidate_pitch_period] + - yy_values[candidate_pitch_secondary_period]); - float candidate_pitch_gain = pitch_gain(xy, yy, xx); + // `alternative_pitch.period` by also looking at its possible sub-harmonic + // `dual_alternative_period`. + const float xy_primary_period = ComputeAutoCorrelation( + kMaxPitch24kHz - alternative_pitch.period, pitch_buffer, vector_math); + // TODO(webrtc:10480): Copy `xy_primary_period` if the secondary period is + // equal to the primary one. + const float xy_secondary_period = ComputeAutoCorrelation( + kMaxPitch24kHz - dual_alternative_period, pitch_buffer, vector_math); + const float xy = 0.5f * (xy_primary_period + xy_secondary_period); + const float yy = + 0.5f * (y_energy[kMaxPitch24kHz - alternative_pitch.period] + + y_energy[kMaxPitch24kHz - dual_alternative_period]); + alternative_pitch.strength = pitch_strength(xy, yy); // Maybe update best period. - float threshold = ComputePitchGainThreshold( - candidate_pitch_period, k, initial_pitch_period, initial_pitch_gain, - prev_pitch_48kHz.period / 2, prev_pitch_48kHz.gain); - if (candidate_pitch_gain > threshold) { - best_pitch = {candidate_pitch_period, candidate_pitch_gain, xy, yy}; + if (IsAlternativePitchStrongerThanInitial( + last_pitch, initial_pitch, alternative_pitch, period_divisor)) { + best_pitch = {alternative_pitch.period, alternative_pitch.strength, xy, + yy}; } } - // Final pitch gain and period. + // Final pitch strength and period. best_pitch.xy = std::max(0.f, best_pitch.xy); - RTC_DCHECK_LE(0.f, best_pitch.yy); - float final_pitch_gain = (best_pitch.yy <= best_pitch.xy) - ? 1.f - : best_pitch.xy / (best_pitch.yy + 1.f); - final_pitch_gain = std::min(best_pitch.gain, final_pitch_gain); + RTC_DCHECK_LE(0.f, best_pitch.y_energy); + float final_pitch_strength = + (best_pitch.y_energy <= best_pitch.xy) + ? 1.f + : best_pitch.xy / (best_pitch.y_energy + 1.f); + final_pitch_strength = std::min(best_pitch.strength, final_pitch_strength); int final_pitch_period_48kHz = std::max( - kMinPitch48kHz, - PitchPseudoInterpolationLagPitchBuf(best_pitch.period_24kHz, pitch_buf)); + kMinPitch48kHz, PitchPseudoInterpolationLagPitchBuf( + best_pitch.period, pitch_buffer, vector_math)); - return {final_pitch_period_48kHz, final_pitch_gain}; + return {final_pitch_period_48kHz, final_pitch_strength}; } } // namespace rnn_vad diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h index 2cc5ce6..aa2dd13 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h @@ -14,10 +14,11 @@ #include #include +#include #include "api/array_view.h" +#include "modules/audio_processing/agc2/cpu_features.h" #include "modules/audio_processing/agc2/rnn_vad/common.h" -#include "modules/audio_processing/agc2/rnn_vad/pitch_info.h" namespace webrtc { namespace rnn_vad { @@ -26,50 +27,86 @@ namespace rnn_vad { void Decimate2x(rtc::ArrayView src, rtc::ArrayView dst); -// Computes a gain threshold for a candidate pitch period given the initial and -// the previous pitch period and gain estimates and the pitch period ratio used -// to derive the candidate pitch period from the initial period. -float ComputePitchGainThreshold(int candidate_pitch_period, - int pitch_period_ratio, - int initial_pitch_period, - float initial_pitch_gain, - int prev_pitch_period, - float prev_pitch_gain); - -// Computes the sum of squared samples for every sliding frame in the pitch -// buffer. |yy_values| indexes are lags. +// Key concepts and keywords used below in this file. // -// The pitch buffer is structured as depicted below: -// |.........|...........| -// a b -// The part on the left, named "a" contains the oldest samples, whereas "b" the -// most recent ones. The size of "a" corresponds to the maximum pitch period, -// that of "b" to the frame size (e.g., 16 ms and 20 ms respectively). -void ComputeSlidingFrameSquareEnergies( - rtc::ArrayView pitch_buf, - rtc::ArrayView yy_values); +// The pitch estimation relies on a pitch buffer, which is an array-like data +// structured designed as follows: +// +// |....A....|.....B.....| +// +// The part on the left, named `A` contains the oldest samples, whereas `B` +// contains the most recent ones. The size of `A` corresponds to the maximum +// pitch period, that of `B` to the analysis frame size (e.g., 16 ms and 20 ms +// respectively). +// +// Pitch estimation is essentially based on the analysis of two 20 ms frames +// extracted from the pitch buffer. One frame, called `x`, is kept fixed and +// corresponds to `B` - i.e., the most recent 20 ms. The other frame, called +// `y`, is extracted from different parts of the buffer instead. +// +// The offset between `x` and `y` corresponds to a specific pitch period. +// For instance, if `y` is positioned at the beginning of the pitch buffer, then +// the cross-correlation between `x` and `y` can be used as an indication of the +// strength for the maximum pitch. +// +// Such an offset can be encoded in two ways: +// - As a lag, which is the index in the pitch buffer for the first item in `y` +// - As an inverted lag, which is the number of samples from the beginning of +// `x` and the end of `y` +// +// |---->| lag +// |....A....|.....B.....| +// |<--| inverted lag +// |.....y.....| `y` 20 ms frame +// +// The inverted lag has the advantage of being directly proportional to the +// corresponding pitch period. -// Given the auto-correlation coefficients stored according to -// ComputePitchAutoCorrelation() (i.e., using inverted lags), returns the best -// and the second best pitch periods. -std::array FindBestPitchPeriods( - rtc::ArrayView auto_corr, - rtc::ArrayView pitch_buf, - size_t max_pitch_period); +// Computes the sum of squared samples for every sliding frame `y` in the pitch +// buffer. The indexes of `y_energy` are inverted lags. +void ComputeSlidingFrameSquareEnergies24kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView y_energy, + AvailableCpuFeatures cpu_features); -// Refines the pitch period estimation given the pitch buffer |pitch_buf| and -// the initial pitch period estimation |inv_lags|. Returns an inverted lag at -// 48 kHz. -size_t RefinePitchPeriod48kHz( - rtc::ArrayView pitch_buf, - rtc::ArrayView inv_lags); +// Top-2 pitch period candidates. Unit: number of samples - i.e., inverted lags. +struct CandidatePitchPeriods { + int best; + int second_best; +}; -// Refines the pitch period estimation and compute the pitch gain. Returns the -// refined pitch estimation data at 48 kHz. -PitchInfo CheckLowerPitchPeriodsAndComputePitchGain( - rtc::ArrayView pitch_buf, +// Computes the candidate pitch periods at 12 kHz given a view on the 12 kHz +// pitch buffer and the auto-correlation values (having inverted lags as +// indexes). +CandidatePitchPeriods ComputePitchPeriod12kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView auto_correlation, + AvailableCpuFeatures cpu_features); + +// Computes the pitch period at 48 kHz given a view on the 24 kHz pitch buffer, +// the energies for the sliding frames `y` at 24 kHz and the pitch period +// candidates at 24 kHz (encoded as inverted lag). +int ComputePitchPeriod48kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView y_energy, + CandidatePitchPeriods pitch_candidates_24kHz, + AvailableCpuFeatures cpu_features); + +struct PitchInfo { + int period; + float strength; +}; + +// Computes the pitch period at 48 kHz searching in an extended pitch range +// given a view on the 24 kHz pitch buffer, the energies for the sliding frames +// `y` at 24 kHz, the initial 48 kHz estimation (computed by +// `ComputePitchPeriod48kHz()`) and the last estimated pitch. +PitchInfo ComputeExtendedPitchPeriod48kHz( + rtc::ArrayView pitch_buffer, + rtc::ArrayView y_energy, int initial_pitch_period_48kHz, - PitchInfo prev_pitch_48kHz); + PitchInfo last_pitch_48kHz, + AvailableCpuFeatures cpu_features); } // namespace rnn_vad } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/ring_buffer.h b/webrtc/modules/audio_processing/agc2/rnn_vad/ring_buffer.h index 294b0c0..a6f7fdd 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/ring_buffer.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/ring_buffer.h @@ -21,7 +21,7 @@ namespace webrtc { namespace rnn_vad { // Ring buffer for N arrays of type T each one with size S. -template +template class RingBuffer { static_assert(S > 0, ""); static_assert(N > 0, ""); @@ -35,7 +35,7 @@ class RingBuffer { ~RingBuffer() = default; // Set the ring buffer values to zero. void Reset() { buffer_.fill(0); } - // Replace the least recently pushed array in the buffer with |new_values|. + // Replace the least recently pushed array in the buffer with `new_values`. void Push(rtc::ArrayView new_values) { std::memcpy(buffer_.data() + S * tail_, new_values.data(), S * sizeof(T)); tail_ += 1; @@ -43,13 +43,12 @@ class RingBuffer { tail_ = 0; } // Return an array view onto the array with a given delay. A view on the last - // and least recently push array is returned when |delay| is 0 and N - 1 + // and least recently push array is returned when `delay` is 0 and N - 1 // respectively. - rtc::ArrayView GetArrayView(size_t delay) const { - const int delay_int = static_cast(delay); - RTC_DCHECK_LE(0, delay_int); - RTC_DCHECK_LT(delay_int, N); - int offset = tail_ - 1 - delay_int; + rtc::ArrayView GetArrayView(int delay) const { + RTC_DCHECK_LE(0, delay); + RTC_DCHECK_LT(delay, N); + int offset = tail_ - 1 - delay; if (offset < 0) offset += N; return {buffer_.data() + S * offset, S}; diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.cc index 55a51ff..475bef9 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.cc @@ -10,415 +10,81 @@ #include "modules/audio_processing/agc2/rnn_vad/rnn.h" -// Defines WEBRTC_ARCH_X86_FAMILY, used below. -#include "rtc_base/system/arch.h" - -#if defined(WEBRTC_HAS_NEON) -#include -#endif -#if defined(WEBRTC_ARCH_X86_FAMILY) -#include -#endif -#include -#include -#include -#include - #include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "third_party/rnnoise/src/rnn_activations.h" #include "third_party/rnnoise/src/rnn_vad_weights.h" namespace webrtc { namespace rnn_vad { namespace { -using rnnoise::kWeightsScale; - -using rnnoise::kInputLayerInputSize; +using ::rnnoise::kInputLayerInputSize; static_assert(kFeatureVectorSize == kInputLayerInputSize, ""); -using rnnoise::kInputDenseBias; -using rnnoise::kInputDenseWeights; -using rnnoise::kInputLayerOutputSize; -static_assert(kInputLayerOutputSize <= kFullyConnectedLayersMaxUnits, - "Increase kFullyConnectedLayersMaxUnits."); +using ::rnnoise::kInputDenseBias; +using ::rnnoise::kInputDenseWeights; +using ::rnnoise::kInputLayerOutputSize; +static_assert(kInputLayerOutputSize <= kFullyConnectedLayerMaxUnits, ""); -using rnnoise::kHiddenGruBias; -using rnnoise::kHiddenGruRecurrentWeights; -using rnnoise::kHiddenGruWeights; -using rnnoise::kHiddenLayerOutputSize; -static_assert(kHiddenLayerOutputSize <= kRecurrentLayersMaxUnits, - "Increase kRecurrentLayersMaxUnits."); +using ::rnnoise::kHiddenGruBias; +using ::rnnoise::kHiddenGruRecurrentWeights; +using ::rnnoise::kHiddenGruWeights; +using ::rnnoise::kHiddenLayerOutputSize; +static_assert(kHiddenLayerOutputSize <= kGruLayerMaxUnits, ""); -using rnnoise::kOutputDenseBias; -using rnnoise::kOutputDenseWeights; -using rnnoise::kOutputLayerOutputSize; -static_assert(kOutputLayerOutputSize <= kFullyConnectedLayersMaxUnits, - "Increase kFullyConnectedLayersMaxUnits."); - -using rnnoise::SigmoidApproximated; -using rnnoise::TansigApproximated; - -inline float RectifiedLinearUnit(float x) { - return x < 0.f ? 0.f : x; -} - -std::vector GetScaledParams(rtc::ArrayView params) { - std::vector scaled_params(params.size()); - std::transform(params.begin(), params.end(), scaled_params.begin(), - [](int8_t x) -> float { - return rnnoise::kWeightsScale * static_cast(x); - }); - return scaled_params; -} - -// TODO(bugs.chromium.org/10480): Hard-code optimized layout and remove this -// function to improve setup time. -// Casts and scales |weights| and re-arranges the layout. -std::vector GetPreprocessedFcWeights( - rtc::ArrayView weights, - size_t output_size) { - if (output_size == 1) { - return GetScaledParams(weights); - } - // Transpose, scale and cast. - const size_t input_size = rtc::CheckedDivExact(weights.size(), output_size); - std::vector w(weights.size()); - for (size_t o = 0; o < output_size; ++o) { - for (size_t i = 0; i < input_size; ++i) { - w[o * input_size + i] = rnnoise::kWeightsScale * - static_cast(weights[i * output_size + o]); - } - } - return w; -} - -constexpr size_t kNumGruGates = 3; // Update, reset, output. - -// TODO(bugs.chromium.org/10480): Hard-coded optimized layout and remove this -// function to improve setup time. -// Casts and scales |tensor_src| for a GRU layer and re-arranges the layout. -// It works both for weights, recurrent weights and bias. -std::vector GetPreprocessedGruTensor( - rtc::ArrayView tensor_src, - size_t output_size) { - // Transpose, cast and scale. - // |n| is the size of the first dimension of the 3-dim tensor |weights|. - const size_t n = - rtc::CheckedDivExact(tensor_src.size(), output_size * kNumGruGates); - const size_t stride_src = kNumGruGates * output_size; - const size_t stride_dst = n * output_size; - std::vector tensor_dst(tensor_src.size()); - for (size_t g = 0; g < kNumGruGates; ++g) { - for (size_t o = 0; o < output_size; ++o) { - for (size_t i = 0; i < n; ++i) { - tensor_dst[g * stride_dst + o * n + i] = - rnnoise::kWeightsScale * - static_cast( - tensor_src[i * stride_src + g * output_size + o]); - } - } - } - return tensor_dst; -} - -void ComputeGruUpdateResetGates(size_t input_size, - size_t output_size, - rtc::ArrayView weights, - rtc::ArrayView recurrent_weights, - rtc::ArrayView bias, - rtc::ArrayView input, - rtc::ArrayView state, - rtc::ArrayView gate) { - for (size_t o = 0; o < output_size; ++o) { - gate[o] = bias[o]; - for (size_t i = 0; i < input_size; ++i) { - gate[o] += input[i] * weights[o * input_size + i]; - } - for (size_t s = 0; s < output_size; ++s) { - gate[o] += state[s] * recurrent_weights[o * output_size + s]; - } - gate[o] = SigmoidApproximated(gate[o]); - } -} - -void ComputeGruOutputGate(size_t input_size, - size_t output_size, - rtc::ArrayView weights, - rtc::ArrayView recurrent_weights, - rtc::ArrayView bias, - rtc::ArrayView input, - rtc::ArrayView state, - rtc::ArrayView reset, - rtc::ArrayView gate) { - for (size_t o = 0; o < output_size; ++o) { - gate[o] = bias[o]; - for (size_t i = 0; i < input_size; ++i) { - gate[o] += input[i] * weights[o * input_size + i]; - } - for (size_t s = 0; s < output_size; ++s) { - gate[o] += state[s] * recurrent_weights[o * output_size + s] * reset[s]; - } - gate[o] = RectifiedLinearUnit(gate[o]); - } -} - -// Gated recurrent unit (GRU) layer un-optimized implementation. -void ComputeGruLayerOutput(size_t input_size, - size_t output_size, - rtc::ArrayView input, - rtc::ArrayView weights, - rtc::ArrayView recurrent_weights, - rtc::ArrayView bias, - rtc::ArrayView state) { - RTC_DCHECK_EQ(input_size, input.size()); - // Stride and offset used to read parameter arrays. - const size_t stride_in = input_size * output_size; - const size_t stride_out = output_size * output_size; - - // Update gate. - std::array update; - ComputeGruUpdateResetGates( - input_size, output_size, weights.subview(0, stride_in), - recurrent_weights.subview(0, stride_out), bias.subview(0, output_size), - input, state, update); - - // Reset gate. - std::array reset; - ComputeGruUpdateResetGates( - input_size, output_size, weights.subview(stride_in, stride_in), - recurrent_weights.subview(stride_out, stride_out), - bias.subview(output_size, output_size), input, state, reset); - - // Output gate. - std::array output; - ComputeGruOutputGate( - input_size, output_size, weights.subview(2 * stride_in, stride_in), - recurrent_weights.subview(2 * stride_out, stride_out), - bias.subview(2 * output_size, output_size), input, state, reset, output); - - // Update output through the update gates and update the state. - for (size_t o = 0; o < output_size; ++o) { - output[o] = update[o] * state[o] + (1.f - update[o]) * output[o]; - state[o] = output[o]; - } -} - -// Fully connected layer un-optimized implementation. -void ComputeFullyConnectedLayerOutput( - size_t input_size, - size_t output_size, - rtc::ArrayView input, - rtc::ArrayView bias, - rtc::ArrayView weights, - rtc::FunctionView activation_function, - rtc::ArrayView output) { - RTC_DCHECK_EQ(input.size(), input_size); - RTC_DCHECK_EQ(bias.size(), output_size); - RTC_DCHECK_EQ(weights.size(), input_size * output_size); - for (size_t o = 0; o < output_size; ++o) { - output[o] = bias[o]; - // TODO(bugs.chromium.org/9076): Benchmark how different layouts for - // |weights_| change the performance across different platforms. - for (size_t i = 0; i < input_size; ++i) { - output[o] += input[i] * weights[o * input_size + i]; - } - output[o] = activation_function(output[o]); - } -} - -#if defined(WEBRTC_ARCH_X86_FAMILY) -// Fully connected layer SSE2 implementation. -void ComputeFullyConnectedLayerOutputSse2( - size_t input_size, - size_t output_size, - rtc::ArrayView input, - rtc::ArrayView bias, - rtc::ArrayView weights, - rtc::FunctionView activation_function, - rtc::ArrayView output) { - RTC_DCHECK_EQ(input.size(), input_size); - RTC_DCHECK_EQ(bias.size(), output_size); - RTC_DCHECK_EQ(weights.size(), input_size * output_size); - const size_t input_size_by_4 = input_size >> 2; - const size_t offset = input_size & ~3; - __m128 sum_wx_128; - const float* v = reinterpret_cast(&sum_wx_128); - for (size_t o = 0; o < output_size; ++o) { - // Perform 128 bit vector operations. - sum_wx_128 = _mm_set1_ps(0); - const float* x_p = input.data(); - const float* w_p = weights.data() + o * input_size; - for (size_t i = 0; i < input_size_by_4; ++i, x_p += 4, w_p += 4) { - sum_wx_128 = _mm_add_ps(sum_wx_128, - _mm_mul_ps(_mm_loadu_ps(x_p), _mm_loadu_ps(w_p))); - } - // Perform non-vector operations for any remaining items, sum up bias term - // and results from the vectorized code, and apply the activation function. - output[o] = activation_function( - std::inner_product(input.begin() + offset, input.end(), - weights.begin() + o * input_size + offset, - bias[o] + v[0] + v[1] + v[2] + v[3])); - } -} -#endif +using ::rnnoise::kOutputDenseBias; +using ::rnnoise::kOutputDenseWeights; +using ::rnnoise::kOutputLayerOutputSize; +static_assert(kOutputLayerOutputSize <= kFullyConnectedLayerMaxUnits, ""); } // namespace -FullyConnectedLayer::FullyConnectedLayer( - const size_t input_size, - const size_t output_size, - const rtc::ArrayView bias, - const rtc::ArrayView weights, - rtc::FunctionView activation_function, - Optimization optimization) - : input_size_(input_size), - output_size_(output_size), - bias_(GetScaledParams(bias)), - weights_(GetPreprocessedFcWeights(weights, output_size)), - activation_function_(activation_function), - optimization_(optimization) { - RTC_DCHECK_LE(output_size_, kFullyConnectedLayersMaxUnits) - << "Static over-allocation of fully-connected layers output vectors is " - "not sufficient."; - RTC_DCHECK_EQ(output_size_, bias_.size()) - << "Mismatching output size and bias terms array size."; - RTC_DCHECK_EQ(input_size_ * output_size_, weights_.size()) - << "Mismatching input-output size and weight coefficients array size."; -} - -FullyConnectedLayer::~FullyConnectedLayer() = default; - -rtc::ArrayView FullyConnectedLayer::GetOutput() const { - return rtc::ArrayView(output_.data(), output_size_); -} - -void FullyConnectedLayer::ComputeOutput(rtc::ArrayView input) { - switch (optimization_) { -#if defined(WEBRTC_ARCH_X86_FAMILY) - case Optimization::kSse2: - ComputeFullyConnectedLayerOutputSse2(input_size_, output_size_, input, - bias_, weights_, - activation_function_, output_); - break; -#endif -#if defined(WEBRTC_HAS_NEON) - case Optimization::kNeon: - // TODO(bugs.chromium.org/10480): Handle Optimization::kNeon. - ComputeFullyConnectedLayerOutput(input_size_, output_size_, input, bias_, - weights_, activation_function_, output_); - break; -#endif - default: - ComputeFullyConnectedLayerOutput(input_size_, output_size_, input, bias_, - weights_, activation_function_, output_); - } -} - -GatedRecurrentLayer::GatedRecurrentLayer( - const size_t input_size, - const size_t output_size, - const rtc::ArrayView bias, - const rtc::ArrayView weights, - const rtc::ArrayView recurrent_weights, - Optimization optimization) - : input_size_(input_size), - output_size_(output_size), - bias_(GetPreprocessedGruTensor(bias, output_size)), - weights_(GetPreprocessedGruTensor(weights, output_size)), - recurrent_weights_( - GetPreprocessedGruTensor(recurrent_weights, output_size)), - optimization_(optimization) { - RTC_DCHECK_LE(output_size_, kRecurrentLayersMaxUnits) - << "Static over-allocation of recurrent layers state vectors is not " - "sufficient."; - RTC_DCHECK_EQ(kNumGruGates * output_size_, bias_.size()) - << "Mismatching output size and bias terms array size."; - RTC_DCHECK_EQ(kNumGruGates * input_size_ * output_size_, weights_.size()) - << "Mismatching input-output size and weight coefficients array size."; - RTC_DCHECK_EQ(kNumGruGates * output_size_ * output_size_, - recurrent_weights_.size()) - << "Mismatching input-output size and recurrent weight coefficients array" - " size."; - Reset(); -} - -GatedRecurrentLayer::~GatedRecurrentLayer() = default; - -rtc::ArrayView GatedRecurrentLayer::GetOutput() const { - return rtc::ArrayView(state_.data(), output_size_); -} - -void GatedRecurrentLayer::Reset() { - state_.fill(0.f); -} - -void GatedRecurrentLayer::ComputeOutput(rtc::ArrayView input) { - switch (optimization_) { -#if defined(WEBRTC_ARCH_X86_FAMILY) - case Optimization::kSse2: - // TODO(bugs.chromium.org/10480): Handle Optimization::kSse2. - ComputeGruLayerOutput(input_size_, output_size_, input, weights_, - recurrent_weights_, bias_, state_); - break; -#endif -#if defined(WEBRTC_HAS_NEON) - case Optimization::kNeon: - // TODO(bugs.chromium.org/10480): Handle Optimization::kNeon. - ComputeGruLayerOutput(input_size_, output_size_, input, weights_, - recurrent_weights_, bias_, state_); - break; -#endif - default: - ComputeGruLayerOutput(input_size_, output_size_, input, weights_, - recurrent_weights_, bias_, state_); - } -} - -RnnBasedVad::RnnBasedVad() - : input_layer_(kInputLayerInputSize, - kInputLayerOutputSize, - kInputDenseBias, - kInputDenseWeights, - TansigApproximated, - DetectOptimization()), - hidden_layer_(kInputLayerOutputSize, - kHiddenLayerOutputSize, - kHiddenGruBias, - kHiddenGruWeights, - kHiddenGruRecurrentWeights, - DetectOptimization()), - output_layer_(kHiddenLayerOutputSize, - kOutputLayerOutputSize, - kOutputDenseBias, - kOutputDenseWeights, - SigmoidApproximated, - DetectOptimization()) { +RnnVad::RnnVad(const AvailableCpuFeatures& cpu_features) + : input_(kInputLayerInputSize, + kInputLayerOutputSize, + kInputDenseBias, + kInputDenseWeights, + ActivationFunction::kTansigApproximated, + cpu_features, + /*layer_name=*/"FC1"), + hidden_(kInputLayerOutputSize, + kHiddenLayerOutputSize, + kHiddenGruBias, + kHiddenGruWeights, + kHiddenGruRecurrentWeights, + cpu_features, + /*layer_name=*/"GRU1"), + output_(kHiddenLayerOutputSize, + kOutputLayerOutputSize, + kOutputDenseBias, + kOutputDenseWeights, + ActivationFunction::kSigmoidApproximated, + // The output layer is just 24x1. The unoptimized code is faster. + NoAvailableCpuFeatures(), + /*layer_name=*/"FC2") { // Input-output chaining size checks. - RTC_DCHECK_EQ(input_layer_.output_size(), hidden_layer_.input_size()) + RTC_DCHECK_EQ(input_.size(), hidden_.input_size()) << "The input and the hidden layers sizes do not match."; - RTC_DCHECK_EQ(hidden_layer_.output_size(), output_layer_.input_size()) + RTC_DCHECK_EQ(hidden_.size(), output_.input_size()) << "The hidden and the output layers sizes do not match."; } -RnnBasedVad::~RnnBasedVad() = default; +RnnVad::~RnnVad() = default; -void RnnBasedVad::Reset() { - hidden_layer_.Reset(); +void RnnVad::Reset() { + hidden_.Reset(); } -float RnnBasedVad::ComputeVadProbability( +float RnnVad::ComputeVadProbability( rtc::ArrayView feature_vector, bool is_silence) { if (is_silence) { Reset(); return 0.f; } - input_layer_.ComputeOutput(feature_vector); - hidden_layer_.ComputeOutput(input_layer_.GetOutput()); - output_layer_.ComputeOutput(hidden_layer_.GetOutput()); - const auto vad_output = output_layer_.GetOutput(); - return vad_output[0]; + input_.ComputeOutput(feature_vector); + hidden_.ComputeOutput(input_); + output_.ComputeOutput(hidden_); + RTC_DCHECK_EQ(output_.size(), 1); + return output_.data()[0]; } } // namespace rnn_vad diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.h b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.h index 58274b2..3148f1b 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn.h @@ -18,106 +18,33 @@ #include #include "api/array_view.h" -#include "api/function_view.h" +#include "modules/audio_processing/agc2/cpu_features.h" #include "modules/audio_processing/agc2/rnn_vad/common.h" -#include "rtc_base/system/arch.h" +#include "modules/audio_processing/agc2/rnn_vad/rnn_fc.h" +#include "modules/audio_processing/agc2/rnn_vad/rnn_gru.h" namespace webrtc { namespace rnn_vad { -// Maximum number of units for a fully-connected layer. This value is used to -// over-allocate space for fully-connected layers output vectors (implemented as -// std::array). The value should equal the number of units of the largest -// fully-connected layer. -constexpr size_t kFullyConnectedLayersMaxUnits = 24; - -// Maximum number of units for a recurrent layer. This value is used to -// over-allocate space for recurrent layers state vectors (implemented as -// std::array). The value should equal the number of units of the largest -// recurrent layer. -constexpr size_t kRecurrentLayersMaxUnits = 24; - -// Fully-connected layer. -class FullyConnectedLayer { +// Recurrent network with hard-coded architecture and weights for voice activity +// detection. +class RnnVad { public: - FullyConnectedLayer(size_t input_size, - size_t output_size, - rtc::ArrayView bias, - rtc::ArrayView weights, - rtc::FunctionView activation_function, - Optimization optimization); - FullyConnectedLayer(const FullyConnectedLayer&) = delete; - FullyConnectedLayer& operator=(const FullyConnectedLayer&) = delete; - ~FullyConnectedLayer(); - size_t input_size() const { return input_size_; } - size_t output_size() const { return output_size_; } - Optimization optimization() const { return optimization_; } - rtc::ArrayView GetOutput() const; - // Computes the fully-connected layer output. - void ComputeOutput(rtc::ArrayView input); - - private: - const size_t input_size_; - const size_t output_size_; - const std::vector bias_; - const std::vector weights_; - rtc::FunctionView activation_function_; - // The output vector of a recurrent layer has length equal to |output_size_|. - // However, for efficiency, over-allocation is used. - std::array output_; - const Optimization optimization_; -}; - -// Recurrent layer with gated recurrent units (GRUs) with sigmoid and ReLU as -// activation functions for the update/reset and output gates respectively. -class GatedRecurrentLayer { - public: - GatedRecurrentLayer(size_t input_size, - size_t output_size, - rtc::ArrayView bias, - rtc::ArrayView weights, - rtc::ArrayView recurrent_weights, - Optimization optimization); - GatedRecurrentLayer(const GatedRecurrentLayer&) = delete; - GatedRecurrentLayer& operator=(const GatedRecurrentLayer&) = delete; - ~GatedRecurrentLayer(); - size_t input_size() const { return input_size_; } - size_t output_size() const { return output_size_; } - Optimization optimization() const { return optimization_; } - rtc::ArrayView GetOutput() const; + explicit RnnVad(const AvailableCpuFeatures& cpu_features); + RnnVad(const RnnVad&) = delete; + RnnVad& operator=(const RnnVad&) = delete; + ~RnnVad(); void Reset(); - // Computes the recurrent layer output and updates the status. - void ComputeOutput(rtc::ArrayView input); - - private: - const size_t input_size_; - const size_t output_size_; - const std::vector bias_; - const std::vector weights_; - const std::vector recurrent_weights_; - // The state vector of a recurrent layer has length equal to |output_size_|. - // However, to avoid dynamic allocation, over-allocation is used. - std::array state_; - const Optimization optimization_; -}; - -// Recurrent network based VAD. -class RnnBasedVad { - public: - RnnBasedVad(); - RnnBasedVad(const RnnBasedVad&) = delete; - RnnBasedVad& operator=(const RnnBasedVad&) = delete; - ~RnnBasedVad(); - void Reset(); - // Compute and returns the probability of voice (range: [0.0, 1.0]). + // Observes `feature_vector` and `is_silence`, updates the RNN and returns the + // current voice probability. float ComputeVadProbability( rtc::ArrayView feature_vector, bool is_silence); private: - FullyConnectedLayer input_layer_; - GatedRecurrentLayer hidden_layer_; - FullyConnectedLayer output_layer_; + FullyConnectedLayer input_; + GatedRecurrentLayer hidden_; + FullyConnectedLayer output_; }; } // namespace rnn_vad diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.cc new file mode 100644 index 0000000..a13e774 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.cc @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/rnn_vad/rnn_fc.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "third_party/rnnoise/src/rnn_activations.h" +#include "third_party/rnnoise/src/rnn_vad_weights.h" + +namespace webrtc { +namespace rnn_vad { +namespace { + +std::vector GetScaledParams(rtc::ArrayView params) { + std::vector scaled_params(params.size()); + std::transform(params.begin(), params.end(), scaled_params.begin(), + [](int8_t x) -> float { + return ::rnnoise::kWeightsScale * static_cast(x); + }); + return scaled_params; +} + +// TODO(bugs.chromium.org/10480): Hard-code optimized layout and remove this +// function to improve setup time. +// Casts and scales `weights` and re-arranges the layout. +std::vector PreprocessWeights(rtc::ArrayView weights, + int output_size) { + if (output_size == 1) { + return GetScaledParams(weights); + } + // Transpose, scale and cast. + const int input_size = rtc::CheckedDivExact( + rtc::dchecked_cast(weights.size()), output_size); + std::vector w(weights.size()); + for (int o = 0; o < output_size; ++o) { + for (int i = 0; i < input_size; ++i) { + w[o * input_size + i] = rnnoise::kWeightsScale * + static_cast(weights[i * output_size + o]); + } + } + return w; +} + +rtc::FunctionView GetActivationFunction( + ActivationFunction activation_function) { + switch (activation_function) { + case ActivationFunction::kTansigApproximated: + return ::rnnoise::TansigApproximated; + case ActivationFunction::kSigmoidApproximated: + return ::rnnoise::SigmoidApproximated; + } +} + +} // namespace + +FullyConnectedLayer::FullyConnectedLayer( + const int input_size, + const int output_size, + const rtc::ArrayView bias, + const rtc::ArrayView weights, + ActivationFunction activation_function, + const AvailableCpuFeatures& cpu_features, + absl::string_view layer_name) + : input_size_(input_size), + output_size_(output_size), + bias_(GetScaledParams(bias)), + weights_(PreprocessWeights(weights, output_size)), + vector_math_(cpu_features), + activation_function_(GetActivationFunction(activation_function)) { + RTC_DCHECK_LE(output_size_, kFullyConnectedLayerMaxUnits) + << "Insufficient FC layer over-allocation (" << layer_name << ")."; + RTC_DCHECK_EQ(output_size_, bias_.size()) + << "Mismatching output size and bias terms array size (" << layer_name + << ")."; + RTC_DCHECK_EQ(input_size_ * output_size_, weights_.size()) + << "Mismatching input-output size and weight coefficients array size (" + << layer_name << ")."; +} + +FullyConnectedLayer::~FullyConnectedLayer() = default; + +void FullyConnectedLayer::ComputeOutput(rtc::ArrayView input) { + RTC_DCHECK_EQ(input.size(), input_size_); + rtc::ArrayView weights(weights_); + for (int o = 0; o < output_size_; ++o) { + output_[o] = activation_function_( + bias_[o] + vector_math_.DotProduct( + input, weights.subview(o * input_size_, input_size_))); + } +} + +} // namespace rnn_vad +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.h b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.h new file mode 100644 index 0000000..d23957a --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_fc.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RNN_FC_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RNN_FC_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/function_view.h" +#include "modules/audio_processing/agc2/cpu_features.h" +#include "modules/audio_processing/agc2/rnn_vad/vector_math.h" + +namespace webrtc { +namespace rnn_vad { + +// Activation function for a neural network cell. +enum class ActivationFunction { kTansigApproximated, kSigmoidApproximated }; + +// Maximum number of units for an FC layer. +constexpr int kFullyConnectedLayerMaxUnits = 24; + +// Fully-connected layer with a custom activation function which owns the output +// buffer. +class FullyConnectedLayer { + public: + // Ctor. `output_size` cannot be greater than `kFullyConnectedLayerMaxUnits`. + FullyConnectedLayer(int input_size, + int output_size, + rtc::ArrayView bias, + rtc::ArrayView weights, + ActivationFunction activation_function, + const AvailableCpuFeatures& cpu_features, + absl::string_view layer_name); + FullyConnectedLayer(const FullyConnectedLayer&) = delete; + FullyConnectedLayer& operator=(const FullyConnectedLayer&) = delete; + ~FullyConnectedLayer(); + + // Returns the size of the input vector. + int input_size() const { return input_size_; } + // Returns the pointer to the first element of the output buffer. + const float* data() const { return output_.data(); } + // Returns the size of the output buffer. + int size() const { return output_size_; } + + // Computes the fully-connected layer output. + void ComputeOutput(rtc::ArrayView input); + + private: + const int input_size_; + const int output_size_; + const std::vector bias_; + const std::vector weights_; + const VectorMath vector_math_; + rtc::FunctionView activation_function_; + // Over-allocated array with size equal to `output_size_`. + std::array output_; +}; + +} // namespace rnn_vad +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RNN_FC_H_ diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.cc new file mode 100644 index 0000000..ef37410 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.cc @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/rnn_vad/rnn_gru.h" + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "third_party/rnnoise/src/rnn_activations.h" +#include "third_party/rnnoise/src/rnn_vad_weights.h" + +namespace webrtc { +namespace rnn_vad { +namespace { + +constexpr int kNumGruGates = 3; // Update, reset, output. + +std::vector PreprocessGruTensor(rtc::ArrayView tensor_src, + int output_size) { + // Transpose, cast and scale. + // `n` is the size of the first dimension of the 3-dim tensor `weights`. + const int n = rtc::CheckedDivExact(rtc::dchecked_cast(tensor_src.size()), + output_size * kNumGruGates); + const int stride_src = kNumGruGates * output_size; + const int stride_dst = n * output_size; + std::vector tensor_dst(tensor_src.size()); + for (int g = 0; g < kNumGruGates; ++g) { + for (int o = 0; o < output_size; ++o) { + for (int i = 0; i < n; ++i) { + tensor_dst[g * stride_dst + o * n + i] = + ::rnnoise::kWeightsScale * + static_cast( + tensor_src[i * stride_src + g * output_size + o]); + } + } + } + return tensor_dst; +} + +// Computes the output for the update or the reset gate. +// Operation: `g = sigmoid(W^T∙i + R^T∙s + b)` where +// - `g`: output gate vector +// - `W`: weights matrix +// - `i`: input vector +// - `R`: recurrent weights matrix +// - `s`: state gate vector +// - `b`: bias vector +void ComputeUpdateResetGate(int input_size, + int output_size, + const VectorMath& vector_math, + rtc::ArrayView input, + rtc::ArrayView state, + rtc::ArrayView bias, + rtc::ArrayView weights, + rtc::ArrayView recurrent_weights, + rtc::ArrayView gate) { + RTC_DCHECK_EQ(input.size(), input_size); + RTC_DCHECK_EQ(state.size(), output_size); + RTC_DCHECK_EQ(bias.size(), output_size); + RTC_DCHECK_EQ(weights.size(), input_size * output_size); + RTC_DCHECK_EQ(recurrent_weights.size(), output_size * output_size); + RTC_DCHECK_GE(gate.size(), output_size); // `gate` is over-allocated. + for (int o = 0; o < output_size; ++o) { + float x = bias[o]; + x += vector_math.DotProduct(input, + weights.subview(o * input_size, input_size)); + x += vector_math.DotProduct( + state, recurrent_weights.subview(o * output_size, output_size)); + gate[o] = ::rnnoise::SigmoidApproximated(x); + } +} + +// Computes the output for the state gate. +// Operation: `s' = u .* s + (1 - u) .* ReLU(W^T∙i + R^T∙(s .* r) + b)` where +// - `s'`: output state gate vector +// - `s`: previous state gate vector +// - `u`: update gate vector +// - `W`: weights matrix +// - `i`: input vector +// - `R`: recurrent weights matrix +// - `r`: reset gate vector +// - `b`: bias vector +// - `.*` element-wise product +void ComputeStateGate(int input_size, + int output_size, + const VectorMath& vector_math, + rtc::ArrayView input, + rtc::ArrayView update, + rtc::ArrayView reset, + rtc::ArrayView bias, + rtc::ArrayView weights, + rtc::ArrayView recurrent_weights, + rtc::ArrayView state) { + RTC_DCHECK_EQ(input.size(), input_size); + RTC_DCHECK_GE(update.size(), output_size); // `update` is over-allocated. + RTC_DCHECK_GE(reset.size(), output_size); // `reset` is over-allocated. + RTC_DCHECK_EQ(bias.size(), output_size); + RTC_DCHECK_EQ(weights.size(), input_size * output_size); + RTC_DCHECK_EQ(recurrent_weights.size(), output_size * output_size); + RTC_DCHECK_EQ(state.size(), output_size); + std::array reset_x_state; + for (int o = 0; o < output_size; ++o) { + reset_x_state[o] = state[o] * reset[o]; + } + for (int o = 0; o < output_size; ++o) { + float x = bias[o]; + x += vector_math.DotProduct(input, + weights.subview(o * input_size, input_size)); + x += vector_math.DotProduct( + {reset_x_state.data(), static_cast(output_size)}, + recurrent_weights.subview(o * output_size, output_size)); + state[o] = update[o] * state[o] + (1.f - update[o]) * std::max(0.f, x); + } +} + +} // namespace + +GatedRecurrentLayer::GatedRecurrentLayer( + const int input_size, + const int output_size, + const rtc::ArrayView bias, + const rtc::ArrayView weights, + const rtc::ArrayView recurrent_weights, + const AvailableCpuFeatures& cpu_features, + absl::string_view layer_name) + : input_size_(input_size), + output_size_(output_size), + bias_(PreprocessGruTensor(bias, output_size)), + weights_(PreprocessGruTensor(weights, output_size)), + recurrent_weights_(PreprocessGruTensor(recurrent_weights, output_size)), + vector_math_(cpu_features) { + RTC_DCHECK_LE(output_size_, kGruLayerMaxUnits) + << "Insufficient GRU layer over-allocation (" << layer_name << ")."; + RTC_DCHECK_EQ(kNumGruGates * output_size_, bias_.size()) + << "Mismatching output size and bias terms array size (" << layer_name + << ")."; + RTC_DCHECK_EQ(kNumGruGates * input_size_ * output_size_, weights_.size()) + << "Mismatching input-output size and weight coefficients array size (" + << layer_name << ")."; + RTC_DCHECK_EQ(kNumGruGates * output_size_ * output_size_, + recurrent_weights_.size()) + << "Mismatching input-output size and recurrent weight coefficients array" + " size (" + << layer_name << ")."; + Reset(); +} + +GatedRecurrentLayer::~GatedRecurrentLayer() = default; + +void GatedRecurrentLayer::Reset() { + state_.fill(0.f); +} + +void GatedRecurrentLayer::ComputeOutput(rtc::ArrayView input) { + RTC_DCHECK_EQ(input.size(), input_size_); + + // The tensors below are organized as a sequence of flattened tensors for the + // `update`, `reset` and `state` gates. + rtc::ArrayView bias(bias_); + rtc::ArrayView weights(weights_); + rtc::ArrayView recurrent_weights(recurrent_weights_); + // Strides to access to the flattened tensors for a specific gate. + const int stride_weights = input_size_ * output_size_; + const int stride_recurrent_weights = output_size_ * output_size_; + + rtc::ArrayView state(state_.data(), output_size_); + + // Update gate. + std::array update; + ComputeUpdateResetGate( + input_size_, output_size_, vector_math_, input, state, + bias.subview(0, output_size_), weights.subview(0, stride_weights), + recurrent_weights.subview(0, stride_recurrent_weights), update); + // Reset gate. + std::array reset; + ComputeUpdateResetGate(input_size_, output_size_, vector_math_, input, state, + bias.subview(output_size_, output_size_), + weights.subview(stride_weights, stride_weights), + recurrent_weights.subview(stride_recurrent_weights, + stride_recurrent_weights), + reset); + // State gate. + ComputeStateGate(input_size_, output_size_, vector_math_, input, update, + reset, bias.subview(2 * output_size_, output_size_), + weights.subview(2 * stride_weights, stride_weights), + recurrent_weights.subview(2 * stride_recurrent_weights, + stride_recurrent_weights), + state); +} + +} // namespace rnn_vad +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.h b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.h new file mode 100644 index 0000000..3407dfc --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/rnn_gru.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RNN_GRU_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RNN_GRU_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "modules/audio_processing/agc2/cpu_features.h" +#include "modules/audio_processing/agc2/rnn_vad/vector_math.h" + +namespace webrtc { +namespace rnn_vad { + +// Maximum number of units for a GRU layer. +constexpr int kGruLayerMaxUnits = 24; + +// Recurrent layer with gated recurrent units (GRUs) with sigmoid and ReLU as +// activation functions for the update/reset and output gates respectively. +class GatedRecurrentLayer { + public: + // Ctor. `output_size` cannot be greater than `kGruLayerMaxUnits`. + GatedRecurrentLayer(int input_size, + int output_size, + rtc::ArrayView bias, + rtc::ArrayView weights, + rtc::ArrayView recurrent_weights, + const AvailableCpuFeatures& cpu_features, + absl::string_view layer_name); + GatedRecurrentLayer(const GatedRecurrentLayer&) = delete; + GatedRecurrentLayer& operator=(const GatedRecurrentLayer&) = delete; + ~GatedRecurrentLayer(); + + // Returns the size of the input vector. + int input_size() const { return input_size_; } + // Returns the pointer to the first element of the output buffer. + const float* data() const { return state_.data(); } + // Returns the size of the output buffer. + int size() const { return output_size_; } + + // Resets the GRU state. + void Reset(); + // Computes the recurrent layer output and updates the status. + void ComputeOutput(rtc::ArrayView input); + + private: + const int input_size_; + const int output_size_; + const std::vector bias_; + const std::vector weights_; + const std::vector recurrent_weights_; + const VectorMath vector_math_; + // Over-allocated array with size equal to `output_size_`. + std::array state_; +}; + +} // namespace rnn_vad +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RNN_GRU_H_ diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h b/webrtc/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h index 75d3d9b..a740278 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h @@ -29,7 +29,7 @@ namespace rnn_vad { // values are written at the end of the buffer. // The class also provides a view on the most recent M values, where 0 < M <= S // and by default M = N. -template +template class SequenceBuffer { static_assert(N <= S, "The new chunk size cannot be larger than the sequence buffer " @@ -45,8 +45,8 @@ class SequenceBuffer { SequenceBuffer(const SequenceBuffer&) = delete; SequenceBuffer& operator=(const SequenceBuffer&) = delete; ~SequenceBuffer() = default; - size_t size() const { return S; } - size_t chunks_size() const { return N; } + int size() const { return S; } + int chunks_size() const { return N; } // Sets the sequence buffer values to zero. void Reset() { std::fill(buffer_.begin(), buffer_.end(), 0); } // Returns a view on the whole buffer. diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features.cc index 81e3339..96086ba 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features.cc @@ -16,6 +16,7 @@ #include #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" namespace webrtc { namespace rnn_vad { @@ -32,11 +33,11 @@ void UpdateCepstralDifferenceStats( RTC_DCHECK(sym_matrix_buf); // Compute the new cepstral distance stats. std::array distances; - for (size_t i = 0; i < kCepstralCoeffsHistorySize - 1; ++i) { - const size_t delay = i + 1; + for (int i = 0; i < kCepstralCoeffsHistorySize - 1; ++i) { + const int delay = i + 1; auto old_cepstral_coeffs = ring_buf.GetArrayView(delay); distances[i] = 0.f; - for (size_t k = 0; k < kNumBands; ++k) { + for (int k = 0; k < kNumBands; ++k) { const float c = new_cepstral_coeffs[k] - old_cepstral_coeffs[k]; distances[i] += c * c; } @@ -48,9 +49,9 @@ void UpdateCepstralDifferenceStats( // Computes the first half of the Vorbis window. std::array ComputeScaledHalfVorbisWindow( float scaling = 1.f) { - constexpr size_t kHalfSize = kFrameSize20ms24kHz / 2; + constexpr int kHalfSize = kFrameSize20ms24kHz / 2; std::array half_window{}; - for (size_t i = 0; i < kHalfSize; ++i) { + for (int i = 0; i < kHalfSize; ++i) { half_window[i] = scaling * std::sin(0.5 * kPi * std::sin(0.5 * kPi * (i + 0.5) / kHalfSize) * @@ -71,8 +72,8 @@ void ComputeWindowedForwardFft( RTC_DCHECK_EQ(frame.size(), 2 * half_window.size()); // Apply windowing. auto in = fft_input_buffer->GetView(); - for (size_t i = 0, j = kFrameSize20ms24kHz - 1; i < half_window.size(); - ++i, --j) { + for (int i = 0, j = kFrameSize20ms24kHz - 1; + rtc::SafeLt(i, half_window.size()); ++i, --j) { in[i] = frame[i] * half_window[i]; in[j] = frame[j] * half_window[i]; } @@ -162,7 +163,7 @@ void SpectralFeaturesExtractor::ComputeAvgAndDerivatives( RTC_DCHECK_EQ(average.size(), first_derivative.size()); RTC_DCHECK_EQ(first_derivative.size(), second_derivative.size()); RTC_DCHECK_LE(average.size(), curr.size()); - for (size_t i = 0; i < average.size(); ++i) { + for (int i = 0; rtc::SafeLt(i, average.size()); ++i) { // Average, kernel: [1, 1, 1]. average[i] = curr[i] + prev1[i] + prev2[i]; // First derivative, kernel: [1, 0, - 1]. @@ -178,7 +179,7 @@ void SpectralFeaturesExtractor::ComputeNormalizedCepstralCorrelation( reference_frame_fft_->GetConstView(), lagged_frame_fft_->GetConstView(), bands_cross_corr_); // Normalize. - for (size_t i = 0; i < bands_cross_corr_.size(); ++i) { + for (int i = 0; rtc::SafeLt(i, bands_cross_corr_.size()); ++i) { bands_cross_corr_[i] = bands_cross_corr_[i] / std::sqrt(0.001f + reference_frame_bands_energy_[i] * @@ -194,9 +195,9 @@ void SpectralFeaturesExtractor::ComputeNormalizedCepstralCorrelation( float SpectralFeaturesExtractor::ComputeVariability() const { // Compute cepstral variability score. float variability = 0.f; - for (size_t delay1 = 0; delay1 < kCepstralCoeffsHistorySize; ++delay1) { + for (int delay1 = 0; delay1 < kCepstralCoeffsHistorySize; ++delay1) { float min_dist = std::numeric_limits::max(); - for (size_t delay2 = 0; delay2 < kCepstralCoeffsHistorySize; ++delay2) { + for (int delay2 = 0; delay2 < kCepstralCoeffsHistorySize; ++delay2) { if (delay1 == delay2) // The distance would be 0. continue; min_dist = diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.cc index 29192a0..a10b0f7 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.cc @@ -15,6 +15,7 @@ #include #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" namespace webrtc { namespace rnn_vad { @@ -22,7 +23,7 @@ namespace { // Weights for each FFT coefficient for each Opus band (Nyquist frequency // excluded). The size of each band is specified in -// |kOpusScaleNumBins24kHz20ms|. +// `kOpusScaleNumBins24kHz20ms`. constexpr std::array kOpusBandWeights24kHz20ms = {{ 0.f, 0.25f, 0.5f, 0.75f, // Band 0 @@ -105,9 +106,9 @@ void SpectralCorrelator::ComputeCrossCorrelation( RTC_DCHECK_EQ(x[1], 0.f) << "The Nyquist coefficient must be zeroed."; RTC_DCHECK_EQ(y[1], 0.f) << "The Nyquist coefficient must be zeroed."; constexpr auto kOpusScaleNumBins24kHz20ms = GetOpusScaleNumBins24kHz20ms(); - size_t k = 0; // Next Fourier coefficient index. + int k = 0; // Next Fourier coefficient index. cross_corr[0] = 0.f; - for (size_t i = 0; i < kOpusBands24kHz - 1; ++i) { + for (int i = 0; i < kOpusBands24kHz - 1; ++i) { cross_corr[i + 1] = 0.f; for (int j = 0; j < kOpusScaleNumBins24kHz20ms[i]; ++j) { // Band size. const float v = x[2 * k] * y[2 * k] + x[2 * k + 1] * y[2 * k + 1]; @@ -137,11 +138,11 @@ void ComputeSmoothedLogMagnitudeSpectrum( return x; }; // Smoothing over the bands for which the band energy is defined. - for (size_t i = 0; i < bands_energy.size(); ++i) { + for (int i = 0; rtc::SafeLt(i, bands_energy.size()); ++i) { log_bands_energy[i] = smooth(std::log10(kOneByHundred + bands_energy[i])); } // Smoothing over the remaining bands (zero energy). - for (size_t i = bands_energy.size(); i < kNumBands; ++i) { + for (int i = bands_energy.size(); i < kNumBands; ++i) { log_bands_energy[i] = smooth(kLogOneByHundred); } } @@ -149,8 +150,8 @@ void ComputeSmoothedLogMagnitudeSpectrum( std::array ComputeDctTable() { std::array dct_table; const double k = std::sqrt(0.5); - for (size_t i = 0; i < kNumBands; ++i) { - for (size_t j = 0; j < kNumBands; ++j) + for (int i = 0; i < kNumBands; ++i) { + for (int j = 0; j < kNumBands; ++j) dct_table[i * kNumBands + j] = std::cos((i + 0.5) * j * kPi / kNumBands); dct_table[i * kNumBands] *= k; } @@ -173,9 +174,9 @@ void ComputeDct(rtc::ArrayView in, RTC_DCHECK_LE(in.size(), kNumBands); RTC_DCHECK_LE(1, out.size()); RTC_DCHECK_LE(out.size(), in.size()); - for (size_t i = 0; i < out.size(); ++i) { + for (int i = 0; rtc::SafeLt(i, out.size()); ++i) { out[i] = 0.f; - for (size_t j = 0; j < in.size(); ++j) { + for (int j = 0; rtc::SafeLt(j, in.size()); ++j) { out[i] += in[j] * dct_table[j * kNumBands + i]; } // TODO(bugs.webrtc.org/10480): Scaling factor in the DCT table. diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.h b/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.h index ed4caad..f4b293a 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/spectral_features_internal.h @@ -25,7 +25,7 @@ namespace rnn_vad { // At a sample rate of 24 kHz, the last 3 Opus bands are beyond the Nyquist // frequency. However, band #19 gets the contributions from band #18 because // of the symmetric triangular filter with peak response at 12 kHz. -constexpr size_t kOpusBands24kHz = 20; +constexpr int kOpusBands24kHz = 20; static_assert(kOpusBands24kHz < kNumBands, "The number of bands at 24 kHz must be less than those defined " "in the Opus scale at 48 kHz."); @@ -50,8 +50,8 @@ class SpectralCorrelator { ~SpectralCorrelator(); // Computes the band-wise spectral auto-correlations. - // |x| must: - // - have size equal to |kFrameSize20ms24kHz|; + // `x` must: + // - have size equal to `kFrameSize20ms24kHz`; // - be encoded as vectors of interleaved real-complex FFT coefficients // where x[1] = y[1] = 0 (the Nyquist frequency coefficient is omitted). void ComputeAutoCorrelation( @@ -59,8 +59,8 @@ class SpectralCorrelator { rtc::ArrayView auto_corr) const; // Computes the band-wise spectral cross-correlations. - // |x| and |y| must: - // - have size equal to |kFrameSize20ms24kHz|; + // `x` and `y` must: + // - have size equal to `kFrameSize20ms24kHz`; // - be encoded as vectors of interleaved real-complex FFT coefficients where // x[1] = y[1] = 0 (the Nyquist frequency coefficient is omitted). void ComputeCrossCorrelation( @@ -82,12 +82,12 @@ void ComputeSmoothedLogMagnitudeSpectrum( // TODO(bugs.webrtc.org/10480): Move to anonymous namespace in // spectral_features.cc. Creates a DCT table for arrays having size equal to -// |kNumBands|. Declared here for unit testing. +// `kNumBands`. Declared here for unit testing. std::array ComputeDctTable(); // TODO(bugs.webrtc.org/10480): Move to anonymous namespace in -// spectral_features.cc. Computes DCT for |in| given a pre-computed DCT table. -// In-place computation is not allowed and |out| can be smaller than |in| in +// spectral_features.cc. Computes DCT for `in` given a pre-computed DCT table. +// In-place computation is not allowed and `out` can be smaller than `in` in // order to only compute the first DCT coefficients. Declared here for unit // testing. void ComputeDct(rtc::ArrayView in, diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h b/webrtc/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h index f0282aa..d186479 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h @@ -18,6 +18,7 @@ #include "api/array_view.h" #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" namespace webrtc { namespace rnn_vad { @@ -29,7 +30,7 @@ namespace rnn_vad { // removed when one of the two corresponding items that have been compared is // removed from the ring buffer. It is assumed that the comparison is symmetric // and that comparing an item with itself is not needed. -template +template class SymmetricMatrixBuffer { static_assert(S > 2, ""); @@ -45,9 +46,9 @@ class SymmetricMatrixBuffer { buf_.fill(0); } // Pushes the results from the comparison between the most recent item and - // those that are still in the ring buffer. The first element in |values| must + // those that are still in the ring buffer. The first element in `values` must // correspond to the comparison between the most recent item and the second - // most recent one in the ring buffer, whereas the last element in |values| + // most recent one in the ring buffer, whereas the last element in `values` // must correspond to the comparison between the most recent item and the // oldest one in the ring buffer. void Push(rtc::ArrayView values) { @@ -55,19 +56,19 @@ class SymmetricMatrixBuffer { // column left. std::memmove(buf_.data(), buf_.data() + S, (buf_.size() - S) * sizeof(T)); // Copy new values in the last column in the right order. - for (size_t i = 0; i < values.size(); ++i) { - const size_t index = (S - 1 - i) * (S - 1) - 1; - RTC_DCHECK_LE(static_cast(0), index); + for (int i = 0; rtc::SafeLt(i, values.size()); ++i) { + const int index = (S - 1 - i) * (S - 1) - 1; + RTC_DCHECK_GE(index, 0); RTC_DCHECK_LT(index, buf_.size()); buf_[index] = values[i]; } } // Reads the value that corresponds to comparison of two items in the ring - // buffer having delay |delay1| and |delay2|. The two arguments must not be + // buffer having delay `delay1` and `delay2`. The two arguments must not be // equal and both must be in {0, ..., S - 1}. - T GetValue(size_t delay1, size_t delay2) const { - int row = S - 1 - static_cast(delay1); - int col = S - 1 - static_cast(delay2); + T GetValue(int delay1, int delay2) const { + int row = S - 1 - delay1; + int col = S - 1 - delay2; RTC_DCHECK_NE(row, col) << "The diagonal cannot be accessed."; if (row > col) std::swap(row, col); // Swap to access the upper-right triangular part. diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.cc index c7bf02e..857a9f2 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.cc +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.cc @@ -10,21 +10,61 @@ #include "modules/audio_processing/agc2/rnn_vad/test_utils.h" +#include +#include #include +#include +#include +#include +#include "absl/strings/string_view.h" #include "rtc_base/checks.h" -#include "rtc_base/system/arch.h" -#include "system_wrappers/include/cpu_features_wrapper.h" +#include "rtc_base/numerics/safe_compare.h" #include "test/gtest.h" #include "test/testsupport/file_utils.h" namespace webrtc { namespace rnn_vad { -namespace test { namespace { -using ReaderPairType = - std::pair>, const size_t>; +// File reader for binary files that contain a sequence of values with +// arithmetic type `T`. The values of type `T` that are read are cast to float. +template +class FloatFileReader : public FileReader { + public: + static_assert(std::is_arithmetic::value, ""); + explicit FloatFileReader(absl::string_view filename) + : is_(std::string(filename), std::ios::binary | std::ios::ate), + size_(is_.tellg() / sizeof(T)) { + RTC_CHECK(is_); + SeekBeginning(); + } + FloatFileReader(const FloatFileReader&) = delete; + FloatFileReader& operator=(const FloatFileReader&) = delete; + ~FloatFileReader() = default; + + int size() const override { return size_; } + bool ReadChunk(rtc::ArrayView dst) override { + const std::streamsize bytes_to_read = dst.size() * sizeof(T); + if (std::is_same::value) { + is_.read(reinterpret_cast(dst.data()), bytes_to_read); + } else { + buffer_.resize(dst.size()); + is_.read(reinterpret_cast(buffer_.data()), bytes_to_read); + std::transform(buffer_.begin(), buffer_.end(), dst.begin(), + [](const T& v) -> float { return static_cast(v); }); + } + return is_.gcount() == bytes_to_read; + } + bool ReadValue(float& dst) override { return ReadChunk({&dst, 1}); } + void SeekForward(int hop) override { is_.seekg(hop * sizeof(T), is_.cur); } + void SeekBeginning() override { is_.seekg(0, is_.beg); } + + private: + std::ifstream is_; + const int size_; + std::vector buffer_; +}; } // namespace @@ -33,7 +73,7 @@ using webrtc::test::ResourcePath; void ExpectEqualFloatArray(rtc::ArrayView expected, rtc::ArrayView computed) { ASSERT_EQ(expected.size(), computed.size()); - for (size_t i = 0; i < expected.size(); ++i) { + for (int i = 0; rtc::SafeLt(i, expected.size()); ++i) { SCOPED_TRACE(i); EXPECT_FLOAT_EQ(expected[i], computed[i]); } @@ -43,87 +83,61 @@ void ExpectNearAbsolute(rtc::ArrayView expected, rtc::ArrayView computed, float tolerance) { ASSERT_EQ(expected.size(), computed.size()); - for (size_t i = 0; i < expected.size(); ++i) { + for (int i = 0; rtc::SafeLt(i, expected.size()); ++i) { SCOPED_TRACE(i); EXPECT_NEAR(expected[i], computed[i], tolerance); } } -std::pair>, const size_t> -CreatePcmSamplesReader(const size_t frame_length) { - auto ptr = std::make_unique>( - test::ResourcePath("audio_processing/agc2/rnn_vad/samples", "pcm"), - frame_length); - // The last incomplete frame is ignored. - return {std::move(ptr), ptr->data_length() / frame_length}; +std::unique_ptr CreatePcmSamplesReader() { + return std::make_unique>( + /*filename=*/test::ResourcePath("audio_processing/agc2/rnn_vad/samples", + "pcm")); } -ReaderPairType CreatePitchBuffer24kHzReader() { - constexpr size_t cols = 864; - auto ptr = std::make_unique>( - ResourcePath("audio_processing/agc2/rnn_vad/pitch_buf_24k", "dat"), cols); - return {std::move(ptr), rtc::CheckedDivExact(ptr->data_length(), cols)}; +ChunksFileReader CreatePitchBuffer24kHzReader() { + auto reader = std::make_unique>( + /*filename=*/test::ResourcePath( + "audio_processing/agc2/rnn_vad/pitch_buf_24k", "dat")); + const int num_chunks = rtc::CheckedDivExact(reader->size(), kBufSize24kHz); + return {/*chunk_size=*/kBufSize24kHz, num_chunks, std::move(reader)}; } -ReaderPairType CreateLpResidualAndPitchPeriodGainReader() { - constexpr size_t num_lp_residual_coeffs = 864; - auto ptr = std::make_unique>( - ResourcePath("audio_processing/agc2/rnn_vad/pitch_lp_res", "dat"), - num_lp_residual_coeffs); - return {std::move(ptr), - rtc::CheckedDivExact(ptr->data_length(), 2 + num_lp_residual_coeffs)}; +ChunksFileReader CreateLpResidualAndPitchInfoReader() { + constexpr int kPitchInfoSize = 2; // Pitch period and strength. + constexpr int kChunkSize = kBufSize24kHz + kPitchInfoSize; + auto reader = std::make_unique>( + /*filename=*/test::ResourcePath( + "audio_processing/agc2/rnn_vad/pitch_lp_res", "dat")); + const int num_chunks = rtc::CheckedDivExact(reader->size(), kChunkSize); + return {kChunkSize, num_chunks, std::move(reader)}; } -ReaderPairType CreateVadProbsReader() { - auto ptr = std::make_unique>( - test::ResourcePath("audio_processing/agc2/rnn_vad/vad_prob", "dat")); - return {std::move(ptr), ptr->data_length()}; +std::unique_ptr CreateGruInputReader() { + return std::make_unique>( + /*filename=*/test::ResourcePath("audio_processing/agc2/rnn_vad/gru_in", + "dat")); +} + +std::unique_ptr CreateVadProbsReader() { + return std::make_unique>( + /*filename=*/test::ResourcePath("audio_processing/agc2/rnn_vad/vad_prob", + "dat")); } PitchTestData::PitchTestData() { - BinaryFileReader test_data_reader( - ResourcePath("audio_processing/agc2/rnn_vad/pitch_search_int", "dat"), - static_cast(1396)); - test_data_reader.ReadChunk(test_data_); + FloatFileReader reader( + /*filename=*/ResourcePath( + "audio_processing/agc2/rnn_vad/pitch_search_int", "dat")); + reader.ReadChunk(pitch_buffer_24k_); + reader.ReadChunk(square_energies_24k_); + reader.ReadChunk(auto_correlation_12k_); + // Reverse the order of the squared energy values. + // Required after the WebRTC CL 191703 which switched to forward computation. + std::reverse(square_energies_24k_.begin(), square_energies_24k_.end()); } PitchTestData::~PitchTestData() = default; -rtc::ArrayView PitchTestData::GetPitchBufView() - const { - return {test_data_.data(), kBufSize24kHz}; -} - -rtc::ArrayView -PitchTestData::GetPitchBufSquareEnergiesView() const { - return {test_data_.data() + kBufSize24kHz, kNumPitchBufSquareEnergies}; -} - -rtc::ArrayView -PitchTestData::GetPitchBufAutoCorrCoeffsView() const { - return {test_data_.data() + kBufSize24kHz + kNumPitchBufSquareEnergies, - kNumPitchBufAutoCorrCoeffs}; -} - -bool IsOptimizationAvailable(Optimization optimization) { - switch (optimization) { - case Optimization::kSse2: -#if defined(WEBRTC_ARCH_X86_FAMILY) - return GetCPUInfo(kSSE2) != 0; -#else - return false; -#endif - case Optimization::kNeon: -#if defined(WEBRTC_HAS_NEON) - return true; -#else - return false; -#endif - case Optimization::kNone: - return true; - } -} - -} // namespace test } // namespace rnn_vad } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.h b/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.h index db155e6..e64b7b7 100644 --- a/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.h +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/test_utils.h @@ -11,23 +11,19 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_TEST_UTILS_H_ #define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_TEST_UTILS_H_ -#include #include #include -#include #include #include -#include -#include -#include +#include "absl/strings/string_view.h" #include "api/array_view.h" #include "modules/audio_processing/agc2/rnn_vad/common.h" #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" namespace webrtc { namespace rnn_vad { -namespace test { constexpr float kFloatMin = std::numeric_limits::min(); @@ -42,98 +38,51 @@ void ExpectNearAbsolute(rtc::ArrayView expected, rtc::ArrayView computed, float tolerance); -// Reader for binary files consisting of an arbitrary long sequence of elements -// having type T. It is possible to read and cast to another type D at once. -template -class BinaryFileReader { +// File reader interface. +class FileReader { public: - explicit BinaryFileReader(const std::string& file_path, size_t chunk_size = 0) - : is_(file_path, std::ios::binary | std::ios::ate), - data_length_(is_.tellg() / sizeof(T)), - chunk_size_(chunk_size) { - RTC_CHECK(is_); - SeekBeginning(); - buf_.resize(chunk_size_); - } - BinaryFileReader(const BinaryFileReader&) = delete; - BinaryFileReader& operator=(const BinaryFileReader&) = delete; - ~BinaryFileReader() = default; - size_t data_length() const { return data_length_; } - bool ReadValue(D* dst) { - if (std::is_same::value) { - is_.read(reinterpret_cast(dst), sizeof(T)); - } else { - T v; - is_.read(reinterpret_cast(&v), sizeof(T)); - *dst = static_cast(v); - } - return is_.gcount() == sizeof(T); - } - // If |chunk_size| was specified in the ctor, it will check that the size of - // |dst| equals |chunk_size|. - bool ReadChunk(rtc::ArrayView dst) { - RTC_DCHECK((chunk_size_ == 0) || (chunk_size_ == dst.size())); - const std::streamsize bytes_to_read = dst.size() * sizeof(T); - if (std::is_same::value) { - is_.read(reinterpret_cast(dst.data()), bytes_to_read); - } else { - is_.read(reinterpret_cast(buf_.data()), bytes_to_read); - std::transform(buf_.begin(), buf_.end(), dst.begin(), - [](const T& v) -> D { return static_cast(v); }); - } - return is_.gcount() == bytes_to_read; - } - void SeekForward(size_t items) { is_.seekg(items * sizeof(T), is_.cur); } - void SeekBeginning() { is_.seekg(0, is_.beg); } - - private: - std::ifstream is_; - const size_t data_length_; - const size_t chunk_size_; - std::vector buf_; + virtual ~FileReader() = default; + // Number of values in the file. + virtual int size() const = 0; + // Reads `dst.size()` float values into `dst`, advances the internal file + // position according to the number of read bytes and returns true if the + // values are correctly read. If the number of remaining bytes in the file is + // not sufficient to read `dst.size()` float values, `dst` is partially + // modified and false is returned. + virtual bool ReadChunk(rtc::ArrayView dst) = 0; + // Reads a single float value, advances the internal file position according + // to the number of read bytes and returns true if the value is correctly + // read. If the number of remaining bytes in the file is not sufficient to + // read one float, `dst` is not modified and false is returned. + virtual bool ReadValue(float& dst) = 0; + // Advances the internal file position by `hop` float values. + virtual void SeekForward(int hop) = 0; + // Resets the internal file position to BOF. + virtual void SeekBeginning() = 0; }; -// Writer for binary files. -template -class BinaryFileWriter { - public: - explicit BinaryFileWriter(const std::string& file_path) - : os_(file_path, std::ios::binary) {} - BinaryFileWriter(const BinaryFileWriter&) = delete; - BinaryFileWriter& operator=(const BinaryFileWriter&) = delete; - ~BinaryFileWriter() = default; - static_assert(std::is_arithmetic::value, ""); - void WriteChunk(rtc::ArrayView value) { - const std::streamsize bytes_to_write = value.size() * sizeof(T); - os_.write(reinterpret_cast(value.data()), bytes_to_write); - } - - private: - std::ofstream os_; +// File reader for files that contain `num_chunks` chunks with size equal to +// `chunk_size`. +struct ChunksFileReader { + const int chunk_size; + const int num_chunks; + std::unique_ptr reader; }; -// Factories for resource file readers. -// The functions below return a pair where the first item is a reader unique -// pointer and the second the number of chunks that can be read from the file. -// Creates a reader for the PCM samples that casts from S16 to float and reads -// chunks with length |frame_length|. -std::pair>, const size_t> -CreatePcmSamplesReader(const size_t frame_length); -// Creates a reader for the pitch buffer content at 24 kHz. -std::pair>, const size_t> -CreatePitchBuffer24kHzReader(); -// Creates a reader for the the LP residual coefficients and the pitch period -// and gain values. -std::pair>, const size_t> -CreateLpResidualAndPitchPeriodGainReader(); -// Creates a reader for the VAD probabilities. -std::pair>, const size_t> -CreateVadProbsReader(); +// Creates a reader for the PCM S16 samples file. +std::unique_ptr CreatePcmSamplesReader(); -constexpr size_t kNumPitchBufAutoCorrCoeffs = 147; -constexpr size_t kNumPitchBufSquareEnergies = 385; -constexpr size_t kPitchTestDataSize = - kBufSize24kHz + kNumPitchBufSquareEnergies + kNumPitchBufAutoCorrCoeffs; +// Creates a reader for the 24 kHz pitch buffer test data. +ChunksFileReader CreatePitchBuffer24kHzReader(); + +// Creates a reader for the LP residual and pitch information test data. +ChunksFileReader CreateLpResidualAndPitchInfoReader(); + +// Creates a reader for the sequence of GRU input vectors. +std::unique_ptr CreateGruInputReader(); + +// Creates a reader for the VAD probabilities test data. +std::unique_ptr CreateVadProbsReader(); // Class to retrieve a test pitch buffer content and the expected output for the // analysis steps. @@ -141,20 +90,40 @@ class PitchTestData { public: PitchTestData(); ~PitchTestData(); - rtc::ArrayView GetPitchBufView() const; - rtc::ArrayView - GetPitchBufSquareEnergiesView() const; - rtc::ArrayView - GetPitchBufAutoCorrCoeffsView() const; + rtc::ArrayView PitchBuffer24kHzView() const { + return pitch_buffer_24k_; + } + rtc::ArrayView SquareEnergies24kHzView() + const { + return square_energies_24k_; + } + rtc::ArrayView AutoCorrelation12kHzView() const { + return auto_correlation_12k_; + } private: - std::array test_data_; + std::array pitch_buffer_24k_; + std::array square_energies_24k_; + std::array auto_correlation_12k_; }; -// Returns true if the given optimization is available. -bool IsOptimizationAvailable(Optimization optimization); +// Writer for binary files. +class FileWriter { + public: + explicit FileWriter(absl::string_view file_path) + : os_(std::string(file_path), std::ios::binary) {} + FileWriter(const FileWriter&) = delete; + FileWriter& operator=(const FileWriter&) = delete; + ~FileWriter() = default; + void WriteChunk(rtc::ArrayView value) { + const std::streamsize bytes_to_write = value.size() * sizeof(float); + os_.write(reinterpret_cast(value.data()), bytes_to_write); + } + + private: + std::ofstream os_; +}; -} // namespace test } // namespace rnn_vad } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/vector_math.h b/webrtc/modules/audio_processing/agc2/rnn_vad/vector_math.h new file mode 100644 index 0000000..47f6811 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/vector_math.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_VECTOR_MATH_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_VECTOR_MATH_H_ + +// Defines WEBRTC_ARCH_X86_FAMILY, used below. +#include "rtc_base/system/arch.h" + +#if defined(WEBRTC_HAS_NEON) +#include +#endif +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include +#endif + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/agc2/cpu_features.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/system/arch.h" + +namespace webrtc { +namespace rnn_vad { + +// Provides optimizations for mathematical operations having vectors as +// operand(s). +class VectorMath { + public: + explicit VectorMath(AvailableCpuFeatures cpu_features) + : cpu_features_(cpu_features) {} + + // Computes the dot product between two equally sized vectors. + float DotProduct(rtc::ArrayView x, + rtc::ArrayView y) const { + RTC_DCHECK_EQ(x.size(), y.size()); +#if defined(WEBRTC_ARCH_X86_FAMILY) + if (cpu_features_.avx2) { + return DotProductAvx2(x, y); + } else if (cpu_features_.sse2) { + __m128 accumulator = _mm_setzero_ps(); + constexpr int kBlockSizeLog2 = 2; + constexpr int kBlockSize = 1 << kBlockSizeLog2; + const int incomplete_block_index = (x.size() >> kBlockSizeLog2) + << kBlockSizeLog2; + for (int i = 0; i < incomplete_block_index; i += kBlockSize) { + RTC_DCHECK_LE(i + kBlockSize, x.size()); + const __m128 x_i = _mm_loadu_ps(&x[i]); + const __m128 y_i = _mm_loadu_ps(&y[i]); + // Multiply-add. + const __m128 z_j = _mm_mul_ps(x_i, y_i); + accumulator = _mm_add_ps(accumulator, z_j); + } + // Reduce `accumulator` by addition. + __m128 high = _mm_movehl_ps(accumulator, accumulator); + accumulator = _mm_add_ps(accumulator, high); + high = _mm_shuffle_ps(accumulator, accumulator, 1); + accumulator = _mm_add_ps(accumulator, high); + float dot_product = _mm_cvtss_f32(accumulator); + // Add the result for the last block if incomplete. + for (int i = incomplete_block_index; + i < rtc::dchecked_cast(x.size()); ++i) { + dot_product += x[i] * y[i]; + } + return dot_product; + } +#elif defined(WEBRTC_HAS_NEON) && defined(WEBRTC_ARCH_ARM64) + if (cpu_features_.neon) { + float32x4_t accumulator = vdupq_n_f32(0.f); + constexpr int kBlockSizeLog2 = 2; + constexpr int kBlockSize = 1 << kBlockSizeLog2; + const int incomplete_block_index = (x.size() >> kBlockSizeLog2) + << kBlockSizeLog2; + for (int i = 0; i < incomplete_block_index; i += kBlockSize) { + RTC_DCHECK_LE(i + kBlockSize, x.size()); + const float32x4_t x_i = vld1q_f32(&x[i]); + const float32x4_t y_i = vld1q_f32(&y[i]); + accumulator = vfmaq_f32(accumulator, x_i, y_i); + } + // Reduce `accumulator` by addition. + const float32x2_t tmp = + vpadd_f32(vget_low_f32(accumulator), vget_high_f32(accumulator)); + float dot_product = vget_lane_f32(vpadd_f32(tmp, vrev64_f32(tmp)), 0); + // Add the result for the last block if incomplete. + for (int i = incomplete_block_index; + i < rtc::dchecked_cast(x.size()); ++i) { + dot_product += x[i] * y[i]; + } + return dot_product; + } +#endif + return std::inner_product(x.begin(), x.end(), y.begin(), 0.f); + } + + private: + float DotProductAvx2(rtc::ArrayView x, + rtc::ArrayView y) const; + + const AvailableCpuFeatures cpu_features_; +}; + +} // namespace rnn_vad +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_VECTOR_MATH_H_ diff --git a/webrtc/modules/audio_processing/agc2/rnn_vad/vector_math_avx2.cc b/webrtc/modules/audio_processing/agc2/rnn_vad/vector_math_avx2.cc new file mode 100644 index 0000000..a875e11 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/rnn_vad/vector_math_avx2.cc @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/agc2/rnn_vad/vector_math.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { +namespace rnn_vad { + +float VectorMath::DotProductAvx2(rtc::ArrayView x, + rtc::ArrayView y) const { + RTC_DCHECK(cpu_features_.avx2); + RTC_DCHECK_EQ(x.size(), y.size()); + __m256 accumulator = _mm256_setzero_ps(); + constexpr int kBlockSizeLog2 = 3; + constexpr int kBlockSize = 1 << kBlockSizeLog2; + const int incomplete_block_index = (x.size() >> kBlockSizeLog2) + << kBlockSizeLog2; + for (int i = 0; i < incomplete_block_index; i += kBlockSize) { + RTC_DCHECK_LE(i + kBlockSize, x.size()); + const __m256 x_i = _mm256_loadu_ps(&x[i]); + const __m256 y_i = _mm256_loadu_ps(&y[i]); + accumulator = _mm256_fmadd_ps(x_i, y_i, accumulator); + } + // Reduce `accumulator` by addition. + __m128 high = _mm256_extractf128_ps(accumulator, 1); + __m128 low = _mm256_extractf128_ps(accumulator, 0); + low = _mm_add_ps(high, low); + high = _mm_movehl_ps(high, low); + low = _mm_add_ps(high, low); + high = _mm_shuffle_ps(low, low, 1); + low = _mm_add_ss(high, low); + float dot_product = _mm_cvtss_f32(low); + // Add the result for the last block if incomplete. + for (int i = incomplete_block_index; i < rtc::dchecked_cast(x.size()); + ++i) { + dot_product += x[i] * y[i]; + } + return dot_product; +} + +} // namespace rnn_vad +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/saturation_protector.cc b/webrtc/modules/audio_processing/agc2/saturation_protector.cc index b64fcdb..961baf4 100644 --- a/webrtc/modules/audio_processing/agc2/saturation_protector.cc +++ b/webrtc/modules/audio_processing/agc2/saturation_protector.cc @@ -10,84 +10,59 @@ #include "modules/audio_processing/agc2/saturation_protector.h" +#include + +#include "modules/audio_processing/agc2/agc2_common.h" +#include "modules/audio_processing/agc2/saturation_protector_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" #include "rtc_base/numerics/safe_minmax.h" namespace webrtc { namespace { -constexpr float kMinLevelDbfs = -90.f; +constexpr int kPeakEnveloperSuperFrameLengthMs = 400; +constexpr float kMinMarginDb = 12.0f; +constexpr float kMaxMarginDb = 25.0f; +constexpr float kAttack = 0.9988493699365052f; +constexpr float kDecay = 0.9997697679981565f; -// Min/max margins are based on speech crest-factor. -constexpr float kMinMarginDb = 12.f; -constexpr float kMaxMarginDb = 25.f; - -using saturation_protector_impl::RingBuffer; - -} // namespace - -bool RingBuffer::operator==(const RingBuffer& b) const { - RTC_DCHECK_LE(size_, buffer_.size()); - RTC_DCHECK_LE(b.size_, b.buffer_.size()); - if (size_ != b.size_) { - return false; +// Saturation protector state. Defined outside of `SaturationProtectorImpl` to +// implement check-point and restore ops. +struct SaturationProtectorState { + bool operator==(const SaturationProtectorState& s) const { + return headroom_db == s.headroom_db && + peak_delay_buffer == s.peak_delay_buffer && + max_peaks_dbfs == s.max_peaks_dbfs && + time_since_push_ms == s.time_since_push_ms; } - for (int i = 0, i0 = FrontIndex(), i1 = b.FrontIndex(); i < size_; - ++i, ++i0, ++i1) { - if (buffer_[i0 % buffer_.size()] != b.buffer_[i1 % b.buffer_.size()]) { - return false; - } + inline bool operator!=(const SaturationProtectorState& s) const { + return !(*this == s); } - return true; -} -void RingBuffer::Reset() { - next_ = 0; - size_ = 0; -} + float headroom_db; + SaturationProtectorBuffer peak_delay_buffer; + float max_peaks_dbfs; + int time_since_push_ms; // Time since the last ring buffer push operation. +}; -void RingBuffer::PushBack(float v) { - RTC_DCHECK_GE(next_, 0); - RTC_DCHECK_GE(size_, 0); - RTC_DCHECK_LT(next_, buffer_.size()); - RTC_DCHECK_LE(size_, buffer_.size()); - buffer_[next_++] = v; - if (rtc::SafeEq(next_, buffer_.size())) { - next_ = 0; - } - if (rtc::SafeLt(size_, buffer_.size())) { - size_++; - } -} - -absl::optional RingBuffer::Front() const { - if (size_ == 0) { - return absl::nullopt; - } - RTC_DCHECK_LT(FrontIndex(), buffer_.size()); - return buffer_[FrontIndex()]; -} - -bool SaturationProtectorState::operator==( - const SaturationProtectorState& b) const { - return margin_db == b.margin_db && peak_delay_buffer == b.peak_delay_buffer && - max_peaks_dbfs == b.max_peaks_dbfs && - time_since_push_ms == b.time_since_push_ms; -} - -void ResetSaturationProtectorState(float initial_margin_db, +// Resets the saturation protector state. +void ResetSaturationProtectorState(float initial_headroom_db, SaturationProtectorState& state) { - state.margin_db = initial_margin_db; + state.headroom_db = initial_headroom_db; state.peak_delay_buffer.Reset(); state.max_peaks_dbfs = kMinLevelDbfs; state.time_since_push_ms = 0; } -void UpdateSaturationProtectorState(float speech_peak_dbfs, +// Updates `state` by analyzing the estimated speech level `speech_level_dbfs` +// and the peak level `peak_dbfs` for an observed frame. `state` must not be +// modified without calling this function. +void UpdateSaturationProtectorState(float peak_dbfs, float speech_level_dbfs, SaturationProtectorState& state) { // Get the max peak over `kPeakEnveloperSuperFrameLengthMs` ms. - state.max_peaks_dbfs = std::max(state.max_peaks_dbfs, speech_peak_dbfs); + state.max_peaks_dbfs = std::max(state.max_peaks_dbfs, peak_dbfs); state.time_since_push_ms += kFrameDurationMs; if (rtc::SafeGt(state.time_since_push_ms, kPeakEnveloperSuperFrameLengthMs)) { // Push `max_peaks_dbfs` back into the ring buffer. @@ -97,25 +72,112 @@ void UpdateSaturationProtectorState(float speech_peak_dbfs, state.time_since_push_ms = 0; } - // Update margin by comparing the estimated speech level and the delayed max - // speech peak power. - // TODO(alessiob): Check with aleloi@ why we use a delay and how to tune it. + // Update the headroom by comparing the estimated speech level and the delayed + // max speech peak. const float delayed_peak_dbfs = state.peak_delay_buffer.Front().value_or(state.max_peaks_dbfs); const float difference_db = delayed_peak_dbfs - speech_level_dbfs; - if (difference_db > state.margin_db) { + if (difference_db > state.headroom_db) { // Attack. - state.margin_db = - state.margin_db * kSaturationProtectorAttackConstant + - difference_db * (1.f - kSaturationProtectorAttackConstant); + state.headroom_db = + state.headroom_db * kAttack + difference_db * (1.0f - kAttack); } else { // Decay. - state.margin_db = state.margin_db * kSaturationProtectorDecayConstant + - difference_db * (1.f - kSaturationProtectorDecayConstant); + state.headroom_db = + state.headroom_db * kDecay + difference_db * (1.0f - kDecay); } - state.margin_db = - rtc::SafeClamp(state.margin_db, kMinMarginDb, kMaxMarginDb); + state.headroom_db = + rtc::SafeClamp(state.headroom_db, kMinMarginDb, kMaxMarginDb); +} + +// Saturation protector which recommends a headroom based on the recent peaks. +class SaturationProtectorImpl : public SaturationProtector { + public: + explicit SaturationProtectorImpl(float initial_headroom_db, + int adjacent_speech_frames_threshold, + ApmDataDumper* apm_data_dumper) + : apm_data_dumper_(apm_data_dumper), + initial_headroom_db_(initial_headroom_db), + adjacent_speech_frames_threshold_(adjacent_speech_frames_threshold) { + Reset(); + } + SaturationProtectorImpl(const SaturationProtectorImpl&) = delete; + SaturationProtectorImpl& operator=(const SaturationProtectorImpl&) = delete; + ~SaturationProtectorImpl() = default; + + float HeadroomDb() override { return headroom_db_; } + + void Analyze(float speech_probability, + float peak_dbfs, + float speech_level_dbfs) override { + if (speech_probability < kVadConfidenceThreshold) { + // Not a speech frame. + if (adjacent_speech_frames_threshold_ > 1) { + // When two or more adjacent speech frames are required in order to + // update the state, we need to decide whether to discard or confirm the + // updates based on the speech sequence length. + if (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_) { + // First non-speech frame after a long enough sequence of speech + // frames. Update the reliable state. + reliable_state_ = preliminary_state_; + } else if (num_adjacent_speech_frames_ > 0) { + // First non-speech frame after a too short sequence of speech frames. + // Reset to the last reliable state. + preliminary_state_ = reliable_state_; + } + } + num_adjacent_speech_frames_ = 0; + } else { + // Speech frame observed. + num_adjacent_speech_frames_++; + + // Update preliminary level estimate. + UpdateSaturationProtectorState(peak_dbfs, speech_level_dbfs, + preliminary_state_); + + if (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_) { + // `preliminary_state_` is now reliable. Update the headroom. + headroom_db_ = preliminary_state_.headroom_db; + } + } + DumpDebugData(); + } + + void Reset() override { + num_adjacent_speech_frames_ = 0; + headroom_db_ = initial_headroom_db_; + ResetSaturationProtectorState(initial_headroom_db_, preliminary_state_); + ResetSaturationProtectorState(initial_headroom_db_, reliable_state_); + } + + private: + void DumpDebugData() { + apm_data_dumper_->DumpRaw( + "agc2_saturation_protector_preliminary_max_peak_dbfs", + preliminary_state_.max_peaks_dbfs); + apm_data_dumper_->DumpRaw( + "agc2_saturation_protector_reliable_max_peak_dbfs", + reliable_state_.max_peaks_dbfs); + } + + ApmDataDumper* const apm_data_dumper_; + const float initial_headroom_db_; + const int adjacent_speech_frames_threshold_; + int num_adjacent_speech_frames_; + float headroom_db_; + SaturationProtectorState preliminary_state_; + SaturationProtectorState reliable_state_; +}; + +} // namespace + +std::unique_ptr CreateSaturationProtector( + float initial_headroom_db, + int adjacent_speech_frames_threshold, + ApmDataDumper* apm_data_dumper) { + return std::make_unique( + initial_headroom_db, adjacent_speech_frames_threshold, apm_data_dumper); } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/saturation_protector.h b/webrtc/modules/audio_processing/agc2/saturation_protector.h index 88be91a..ef22145 100644 --- a/webrtc/modules/audio_processing/agc2/saturation_protector.h +++ b/webrtc/modules/audio_processing/agc2/saturation_protector.h @@ -11,71 +11,35 @@ #ifndef MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_H_ #define MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_H_ -#include - -#include "absl/types/optional.h" -#include "modules/audio_processing/agc2/agc2_common.h" -#include "rtc_base/numerics/safe_compare.h" +#include namespace webrtc { -namespace saturation_protector_impl { +class ApmDataDumper; -// Ring buffer which only supports (i) push back and (ii) read oldest item. -class RingBuffer { +// Saturation protector. Analyzes peak levels and recommends a headroom to +// reduce the chances of clipping. +class SaturationProtector { public: - bool operator==(const RingBuffer& b) const; - inline bool operator!=(const RingBuffer& b) const { return !(*this == b); } + virtual ~SaturationProtector() = default; - // Maximum number of values that the buffer can contain. - int Capacity() const { return buffer_.size(); } - // Number of values in the buffer. - int Size() const { return size_; } + // Returns the recommended headroom in dB. + virtual float HeadroomDb() = 0; - void Reset(); - // Pushes back `v`. If the buffer is full, the oldest value is replaced. - void PushBack(float v); - // Returns the oldest item in the buffer. Returns an empty value if the - // buffer is empty. - absl::optional Front() const; + // Analyzes the peak level of a 10 ms frame along with its speech probability + // and the current speech level estimate to update the recommended headroom. + virtual void Analyze(float speech_probability, + float peak_dbfs, + float speech_level_dbfs) = 0; - private: - inline int FrontIndex() const { - return rtc::SafeEq(size_, buffer_.size()) ? next_ : 0; - } - // `buffer_` has `size_` elements (up to the size of `buffer_`) and `next_` is - // the position where the next new value is written in `buffer_`. - std::array buffer_; - int next_ = 0; - int size_ = 0; + // Resets the internal state. + virtual void Reset() = 0; }; -} // namespace saturation_protector_impl - -// Saturation protector state. Exposed publicly for check-pointing and restore -// ops. -struct SaturationProtectorState { - bool operator==(const SaturationProtectorState& s) const; - inline bool operator!=(const SaturationProtectorState& s) const { - return !(*this == s); - } - - float margin_db; // Recommended margin. - saturation_protector_impl::RingBuffer peak_delay_buffer; - float max_peaks_dbfs; - int time_since_push_ms; // Time since the last ring buffer push operation. -}; - -// Resets the saturation protector state. -void ResetSaturationProtectorState(float initial_margin_db, - SaturationProtectorState& state); - -// Updates `state` by analyzing the estimated speech level `speech_level_dbfs` -// and the peak power `speech_peak_dbfs` for an observed frame which is -// reliably classified as "speech". `state` must not be modified without calling -// this function. -void UpdateSaturationProtectorState(float speech_peak_dbfs, - float speech_level_dbfs, - SaturationProtectorState& state); +// Creates a saturation protector that starts at `initial_headroom_db`. +std::unique_ptr CreateSaturationProtector( + float initial_headroom_db, + int adjacent_speech_frames_threshold, + ApmDataDumper* apm_data_dumper); } // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/saturation_protector_buffer.cc b/webrtc/modules/audio_processing/agc2/saturation_protector_buffer.cc new file mode 100644 index 0000000..41efdad --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/saturation_protector_buffer.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/saturation_protector_buffer.h" + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" + +namespace webrtc { + +SaturationProtectorBuffer::SaturationProtectorBuffer() = default; + +SaturationProtectorBuffer::~SaturationProtectorBuffer() = default; + +bool SaturationProtectorBuffer::operator==( + const SaturationProtectorBuffer& b) const { + RTC_DCHECK_LE(size_, buffer_.size()); + RTC_DCHECK_LE(b.size_, b.buffer_.size()); + if (size_ != b.size_) { + return false; + } + for (int i = 0, i0 = FrontIndex(), i1 = b.FrontIndex(); i < size_; + ++i, ++i0, ++i1) { + if (buffer_[i0 % buffer_.size()] != b.buffer_[i1 % b.buffer_.size()]) { + return false; + } + } + return true; +} + +int SaturationProtectorBuffer::Capacity() const { + return buffer_.size(); +} + +int SaturationProtectorBuffer::Size() const { + return size_; +} + +void SaturationProtectorBuffer::Reset() { + next_ = 0; + size_ = 0; +} + +void SaturationProtectorBuffer::PushBack(float v) { + RTC_DCHECK_GE(next_, 0); + RTC_DCHECK_GE(size_, 0); + RTC_DCHECK_LT(next_, buffer_.size()); + RTC_DCHECK_LE(size_, buffer_.size()); + buffer_[next_++] = v; + if (rtc::SafeEq(next_, buffer_.size())) { + next_ = 0; + } + if (rtc::SafeLt(size_, buffer_.size())) { + size_++; + } +} + +absl::optional SaturationProtectorBuffer::Front() const { + if (size_ == 0) { + return absl::nullopt; + } + RTC_DCHECK_LT(FrontIndex(), buffer_.size()); + return buffer_[FrontIndex()]; +} + +int SaturationProtectorBuffer::FrontIndex() const { + return rtc::SafeEq(size_, buffer_.size()) ? next_ : 0; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/saturation_protector_buffer.h b/webrtc/modules/audio_processing/agc2/saturation_protector_buffer.h new file mode 100644 index 0000000..e17d099 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/saturation_protector_buffer.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_BUFFER_H_ + +#include + +#include "absl/types/optional.h" +#include "modules/audio_processing/agc2/agc2_common.h" + +namespace webrtc { + +// Ring buffer for the saturation protector which only supports (i) push back +// and (ii) read oldest item. +class SaturationProtectorBuffer { + public: + SaturationProtectorBuffer(); + ~SaturationProtectorBuffer(); + + bool operator==(const SaturationProtectorBuffer& b) const; + inline bool operator!=(const SaturationProtectorBuffer& b) const { + return !(*this == b); + } + + // Maximum number of values that the buffer can contain. + int Capacity() const; + + // Number of values in the buffer. + int Size() const; + + void Reset(); + + // Pushes back `v`. If the buffer is full, the oldest value is replaced. + void PushBack(float v); + + // Returns the oldest item in the buffer. Returns an empty value if the + // buffer is empty. + absl::optional Front() const; + + private: + int FrontIndex() const; + // `buffer_` has `size_` elements (up to the size of `buffer_`) and `next_` is + // the position where the next new value is written in `buffer_`. + std::array buffer_; + int next_ = 0; + int size_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/agc2/signal_classifier.cc b/webrtc/modules/audio_processing/agc2/signal_classifier.cc deleted file mode 100644 index a06413d..0000000 --- a/webrtc/modules/audio_processing/agc2/signal_classifier.cc +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/signal_classifier.h" - -#include -#include -#include - -#include "api/array_view.h" -#include "modules/audio_processing/agc2/down_sampler.h" -#include "modules/audio_processing/agc2/noise_spectrum_estimator.h" -#include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/checks.h" -#include "system_wrappers/include/cpu_features_wrapper.h" - -namespace webrtc { -namespace { - -bool IsSse2Available() { -#if defined(WEBRTC_ARCH_X86_FAMILY) - return GetCPUInfo(kSSE2) != 0; -#else - return false; -#endif -} - -void RemoveDcLevel(rtc::ArrayView x) { - RTC_DCHECK_LT(0, x.size()); - float mean = std::accumulate(x.data(), x.data() + x.size(), 0.f); - mean /= x.size(); - - for (float& v : x) { - v -= mean; - } -} - -void PowerSpectrum(const OouraFft* ooura_fft, - rtc::ArrayView x, - rtc::ArrayView spectrum) { - RTC_DCHECK_EQ(65, spectrum.size()); - RTC_DCHECK_EQ(128, x.size()); - float X[128]; - std::copy(x.data(), x.data() + x.size(), X); - ooura_fft->Fft(X); - - float* X_p = X; - RTC_DCHECK_EQ(X_p, &X[0]); - spectrum[0] = (*X_p) * (*X_p); - ++X_p; - RTC_DCHECK_EQ(X_p, &X[1]); - spectrum[64] = (*X_p) * (*X_p); - for (int k = 1; k < 64; ++k) { - ++X_p; - RTC_DCHECK_EQ(X_p, &X[2 * k]); - spectrum[k] = (*X_p) * (*X_p); - ++X_p; - RTC_DCHECK_EQ(X_p, &X[2 * k + 1]); - spectrum[k] += (*X_p) * (*X_p); - } -} - -webrtc::SignalClassifier::SignalType ClassifySignal( - rtc::ArrayView signal_spectrum, - rtc::ArrayView noise_spectrum, - ApmDataDumper* data_dumper) { - int num_stationary_bands = 0; - int num_highly_nonstationary_bands = 0; - - // Detect stationary and highly nonstationary bands. - for (size_t k = 1; k < 40; k++) { - if (signal_spectrum[k] < 3 * noise_spectrum[k] && - signal_spectrum[k] * 3 > noise_spectrum[k]) { - ++num_stationary_bands; - } else if (signal_spectrum[k] > 9 * noise_spectrum[k]) { - ++num_highly_nonstationary_bands; - } - } - - data_dumper->DumpRaw("lc_num_stationary_bands", 1, &num_stationary_bands); - data_dumper->DumpRaw("lc_num_highly_nonstationary_bands", 1, - &num_highly_nonstationary_bands); - - // Use the detected number of bands to classify the overall signal - // stationarity. - if (num_stationary_bands > 15) { - return SignalClassifier::SignalType::kStationary; - } else { - return SignalClassifier::SignalType::kNonStationary; - } -} - -} // namespace - -SignalClassifier::FrameExtender::FrameExtender(size_t frame_size, - size_t extended_frame_size) - : x_old_(extended_frame_size - frame_size, 0.f) {} - -SignalClassifier::FrameExtender::~FrameExtender() = default; - -void SignalClassifier::FrameExtender::ExtendFrame( - rtc::ArrayView x, - rtc::ArrayView x_extended) { - RTC_DCHECK_EQ(x_old_.size() + x.size(), x_extended.size()); - std::copy(x_old_.data(), x_old_.data() + x_old_.size(), x_extended.data()); - std::copy(x.data(), x.data() + x.size(), x_extended.data() + x_old_.size()); - std::copy(x_extended.data() + x_extended.size() - x_old_.size(), - x_extended.data() + x_extended.size(), x_old_.data()); -} - -SignalClassifier::SignalClassifier(ApmDataDumper* data_dumper) - : data_dumper_(data_dumper), - down_sampler_(data_dumper_), - noise_spectrum_estimator_(data_dumper_), - ooura_fft_(IsSse2Available()) { - Initialize(48000); -} -SignalClassifier::~SignalClassifier() {} - -void SignalClassifier::Initialize(int sample_rate_hz) { - down_sampler_.Initialize(sample_rate_hz); - noise_spectrum_estimator_.Initialize(); - frame_extender_.reset(new FrameExtender(80, 128)); - sample_rate_hz_ = sample_rate_hz; - initialization_frames_left_ = 2; - consistent_classification_counter_ = 3; - last_signal_type_ = SignalClassifier::SignalType::kNonStationary; -} - -SignalClassifier::SignalType SignalClassifier::Analyze( - rtc::ArrayView signal) { - RTC_DCHECK_EQ(signal.size(), sample_rate_hz_ / 100); - - // Compute the signal power spectrum. - float downsampled_frame[80]; - down_sampler_.DownSample(signal, downsampled_frame); - float extended_frame[128]; - frame_extender_->ExtendFrame(downsampled_frame, extended_frame); - RemoveDcLevel(extended_frame); - float signal_spectrum[65]; - PowerSpectrum(&ooura_fft_, extended_frame, signal_spectrum); - - // Classify the signal based on the estimate of the noise spectrum and the - // signal spectrum estimate. - const SignalType signal_type = ClassifySignal( - signal_spectrum, noise_spectrum_estimator_.GetNoiseSpectrum(), - data_dumper_); - - // Update the noise spectrum based on the signal spectrum. - noise_spectrum_estimator_.Update(signal_spectrum, - initialization_frames_left_ > 0); - - // Update the number of frames until a reliable signal spectrum is achieved. - initialization_frames_left_ = std::max(0, initialization_frames_left_ - 1); - - if (last_signal_type_ == signal_type) { - consistent_classification_counter_ = - std::max(0, consistent_classification_counter_ - 1); - } else { - last_signal_type_ = signal_type; - consistent_classification_counter_ = 3; - } - - if (consistent_classification_counter_ > 0) { - return SignalClassifier::SignalType::kNonStationary; - } - return signal_type; -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/signal_classifier.h b/webrtc/modules/audio_processing/agc2/signal_classifier.h deleted file mode 100644 index 20cce92..0000000 --- a/webrtc/modules/audio_processing/agc2/signal_classifier.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_SIGNAL_CLASSIFIER_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_SIGNAL_CLASSIFIER_H_ - -#include -#include - -#include "api/array_view.h" -#include "common_audio/third_party/ooura/fft_size_128/ooura_fft.h" -#include "modules/audio_processing/agc2/down_sampler.h" -#include "modules/audio_processing/agc2/noise_spectrum_estimator.h" - -namespace webrtc { - -class ApmDataDumper; -class AudioBuffer; - -class SignalClassifier { - public: - enum class SignalType { kNonStationary, kStationary }; - - explicit SignalClassifier(ApmDataDumper* data_dumper); - - SignalClassifier() = delete; - SignalClassifier(const SignalClassifier&) = delete; - SignalClassifier& operator=(const SignalClassifier&) = delete; - - ~SignalClassifier(); - - void Initialize(int sample_rate_hz); - SignalType Analyze(rtc::ArrayView signal); - - private: - class FrameExtender { - public: - FrameExtender(size_t frame_size, size_t extended_frame_size); - - FrameExtender() = delete; - FrameExtender(const FrameExtender&) = delete; - FrameExtender& operator=(const FrameExtender&) = delete; - - ~FrameExtender(); - - void ExtendFrame(rtc::ArrayView x, - rtc::ArrayView x_extended); - - private: - std::vector x_old_; - }; - - ApmDataDumper* const data_dumper_; - DownSampler down_sampler_; - std::unique_ptr frame_extender_; - NoiseSpectrumEstimator noise_spectrum_estimator_; - int sample_rate_hz_; - int initialization_frames_left_; - int consistent_classification_counter_; - SignalType last_signal_type_; - const OouraFft ooura_fft_; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_SIGNAL_CLASSIFIER_H_ diff --git a/webrtc/modules/audio_processing/agc2/speech_level_estimator.cc b/webrtc/modules/audio_processing/agc2/speech_level_estimator.cc new file mode 100644 index 0000000..7bf3252 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/speech_level_estimator.cc @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/speech_level_estimator.h" + +#include "modules/audio_processing/agc2/agc2_common.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { +namespace { + +float ClampLevelEstimateDbfs(float level_estimate_dbfs) { + return rtc::SafeClamp(level_estimate_dbfs, -90.0f, 30.0f); +} + +// Returns the initial speech level estimate needed to apply the initial gain. +float GetInitialSpeechLevelEstimateDbfs( + const AudioProcessing::Config::GainController2::AdaptiveDigital& config) { + return ClampLevelEstimateDbfs(-kSaturationProtectorInitialHeadroomDb - + config.initial_gain_db - config.headroom_db); +} + +} // namespace + +bool SpeechLevelEstimator::LevelEstimatorState::operator==( + const SpeechLevelEstimator::LevelEstimatorState& b) const { + return time_to_confidence_ms == b.time_to_confidence_ms && + level_dbfs.numerator == b.level_dbfs.numerator && + level_dbfs.denominator == b.level_dbfs.denominator; +} + +float SpeechLevelEstimator::LevelEstimatorState::Ratio::GetRatio() const { + RTC_DCHECK_NE(denominator, 0.f); + return numerator / denominator; +} + +SpeechLevelEstimator::SpeechLevelEstimator( + ApmDataDumper* apm_data_dumper, + const AudioProcessing::Config::GainController2::AdaptiveDigital& config, + int adjacent_speech_frames_threshold) + : apm_data_dumper_(apm_data_dumper), + initial_speech_level_dbfs_(GetInitialSpeechLevelEstimateDbfs(config)), + adjacent_speech_frames_threshold_(adjacent_speech_frames_threshold), + level_dbfs_(initial_speech_level_dbfs_), + // TODO(bugs.webrtc.org/7494): Remove init below when AGC2 input volume + // controller temporal dependency removed. + is_confident_(false) { + RTC_DCHECK(apm_data_dumper_); + RTC_DCHECK_GE(adjacent_speech_frames_threshold_, 1); + Reset(); +} + +void SpeechLevelEstimator::Update(float rms_dbfs, + float peak_dbfs, + float speech_probability) { + RTC_DCHECK_GT(rms_dbfs, -150.0f); + RTC_DCHECK_LT(rms_dbfs, 50.0f); + RTC_DCHECK_GT(peak_dbfs, -150.0f); + RTC_DCHECK_LT(peak_dbfs, 50.0f); + RTC_DCHECK_GE(speech_probability, 0.0f); + RTC_DCHECK_LE(speech_probability, 1.0f); + if (speech_probability < kVadConfidenceThreshold) { + // Not a speech frame. + if (adjacent_speech_frames_threshold_ > 1) { + // When two or more adjacent speech frames are required in order to update + // the state, we need to decide whether to discard or confirm the updates + // based on the speech sequence length. + if (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_) { + // First non-speech frame after a long enough sequence of speech frames. + // Update the reliable state. + reliable_state_ = preliminary_state_; + } else if (num_adjacent_speech_frames_ > 0) { + // First non-speech frame after a too short sequence of speech frames. + // Reset to the last reliable state. + preliminary_state_ = reliable_state_; + } + } + num_adjacent_speech_frames_ = 0; + } else { + // Speech frame observed. + num_adjacent_speech_frames_++; + + // Update preliminary level estimate. + RTC_DCHECK_GE(preliminary_state_.time_to_confidence_ms, 0); + const bool buffer_is_full = preliminary_state_.time_to_confidence_ms == 0; + if (!buffer_is_full) { + preliminary_state_.time_to_confidence_ms -= kFrameDurationMs; + } + // Weighted average of levels with speech probability as weight. + RTC_DCHECK_GT(speech_probability, 0.0f); + const float leak_factor = buffer_is_full ? kLevelEstimatorLeakFactor : 1.0f; + preliminary_state_.level_dbfs.numerator = + preliminary_state_.level_dbfs.numerator * leak_factor + + rms_dbfs * speech_probability; + preliminary_state_.level_dbfs.denominator = + preliminary_state_.level_dbfs.denominator * leak_factor + + speech_probability; + + const float level_dbfs = preliminary_state_.level_dbfs.GetRatio(); + + if (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_) { + // `preliminary_state_` is now reliable. Update the last level estimation. + level_dbfs_ = ClampLevelEstimateDbfs(level_dbfs); + } + } + UpdateIsConfident(); + DumpDebugData(); +} + +void SpeechLevelEstimator::UpdateIsConfident() { + if (adjacent_speech_frames_threshold_ == 1) { + // Ignore `reliable_state_` when a single frame is enough to update the + // level estimate (because it is not used). + is_confident_ = preliminary_state_.time_to_confidence_ms == 0; + return; + } + // Once confident, it remains confident. + RTC_DCHECK(reliable_state_.time_to_confidence_ms != 0 || + preliminary_state_.time_to_confidence_ms == 0); + // During the first long enough speech sequence, `reliable_state_` must be + // ignored since `preliminary_state_` is used. + is_confident_ = + reliable_state_.time_to_confidence_ms == 0 || + (num_adjacent_speech_frames_ >= adjacent_speech_frames_threshold_ && + preliminary_state_.time_to_confidence_ms == 0); +} + +void SpeechLevelEstimator::Reset() { + ResetLevelEstimatorState(preliminary_state_); + ResetLevelEstimatorState(reliable_state_); + level_dbfs_ = initial_speech_level_dbfs_; + num_adjacent_speech_frames_ = 0; +} + +void SpeechLevelEstimator::ResetLevelEstimatorState( + LevelEstimatorState& state) const { + state.time_to_confidence_ms = kLevelEstimatorTimeToConfidenceMs; + state.level_dbfs.numerator = initial_speech_level_dbfs_; + state.level_dbfs.denominator = 1.0f; +} + +void SpeechLevelEstimator::DumpDebugData() const { + if (!apm_data_dumper_) + return; + apm_data_dumper_->DumpRaw("agc2_speech_level_dbfs", level_dbfs_); + apm_data_dumper_->DumpRaw("agc2_speech_level_is_confident", is_confident_); + apm_data_dumper_->DumpRaw( + "agc2_adaptive_level_estimator_num_adjacent_speech_frames", + num_adjacent_speech_frames_); + apm_data_dumper_->DumpRaw( + "agc2_adaptive_level_estimator_preliminary_level_estimate_num", + preliminary_state_.level_dbfs.numerator); + apm_data_dumper_->DumpRaw( + "agc2_adaptive_level_estimator_preliminary_level_estimate_den", + preliminary_state_.level_dbfs.denominator); + apm_data_dumper_->DumpRaw( + "agc2_adaptive_level_estimator_preliminary_time_to_confidence_ms", + preliminary_state_.time_to_confidence_ms); + apm_data_dumper_->DumpRaw( + "agc2_adaptive_level_estimator_reliable_time_to_confidence_ms", + reliable_state_.time_to_confidence_ms); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.h b/webrtc/modules/audio_processing/agc2/speech_level_estimator.h similarity index 55% rename from webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.h rename to webrtc/modules/audio_processing/agc2/speech_level_estimator.h index 213fc0f..4d9f106 100644 --- a/webrtc/modules/audio_processing/agc2/adaptive_mode_level_estimator.h +++ b/webrtc/modules/audio_processing/agc2/speech_level_estimator.h @@ -8,40 +8,37 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_H_ +#ifndef MODULES_AUDIO_PROCESSING_AGC2_SPEECH_LEVEL_ESTIMATOR_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_SPEECH_LEVEL_ESTIMATOR_H_ #include + #include #include "modules/audio_processing/agc2/agc2_common.h" -#include "modules/audio_processing/agc2/saturation_protector.h" -#include "modules/audio_processing/agc2/vad_with_level.h" #include "modules/audio_processing/include/audio_processing.h" namespace webrtc { class ApmDataDumper; -// Level estimator for the digital adaptive gain controller. -class AdaptiveModeLevelEstimator { +// Active speech level estimator based on the analysis of the following +// framewise properties: RMS level (dBFS), peak level (dBFS), speech +// probability. +class SpeechLevelEstimator { public: - explicit AdaptiveModeLevelEstimator(ApmDataDumper* apm_data_dumper); - AdaptiveModeLevelEstimator(const AdaptiveModeLevelEstimator&) = delete; - AdaptiveModeLevelEstimator& operator=(const AdaptiveModeLevelEstimator&) = - delete; - AdaptiveModeLevelEstimator( + SpeechLevelEstimator( ApmDataDumper* apm_data_dumper, - AudioProcessing::Config::GainController2::LevelEstimator level_estimator, - int adjacent_speech_frames_threshold, - float initial_saturation_margin_db, - float extra_saturation_margin_db); + const AudioProcessing::Config::GainController2::AdaptiveDigital& config, + int adjacent_speech_frames_threshold); + SpeechLevelEstimator(const SpeechLevelEstimator&) = delete; + SpeechLevelEstimator& operator=(const SpeechLevelEstimator&) = delete; // Updates the level estimation. - void Update(const VadLevelAnalyzer::Result& vad_data); + void Update(float rms_dbfs, float peak_dbfs, float speech_probability); // Returns the estimated speech plus noise level. float level_dbfs() const { return level_dbfs_; } // Returns true if the estimator is confident on its current estimate. - bool IsConfident() const; + bool is_confident() const { return is_confident_; } void Reset(); @@ -52,35 +49,33 @@ class AdaptiveModeLevelEstimator { inline bool operator!=(const LevelEstimatorState& s) const { return !(*this == s); } + // TODO(bugs.webrtc.org/7494): Remove `time_to_confidence_ms` if redundant. + int time_to_confidence_ms; struct Ratio { float numerator; float denominator; float GetRatio() const; - }; - // TODO(crbug.com/webrtc/7494): Remove time_to_full_buffer_ms if redundant. - int time_to_full_buffer_ms; - Ratio level_dbfs; - SaturationProtectorState saturation_protector; + } level_dbfs; }; static_assert(std::is_trivially_copyable::value, ""); + void UpdateIsConfident(); + void ResetLevelEstimatorState(LevelEstimatorState& state) const; void DumpDebugData() const; ApmDataDumper* const apm_data_dumper_; - const AudioProcessing::Config::GainController2::LevelEstimator - level_estimator_type_; + const float initial_speech_level_dbfs_; const int adjacent_speech_frames_threshold_; - const float initial_saturation_margin_db_; - const float extra_saturation_margin_db_; LevelEstimatorState preliminary_state_; LevelEstimatorState reliable_state_; float level_dbfs_; + bool is_confident_; int num_adjacent_speech_frames_; }; } // namespace webrtc -#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_H_ +#endif // MODULES_AUDIO_PROCESSING_AGC2_SPEECH_LEVEL_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/agc2/speech_probability_buffer.cc b/webrtc/modules/audio_processing/agc2/speech_probability_buffer.cc new file mode 100644 index 0000000..7746f6c --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/speech_probability_buffer.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/speech_probability_buffer.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +constexpr float kActivityThreshold = 0.9f; +constexpr int kNumAnalysisFrames = 100; +// We use 12 in AGC2 adaptive digital, but with a slightly different logic. +constexpr int kTransientWidthThreshold = 7; + +} // namespace + +SpeechProbabilityBuffer::SpeechProbabilityBuffer( + float low_probability_threshold) + : low_probability_threshold_(low_probability_threshold), + probabilities_(kNumAnalysisFrames) { + RTC_DCHECK_GE(low_probability_threshold, 0.0f); + RTC_DCHECK_LE(low_probability_threshold, 1.0f); + RTC_DCHECK(!probabilities_.empty()); +} + +void SpeechProbabilityBuffer::Update(float probability) { + // Remove the oldest entry if the circular buffer is full. + if (buffer_is_full_) { + const float oldest_probability = probabilities_[buffer_index_]; + sum_probabilities_ -= oldest_probability; + } + + // Check for transients. + if (probability <= low_probability_threshold_) { + // Set a probability lower than the threshold to zero. + probability = 0.0f; + + // Check if this has been a transient. + if (num_high_probability_observations_ <= kTransientWidthThreshold) { + RemoveTransient(); + } + num_high_probability_observations_ = 0; + } else if (num_high_probability_observations_ <= kTransientWidthThreshold) { + ++num_high_probability_observations_; + } + + // Update the circular buffer and the current sum. + probabilities_[buffer_index_] = probability; + sum_probabilities_ += probability; + + // Increment the buffer index and check for wrap-around. + if (++buffer_index_ >= kNumAnalysisFrames) { + buffer_index_ = 0; + buffer_is_full_ = true; + } +} + +void SpeechProbabilityBuffer::RemoveTransient() { + // Don't expect to be here if high-activity region is longer than + // `kTransientWidthThreshold` or there has not been any transient. + RTC_DCHECK_LE(num_high_probability_observations_, kTransientWidthThreshold); + + // Replace previously added probabilities with zero. + int index = + (buffer_index_ > 0) ? (buffer_index_ - 1) : (kNumAnalysisFrames - 1); + + while (num_high_probability_observations_-- > 0) { + sum_probabilities_ -= probabilities_[index]; + probabilities_[index] = 0.0f; + + // Update the circular buffer index. + index = (index > 0) ? (index - 1) : (kNumAnalysisFrames - 1); + } +} + +bool SpeechProbabilityBuffer::IsActiveSegment() const { + if (!buffer_is_full_) { + return false; + } + if (sum_probabilities_ < kActivityThreshold * kNumAnalysisFrames) { + return false; + } + return true; +} + +void SpeechProbabilityBuffer::Reset() { + sum_probabilities_ = 0.0f; + + // Empty the circular buffer. + buffer_index_ = 0; + buffer_is_full_ = false; + num_high_probability_observations_ = 0; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/speech_probability_buffer.h b/webrtc/modules/audio_processing/agc2/speech_probability_buffer.h new file mode 100644 index 0000000..3056a3e --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/speech_probability_buffer.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_SPEECH_PROBABILITY_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_SPEECH_PROBABILITY_BUFFER_H_ + +#include + +#include "rtc_base/gtest_prod_util.h" + +namespace webrtc { + +// This class implements a circular buffer that stores speech probabilities +// for a speech segment and estimates speech activity for that segment. +class SpeechProbabilityBuffer { + public: + // Ctor. The value of `low_probability_threshold` is required to be on the + // range [0.0f, 1.0f]. + explicit SpeechProbabilityBuffer(float low_probability_threshold); + ~SpeechProbabilityBuffer() {} + SpeechProbabilityBuffer(const SpeechProbabilityBuffer&) = delete; + SpeechProbabilityBuffer& operator=(const SpeechProbabilityBuffer&) = delete; + + // Adds `probability` in the buffer and computes an updatds sum of the buffer + // probabilities. Value of `probability` is required to be on the range + // [0.0f, 1.0f]. + void Update(float probability); + + // Resets the histogram, forgets the past. + void Reset(); + + // Returns true if the segment is active (a long enough segment with an + // average speech probability above `low_probability_threshold`). + bool IsActiveSegment() const; + + private: + void RemoveTransient(); + + // Use only for testing. + float GetSumProbabilities() const { return sum_probabilities_; } + + FRIEND_TEST_ALL_PREFIXES(SpeechProbabilityBufferTest, + CheckSumAfterInitialization); + FRIEND_TEST_ALL_PREFIXES(SpeechProbabilityBufferTest, CheckSumAfterUpdate); + FRIEND_TEST_ALL_PREFIXES(SpeechProbabilityBufferTest, CheckSumAfterReset); + FRIEND_TEST_ALL_PREFIXES(SpeechProbabilityBufferTest, + CheckSumAfterTransientNotRemoved); + FRIEND_TEST_ALL_PREFIXES(SpeechProbabilityBufferTest, + CheckSumAfterTransientRemoved); + + const float low_probability_threshold_; + + // Sum of probabilities stored in `probabilities_`. Must be updated if + // `probabilities_` is updated. + float sum_probabilities_ = 0.0f; + + // Circular buffer for probabilities. + std::vector probabilities_; + + // Current index of the circular buffer, where the newest data will be written + // to, therefore, pointing to the oldest data if buffer is full. + int buffer_index_ = 0; + + // Indicates if the buffer is full and adding a new value removes the oldest + // value. + int buffer_is_full_ = false; + + int num_high_probability_observations_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_SPEECH_PROBABILITY_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/agc2/vad_with_level.cc b/webrtc/modules/audio_processing/agc2/vad_with_level.cc deleted file mode 100644 index 3dbb557..0000000 --- a/webrtc/modules/audio_processing/agc2/vad_with_level.cc +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/agc2/vad_with_level.h" - -#include -#include -#include - -#include "api/array_view.h" -#include "common_audio/include/audio_util.h" -#include "common_audio/resampler/include/push_resampler.h" -#include "modules/audio_processing/agc2/agc2_common.h" -#include "modules/audio_processing/agc2/rnn_vad/common.h" -#include "modules/audio_processing/agc2/rnn_vad/features_extraction.h" -#include "modules/audio_processing/agc2/rnn_vad/rnn.h" -#include "rtc_base/checks.h" - -namespace webrtc { -namespace { - -using VoiceActivityDetector = VadLevelAnalyzer::VoiceActivityDetector; - -// Default VAD that combines a resampler and the RNN VAD. -// Computes the speech probability on the first channel. -class Vad : public VoiceActivityDetector { - public: - Vad() = default; - Vad(const Vad&) = delete; - Vad& operator=(const Vad&) = delete; - ~Vad() = default; - - float ComputeProbability(AudioFrameView frame) override { - // The source number of channels is 1, because we always use the 1st - // channel. - resampler_.InitializeIfNeeded( - /*sample_rate_hz=*/static_cast(frame.samples_per_channel() * 100), - rnn_vad::kSampleRate24kHz, - /*num_channels=*/1); - - std::array work_frame; - // Feed the 1st channel to the resampler. - resampler_.Resample(frame.channel(0).data(), frame.samples_per_channel(), - work_frame.data(), rnn_vad::kFrameSize10ms24kHz); - - std::array feature_vector; - const bool is_silence = features_extractor_.CheckSilenceComputeFeatures( - work_frame, feature_vector); - return rnn_vad_.ComputeVadProbability(feature_vector, is_silence); - } - - private: - PushResampler resampler_; - rnn_vad::FeaturesExtractor features_extractor_; - rnn_vad::RnnBasedVad rnn_vad_; -}; - -// Returns an updated version of `p_old` by using instant decay and the given -// `attack` on a new VAD probability value `p_new`. -float SmoothedVadProbability(float p_old, float p_new, float attack) { - RTC_DCHECK_GT(attack, 0.f); - RTC_DCHECK_LE(attack, 1.f); - if (p_new < p_old || attack == 1.f) { - // Instant decay (or no smoothing). - return p_new; - } else { - // Attack phase. - return attack * p_new + (1.f - attack) * p_old; - } -} - -} // namespace - -VadLevelAnalyzer::VadLevelAnalyzer() - : VadLevelAnalyzer(kDefaultSmoothedVadProbabilityAttack, - std::make_unique()) {} - -VadLevelAnalyzer::VadLevelAnalyzer(float vad_probability_attack) - : VadLevelAnalyzer(vad_probability_attack, std::make_unique()) {} - -VadLevelAnalyzer::VadLevelAnalyzer(float vad_probability_attack, - std::unique_ptr vad) - : vad_(std::move(vad)), vad_probability_attack_(vad_probability_attack) { - RTC_DCHECK(vad_); -} - -VadLevelAnalyzer::~VadLevelAnalyzer() = default; - -VadLevelAnalyzer::Result VadLevelAnalyzer::AnalyzeFrame( - AudioFrameView frame) { - // Compute levels. - float peak = 0.f; - float rms = 0.f; - for (const auto& x : frame.channel(0)) { - peak = std::max(std::fabs(x), peak); - rms += x * x; - } - // Compute smoothed speech probability. - vad_probability_ = SmoothedVadProbability( - /*p_old=*/vad_probability_, /*p_new=*/vad_->ComputeProbability(frame), - vad_probability_attack_); - return {vad_probability_, - FloatS16ToDbfs(std::sqrt(rms / frame.samples_per_channel())), - FloatS16ToDbfs(peak)}; -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/vad_with_level.h b/webrtc/modules/audio_processing/agc2/vad_with_level.h deleted file mode 100644 index ce72cdc..0000000 --- a/webrtc/modules/audio_processing/agc2/vad_with_level.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_AGC2_VAD_WITH_LEVEL_H_ -#define MODULES_AUDIO_PROCESSING_AGC2_VAD_WITH_LEVEL_H_ - -#include - -#include "modules/audio_processing/include/audio_frame_view.h" - -namespace webrtc { - -// Class to analyze voice activity and audio levels. -class VadLevelAnalyzer { - public: - struct Result { - float speech_probability; // Range: [0, 1]. - float rms_dbfs; // Root mean square power (dBFS). - float peak_dbfs; // Peak power (dBFS). - }; - - // Voice Activity Detector (VAD) interface. - class VoiceActivityDetector { - public: - virtual ~VoiceActivityDetector() = default; - // Analyzes an audio frame and returns the speech probability. - virtual float ComputeProbability(AudioFrameView frame) = 0; - }; - - // Ctor. Uses the default VAD. - VadLevelAnalyzer(); - explicit VadLevelAnalyzer(float vad_probability_attack); - // Ctor. Uses a custom `vad`. - VadLevelAnalyzer(float vad_probability_attack, - std::unique_ptr vad); - VadLevelAnalyzer(const VadLevelAnalyzer&) = delete; - VadLevelAnalyzer& operator=(const VadLevelAnalyzer&) = delete; - ~VadLevelAnalyzer(); - - // Computes the speech probability and the level for `frame`. - Result AnalyzeFrame(AudioFrameView frame); - - private: - std::unique_ptr vad_; - const float vad_probability_attack_; - float vad_probability_ = 0.f; -}; - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_AGC2_VAD_WITH_LEVEL_H_ diff --git a/webrtc/modules/audio_processing/agc2/vad_wrapper.cc b/webrtc/modules/audio_processing/agc2/vad_wrapper.cc new file mode 100644 index 0000000..af6325d --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/vad_wrapper.cc @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/agc2/vad_wrapper.h" + +#include +#include + +#include "api/array_view.h" +#include "common_audio/resampler/include/push_resampler.h" +#include "modules/audio_processing/agc2/agc2_common.h" +#include "modules/audio_processing/agc2/rnn_vad/common.h" +#include "modules/audio_processing/agc2/rnn_vad/features_extraction.h" +#include "modules/audio_processing/agc2/rnn_vad/rnn.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +constexpr int kNumFramesPerSecond = 100; + +class MonoVadImpl : public VoiceActivityDetectorWrapper::MonoVad { + public: + explicit MonoVadImpl(const AvailableCpuFeatures& cpu_features) + : features_extractor_(cpu_features), rnn_vad_(cpu_features) {} + MonoVadImpl(const MonoVadImpl&) = delete; + MonoVadImpl& operator=(const MonoVadImpl&) = delete; + ~MonoVadImpl() = default; + + int SampleRateHz() const override { return rnn_vad::kSampleRate24kHz; } + void Reset() override { rnn_vad_.Reset(); } + float Analyze(rtc::ArrayView frame) override { + RTC_DCHECK_EQ(frame.size(), rnn_vad::kFrameSize10ms24kHz); + std::array feature_vector; + const bool is_silence = features_extractor_.CheckSilenceComputeFeatures( + /*samples=*/{frame.data(), rnn_vad::kFrameSize10ms24kHz}, + feature_vector); + return rnn_vad_.ComputeVadProbability(feature_vector, is_silence); + } + + private: + rnn_vad::FeaturesExtractor features_extractor_; + rnn_vad::RnnVad rnn_vad_; +}; + +} // namespace + +VoiceActivityDetectorWrapper::VoiceActivityDetectorWrapper( + const AvailableCpuFeatures& cpu_features, + int sample_rate_hz) + : VoiceActivityDetectorWrapper(kVadResetPeriodMs, + cpu_features, + sample_rate_hz) {} + +VoiceActivityDetectorWrapper::VoiceActivityDetectorWrapper( + int vad_reset_period_ms, + const AvailableCpuFeatures& cpu_features, + int sample_rate_hz) + : VoiceActivityDetectorWrapper(vad_reset_period_ms, + std::make_unique(cpu_features), + sample_rate_hz) {} + +VoiceActivityDetectorWrapper::VoiceActivityDetectorWrapper( + int vad_reset_period_ms, + std::unique_ptr vad, + int sample_rate_hz) + : vad_reset_period_frames_( + rtc::CheckedDivExact(vad_reset_period_ms, kFrameDurationMs)), + time_to_vad_reset_(vad_reset_period_frames_), + vad_(std::move(vad)) { + RTC_DCHECK(vad_); + RTC_DCHECK_GT(vad_reset_period_frames_, 1); + resampled_buffer_.resize( + rtc::CheckedDivExact(vad_->SampleRateHz(), kNumFramesPerSecond)); + Initialize(sample_rate_hz); +} + +VoiceActivityDetectorWrapper::~VoiceActivityDetectorWrapper() = default; + +void VoiceActivityDetectorWrapper::Initialize(int sample_rate_hz) { + RTC_DCHECK_GT(sample_rate_hz, 0); + frame_size_ = rtc::CheckedDivExact(sample_rate_hz, kNumFramesPerSecond); + int status = + resampler_.InitializeIfNeeded(sample_rate_hz, vad_->SampleRateHz(), + /*num_channels=*/1); + constexpr int kStatusOk = 0; + RTC_DCHECK_EQ(status, kStatusOk); + vad_->Reset(); +} + +float VoiceActivityDetectorWrapper::Analyze(AudioFrameView frame) { + // Periodically reset the VAD. + time_to_vad_reset_--; + if (time_to_vad_reset_ <= 0) { + vad_->Reset(); + time_to_vad_reset_ = vad_reset_period_frames_; + } + // Resample the first channel of `frame`. + RTC_DCHECK_EQ(frame.samples_per_channel(), frame_size_); + resampler_.Resample(frame.channel(0).data(), frame_size_, + resampled_buffer_.data(), resampled_buffer_.size()); + + return vad_->Analyze(resampled_buffer_); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/agc2/vad_wrapper.h b/webrtc/modules/audio_processing/agc2/vad_wrapper.h new file mode 100644 index 0000000..459c471 --- /dev/null +++ b/webrtc/modules/audio_processing/agc2/vad_wrapper.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AGC2_VAD_WRAPPER_H_ +#define MODULES_AUDIO_PROCESSING_AGC2_VAD_WRAPPER_H_ + +#include +#include + +#include "api/array_view.h" +#include "common_audio/resampler/include/push_resampler.h" +#include "modules/audio_processing/agc2/cpu_features.h" +#include "modules/audio_processing/include/audio_frame_view.h" + +namespace webrtc { + +// Wraps a single-channel Voice Activity Detector (VAD) which is used to analyze +// the first channel of the input audio frames. Takes care of resampling the +// input frames to match the sample rate of the wrapped VAD and periodically +// resets the VAD. +class VoiceActivityDetectorWrapper { + public: + // Single channel VAD interface. + class MonoVad { + public: + virtual ~MonoVad() = default; + // Returns the sample rate (Hz) required for the input frames analyzed by + // `ComputeProbability`. + virtual int SampleRateHz() const = 0; + // Resets the internal state. + virtual void Reset() = 0; + // Analyzes an audio frame and returns the speech probability. + virtual float Analyze(rtc::ArrayView frame) = 0; + }; + + // Ctor. Uses `cpu_features` to instantiate the default VAD. + VoiceActivityDetectorWrapper(const AvailableCpuFeatures& cpu_features, + int sample_rate_hz); + + // Ctor. `vad_reset_period_ms` indicates the period in milliseconds to call + // `MonoVad::Reset()`; it must be equal to or greater than the duration of two + // frames. Uses `cpu_features` to instantiate the default VAD. + VoiceActivityDetectorWrapper(int vad_reset_period_ms, + const AvailableCpuFeatures& cpu_features, + int sample_rate_hz); + // Ctor. Uses a custom `vad`. + VoiceActivityDetectorWrapper(int vad_reset_period_ms, + std::unique_ptr vad, + int sample_rate_hz); + + VoiceActivityDetectorWrapper(const VoiceActivityDetectorWrapper&) = delete; + VoiceActivityDetectorWrapper& operator=(const VoiceActivityDetectorWrapper&) = + delete; + ~VoiceActivityDetectorWrapper(); + + // Initializes the VAD wrapper. + void Initialize(int sample_rate_hz); + + // Analyzes the first channel of `frame` and returns the speech probability. + // `frame` must be a 10 ms frame with the sample rate specified in the last + // `Initialize()` call. + float Analyze(AudioFrameView frame); + + private: + const int vad_reset_period_frames_; + int frame_size_; + int time_to_vad_reset_; + PushResampler resampler_; + std::unique_ptr vad_; + std::vector resampled_buffer_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AGC2_VAD_WRAPPER_H_ diff --git a/webrtc/modules/audio_processing/audio_buffer.cc b/webrtc/modules/audio_processing/audio_buffer.cc index ff6636d..3dbe1fe 100644 --- a/webrtc/modules/audio_processing/audio_buffer.cc +++ b/webrtc/modules/audio_processing/audio_buffer.cc @@ -45,22 +45,11 @@ AudioBuffer::AudioBuffer(size_t input_rate, size_t buffer_num_channels, size_t output_rate, size_t output_num_channels) - : AudioBuffer(static_cast(input_rate) / 100, - input_num_channels, - static_cast(buffer_rate) / 100, - buffer_num_channels, - static_cast(output_rate) / 100) {} - -AudioBuffer::AudioBuffer(size_t input_num_frames, - size_t input_num_channels, - size_t buffer_num_frames, - size_t buffer_num_channels, - size_t output_num_frames) - : input_num_frames_(input_num_frames), + : input_num_frames_(static_cast(input_rate) / 100), input_num_channels_(input_num_channels), - buffer_num_frames_(buffer_num_frames), + buffer_num_frames_(static_cast(buffer_rate) / 100), buffer_num_channels_(buffer_num_channels), - output_num_frames_(output_num_frames), + output_num_frames_(static_cast(output_rate) / 100), output_num_channels_(0), num_channels_(buffer_num_channels), num_bands_(NumBandsFromFramesPerChannel(buffer_num_frames_)), diff --git a/webrtc/modules/audio_processing/audio_buffer.h b/webrtc/modules/audio_processing/audio_buffer.h index 3eecf0d..b9ea300 100644 --- a/webrtc/modules/audio_processing/audio_buffer.h +++ b/webrtc/modules/audio_processing/audio_buffer.h @@ -32,7 +32,7 @@ enum Band { kBand0To8kHz = 0, kBand8To16kHz = 1, kBand16To24kHz = 2 }; class AudioBuffer { public: static const int kSplitBandSize = 160; - static const size_t kMaxSampleRate = 384000; + static const int kMaxSampleRate = 384000; AudioBuffer(size_t input_rate, size_t input_num_channels, size_t buffer_rate, @@ -40,12 +40,6 @@ class AudioBuffer { size_t output_rate, size_t output_num_channels); - // The constructor below will be deprecated. - AudioBuffer(size_t input_num_frames, - size_t input_num_channels, - size_t buffer_num_frames, - size_t buffer_num_channels, - size_t output_num_frames); virtual ~AudioBuffer(); AudioBuffer(const AudioBuffer&) = delete; @@ -71,8 +65,8 @@ class AudioBuffer { // Usage: // channels()[channel][sample]. // Where: - // 0 <= channel < |buffer_num_channels_| - // 0 <= sample < |buffer_num_frames_| + // 0 <= channel < `buffer_num_channels_` + // 0 <= sample < `buffer_num_frames_` float* const* channels() { return data_->channels(); } const float* const* channels_const() const { return data_->channels(); } @@ -80,9 +74,9 @@ class AudioBuffer { // Usage: // split_bands(channel)[band][sample]. // Where: - // 0 <= channel < |buffer_num_channels_| - // 0 <= band < |num_bands_| - // 0 <= sample < |num_split_frames_| + // 0 <= channel < `buffer_num_channels_` + // 0 <= band < `num_bands_` + // 0 <= sample < `num_split_frames_` const float* const* split_bands_const(size_t channel) const { return split_data_.get() ? split_data_->bands(channel) : data_->bands(channel); @@ -96,9 +90,9 @@ class AudioBuffer { // Usage: // split_channels(band)[channel][sample]. // Where: - // 0 <= band < |num_bands_| - // 0 <= channel < |buffer_num_channels_| - // 0 <= sample < |num_split_frames_| + // 0 <= band < `num_bands_` + // 0 <= channel < `buffer_num_channels_` + // 0 <= sample < `num_split_frames_` const float* const* split_channels_const(Band band) const { if (split_data_.get()) { return split_data_->channels(band); diff --git a/webrtc/modules/audio_processing/audio_processing_builder_impl.cc b/webrtc/modules/audio_processing/audio_processing_builder_impl.cc index f55c915..a246448 100644 --- a/webrtc/modules/audio_processing/audio_processing_builder_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_builder_impl.cc @@ -8,35 +8,24 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/audio_processing/include/audio_processing.h" - #include +#include "api/make_ref_counted.h" #include "modules/audio_processing/audio_processing_impl.h" -#include "rtc_base/ref_counted_object.h" +#include "modules/audio_processing/include/audio_processing.h" namespace webrtc { AudioProcessingBuilder::AudioProcessingBuilder() = default; AudioProcessingBuilder::~AudioProcessingBuilder() = default; -AudioProcessing* AudioProcessingBuilder::Create() { - webrtc::Config config; - return Create(config); -} - -AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) { +rtc::scoped_refptr AudioProcessingBuilder::Create() { #ifdef WEBRTC_EXCLUDE_AUDIO_PROCESSING_MODULE - - // Implementation returning a null pointer for using when the APM is excluded - // from the build.. + // Return a null pointer when the APM is excluded from the build. return nullptr; - -#else - - // Standard implementation. - return new rtc::RefCountedObject( - config, std::move(capture_post_processing_), +#else // WEBRTC_EXCLUDE_AUDIO_PROCESSING_MODULE + return rtc::make_ref_counted( + config_, std::move(capture_post_processing_), std::move(render_pre_processing_), std::move(echo_control_factory_), std::move(echo_detector_), std::move(capture_analyzer_)); #endif diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index 67208df..c80cc76 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -12,30 +12,30 @@ #include #include +#include #include #include #include #include +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/array_view.h" #include "api/audio/audio_frame.h" #include "common_audio/audio_converter.h" #include "common_audio/include/audio_util.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" -#include "modules/audio_processing/agc2/gain_applier.h" #include "modules/audio_processing/audio_buffer.h" -#include "modules/audio_processing/common.h" #include "modules/audio_processing/include/audio_frame_view.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "modules/audio_processing/optionally_built_submodule_creators.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" -#include "rtc_base/constructor_magic.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" -#include "rtc_base/ref_counted_object.h" #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" +#include "system_wrappers/include/denormal_disabler.h" #include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" @@ -49,24 +49,8 @@ namespace webrtc { -constexpr int kRuntimeSettingQueueSize = 100; - namespace { -static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { - switch (layout) { - case AudioProcessing::kMono: - case AudioProcessing::kStereo: - return false; - case AudioProcessing::kMonoAndKeyboard: - case AudioProcessing::kStereoAndKeyboard: - return true; - } - - RTC_NOTREACHED(); - return false; -} - bool SampleRateSupportsMultiBand(int sample_rate_hz) { return sample_rate_hz == AudioProcessing::kSampleRate32kHz || sample_rate_hz == AudioProcessing::kSampleRate48kHz; @@ -99,7 +83,7 @@ int SuitableProcessRate(int minimum_rate, return rate; } } - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return uppermost_native_rate; } @@ -114,6 +98,11 @@ GainControl::Mode Agc1ConfigModeToInterfaceMode( case Agc1Config::kFixedDigital: return GainControl::kFixedDigital; } + RTC_CHECK_NOTREACHED(); +} + +bool MinimizeProcessingForUnusedOutput() { + return !field_trial::IsEnabled("WebRTC-MutedStateKillSwitch"); } // Maximum lengths that frame of samples being passed from the render side to @@ -125,11 +114,439 @@ static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480; // TODO(peah): Decrease this once we properly handle hugely unbalanced // reverse and forward call numbers. static const size_t kMaxNumFramesToBuffer = 100; + +void PackRenderAudioBufferForEchoDetector(const AudioBuffer& audio, + std::vector& packed_buffer) { + packed_buffer.clear(); + packed_buffer.insert(packed_buffer.end(), audio.channels_const()[0], + audio.channels_const()[0] + audio.num_frames()); +} + +// Options for gracefully handling processing errors. +enum class FormatErrorOutputOption { + kOutputExactCopyOfInput, + kOutputBroadcastCopyOfFirstInputChannel, + kOutputSilence, + kDoNothing +}; + +enum class AudioFormatValidity { + // Format is supported by APM. + kValidAndSupported, + // Format has a reasonable interpretation but is not supported. + kValidButUnsupportedSampleRate, + // The remaining enums values signal that the audio does not have a reasonable + // interpretation and cannot be used. + kInvalidSampleRate, + kInvalidChannelCount +}; + +AudioFormatValidity ValidateAudioFormat(const StreamConfig& config) { + if (config.sample_rate_hz() < 0) + return AudioFormatValidity::kInvalidSampleRate; + if (config.num_channels() == 0) + return AudioFormatValidity::kInvalidChannelCount; + + // Format has a reasonable interpretation, but may still be unsupported. + if (config.sample_rate_hz() < 8000 || + config.sample_rate_hz() > AudioBuffer::kMaxSampleRate) + return AudioFormatValidity::kValidButUnsupportedSampleRate; + + // Format is fully supported. + return AudioFormatValidity::kValidAndSupported; +} + +int AudioFormatValidityToErrorCode(AudioFormatValidity validity) { + switch (validity) { + case AudioFormatValidity::kValidAndSupported: + return AudioProcessing::kNoError; + case AudioFormatValidity::kValidButUnsupportedSampleRate: // fall-through + case AudioFormatValidity::kInvalidSampleRate: + return AudioProcessing::kBadSampleRateError; + case AudioFormatValidity::kInvalidChannelCount: + return AudioProcessing::kBadNumberChannelsError; + } + RTC_DCHECK(false); +} + +// Returns an AudioProcessing::Error together with the best possible option for +// output audio content. +std::pair ChooseErrorOutputOption( + const StreamConfig& input_config, + const StreamConfig& output_config) { + AudioFormatValidity input_validity = ValidateAudioFormat(input_config); + AudioFormatValidity output_validity = ValidateAudioFormat(output_config); + + if (input_validity == AudioFormatValidity::kValidAndSupported && + output_validity == AudioFormatValidity::kValidAndSupported && + (output_config.num_channels() == 1 || + output_config.num_channels() == input_config.num_channels())) { + return {AudioProcessing::kNoError, FormatErrorOutputOption::kDoNothing}; + } + + int error_code = AudioFormatValidityToErrorCode(input_validity); + if (error_code == AudioProcessing::kNoError) { + error_code = AudioFormatValidityToErrorCode(output_validity); + } + if (error_code == AudioProcessing::kNoError) { + // The individual formats are valid but there is some error - must be + // channel mismatch. + error_code = AudioProcessing::kBadNumberChannelsError; + } + + FormatErrorOutputOption output_option; + if (output_validity != AudioFormatValidity::kValidAndSupported && + output_validity != AudioFormatValidity::kValidButUnsupportedSampleRate) { + // The output format is uninterpretable: cannot do anything. + output_option = FormatErrorOutputOption::kDoNothing; + } else if (input_validity != AudioFormatValidity::kValidAndSupported && + input_validity != + AudioFormatValidity::kValidButUnsupportedSampleRate) { + // The input format is uninterpretable: cannot use it, must output silence. + output_option = FormatErrorOutputOption::kOutputSilence; + } else if (input_config.sample_rate_hz() != output_config.sample_rate_hz()) { + // Sample rates do not match: Cannot copy input into output, output silence. + // Note: If the sample rates are in a supported range, we could resample. + // However, that would significantly increase complexity of this error + // handling code. + output_option = FormatErrorOutputOption::kOutputSilence; + } else if (input_config.num_channels() != output_config.num_channels()) { + // Channel counts do not match: We cannot easily map input channels to + // output channels. + output_option = + FormatErrorOutputOption::kOutputBroadcastCopyOfFirstInputChannel; + } else { + // The formats match exactly. + RTC_DCHECK(input_config == output_config); + output_option = FormatErrorOutputOption::kOutputExactCopyOfInput; + } + return std::make_pair(error_code, output_option); +} + +// Checks if the audio format is supported. If not, the output is populated in a +// best-effort manner and an APM error code is returned. +int HandleUnsupportedAudioFormats(const int16_t* const src, + const StreamConfig& input_config, + const StreamConfig& output_config, + int16_t* const dest) { + RTC_DCHECK(src); + RTC_DCHECK(dest); + + auto [error_code, output_option] = + ChooseErrorOutputOption(input_config, output_config); + if (error_code == AudioProcessing::kNoError) + return AudioProcessing::kNoError; + + const size_t num_output_channels = output_config.num_channels(); + switch (output_option) { + case FormatErrorOutputOption::kOutputSilence: + memset(dest, 0, output_config.num_samples() * sizeof(int16_t)); + break; + case FormatErrorOutputOption::kOutputBroadcastCopyOfFirstInputChannel: + for (size_t i = 0; i < output_config.num_frames(); ++i) { + int16_t sample = src[input_config.num_channels() * i]; + for (size_t ch = 0; ch < num_output_channels; ++ch) { + dest[ch + num_output_channels * i] = sample; + } + } + break; + case FormatErrorOutputOption::kOutputExactCopyOfInput: + memcpy(dest, src, output_config.num_samples() * sizeof(int16_t)); + break; + case FormatErrorOutputOption::kDoNothing: + break; + } + return error_code; +} + +// Checks if the audio format is supported. If not, the output is populated in a +// best-effort manner and an APM error code is returned. +int HandleUnsupportedAudioFormats(const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest) { + RTC_DCHECK(src); + RTC_DCHECK(dest); + for (size_t i = 0; i < input_config.num_channels(); ++i) { + RTC_DCHECK(src[i]); + } + for (size_t i = 0; i < output_config.num_channels(); ++i) { + RTC_DCHECK(dest[i]); + } + + auto [error_code, output_option] = + ChooseErrorOutputOption(input_config, output_config); + if (error_code == AudioProcessing::kNoError) + return AudioProcessing::kNoError; + + const size_t num_output_channels = output_config.num_channels(); + switch (output_option) { + case FormatErrorOutputOption::kOutputSilence: + for (size_t ch = 0; ch < num_output_channels; ++ch) { + memset(dest[ch], 0, output_config.num_frames() * sizeof(float)); + } + break; + case FormatErrorOutputOption::kOutputBroadcastCopyOfFirstInputChannel: + for (size_t ch = 0; ch < num_output_channels; ++ch) { + memcpy(dest[ch], src[0], output_config.num_frames() * sizeof(float)); + } + break; + case FormatErrorOutputOption::kOutputExactCopyOfInput: + for (size_t ch = 0; ch < num_output_channels; ++ch) { + memcpy(dest[ch], src[ch], output_config.num_frames() * sizeof(float)); + } + break; + case FormatErrorOutputOption::kDoNothing: + break; + } + return error_code; +} + +using DownmixMethod = AudioProcessing::Config::Pipeline::DownmixMethod; + +void SetDownmixMethod(AudioBuffer& buffer, DownmixMethod method) { + switch (method) { + case DownmixMethod::kAverageChannels: + buffer.set_downmixing_by_averaging(); + break; + case DownmixMethod::kUseFirstChannel: + buffer.set_downmixing_to_specific_channel(/*channel=*/0); + break; + } +} + +constexpr int kUnspecifiedDataDumpInputVolume = -100; + } // namespace // Throughout webrtc, it's assumed that success is represented by zero. static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); +absl::optional +AudioProcessingImpl::GetGainController2ExperimentParams() { + constexpr char kFieldTrialName[] = "WebRTC-Audio-GainController2"; + + if (!field_trial::IsEnabled(kFieldTrialName)) { + return absl::nullopt; + } + + FieldTrialFlag enabled("Enabled", false); + + // Whether the gain control should switch to AGC2. Enabled by default. + FieldTrialParameter switch_to_agc2("switch_to_agc2", true); + + // AGC2 input volume controller configuration. + constexpr InputVolumeController::Config kDefaultInputVolumeControllerConfig; + FieldTrialConstrained min_input_volume( + "min_input_volume", kDefaultInputVolumeControllerConfig.min_input_volume, + 0, 255); + FieldTrialConstrained clipped_level_min( + "clipped_level_min", + kDefaultInputVolumeControllerConfig.clipped_level_min, 0, 255); + FieldTrialConstrained clipped_level_step( + "clipped_level_step", + kDefaultInputVolumeControllerConfig.clipped_level_step, 0, 255); + FieldTrialConstrained clipped_ratio_threshold( + "clipped_ratio_threshold", + kDefaultInputVolumeControllerConfig.clipped_ratio_threshold, 0, 1); + FieldTrialConstrained clipped_wait_frames( + "clipped_wait_frames", + kDefaultInputVolumeControllerConfig.clipped_wait_frames, 0, + absl::nullopt); + FieldTrialParameter enable_clipping_predictor( + "enable_clipping_predictor", + kDefaultInputVolumeControllerConfig.enable_clipping_predictor); + FieldTrialConstrained target_range_max_dbfs( + "target_range_max_dbfs", + kDefaultInputVolumeControllerConfig.target_range_max_dbfs, -90, 30); + FieldTrialConstrained target_range_min_dbfs( + "target_range_min_dbfs", + kDefaultInputVolumeControllerConfig.target_range_min_dbfs, -90, 30); + FieldTrialConstrained update_input_volume_wait_frames( + "update_input_volume_wait_frames", + kDefaultInputVolumeControllerConfig.update_input_volume_wait_frames, 0, + absl::nullopt); + FieldTrialConstrained speech_probability_threshold( + "speech_probability_threshold", + kDefaultInputVolumeControllerConfig.speech_probability_threshold, 0, 1); + FieldTrialConstrained speech_ratio_threshold( + "speech_ratio_threshold", + kDefaultInputVolumeControllerConfig.speech_ratio_threshold, 0, 1); + + // AGC2 adaptive digital controller configuration. + constexpr AudioProcessing::Config::GainController2::AdaptiveDigital + kDefaultAdaptiveDigitalConfig; + FieldTrialConstrained headroom_db( + "headroom_db", kDefaultAdaptiveDigitalConfig.headroom_db, 0, + absl::nullopt); + FieldTrialConstrained max_gain_db( + "max_gain_db", kDefaultAdaptiveDigitalConfig.max_gain_db, 0, + absl::nullopt); + FieldTrialConstrained initial_gain_db( + "initial_gain_db", kDefaultAdaptiveDigitalConfig.initial_gain_db, 0, + absl::nullopt); + FieldTrialConstrained max_gain_change_db_per_second( + "max_gain_change_db_per_second", + kDefaultAdaptiveDigitalConfig.max_gain_change_db_per_second, 0, + absl::nullopt); + FieldTrialConstrained max_output_noise_level_dbfs( + "max_output_noise_level_dbfs", + kDefaultAdaptiveDigitalConfig.max_output_noise_level_dbfs, absl::nullopt, + 0); + + // Transient suppressor. + FieldTrialParameter disallow_transient_suppressor_usage( + "disallow_transient_suppressor_usage", false); + + // Field-trial based override for the input volume controller and adaptive + // digital configs. + ParseFieldTrial( + {&enabled, &switch_to_agc2, &min_input_volume, &clipped_level_min, + &clipped_level_step, &clipped_ratio_threshold, &clipped_wait_frames, + &enable_clipping_predictor, &target_range_max_dbfs, + &target_range_min_dbfs, &update_input_volume_wait_frames, + &speech_probability_threshold, &speech_ratio_threshold, &headroom_db, + &max_gain_db, &initial_gain_db, &max_gain_change_db_per_second, + &max_output_noise_level_dbfs, &disallow_transient_suppressor_usage}, + field_trial::FindFullName(kFieldTrialName)); + // Checked already by `IsEnabled()` before parsing, therefore always true. + RTC_DCHECK(enabled); + + const bool do_not_change_agc_config = !switch_to_agc2.Get(); + if (do_not_change_agc_config && !disallow_transient_suppressor_usage.Get()) { + // Return an unspecifed value since, in this case, both the AGC2 and TS + // configurations won't be adjusted. + return absl::nullopt; + } + using Params = AudioProcessingImpl::GainController2ExperimentParams; + if (do_not_change_agc_config) { + // Return a value that leaves the AGC2 config unchanged and that always + // disables TS. + return Params{.agc2_config = absl::nullopt, + .disallow_transient_suppressor_usage = true}; + } + // Return a value that switches all the gain control to AGC2. + return Params{ + .agc2_config = + Params::Agc2Config{ + .input_volume_controller = + { + .min_input_volume = min_input_volume.Get(), + .clipped_level_min = clipped_level_min.Get(), + .clipped_level_step = clipped_level_step.Get(), + .clipped_ratio_threshold = + static_cast(clipped_ratio_threshold.Get()), + .clipped_wait_frames = clipped_wait_frames.Get(), + .enable_clipping_predictor = + enable_clipping_predictor.Get(), + .target_range_max_dbfs = target_range_max_dbfs.Get(), + .target_range_min_dbfs = target_range_min_dbfs.Get(), + .update_input_volume_wait_frames = + update_input_volume_wait_frames.Get(), + .speech_probability_threshold = static_cast( + speech_probability_threshold.Get()), + .speech_ratio_threshold = + static_cast(speech_ratio_threshold.Get()), + }, + .adaptive_digital_controller = + { + .headroom_db = static_cast(headroom_db.Get()), + .max_gain_db = static_cast(max_gain_db.Get()), + .initial_gain_db = + static_cast(initial_gain_db.Get()), + .max_gain_change_db_per_second = static_cast( + max_gain_change_db_per_second.Get()), + .max_output_noise_level_dbfs = + static_cast(max_output_noise_level_dbfs.Get()), + }}, + .disallow_transient_suppressor_usage = + disallow_transient_suppressor_usage.Get()}; +} + +AudioProcessing::Config AudioProcessingImpl::AdjustConfig( + const AudioProcessing::Config& config, + const absl::optional& + experiment_params) { + if (!experiment_params.has_value() || + (!experiment_params->agc2_config.has_value() && + !experiment_params->disallow_transient_suppressor_usage)) { + // When the experiment parameters are unspecified or when the AGC and TS + // configuration are not overridden, return the unmodified configuration. + return config; + } + + AudioProcessing::Config adjusted_config = config; + + // Override the transient suppressor configuration. + if (experiment_params->disallow_transient_suppressor_usage) { + adjusted_config.transient_suppression.enabled = false; + } + + // Override the auto gain control configuration if the AGC1 analog gain + // controller is active and `experiment_params->agc2_config` is specified. + const bool agc1_analog_enabled = + config.gain_controller1.enabled && + (config.gain_controller1.mode == + AudioProcessing::Config::GainController1::kAdaptiveAnalog || + config.gain_controller1.analog_gain_controller.enabled); + if (agc1_analog_enabled && experiment_params->agc2_config.has_value()) { + // Check that the unadjusted AGC config meets the preconditions. + const bool hybrid_agc_config_detected = + config.gain_controller1.enabled && + config.gain_controller1.analog_gain_controller.enabled && + !config.gain_controller1.analog_gain_controller + .enable_digital_adaptive && + config.gain_controller2.enabled && + config.gain_controller2.adaptive_digital.enabled; + const bool full_agc1_config_detected = + config.gain_controller1.enabled && + config.gain_controller1.analog_gain_controller.enabled && + config.gain_controller1.analog_gain_controller + .enable_digital_adaptive && + !config.gain_controller2.enabled; + const bool one_and_only_one_input_volume_controller = + hybrid_agc_config_detected != full_agc1_config_detected; + const bool agc2_input_volume_controller_enabled = + config.gain_controller2.enabled && + config.gain_controller2.input_volume_controller.enabled; + if (!one_and_only_one_input_volume_controller || + agc2_input_volume_controller_enabled) { + RTC_LOG(LS_ERROR) << "Cannot adjust AGC config (precondition failed)"; + if (!one_and_only_one_input_volume_controller) + RTC_LOG(LS_ERROR) + << "One and only one input volume controller must be enabled."; + if (agc2_input_volume_controller_enabled) + RTC_LOG(LS_ERROR) + << "The AGC2 input volume controller must be disabled."; + } else { + adjusted_config.gain_controller1.enabled = false; + adjusted_config.gain_controller1.analog_gain_controller.enabled = false; + + adjusted_config.gain_controller2.enabled = true; + adjusted_config.gain_controller2.input_volume_controller.enabled = true; + adjusted_config.gain_controller2.adaptive_digital = + experiment_params->agc2_config->adaptive_digital_controller; + adjusted_config.gain_controller2.adaptive_digital.enabled = true; + } + } + + return adjusted_config; +} + +bool AudioProcessingImpl::UseApmVadSubModule( + const AudioProcessing::Config& config, + const absl::optional& experiment_params) { + // The VAD as an APM sub-module is needed only in one case, that is when TS + // and AGC2 are both enabled and when the AGC2 experiment is running and its + // parameters require to fully switch the gain control to AGC2. + return config.transient_suppression.enabled && + config.gain_controller2.enabled && + (config.gain_controller2.input_volume_controller.enabled || + config.gain_controller2.adaptive_digital.enabled) && + experiment_params.has_value() && + experiment_params->agc2_config.has_value(); +} + AudioProcessingImpl::SubmoduleStates::SubmoduleStates( bool capture_post_processor_enabled, bool render_pre_processor_enabled, @@ -141,38 +558,35 @@ AudioProcessingImpl::SubmoduleStates::SubmoduleStates( bool AudioProcessingImpl::SubmoduleStates::Update( bool high_pass_filter_enabled, bool mobile_echo_controller_enabled, - bool residual_echo_detector_enabled, bool noise_suppressor_enabled, bool adaptive_gain_controller_enabled, bool gain_controller2_enabled, - bool pre_amplifier_enabled, + bool voice_activity_detector_enabled, + bool gain_adjustment_enabled, bool echo_controller_enabled, - bool voice_detector_enabled, bool transient_suppressor_enabled) { bool changed = false; changed |= (high_pass_filter_enabled != high_pass_filter_enabled_); changed |= (mobile_echo_controller_enabled != mobile_echo_controller_enabled_); - changed |= - (residual_echo_detector_enabled != residual_echo_detector_enabled_); changed |= (noise_suppressor_enabled != noise_suppressor_enabled_); changed |= (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_); changed |= (gain_controller2_enabled != gain_controller2_enabled_); - changed |= (pre_amplifier_enabled_ != pre_amplifier_enabled); + changed |= + (voice_activity_detector_enabled != voice_activity_detector_enabled_); + changed |= (gain_adjustment_enabled != gain_adjustment_enabled_); changed |= (echo_controller_enabled != echo_controller_enabled_); - changed |= (voice_detector_enabled != voice_detector_enabled_); changed |= (transient_suppressor_enabled != transient_suppressor_enabled_); if (changed) { high_pass_filter_enabled_ = high_pass_filter_enabled; mobile_echo_controller_enabled_ = mobile_echo_controller_enabled; - residual_echo_detector_enabled_ = residual_echo_detector_enabled; noise_suppressor_enabled_ = noise_suppressor_enabled; adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled; gain_controller2_enabled_ = gain_controller2_enabled; - pre_amplifier_enabled_ = pre_amplifier_enabled; + voice_activity_detector_enabled_ = voice_activity_detector_enabled; + gain_adjustment_enabled_ = gain_adjustment_enabled; echo_controller_enabled_ = echo_controller_enabled; - voice_detector_enabled_ = voice_detector_enabled; transient_suppressor_enabled_ = transient_suppressor_enabled; } @@ -183,7 +597,7 @@ bool AudioProcessingImpl::SubmoduleStates::Update( bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive() const { - return CaptureMultiBandProcessingPresent() || voice_detector_enabled_; + return CaptureMultiBandProcessingPresent(); } bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent() @@ -202,7 +616,7 @@ bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive( bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive() const { return gain_controller2_enabled_ || capture_post_processor_enabled_ || - pre_amplifier_enabled_; + gain_adjustment_enabled_; } bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const { @@ -230,32 +644,34 @@ bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const { noise_suppressor_enabled_; } -AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config) - : AudioProcessingImpl(config, +AudioProcessingImpl::AudioProcessingImpl() + : AudioProcessingImpl(/*config=*/{}, /*capture_post_processor=*/nullptr, /*render_pre_processor=*/nullptr, /*echo_control_factory=*/nullptr, /*echo_detector=*/nullptr, /*capture_analyzer=*/nullptr) {} -int AudioProcessingImpl::instance_count_ = 0; +std::atomic AudioProcessingImpl::instance_count_(0); AudioProcessingImpl::AudioProcessingImpl( - const webrtc::Config& config, + const AudioProcessing::Config& config, std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, std::unique_ptr echo_control_factory, rtc::scoped_refptr echo_detector, std::unique_ptr capture_analyzer) - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), use_setup_specific_default_aec3_config_( UseSetupSpecificDefaultAec3Congfig()), - capture_runtime_settings_(kRuntimeSettingQueueSize), - render_runtime_settings_(kRuntimeSettingQueueSize), + gain_controller2_experiment_params_(GetGainController2ExperimentParams()), + transient_suppressor_vad_mode_(TransientSuppressor::VadMode::kDefault), + capture_runtime_settings_(RuntimeSettingQueueSize()), + render_runtime_settings_(RuntimeSettingQueueSize()), capture_runtime_settings_enqueuer_(&capture_runtime_settings_), render_runtime_settings_enqueuer_(&render_runtime_settings_), echo_control_factory_(std::move(echo_control_factory)), + config_(AdjustConfig(config, gain_controller2_experiment_params_)), submodule_states_(!!capture_post_processor, !!render_pre_processor, !!capture_analyzer), @@ -267,8 +683,15 @@ AudioProcessingImpl::AudioProcessingImpl( "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"), !field_trial::IsEnabled( "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"), - EnforceSplitBandHpf()), - capture_nonlocked_() { + EnforceSplitBandHpf(), + MinimizeProcessingForUnusedOutput(), + field_trial::IsEnabled("WebRTC-TransientSuppressorForcedOff")), + capture_(), + capture_nonlocked_(), + applied_input_volume_stats_reporter_( + InputVolumeStatsReporter::InputVolumeType::kApplied), + recommended_input_volume_stats_reporter_( + InputVolumeStatsReporter::InputVolumeType::kRecommended) { RTC_LOG(LS_INFO) << "Injected APM submodules:" "\nEcho control factory: " << !!echo_control_factory_ @@ -278,36 +701,16 @@ AudioProcessingImpl::AudioProcessingImpl( << !!submodules_.capture_post_processor << "\nRender pre processor: " << !!submodules_.render_pre_processor; + if (!DenormalDisabler::IsSupported()) { + RTC_LOG(LS_INFO) << "Denormal disabler unsupported"; + } + + RTC_LOG(LS_INFO) << "AudioProcessing: " << config_.ToString(); // Mark Echo Controller enabled if a factory is injected. capture_nonlocked_.echo_controller_enabled = static_cast(echo_control_factory_); - // If no echo detector is injected, use the ResidualEchoDetector. - if (!submodules_.echo_detector) { - submodules_.echo_detector = - new rtc::RefCountedObject(); - } - -#if !(defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)) - // TODO(webrtc:5298): Remove once the use of ExperimentalNs has been - // deprecated. - config_.transient_suppression.enabled = config.Get().enabled; - - // TODO(webrtc:5298): Remove once the use of ExperimentalAgc has been - // deprecated. - config_.gain_controller1.analog_gain_controller.enabled = - config.Get().enabled; - config_.gain_controller1.analog_gain_controller.startup_min_volume = - config.Get().startup_min_volume; - config_.gain_controller1.analog_gain_controller.clipped_level_min = - config.Get().clipped_level_min; - config_.gain_controller1.analog_gain_controller.enable_agc2_level_estimator = - config.Get().enabled_agc2_level_estimator; - config_.gain_controller1.analog_gain_controller.enable_digital_adaptive = - !config.Get().digital_adaptive_disabled; -#endif - Initialize(); } @@ -321,42 +724,27 @@ int AudioProcessingImpl::Initialize() { return kNoError; } -int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz, - int capture_output_sample_rate_hz, - int render_input_sample_rate_hz, - ChannelLayout capture_input_layout, - ChannelLayout capture_output_layout, - ChannelLayout render_input_layout) { - const ProcessingConfig processing_config = { - {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout), - LayoutHasKeyboard(capture_input_layout)}, - {capture_output_sample_rate_hz, - ChannelsFromLayout(capture_output_layout), - LayoutHasKeyboard(capture_output_layout)}, - {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), - LayoutHasKeyboard(render_input_layout)}, - {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), - LayoutHasKeyboard(render_input_layout)}}}; - - return Initialize(processing_config); -} - int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { // Run in a single-threaded manner during initialization. MutexLock lock_render(&mutex_render_); MutexLock lock_capture(&mutex_capture_); - return InitializeLocked(processing_config); + InitializeLocked(processing_config); + return kNoError; } -int AudioProcessingImpl::MaybeInitializeRender( - const ProcessingConfig& processing_config) { - // Called from both threads. Thread check is therefore not possible. +void AudioProcessingImpl::MaybeInitializeRender( + const StreamConfig& input_config, + const StreamConfig& output_config) { + ProcessingConfig processing_config = formats_.api_format; + processing_config.reverse_input_stream() = input_config; + processing_config.reverse_output_stream() = output_config; + if (processing_config == formats_.api_format) { - return kNoError; + return; } MutexLock lock_capture(&mutex_capture_); - return InitializeLocked(processing_config); + InitializeLocked(processing_config); } void AudioProcessingImpl::InitializeLocked() { @@ -396,6 +784,8 @@ void AudioProcessingImpl::InitializeLocked() { formats_.api_format.output_stream().num_channels(), formats_.api_format.output_stream().sample_rate_hz(), formats_.api_format.output_stream().num_channels())); + SetDownmixMethod(*capture_.capture_audio, + config_.pipeline.capture_downmix_method); if (capture_nonlocked_.capture_processing_format.sample_rate_hz() < formats_.api_format.output_stream().sample_rate_hz() && @@ -407,6 +797,8 @@ void AudioProcessingImpl::InitializeLocked() { formats_.api_format.output_stream().num_channels(), formats_.api_format.output_stream().sample_rate_hz(), formats_.api_format.output_stream().num_channels())); + SetDownmixMethod(*capture_.capture_fullband_audio, + config_.pipeline.capture_downmix_method); } else { capture_.capture_fullband_audio.reset(); } @@ -416,39 +808,24 @@ void AudioProcessingImpl::InitializeLocked() { InitializeGainController1(); InitializeTransientSuppressor(); InitializeHighPassFilter(true); - InitializeVoiceDetector(); InitializeResidualEchoDetector(); InitializeEchoController(); InitializeGainController2(); + InitializeVoiceActivityDetector(); InitializeNoiseSuppressor(); InitializeAnalyzer(); InitializePostProcessor(); InitializePreProcessor(); + InitializeCaptureLevelsAdjuster(); if (aec_dump_) { aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis()); } } -int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { +void AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { UpdateActiveSubmoduleStates(); - for (const auto& stream : config.streams) { - if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { - return kBadSampleRateError; - } - } - - const size_t num_in_channels = config.input_stream().num_channels(); - const size_t num_out_channels = config.output_stream().num_channels(); - - // Need at least one input channel. - // Need either one output channel or as many outputs as there are inputs. - if (num_in_channels == 0 || - !(num_out_channels == 1 || num_out_channels == num_in_channels)) { - return kBadNumberChannelsError; - } - formats_.api_format = config; // Choose maximum rate to use for the split filtering. @@ -522,74 +899,59 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { } InitializeLocked(); - return kNoError; } void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { - RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString(); - // Run in a single-threaded manner when applying the settings. MutexLock lock_render(&mutex_render_); MutexLock lock_capture(&mutex_capture_); + const auto adjusted_config = + AdjustConfig(config, gain_controller2_experiment_params_); + RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " + << adjusted_config.ToString(); + const bool pipeline_config_changed = config_.pipeline.multi_channel_render != - config.pipeline.multi_channel_render || + adjusted_config.pipeline.multi_channel_render || config_.pipeline.multi_channel_capture != - config.pipeline.multi_channel_capture || + adjusted_config.pipeline.multi_channel_capture || config_.pipeline.maximum_internal_processing_rate != - config.pipeline.maximum_internal_processing_rate; + adjusted_config.pipeline.maximum_internal_processing_rate || + config_.pipeline.capture_downmix_method != + adjusted_config.pipeline.capture_downmix_method; const bool aec_config_changed = - config_.echo_canceller.enabled != config.echo_canceller.enabled || - config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode; + config_.echo_canceller.enabled != + adjusted_config.echo_canceller.enabled || + config_.echo_canceller.mobile_mode != + adjusted_config.echo_canceller.mobile_mode; const bool agc1_config_changed = - config_.gain_controller1.enabled != config.gain_controller1.enabled || - config_.gain_controller1.mode != config.gain_controller1.mode || - config_.gain_controller1.target_level_dbfs != - config.gain_controller1.target_level_dbfs || - config_.gain_controller1.compression_gain_db != - config.gain_controller1.compression_gain_db || - config_.gain_controller1.enable_limiter != - config.gain_controller1.enable_limiter || - config_.gain_controller1.analog_level_minimum != - config.gain_controller1.analog_level_minimum || - config_.gain_controller1.analog_level_maximum != - config.gain_controller1.analog_level_maximum || - config_.gain_controller1.analog_gain_controller.enabled != - config.gain_controller1.analog_gain_controller.enabled || - config_.gain_controller1.analog_gain_controller.startup_min_volume != - config.gain_controller1.analog_gain_controller.startup_min_volume || - config_.gain_controller1.analog_gain_controller.clipped_level_min != - config.gain_controller1.analog_gain_controller.clipped_level_min || - config_.gain_controller1.analog_gain_controller - .enable_agc2_level_estimator != - config.gain_controller1.analog_gain_controller - .enable_agc2_level_estimator || - config_.gain_controller1.analog_gain_controller.enable_digital_adaptive != - config.gain_controller1.analog_gain_controller - .enable_digital_adaptive; + config_.gain_controller1 != adjusted_config.gain_controller1; const bool agc2_config_changed = - config_.gain_controller2.enabled != config.gain_controller2.enabled; - - const bool voice_detection_config_changed = - config_.voice_detection.enabled != config.voice_detection.enabled; + config_.gain_controller2 != adjusted_config.gain_controller2; const bool ns_config_changed = - config_.noise_suppression.enabled != config.noise_suppression.enabled || - config_.noise_suppression.level != config.noise_suppression.level; + config_.noise_suppression.enabled != + adjusted_config.noise_suppression.enabled || + config_.noise_suppression.level != + adjusted_config.noise_suppression.level; const bool ts_config_changed = config_.transient_suppression.enabled != - config.transient_suppression.enabled; + adjusted_config.transient_suppression.enabled; const bool pre_amplifier_config_changed = - config_.pre_amplifier.enabled != config.pre_amplifier.enabled || + config_.pre_amplifier.enabled != adjusted_config.pre_amplifier.enabled || config_.pre_amplifier.fixed_gain_factor != - config.pre_amplifier.fixed_gain_factor; + adjusted_config.pre_amplifier.fixed_gain_factor; - config_ = config; + const bool gain_adjustment_config_changed = + config_.capture_level_adjustment != + adjusted_config.capture_level_adjustment; + + config_ = adjusted_config; if (aec_config_changed) { InitializeEchoController(); @@ -611,27 +973,20 @@ void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { const bool config_ok = GainController2::Validate(config_.gain_controller2); if (!config_ok) { - RTC_LOG(LS_ERROR) << "AudioProcessing module config error\n" - "Gain Controller 2: " - << GainController2::ToString(config_.gain_controller2) - << "\nReverting to default parameter set"; + RTC_LOG(LS_ERROR) + << "Invalid Gain Controller 2 config; using the default config."; config_.gain_controller2 = AudioProcessing::Config::GainController2(); } - if (agc2_config_changed) { + if (agc2_config_changed || ts_config_changed) { + // AGC2 also depends on TS because of the possible dependency on the APM VAD + // sub-module. InitializeGainController2(); + InitializeVoiceActivityDetector(); } - if (pre_amplifier_config_changed) { - InitializePreAmplifier(); - } - - if (config_.level_estimation.enabled && !submodules_.output_level_estimator) { - submodules_.output_level_estimator = std::make_unique(); - } - - if (voice_detection_config_changed) { - InitializeVoiceDetector(); + if (pre_amplifier_config_changed || gain_adjustment_config_changed) { + InitializeCaptureLevelsAdjuster(); } // Reinitialization must happen after all submodule configuration to avoid @@ -690,35 +1045,64 @@ size_t AudioProcessingImpl::num_output_channels() const { void AudioProcessingImpl::set_output_will_be_muted(bool muted) { MutexLock lock(&mutex_capture_); - capture_.output_will_be_muted = muted; + HandleCaptureOutputUsedSetting(!muted); +} + +void AudioProcessingImpl::HandleCaptureOutputUsedSetting( + bool capture_output_used) { + capture_.capture_output_used = + capture_output_used || !constants_.minimize_processing_for_unused_output; + if (submodules_.agc_manager.get()) { - submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted); + submodules_.agc_manager->HandleCaptureOutputUsedChange( + capture_.capture_output_used); + } + if (submodules_.echo_controller) { + submodules_.echo_controller->SetCaptureOutputUsage( + capture_.capture_output_used); + } + if (submodules_.noise_suppressor) { + submodules_.noise_suppressor->SetCaptureOutputUsage( + capture_.capture_output_used); + } + if (submodules_.gain_controller2) { + submodules_.gain_controller2->SetCaptureOutputUsed( + capture_.capture_output_used); } } void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) { + PostRuntimeSetting(setting); +} + +bool AudioProcessingImpl::PostRuntimeSetting(RuntimeSetting setting) { switch (setting.type()) { case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting: case RuntimeSetting::Type::kPlayoutAudioDeviceChange: - render_runtime_settings_enqueuer_.Enqueue(setting); - return; + return render_runtime_settings_enqueuer_.Enqueue(setting); case RuntimeSetting::Type::kCapturePreGain: + case RuntimeSetting::Type::kCapturePostGain: case RuntimeSetting::Type::kCaptureCompressionGain: case RuntimeSetting::Type::kCaptureFixedPostGain: case RuntimeSetting::Type::kCaptureOutputUsed: - capture_runtime_settings_enqueuer_.Enqueue(setting); - return; - case RuntimeSetting::Type::kPlayoutVolumeChange: - capture_runtime_settings_enqueuer_.Enqueue(setting); - render_runtime_settings_enqueuer_.Enqueue(setting); - return; + return capture_runtime_settings_enqueuer_.Enqueue(setting); + case RuntimeSetting::Type::kPlayoutVolumeChange: { + bool enqueueing_successful; + enqueueing_successful = + capture_runtime_settings_enqueuer_.Enqueue(setting); + enqueueing_successful = + render_runtime_settings_enqueuer_.Enqueue(setting) && + enqueueing_successful; + return enqueueing_successful; + } case RuntimeSetting::Type::kNotSpecified: - RTC_NOTREACHED(); - return; + RTC_DCHECK_NOTREACHED(); + return true; } // The language allows the enum to have a non-enumerator // value. Check that this doesn't happen. - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); + return true; } AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer( @@ -730,20 +1114,17 @@ AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer( AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() = default; -void AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue( +bool AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue( RuntimeSetting setting) { - size_t remaining_attempts = 10; - while (!runtime_settings_.Insert(&setting) && remaining_attempts-- > 0) { - RuntimeSetting setting_to_discard; - if (runtime_settings_.Remove(&setting_to_discard)) - RTC_LOG(LS_ERROR) - << "The runtime settings queue is full. Oldest setting discarded."; - } - if (remaining_attempts == 0) + const bool successful_insert = runtime_settings_.Insert(&setting); + + if (!successful_insert) { RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting."; + } + return successful_insert; } -int AudioProcessingImpl::MaybeInitializeCapture( +void AudioProcessingImpl::MaybeInitializeCapture( const StreamConfig& input_config, const StreamConfig& output_config) { ProcessingConfig processing_config; @@ -758,21 +1139,22 @@ int AudioProcessingImpl::MaybeInitializeCapture( } if (processing_config.input_stream() != input_config) { - processing_config.input_stream() = input_config; reinitialization_required = true; } if (processing_config.output_stream() != output_config) { - processing_config.output_stream() = output_config; reinitialization_required = true; } if (reinitialization_required) { MutexLock lock_render(&mutex_render_); MutexLock lock_capture(&mutex_capture_); - RETURN_ON_ERR(InitializeLocked(processing_config)); + // Reread the API format since the render format may have changed. + processing_config = formats_.api_format; + processing_config.input_stream() = input_config; + processing_config.output_stream() = output_config; + InitializeLocked(processing_config); } - return kNoError; } int AudioProcessingImpl::ProcessStream(const float* const* src, @@ -780,11 +1162,10 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, const StreamConfig& output_config, float* const* dest) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig"); - if (!src || !dest) { - return kNullPointerError; - } - - RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config)); + DenormalDisabler denormal_disabler; + RETURN_ON_ERR( + HandleUnsupportedAudioFormats(src, input_config, output_config, dest)); + MaybeInitializeCapture(input_config, output_config); MutexLock lock_capture(&mutex_capture_); @@ -792,7 +1173,6 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, RecordUnprocessedCaptureStream(src); } - capture_.keyboard_info.Extract(src, formats_.api_format.input_stream()); capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); if (capture_.capture_fullband_audio) { capture_.capture_fullband_audio->CopyFrom( @@ -814,22 +1194,55 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, void AudioProcessingImpl::HandleCaptureRuntimeSettings() { RuntimeSetting setting; + int num_settings_processed = 0; while (capture_runtime_settings_.Remove(&setting)) { if (aec_dump_) { aec_dump_->WriteRuntimeSetting(setting); } switch (setting.type()) { case RuntimeSetting::Type::kCapturePreGain: - if (config_.pre_amplifier.enabled) { + if (config_.pre_amplifier.enabled || + config_.capture_level_adjustment.enabled) { float value; setting.GetFloat(&value); - config_.pre_amplifier.fixed_gain_factor = value; - submodules_.pre_amplifier->SetGainFactor(value); + // If the pre-amplifier is used, apply the new gain to the + // pre-amplifier regardless if the capture level adjustment is + // activated. This approach allows both functionalities to coexist + // until they have been properly merged. + if (config_.pre_amplifier.enabled) { + config_.pre_amplifier.fixed_gain_factor = value; + } else { + config_.capture_level_adjustment.pre_gain_factor = value; + } + + // Use both the pre-amplifier and the capture level adjustment gains + // as pre-gains. + float gain = 1.f; + if (config_.pre_amplifier.enabled) { + gain *= config_.pre_amplifier.fixed_gain_factor; + } + if (config_.capture_level_adjustment.enabled) { + gain *= config_.capture_level_adjustment.pre_gain_factor; + } + + submodules_.capture_levels_adjuster->SetPreGain(gain); + } + // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump. + break; + case RuntimeSetting::Type::kCapturePostGain: + if (config_.capture_level_adjustment.enabled) { + float value; + setting.GetFloat(&value); + config_.capture_level_adjustment.post_gain_factor = value; + submodules_.capture_levels_adjuster->SetPostGain( + config_.capture_level_adjustment.post_gain_factor); } // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump. break; case RuntimeSetting::Type::kCaptureCompressionGain: { - if (!submodules_.agc_manager) { + if (!submodules_.agc_manager && + !(submodules_.gain_controller2 && + config_.gain_controller2.input_volume_controller.enabled)) { float value; setting.GetFloat(&value); int int_value = static_cast(value + .5f); @@ -847,7 +1260,7 @@ void AudioProcessingImpl::HandleCaptureRuntimeSettings() { float value; setting.GetFloat(&value); config_.gain_controller2.fixed_digital.gain_db = value; - submodules_.gain_controller2->ApplyConfig(config_.gain_controller2); + submodules_.gain_controller2->SetFixedGainDb(value); } break; } @@ -858,20 +1271,34 @@ void AudioProcessingImpl::HandleCaptureRuntimeSettings() { break; } case RuntimeSetting::Type::kPlayoutAudioDeviceChange: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; case RuntimeSetting::Type::kNotSpecified: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; case RuntimeSetting::Type::kCaptureOutputUsed: - // TODO(b/154437967): Add support for reducing complexity when it is - // known that the capture output will not be used. + bool value; + setting.GetBool(&value); + HandleCaptureOutputUsedSetting(value); break; } + ++num_settings_processed; } + + if (num_settings_processed >= RuntimeSettingQueueSize()) { + // Handle overrun of the runtime settings queue, which likely will has + // caused settings to be discarded. + HandleOverrunInCaptureRuntimeSettingsQueue(); + } +} + +void AudioProcessingImpl::HandleOverrunInCaptureRuntimeSettingsQueue() { + // Fall back to a safe state for the case when a setting for capture output + // usage setting has been missed. + HandleCaptureOutputUsedSetting(/*capture_output_used=*/true); } void AudioProcessingImpl::HandleRenderRuntimeSettings() { @@ -889,11 +1316,12 @@ void AudioProcessingImpl::HandleRenderRuntimeSettings() { } break; case RuntimeSetting::Type::kCapturePreGain: // fall-through + case RuntimeSetting::Type::kCapturePostGain: // fall-through case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through case RuntimeSetting::Type::kCaptureOutputUsed: // fall-through case RuntimeSetting::Type::kNotSpecified: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); break; } } @@ -934,16 +1362,18 @@ void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) { } void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) { - ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_); + if (submodules_.echo_detector) { + PackRenderAudioBufferForEchoDetector(*audio, red_render_queue_buffer_); + RTC_DCHECK(red_render_signal_queue_); + // Insert the samples into the queue. + if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); - // Insert the samples into the queue. - if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - EmptyQueuedRenderAudio(); - - // Retry the insert (should always work). - bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); - RTC_DCHECK(result); + // Retry the insert (should always work). + bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); + RTC_DCHECK(result); + } } } @@ -976,23 +1406,26 @@ void AudioProcessingImpl::AllocateRenderQueue() { agc_render_signal_queue_->Clear(); } - if (red_render_queue_element_max_size_ < - new_red_render_queue_element_max_size) { - red_render_queue_element_max_size_ = new_red_render_queue_element_max_size; + if (submodules_.echo_detector) { + if (red_render_queue_element_max_size_ < + new_red_render_queue_element_max_size) { + red_render_queue_element_max_size_ = + new_red_render_queue_element_max_size; - std::vector template_queue_element( - red_render_queue_element_max_size_); + std::vector template_queue_element( + red_render_queue_element_max_size_); - red_render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier( - red_render_queue_element_max_size_))); + red_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + red_render_queue_element_max_size_))); - red_render_queue_buffer_.resize(red_render_queue_element_max_size_); - red_capture_queue_buffer_.resize(red_render_queue_element_max_size_); - } else { - red_render_signal_queue_->Clear(); + red_render_queue_buffer_.resize(red_render_queue_element_max_size_); + red_capture_queue_buffer_.resize(red_render_queue_element_max_size_); + } else { + red_render_signal_queue_->Clear(); + } } } @@ -1016,9 +1449,10 @@ void AudioProcessingImpl::EmptyQueuedRenderAudioLocked() { } } - while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) { - RTC_DCHECK(submodules_.echo_detector); - submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_); + if (submodules_.echo_detector) { + while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) { + submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_); + } } } @@ -1027,9 +1461,13 @@ int AudioProcessingImpl::ProcessStream(const int16_t* const src, const StreamConfig& output_config, int16_t* const dest) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame"); - RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config)); + + RETURN_ON_ERR( + HandleUnsupportedAudioFormats(src, input_config, output_config, dest)); + MaybeInitializeCapture(input_config, output_config); MutexLock lock_capture(&mutex_capture_); + DenormalDisabler denormal_disabler; if (aec_dump_) { RecordUnprocessedCaptureStream(src, input_config); @@ -1052,13 +1490,13 @@ int AudioProcessingImpl::ProcessStream(const int16_t* const src, if (aec_dump_) { RecordProcessedCaptureStream(dest, output_config); } - return kNoError; } int AudioProcessingImpl::ProcessCaptureStreamLocked() { EmptyQueuedRenderAudioLocked(); HandleCaptureRuntimeSettings(); + DenormalDisabler denormal_disabler; // Ensure that not both the AEC and AECM are active at the same time. // TODO(peah): Simplify once the public API Enable functions for these @@ -1066,6 +1504,10 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { RTC_DCHECK_LE( !!submodules_.echo_controller + !!submodules_.echo_control_mobile, 1); + data_dumper_->DumpRaw( + "applied_input_volume", + capture_.applied_input_volume.value_or(kUnspecifiedDataDumpInputVolume)); + AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity. AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get(); @@ -1076,10 +1518,16 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { /*use_split_band_data=*/false); } - if (submodules_.pre_amplifier) { - submodules_.pre_amplifier->ApplyGain(AudioFrameView( - capture_buffer->channels(), capture_buffer->num_channels(), - capture_buffer->num_frames())); + if (submodules_.capture_levels_adjuster) { + if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) { + // When the input volume is emulated, retrieve the volume applied to the + // input audio and notify that to APM so that the volume is passed to the + // active AGC. + set_stream_analog_level_locked( + submodules_.capture_levels_adjuster->GetAnalogMicGainLevel()); + } + submodules_.capture_levels_adjuster->ApplyPreLevelAdjustment( + *capture_buffer); } capture_input_rms_.Analyze(rtc::ArrayView( @@ -1095,22 +1543,25 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { levels.peak, 1, RmsLevel::kMinLevelDb, 64); } - if (submodules_.echo_controller) { - // Detect and flag any change in the analog gain. - int analog_mic_level = recommended_stream_analog_level_locked(); - capture_.echo_path_gain_change = - capture_.prev_analog_mic_level != analog_mic_level && - capture_.prev_analog_mic_level != -1; - capture_.prev_analog_mic_level = analog_mic_level; + if (capture_.applied_input_volume.has_value()) { + applied_input_volume_stats_reporter_.UpdateStatistics( + *capture_.applied_input_volume); + } - // Detect and flag any change in the pre-amplifier gain. - if (submodules_.pre_amplifier) { - float pre_amp_gain = submodules_.pre_amplifier->GetGainFactor(); + if (submodules_.echo_controller) { + // Determine if the echo path gain has changed by checking all the gains + // applied before AEC. + capture_.echo_path_gain_change = capture_.applied_input_volume_changed; + + // Detect and flag any change in the capture level adjustment pre-gain. + if (submodules_.capture_levels_adjuster) { + float pre_adjustment_gain = + submodules_.capture_levels_adjuster->GetPreAdjustmentGain(); capture_.echo_path_gain_change = capture_.echo_path_gain_change || - (capture_.prev_pre_amp_gain != pre_amp_gain && - capture_.prev_pre_amp_gain >= 0.f); - capture_.prev_pre_amp_gain = pre_amp_gain; + (capture_.prev_pre_adjustment_gain != pre_adjustment_gain && + capture_.prev_pre_adjustment_gain >= 0.0f); + capture_.prev_pre_adjustment_gain = pre_adjustment_gain; } // Detect volume change. @@ -1124,7 +1575,17 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { } if (submodules_.agc_manager) { - submodules_.agc_manager->AnalyzePreProcess(capture_buffer); + submodules_.agc_manager->AnalyzePreProcess(*capture_buffer); + } + + if (submodules_.gain_controller2 && + config_.gain_controller2.input_volume_controller.enabled) { + // Expect the volume to be available if the input controller is enabled. + RTC_DCHECK(capture_.applied_input_volume.has_value()); + if (capture_.applied_input_volume.has_value()) { + submodules_.gain_controller2->Analyze(*capture_.applied_input_volume, + *capture_buffer); + } } if (submodule_states_.CaptureMultiBandSubModulesActive() && @@ -1196,15 +1657,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { } } - if (config_.voice_detection.enabled) { - capture_.stats.voice_detected = - submodules_.voice_detector->ProcessCaptureAudio(capture_buffer); - } else { - capture_.stats.voice_detected = absl::nullopt; - } - if (submodules_.agc_manager) { - submodules_.agc_manager->Process(capture_buffer); + submodules_.agc_manager->Process(*capture_buffer); absl::optional new_digital_gain = submodules_.agc_manager->GetDigitalComressionGain(); @@ -1225,81 +1679,101 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { capture_buffer->MergeFrequencyBands(); } - if (capture_.capture_fullband_audio) { - const auto& ec = submodules_.echo_controller; - bool ec_active = ec ? ec->ActiveProcessing() : false; - // Only update the fullband buffer if the multiband processing has changed - // the signal. Keep the original signal otherwise. - if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) { - capture_buffer->CopyTo(capture_.capture_fullband_audio.get()); + if (capture_.capture_output_used) { + if (capture_.capture_fullband_audio) { + const auto& ec = submodules_.echo_controller; + bool ec_active = ec ? ec->ActiveProcessing() : false; + // Only update the fullband buffer if the multiband processing has changed + // the signal. Keep the original signal otherwise. + if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) { + capture_buffer->CopyTo(capture_.capture_fullband_audio.get()); + } + capture_buffer = capture_.capture_fullband_audio.get(); + } + + if (submodules_.echo_detector) { + submodules_.echo_detector->AnalyzeCaptureAudio( + rtc::ArrayView(capture_buffer->channels()[0], + capture_buffer->num_frames())); + } + + absl::optional voice_probability; + if (!!submodules_.voice_activity_detector) { + voice_probability = submodules_.voice_activity_detector->Analyze( + AudioFrameView(capture_buffer->channels(), + capture_buffer->num_channels(), + capture_buffer->num_frames())); + } + + if (submodules_.transient_suppressor) { + float transient_suppressor_voice_probability = 1.0f; + switch (transient_suppressor_vad_mode_) { + case TransientSuppressor::VadMode::kDefault: + if (submodules_.agc_manager) { + transient_suppressor_voice_probability = + submodules_.agc_manager->voice_probability(); + } + break; + case TransientSuppressor::VadMode::kRnnVad: + RTC_DCHECK(voice_probability.has_value()); + transient_suppressor_voice_probability = *voice_probability; + break; + case TransientSuppressor::VadMode::kNoVad: + // The transient suppressor will ignore `voice_probability`. + break; + } + float delayed_voice_probability = + submodules_.transient_suppressor->Suppress( + capture_buffer->channels()[0], capture_buffer->num_frames(), + capture_buffer->num_channels(), + capture_buffer->split_bands_const(0)[kBand0To8kHz], + capture_buffer->num_frames_per_band(), + /*reference_data=*/nullptr, /*reference_length=*/0, + transient_suppressor_voice_probability, capture_.key_pressed); + if (voice_probability.has_value()) { + *voice_probability = delayed_voice_probability; + } + } + + // Experimental APM sub-module that analyzes `capture_buffer`. + if (submodules_.capture_analyzer) { + submodules_.capture_analyzer->Analyze(capture_buffer); + } + + if (submodules_.gain_controller2) { + // TODO(bugs.webrtc.org/7494): Let AGC2 detect applied input volume + // changes. + submodules_.gain_controller2->Process( + voice_probability, capture_.applied_input_volume_changed, + capture_buffer); + } + + if (submodules_.capture_post_processor) { + submodules_.capture_post_processor->Process(capture_buffer); + } + + capture_output_rms_.Analyze(rtc::ArrayView( + capture_buffer->channels_const()[0], + capture_nonlocked_.capture_processing_format.num_frames())); + if (log_rms) { + RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak(); + RTC_HISTOGRAM_COUNTS_LINEAR( + "WebRTC.Audio.ApmCaptureOutputLevelAverageRms", levels.average, 1, + RmsLevel::kMinLevelDb, 64); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms", + levels.peak, 1, RmsLevel::kMinLevelDb, 64); + } + + // Compute echo-detector stats. + if (submodules_.echo_detector) { + auto ed_metrics = submodules_.echo_detector->GetMetrics(); + capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood; + capture_.stats.residual_echo_likelihood_recent_max = + ed_metrics.echo_likelihood_recent_max; } - capture_buffer = capture_.capture_fullband_audio.get(); } - if (config_.residual_echo_detector.enabled) { - RTC_DCHECK(submodules_.echo_detector); - submodules_.echo_detector->AnalyzeCaptureAudio(rtc::ArrayView( - capture_buffer->channels()[0], capture_buffer->num_frames())); - } - - // TODO(aluebs): Investigate if the transient suppression placement should be - // before or after the AGC. - if (submodules_.transient_suppressor) { - float voice_probability = submodules_.agc_manager.get() - ? submodules_.agc_manager->voice_probability() - : 1.f; - - submodules_.transient_suppressor->Suppress( - capture_buffer->channels()[0], capture_buffer->num_frames(), - capture_buffer->num_channels(), - capture_buffer->split_bands_const(0)[kBand0To8kHz], - capture_buffer->num_frames_per_band(), - capture_.keyboard_info.keyboard_data, - capture_.keyboard_info.num_keyboard_frames, voice_probability, - capture_.key_pressed); - } - - // Experimental APM sub-module that analyzes |capture_buffer|. - if (submodules_.capture_analyzer) { - submodules_.capture_analyzer->Analyze(capture_buffer); - } - - if (submodules_.gain_controller2) { - submodules_.gain_controller2->NotifyAnalogLevel( - recommended_stream_analog_level_locked()); - submodules_.gain_controller2->Process(capture_buffer); - } - - if (submodules_.capture_post_processor) { - submodules_.capture_post_processor->Process(capture_buffer); - } - - // The level estimator operates on the recombined data. - if (config_.level_estimation.enabled) { - submodules_.output_level_estimator->ProcessStream(*capture_buffer); - capture_.stats.output_rms_dbfs = submodules_.output_level_estimator->RMS(); - } else { - capture_.stats.output_rms_dbfs = absl::nullopt; - } - - capture_output_rms_.Analyze(rtc::ArrayView( - capture_buffer->channels_const()[0], - capture_nonlocked_.capture_processing_format.num_frames())); - if (log_rms) { - RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak(); - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelAverageRms", - levels.average, 1, RmsLevel::kMinLevelDb, 64); - RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms", - levels.peak, 1, RmsLevel::kMinLevelDb, 64); - } - - if (submodules_.agc_manager) { - int level = recommended_stream_analog_level_locked(); - data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1, - &level); - } - - // Compute echo-related stats. + // Compute echo-controller stats. if (submodules_.echo_controller) { auto ec_metrics = submodules_.echo_controller->GetMetrics(); capture_.stats.echo_return_loss = ec_metrics.echo_return_loss; @@ -1307,18 +1781,50 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { ec_metrics.echo_return_loss_enhancement; capture_.stats.delay_ms = ec_metrics.delay_ms; } - if (config_.residual_echo_detector.enabled) { - RTC_DCHECK(submodules_.echo_detector); - auto ed_metrics = submodules_.echo_detector->GetMetrics(); - capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood; - capture_.stats.residual_echo_likelihood_recent_max = - ed_metrics.echo_likelihood_recent_max; - } // Pass stats for reporting. stats_reporter_.UpdateStatistics(capture_.stats); + UpdateRecommendedInputVolumeLocked(); + if (capture_.recommended_input_volume.has_value()) { + recommended_input_volume_stats_reporter_.UpdateStatistics( + *capture_.recommended_input_volume); + } + + if (submodules_.capture_levels_adjuster) { + submodules_.capture_levels_adjuster->ApplyPostLevelAdjustment( + *capture_buffer); + + if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) { + // If the input volume emulation is used, retrieve the recommended input + // volume and set that to emulate the input volume on the next processed + // audio frame. + RTC_DCHECK(capture_.recommended_input_volume.has_value()); + submodules_.capture_levels_adjuster->SetAnalogMicGainLevel( + *capture_.recommended_input_volume); + } + } + + // Temporarily set the output to zero after the stream has been unmuted + // (capture output is again used). The purpose of this is to avoid clicks and + // artefacts in the audio that results when the processing again is + // reactivated after unmuting. + if (!capture_.capture_output_used_last_frame && + capture_.capture_output_used) { + for (size_t ch = 0; ch < capture_buffer->num_channels(); ++ch) { + rtc::ArrayView channel_view(capture_buffer->channels()[ch], + capture_buffer->num_frames()); + std::fill(channel_view.begin(), channel_view.end(), 0.f); + } + } + capture_.capture_output_used_last_frame = capture_.capture_output_used; + capture_.was_stream_delay_set = false; + + data_dumper_->DumpRaw("recommended_input_volume", + capture_.recommended_input_volume.value_or( + kUnspecifiedDataDumpInputVolume)); + return kNoError; } @@ -1327,6 +1833,15 @@ int AudioProcessingImpl::AnalyzeReverseStream( const StreamConfig& reverse_config) { TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig"); MutexLock lock(&mutex_render_); + DenormalDisabler denormal_disabler; + RTC_DCHECK(data); + for (size_t i = 0; i < reverse_config.num_channels(); ++i) { + RTC_DCHECK(data[i]); + } + RETURN_ON_ERR( + AudioFormatValidityToErrorCode(ValidateAudioFormat(reverse_config))); + + MaybeInitializeRender(reverse_config, reverse_config); return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); } @@ -1336,7 +1851,14 @@ int AudioProcessingImpl::ProcessReverseStream(const float* const* src, float* const* dest) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); MutexLock lock(&mutex_render_); + DenormalDisabler denormal_disabler; + RETURN_ON_ERR( + HandleUnsupportedAudioFormats(src, input_config, output_config, dest)); + + MaybeInitializeRender(input_config, output_config); + RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config)); + if (submodule_states_.RenderMultiBandProcessingActive() || submodule_states_.RenderFullBandProcessingActive()) { render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), @@ -1357,22 +1879,6 @@ int AudioProcessingImpl::AnalyzeReverseStreamLocked( const float* const* src, const StreamConfig& input_config, const StreamConfig& output_config) { - if (src == nullptr) { - return kNullPointerError; - } - - if (input_config.num_channels() == 0) { - return kBadNumberChannelsError; - } - - ProcessingConfig processing_config = formats_.api_format; - processing_config.reverse_input_stream() = input_config; - processing_config.reverse_output_stream() = output_config; - - RETURN_ON_ERR(MaybeInitializeRender(processing_config)); - RTC_DCHECK_EQ(input_config.num_frames(), - formats_.api_format.reverse_input_stream().num_frames()); - if (aec_dump_) { const size_t channel_size = formats_.api_format.reverse_input_stream().num_frames(); @@ -1392,26 +1898,12 @@ int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src, int16_t* const dest) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); - if (input_config.num_channels() <= 0) { - return AudioProcessing::Error::kBadNumberChannelsError; - } - MutexLock lock(&mutex_render_); - ProcessingConfig processing_config = formats_.api_format; - processing_config.reverse_input_stream().set_sample_rate_hz( - input_config.sample_rate_hz()); - processing_config.reverse_input_stream().set_num_channels( - input_config.num_channels()); - processing_config.reverse_output_stream().set_sample_rate_hz( - output_config.sample_rate_hz()); - processing_config.reverse_output_stream().set_num_channels( - output_config.num_channels()); + DenormalDisabler denormal_disabler; - RETURN_ON_ERR(MaybeInitializeRender(processing_config)); - if (input_config.num_frames() != - formats_.api_format.reverse_input_stream().num_frames()) { - return kBadDataLengthError; - } + RETURN_ON_ERR( + HandleUnsupportedAudioFormats(src, input_config, output_config, dest)); + MaybeInitializeRender(input_config, output_config); if (aec_dump_) { aec_dump_->WriteRenderStreamMessage(src, input_config.num_frames(), @@ -1431,6 +1923,7 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. HandleRenderRuntimeSettings(); + DenormalDisabler denormal_disabler; if (submodules_.render_pre_processor) { submodules_.render_pre_processor->Process(render_buffer); @@ -1497,13 +1990,13 @@ bool AudioProcessingImpl::GetLinearAecOutput( rtc::ArrayView channel_view = rtc::ArrayView(linear_aec_buffer->channels_const()[ch], linear_aec_buffer->num_frames()); - std::copy(channel_view.begin(), channel_view.end(), - linear_output[ch].begin()); + FloatS16ToFloat(channel_view.data(), channel_view.size(), + linear_output[ch].data()); } return true; } RTC_LOG(LS_ERROR) << "No linear AEC output available"; - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return false; } @@ -1519,35 +2012,77 @@ void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { void AudioProcessingImpl::set_stream_analog_level(int level) { MutexLock lock_capture(&mutex_capture_); + set_stream_analog_level_locked(level); +} + +void AudioProcessingImpl::set_stream_analog_level_locked(int level) { + capture_.applied_input_volume_changed = + capture_.applied_input_volume.has_value() && + *capture_.applied_input_volume != level; + capture_.applied_input_volume = level; + + // Invalidate any previously recommended input volume which will be updated by + // `ProcessStream()`. + capture_.recommended_input_volume = absl::nullopt; if (submodules_.agc_manager) { submodules_.agc_manager->set_stream_analog_level(level); - data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level", - 1, &level); - } else if (submodules_.gain_control) { + return; + } + + if (submodules_.gain_control) { int error = submodules_.gain_control->set_stream_analog_level(level); RTC_DCHECK_EQ(kNoError, error); - } else { - capture_.cached_stream_analog_level_ = level; + return; } } int AudioProcessingImpl::recommended_stream_analog_level() const { MutexLock lock_capture(&mutex_capture_); - return recommended_stream_analog_level_locked(); -} - -int AudioProcessingImpl::recommended_stream_analog_level_locked() const { - if (submodules_.agc_manager) { - return submodules_.agc_manager->stream_analog_level(); - } else if (submodules_.gain_control) { - return submodules_.gain_control->stream_analog_level(); - } else { - return capture_.cached_stream_analog_level_; + if (!capture_.applied_input_volume.has_value()) { + RTC_LOG(LS_ERROR) << "set_stream_analog_level has not been called"; } + // Input volume to recommend when `set_stream_analog_level()` is not called. + constexpr int kFallBackInputVolume = 255; + // When APM has no input volume to recommend, return the latest applied input + // volume that has been observed in order to possibly produce no input volume + // change. If no applied input volume has been observed, return a fall-back + // value. + return capture_.recommended_input_volume.value_or( + capture_.applied_input_volume.value_or(kFallBackInputVolume)); } -bool AudioProcessingImpl::CreateAndAttachAecDump(const std::string& file_name, +void AudioProcessingImpl::UpdateRecommendedInputVolumeLocked() { + if (!capture_.applied_input_volume.has_value()) { + // When `set_stream_analog_level()` is not called, no input level can be + // recommended. + capture_.recommended_input_volume = absl::nullopt; + return; + } + + if (submodules_.agc_manager) { + capture_.recommended_input_volume = + submodules_.agc_manager->recommended_analog_level(); + return; + } + + if (submodules_.gain_control) { + capture_.recommended_input_volume = + submodules_.gain_control->stream_analog_level(); + return; + } + + if (submodules_.gain_controller2 && + config_.gain_controller2.input_volume_controller.enabled) { + capture_.recommended_input_volume = + submodules_.gain_controller2->recommended_input_volume(); + return; + } + + capture_.recommended_input_volume = capture_.applied_input_volume; +} + +bool AudioProcessingImpl::CreateAndAttachAecDump(absl::string_view file_name, int64_t max_log_size_bytes, rtc::TaskQueue* worker_queue) { std::unique_ptr aec_dump = @@ -1597,14 +2132,6 @@ void AudioProcessingImpl::DetachAecDump() { } } -void AudioProcessingImpl::MutateConfig( - rtc::FunctionView mutator) { - MutexLock lock_render(&mutex_render_); - MutexLock lock_capture(&mutex_capture_); - mutator(&config_); - ApplyConfig(config_); -} - AudioProcessing::Config AudioProcessingImpl::GetConfig() const { MutexLock lock_render(&mutex_render_); MutexLock lock_capture(&mutex_capture_); @@ -1614,26 +2141,40 @@ AudioProcessing::Config AudioProcessingImpl::GetConfig() const { bool AudioProcessingImpl::UpdateActiveSubmoduleStates() { return submodule_states_.Update( config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile, - config_.residual_echo_detector.enabled, !!submodules_.noise_suppressor, - !!submodules_.gain_control, !!submodules_.gain_controller2, - config_.pre_amplifier.enabled, capture_nonlocked_.echo_controller_enabled, - config_.voice_detection.enabled, !!submodules_.transient_suppressor); + !!submodules_.noise_suppressor, !!submodules_.gain_control, + !!submodules_.gain_controller2, !!submodules_.voice_activity_detector, + config_.pre_amplifier.enabled || config_.capture_level_adjustment.enabled, + capture_nonlocked_.echo_controller_enabled, + !!submodules_.transient_suppressor); } void AudioProcessingImpl::InitializeTransientSuppressor() { - if (config_.transient_suppression.enabled) { + // Choose the VAD mode for TS and detect a VAD mode change. + const TransientSuppressor::VadMode previous_vad_mode = + transient_suppressor_vad_mode_; + transient_suppressor_vad_mode_ = TransientSuppressor::VadMode::kDefault; + if (UseApmVadSubModule(config_, gain_controller2_experiment_params_)) { + transient_suppressor_vad_mode_ = TransientSuppressor::VadMode::kRnnVad; + } + const bool vad_mode_changed = + previous_vad_mode != transient_suppressor_vad_mode_; + + if (config_.transient_suppression.enabled && + !constants_.transient_suppressor_forced_off) { // Attempt to create a transient suppressor, if one is not already created. - if (!submodules_.transient_suppressor) { - submodules_.transient_suppressor = - CreateTransientSuppressor(submodule_creation_overrides_); - } - if (submodules_.transient_suppressor) { + if (!submodules_.transient_suppressor || vad_mode_changed) { + submodules_.transient_suppressor = CreateTransientSuppressor( + submodule_creation_overrides_, transient_suppressor_vad_mode_, + proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate, + num_proc_channels()); + if (!submodules_.transient_suppressor) { + RTC_LOG(LS_WARNING) + << "No transient suppressor created (probably disabled)"; + } + } else { submodules_.transient_suppressor->Initialize( proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate, num_proc_channels()); - } else { - RTC_LOG(LS_WARNING) - << "No transient suppressor created (probably disabled)"; } } else { submodules_.transient_suppressor.reset(); @@ -1666,14 +2207,6 @@ void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) { } } -void AudioProcessingImpl::InitializeVoiceDetector() { - if (config_.voice_detection.enabled) { - submodules_.voice_detector = std::make_unique( - proc_split_sample_rate_hz(), VoiceDetection::kVeryLowLikelihood); - } else { - submodules_.voice_detector.reset(); - } -} void AudioProcessingImpl::InitializeEchoController() { bool use_echo_controller = echo_control_factory_ || @@ -1686,14 +2219,14 @@ void AudioProcessingImpl::InitializeEchoController() { proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels()); RTC_DCHECK(submodules_.echo_controller); } else { - EchoCanceller3Config config = - use_setup_specific_default_aec3_config_ - ? EchoCanceller3::CreateDefaultConfig(num_reverse_channels(), - num_proc_channels()) - : EchoCanceller3Config(); + EchoCanceller3Config config; + absl::optional multichannel_config; + if (use_setup_specific_default_aec3_config_) { + multichannel_config = EchoCanceller3::CreateDefaultMultichannelConfig(); + } submodules_.echo_controller = std::make_unique( - config, proc_sample_rate_hz(), num_reverse_channels(), - num_proc_channels()); + config, multichannel_config, proc_sample_rate_hz(), + num_reverse_channels(), num_proc_channels()); } // Setup the storage for returning the linear AEC output. @@ -1754,19 +2287,32 @@ void AudioProcessingImpl::InitializeEchoController() { } void AudioProcessingImpl::InitializeGainController1() { + if (config_.gain_controller2.enabled && + config_.gain_controller2.input_volume_controller.enabled && + config_.gain_controller1.enabled && + (config_.gain_controller1.mode == + AudioProcessing::Config::GainController1::kAdaptiveAnalog || + config_.gain_controller1.analog_gain_controller.enabled)) { + RTC_LOG(LS_ERROR) << "APM configuration not valid: " + << "Multiple input volume controllers enabled."; + } + if (!config_.gain_controller1.enabled) { submodules_.agc_manager.reset(); submodules_.gain_control.reset(); return; } + RTC_HISTOGRAM_BOOLEAN( + "WebRTC.Audio.GainController.Analog.Enabled", + config_.gain_controller1.analog_gain_controller.enabled); + if (!submodules_.gain_control) { submodules_.gain_control.reset(new GainControlImpl()); } submodules_.gain_control->Initialize(num_proc_channels(), proc_sample_rate_hz()); - if (!config_.gain_controller1.analog_gain_controller.enabled) { int error = submodules_.gain_control->set_mode( Agc1ConfigModeToInterfaceMode(config_.gain_controller1.mode)); @@ -1780,9 +2326,10 @@ void AudioProcessingImpl::InitializeGainController1() { error = submodules_.gain_control->enable_limiter( config_.gain_controller1.enable_limiter); RTC_DCHECK_EQ(kNoError, error); + constexpr int kAnalogLevelMinimum = 0; + constexpr int kAnalogLevelMaximum = 255; error = submodules_.gain_control->set_analog_level_limits( - config_.gain_controller1.analog_level_minimum, - config_.gain_controller1.analog_level_maximum); + kAnalogLevelMinimum, kAnalogLevelMaximum); RTC_DCHECK_EQ(kNoError, error); submodules_.agc_manager.reset(); @@ -1791,45 +2338,66 @@ void AudioProcessingImpl::InitializeGainController1() { if (!submodules_.agc_manager.get() || submodules_.agc_manager->num_channels() != - static_cast(num_proc_channels()) || - submodules_.agc_manager->sample_rate_hz() != - capture_nonlocked_.split_rate) { + static_cast(num_proc_channels())) { int stream_analog_level = -1; const bool re_creation = !!submodules_.agc_manager; if (re_creation) { - stream_analog_level = submodules_.agc_manager->stream_analog_level(); + stream_analog_level = submodules_.agc_manager->recommended_analog_level(); } submodules_.agc_manager.reset(new AgcManagerDirect( - num_proc_channels(), - config_.gain_controller1.analog_gain_controller.startup_min_volume, - config_.gain_controller1.analog_gain_controller.clipped_level_min, - config_.gain_controller1.analog_gain_controller - .enable_agc2_level_estimator, - !config_.gain_controller1.analog_gain_controller - .enable_digital_adaptive, - capture_nonlocked_.split_rate)); + num_proc_channels(), config_.gain_controller1.analog_gain_controller)); if (re_creation) { submodules_.agc_manager->set_stream_analog_level(stream_analog_level); } } submodules_.agc_manager->Initialize(); - submodules_.agc_manager->SetupDigitalGainControl( - submodules_.gain_control.get()); - submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted); + submodules_.agc_manager->SetupDigitalGainControl(*submodules_.gain_control); + submodules_.agc_manager->HandleCaptureOutputUsedChange( + capture_.capture_output_used); } void AudioProcessingImpl::InitializeGainController2() { - if (config_.gain_controller2.enabled) { - if (!submodules_.gain_controller2) { - // TODO(alessiob): Move the injected gain controller once injection is - // implemented. - submodules_.gain_controller2.reset(new GainController2()); - } - - submodules_.gain_controller2->Initialize(proc_fullband_sample_rate_hz()); - submodules_.gain_controller2->ApplyConfig(config_.gain_controller2); - } else { + if (!config_.gain_controller2.enabled) { submodules_.gain_controller2.reset(); + return; + } + // Override the input volume controller configuration if the AGC2 experiment + // is running and its parameters require to fully switch the gain control to + // AGC2. + const bool input_volume_controller_config_overridden = + gain_controller2_experiment_params_.has_value() && + gain_controller2_experiment_params_->agc2_config.has_value(); + const InputVolumeController::Config input_volume_controller_config = + input_volume_controller_config_overridden + ? gain_controller2_experiment_params_->agc2_config + ->input_volume_controller + : InputVolumeController::Config{}; + // If the APM VAD sub-module is not used, let AGC2 use its internal VAD. + const bool use_internal_vad = + !UseApmVadSubModule(config_, gain_controller2_experiment_params_); + submodules_.gain_controller2 = std::make_unique( + config_.gain_controller2, input_volume_controller_config, + proc_fullband_sample_rate_hz(), num_proc_channels(), use_internal_vad); + submodules_.gain_controller2->SetCaptureOutputUsed( + capture_.capture_output_used); +} + +void AudioProcessingImpl::InitializeVoiceActivityDetector() { + if (!UseApmVadSubModule(config_, gain_controller2_experiment_params_)) { + submodules_.voice_activity_detector.reset(); + return; + } + + if (!submodules_.voice_activity_detector) { + RTC_DCHECK(!!submodules_.gain_controller2); + // TODO(bugs.webrtc.org/13663): Cache CPU features in APM and use here. + submodules_.voice_activity_detector = + std::make_unique( + submodules_.gain_controller2->GetCpuFeatures(), + proc_fullband_sample_rate_hz()); + } else { + submodules_.voice_activity_detector->Initialize( + proc_fullband_sample_rate_hz()); } } @@ -1850,9 +2418,8 @@ void AudioProcessingImpl::InitializeNoiseSuppressor() { return NsConfig::SuppressionLevel::k18dB; case NoiseSuppresionConfig::kVeryHigh: return NsConfig::SuppressionLevel::k21dB; - default: - RTC_NOTREACHED(); } + RTC_CHECK_NOTREACHED(); }; NsConfig cfg; @@ -1862,20 +2429,36 @@ void AudioProcessingImpl::InitializeNoiseSuppressor() { } } -void AudioProcessingImpl::InitializePreAmplifier() { - if (config_.pre_amplifier.enabled) { - submodules_.pre_amplifier.reset( - new GainApplier(true, config_.pre_amplifier.fixed_gain_factor)); +void AudioProcessingImpl::InitializeCaptureLevelsAdjuster() { + if (config_.pre_amplifier.enabled || + config_.capture_level_adjustment.enabled) { + // Use both the pre-amplifier and the capture level adjustment gains as + // pre-gains. + float pre_gain = 1.f; + if (config_.pre_amplifier.enabled) { + pre_gain *= config_.pre_amplifier.fixed_gain_factor; + } + if (config_.capture_level_adjustment.enabled) { + pre_gain *= config_.capture_level_adjustment.pre_gain_factor; + } + + submodules_.capture_levels_adjuster = + std::make_unique( + config_.capture_level_adjustment.analog_mic_gain_emulation.enabled, + config_.capture_level_adjustment.analog_mic_gain_emulation + .initial_level, + pre_gain, config_.capture_level_adjustment.post_gain_factor); } else { - submodules_.pre_amplifier.reset(); + submodules_.capture_levels_adjuster.reset(); } } void AudioProcessingImpl::InitializeResidualEchoDetector() { - RTC_DCHECK(submodules_.echo_detector); - submodules_.echo_detector->Initialize( - proc_fullband_sample_rate_hz(), 1, - formats_.render_processing_format.sample_rate_hz(), 1); + if (submodules_.echo_detector) { + submodules_.echo_detector->Initialize( + proc_fullband_sample_rate_hz(), 1, + formats_.render_processing_format.sample_rate_hz(), 1); + } } void AudioProcessingImpl::InitializeAnalyzer() { @@ -1908,10 +2491,6 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) { std::string experiments_description = ""; // TODO(peah): Add semicolon-separated concatenations of experiment // descriptions for other submodules. - if (config_.gain_controller1.analog_gain_controller.clipped_level_min != - kClippedLevelMin) { - experiments_description += "AgcClippingLevelExperiment;"; - } if (!!submodules_.capture_post_processor) { experiments_description += "CapturePostProcessor;"; } @@ -2020,36 +2599,26 @@ void AudioProcessingImpl::RecordAudioProcessingState() { AecDump::AudioProcessingState audio_proc_state; audio_proc_state.delay = capture_nonlocked_.stream_delay_ms; audio_proc_state.drift = 0; - audio_proc_state.level = recommended_stream_analog_level_locked(); + audio_proc_state.applied_input_volume = capture_.applied_input_volume; audio_proc_state.keypress = capture_.key_pressed; aec_dump_->AddAudioProcessingState(audio_proc_state); } AudioProcessingImpl::ApmCaptureState::ApmCaptureState() : was_stream_delay_set(false), - output_will_be_muted(false), + capture_output_used(true), + capture_output_used_last_frame(true), key_pressed(false), capture_processing_format(kSampleRate16kHz), split_rate(kSampleRate16kHz), echo_path_gain_change(false), - prev_analog_mic_level(-1), - prev_pre_amp_gain(-1.f), + prev_pre_adjustment_gain(-1.0f), playout_volume(-1), - prev_playout_volume(-1) {} + prev_playout_volume(-1), + applied_input_volume_changed(false) {} AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; -void AudioProcessingImpl::ApmCaptureState::KeyboardInfo::Extract( - const float* const* data, - const StreamConfig& stream_config) { - if (stream_config.has_keyboard()) { - keyboard_data = data[stream_config.num_channels()]; - } else { - keyboard_data = NULL; - } - num_keyboard_frames = stream_config.num_frames(); -} - AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index d0eec0e..fe80e0d 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -13,16 +13,22 @@ #include +#include #include #include #include #include +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" #include "api/function_view.h" #include "modules/audio_processing/aec3/echo_canceller3.h" #include "modules/audio_processing/agc/agc_manager_direct.h" #include "modules/audio_processing/agc/gain_control.h" +#include "modules/audio_processing/agc2/input_volume_stats_reporter.h" #include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h" #include "modules/audio_processing/echo_control_mobile_impl.h" #include "modules/audio_processing/gain_control_impl.h" #include "modules/audio_processing/gain_controller2.h" @@ -31,14 +37,11 @@ #include "modules/audio_processing/include/audio_frame_proxies.h" #include "modules/audio_processing/include/audio_processing.h" #include "modules/audio_processing/include/audio_processing_statistics.h" -#include "modules/audio_processing/level_estimator.h" #include "modules/audio_processing/ns/noise_suppressor.h" #include "modules/audio_processing/optionally_built_submodule_creators.h" #include "modules/audio_processing/render_queue_item_verifier.h" -#include "modules/audio_processing/residual_echo_detector.h" #include "modules/audio_processing/rms_level.h" #include "modules/audio_processing/transient/transient_suppressor.h" -#include "modules/audio_processing/voice_detection.h" #include "rtc_base/gtest_prod_util.h" #include "rtc_base/ignore_wundef.h" #include "rtc_base/swap_queue.h" @@ -50,13 +53,16 @@ namespace webrtc { class ApmDataDumper; class AudioConverter; +constexpr int RuntimeSettingQueueSize() { + return 100; +} + class AudioProcessingImpl : public AudioProcessing { public: // Methods forcing APM to run in a single-threaded manner. // Acquires both the render and capture locks. - explicit AudioProcessingImpl(const webrtc::Config& config); - // AudioProcessingImpl takes ownership of capture post processor. - AudioProcessingImpl(const webrtc::Config& config, + AudioProcessingImpl(); + AudioProcessingImpl(const AudioProcessing::Config& config, std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, std::unique_ptr echo_control_factory, @@ -64,15 +70,9 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr capture_analyzer); ~AudioProcessingImpl() override; int Initialize() override; - int Initialize(int capture_input_sample_rate_hz, - int capture_output_sample_rate_hz, - int render_sample_rate_hz, - ChannelLayout capture_input_layout, - ChannelLayout capture_output_layout, - ChannelLayout render_input_layout) override; int Initialize(const ProcessingConfig& processing_config) override; void ApplyConfig(const AudioProcessing::Config& config) override; - bool CreateAndAttachAecDump(const std::string& file_name, + bool CreateAndAttachAecDump(absl::string_view file_name, int64_t max_log_size_bytes, rtc::TaskQueue* worker_queue) override; bool CreateAndAttachAecDump(FILE* handle, @@ -82,6 +82,7 @@ class AudioProcessingImpl : public AudioProcessing { void AttachAecDump(std::unique_ptr aec_dump) override; void DetachAecDump() override; void SetRuntimeSetting(RuntimeSetting setting) override; + bool PostRuntimeSetting(RuntimeSetting setting) override; // Capture-side exclusive methods possibly running APM in a // multi-threaded manner. Acquire the capture lock. @@ -96,6 +97,8 @@ class AudioProcessingImpl : public AudioProcessing { bool GetLinearAecOutput( rtc::ArrayView> linear_output) const override; void set_output_will_be_muted(bool muted) override; + void HandleCaptureOutputUsedSetting(bool capture_output_used) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); int set_stream_delay_ms(int delay) override; void set_stream_key_pressed(bool key_pressed) override; void set_stream_analog_level(int level) override; @@ -133,14 +136,17 @@ class AudioProcessingImpl : public AudioProcessing { return stats_reporter_.GetStatistics(); } - // TODO(peah): Remove MutateConfig once the new API allows that. - void MutateConfig(rtc::FunctionView mutator); AudioProcessing::Config GetConfig() const override; protected: // Overridden in a mock. virtual void InitializeLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_); + void AssertLockedForTest() + RTC_ASSERT_EXCLUSIVE_LOCK(mutex_render_, mutex_capture_) { + mutex_render_.AssertHeld(); + mutex_capture_.AssertHeld(); + } private: // TODO(peah): These friend classes should be removed as soon as the new @@ -154,30 +160,80 @@ class AudioProcessingImpl : public AudioProcessing { ReinitializeTransientSuppressor); FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest, BitexactWithDisabledModules); + FRIEND_TEST_ALL_PREFIXES( + AudioProcessingImplGainController2FieldTrialParametrizedTest, + ConfigAdjustedWhenExperimentEnabled); - int recommended_stream_analog_level_locked() const + void set_stream_analog_level_locked(int level) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); + void UpdateRecommendedInputVolumeLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); void OverrideSubmoduleCreationForTesting( const ApmSubmoduleCreationOverrides& overrides); // Class providing thread-safe message pipe functionality for - // |runtime_settings_|. + // `runtime_settings_`. class RuntimeSettingEnqueuer { public: explicit RuntimeSettingEnqueuer( SwapQueue* runtime_settings); ~RuntimeSettingEnqueuer(); - void Enqueue(RuntimeSetting setting); + + // Enqueue setting and return whether the setting was successfully enqueued. + bool Enqueue(RuntimeSetting setting); private: SwapQueue& runtime_settings_; }; - std::unique_ptr data_dumper_; - static int instance_count_; + const std::unique_ptr data_dumper_; + static std::atomic instance_count_; const bool use_setup_specific_default_aec3_config_; + // Parameters for the "GainController2" experiment which determines whether + // the following APM sub-modules are created and, if so, their configurations: + // AGC2 (`gain_controller2`), AGC1 (`gain_control`, `agc_manager`) and TS + // (`transient_suppressor`). + // TODO(bugs.webrtc.org/7494): Remove when the "WebRTC-Audio-GainController2" + // field trial is removed. + struct GainController2ExperimentParams { + struct Agc2Config { + InputVolumeController::Config input_volume_controller; + AudioProcessing::Config::GainController2::AdaptiveDigital + adaptive_digital_controller; + }; + // When `agc2_config` is specified, all gain control switches to AGC2 and + // the configuration is overridden. + absl::optional agc2_config; + // When true, the transient suppressor submodule is never created regardless + // of the APM configuration. + bool disallow_transient_suppressor_usage; + }; + // Specified when the "WebRTC-Audio-GainController2" field trial is specified. + // TODO(bugs.webrtc.org/7494): Remove when the "WebRTC-Audio-GainController2" + // field trial is removed. + const absl::optional + gain_controller2_experiment_params_; + + // Parses the "WebRTC-Audio-GainController2" field trial. If disabled, returns + // an unspecified value. + static absl::optional + GetGainController2ExperimentParams(); + + // When `experiment_params` is specified, returns an APM configuration + // modified according to the experiment parameters. Otherwise returns + // `config`. + static AudioProcessing::Config AdjustConfig( + const AudioProcessing::Config& config, + const absl::optional& experiment_params); + // Returns true if the APM VAD sub-module should be used. + static bool UseApmVadSubModule( + const AudioProcessing::Config& config, + const absl::optional& experiment_params); + + TransientSuppressor::VadMode transient_suppressor_vad_mode_; + SwapQueue capture_runtime_settings_; SwapQueue render_runtime_settings_; @@ -185,7 +241,7 @@ class AudioProcessingImpl : public AudioProcessing { RuntimeSettingEnqueuer render_runtime_settings_enqueuer_; // EchoControl factory. - std::unique_ptr echo_control_factory_; + const std::unique_ptr echo_control_factory_; class SubmoduleStates { public: @@ -195,13 +251,12 @@ class AudioProcessingImpl : public AudioProcessing { // Updates the submodule state and returns true if it has changed. bool Update(bool high_pass_filter_enabled, bool mobile_echo_controller_enabled, - bool residual_echo_detector_enabled, bool noise_suppressor_enabled, bool adaptive_gain_controller_enabled, bool gain_controller2_enabled, - bool pre_amplifier_enabled, + bool voice_activity_detector_enabled, + bool gain_adjustment_enabled, bool echo_controller_enabled, - bool voice_detector_enabled, bool transient_suppressor_enabled); bool CaptureMultiBandSubModulesActive() const; bool CaptureMultiBandProcessingPresent() const; @@ -219,13 +274,12 @@ class AudioProcessingImpl : public AudioProcessing { const bool capture_analyzer_enabled_ = false; bool high_pass_filter_enabled_ = false; bool mobile_echo_controller_enabled_ = false; - bool residual_echo_detector_enabled_ = false; bool noise_suppressor_enabled_ = false; bool adaptive_gain_controller_enabled_ = false; + bool voice_activity_detector_enabled_ = false; bool gain_controller2_enabled_ = false; - bool pre_amplifier_enabled_ = false; + bool gain_adjustment_enabled_ = false; bool echo_controller_enabled_ = false; - bool voice_detector_enabled_ = false; bool transient_suppressor_enabled_ = false; bool first_update_ = true; }; @@ -236,12 +290,13 @@ class AudioProcessingImpl : public AudioProcessing { // capture thread blocks the render thread. // Called by render: Holds the render lock when reading the format struct and // acquires both locks if reinitialization is required. - int MaybeInitializeRender(const ProcessingConfig& processing_config) + void MaybeInitializeRender(const StreamConfig& input_config, + const StreamConfig& output_config) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_); - // Called by capture: Holds the capture lock when reading the format struct - // and acquires both locks if reinitialization is needed. - int MaybeInitializeCapture(const StreamConfig& input_config, - const StreamConfig& output_config); + // Called by capture: Acquires and releases the capture lock to read the + // format struct and acquires both locks if reinitialization is needed. + void MaybeInitializeCapture(const StreamConfig& input_config, + const StreamConfig& output_config); // Method for updating the state keeping track of the active submodules. // Returns a bool indicating whether the state has changed. @@ -250,24 +305,33 @@ class AudioProcessingImpl : public AudioProcessing { // Methods requiring APM running in a single-threaded manner, requiring both // the render and capture lock to be acquired. - int InitializeLocked(const ProcessingConfig& config) + void InitializeLocked(const ProcessingConfig& config) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_); void InitializeResidualEchoDetector() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_); void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_); - // Initializations of capture-only submodules, requiring the capture lock + // Initializations of capture-only sub-modules, requiring the capture lock // already acquired. void InitializeHighPassFilter(bool forced_reset) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); - void InitializeVoiceDetector() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); void InitializeGainController1() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); void InitializeTransientSuppressor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); + // Initializes the `GainController2` sub-module. If the sub-module is enabled, + // recreates it. void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); + // Initializes the `VoiceActivityDetectorWrapper` sub-module. If the + // sub-module is enabled, recreates it. Call `InitializeGainController2()` + // first. + // TODO(bugs.webrtc.org/13663): Remove if TS is removed otherwise remove call + // order requirement - i.e., decouple from `InitializeGainController2()`. + void InitializeVoiceActivityDetector() + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); void InitializeNoiseSuppressor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); - void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); + void InitializeCaptureLevelsAdjuster() + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); @@ -301,7 +365,6 @@ class AudioProcessingImpl : public AudioProcessing { // Render-side exclusive methods possibly running APM in a multi-threaded // manner that are called with the render lock already acquired. - // TODO(ekm): Remove once all clients updated to new interface. int AnalyzeReverseStreamLocked(const float* const* src, const StreamConfig& input_config, const StreamConfig& output_config) @@ -310,8 +373,8 @@ class AudioProcessingImpl : public AudioProcessing { // Collects configuration settings from public and private // submodules to be saved as an audioproc::Config message on the - // AecDump if it is attached. If not |forced|, only writes the current - // config if it is different from the last saved one; if |forced|, + // AecDump if it is attached. If not `forced`, only writes the current + // config if it is different from the last saved one; if `forced`, // writes the config regardless of the last saved. void WriteAecDumpConfigMessage(bool forced) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); @@ -339,6 +402,12 @@ class AudioProcessingImpl : public AudioProcessing { void RecordAudioProcessingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); + // Ensures that overruns in the capture runtime settings queue is properly + // handled by the code, providing safe-fallbacks to mitigate the implications + // of any settings being missed. + void HandleOverrunInCaptureRuntimeSettingsQueue() + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_); + // AecDump instance used for optionally logging APM config, input // and output to file in the AEC-dump format defined in debug.proto. std::unique_ptr aec_dump_; @@ -372,21 +441,20 @@ class AudioProcessingImpl : public AudioProcessing { render_pre_processor(std::move(render_pre_processor)), capture_analyzer(std::move(capture_analyzer)) {} // Accessed internally from capture or during initialization. + const rtc::scoped_refptr echo_detector; + const std::unique_ptr capture_post_processor; + const std::unique_ptr render_pre_processor; + const std::unique_ptr capture_analyzer; std::unique_ptr agc_manager; std::unique_ptr gain_control; std::unique_ptr gain_controller2; + std::unique_ptr voice_activity_detector; std::unique_ptr high_pass_filter; - rtc::scoped_refptr echo_detector; std::unique_ptr echo_controller; std::unique_ptr echo_control_mobile; std::unique_ptr noise_suppressor; std::unique_ptr transient_suppressor; - std::unique_ptr capture_post_processor; - std::unique_ptr render_pre_processor; - std::unique_ptr pre_amplifier; - std::unique_ptr capture_analyzer; - std::unique_ptr output_level_estimator; - std::unique_ptr voice_detector; + std::unique_ptr capture_levels_adjuster; } submodules_; // State that is written to while holding both the render and capture locks @@ -397,10 +465,10 @@ class AudioProcessingImpl : public AudioProcessing { struct ApmFormatState { ApmFormatState() : // Format of processing streams at input/output call sites. - api_format({{{kSampleRate16kHz, 1, false}, - {kSampleRate16kHz, 1, false}, - {kSampleRate16kHz, 1, false}, - {kSampleRate16kHz, 1, false}}}), + api_format({{{kSampleRate16kHz, 1}, + {kSampleRate16kHz, 1}, + {kSampleRate16kHz, 1}, + {kSampleRate16kHz, 1}}}), render_processing_format(kSampleRate16kHz, 1) {} ProcessingConfig api_format; StreamConfig render_processing_format; @@ -410,20 +478,28 @@ class AudioProcessingImpl : public AudioProcessing { const struct ApmConstants { ApmConstants(bool multi_channel_render_support, bool multi_channel_capture_support, - bool enforce_split_band_hpf) + bool enforce_split_band_hpf, + bool minimize_processing_for_unused_output, + bool transient_suppressor_forced_off) : multi_channel_render_support(multi_channel_render_support), multi_channel_capture_support(multi_channel_capture_support), - enforce_split_band_hpf(enforce_split_band_hpf) {} + enforce_split_band_hpf(enforce_split_band_hpf), + minimize_processing_for_unused_output( + minimize_processing_for_unused_output), + transient_suppressor_forced_off(transient_suppressor_forced_off) {} bool multi_channel_render_support; bool multi_channel_capture_support; bool enforce_split_band_hpf; + bool minimize_processing_for_unused_output; + bool transient_suppressor_forced_off; } constants_; struct ApmCaptureState { ApmCaptureState(); ~ApmCaptureState(); bool was_stream_delay_set; - bool output_will_be_muted; + bool capture_output_used; + bool capture_output_used_last_frame; bool key_pressed; std::unique_ptr capture_audio; std::unique_ptr capture_fullband_audio; @@ -434,17 +510,18 @@ class AudioProcessingImpl : public AudioProcessing { StreamConfig capture_processing_format; int split_rate; bool echo_path_gain_change; - int prev_analog_mic_level; - float prev_pre_amp_gain; + float prev_pre_adjustment_gain; int playout_volume; int prev_playout_volume; AudioProcessingStats stats; - struct KeyboardInfo { - void Extract(const float* const* data, const StreamConfig& stream_config); - size_t num_keyboard_frames = 0; - const float* keyboard_data = nullptr; - } keyboard_info; - int cached_stream_analog_level_ = 0; + // Input volume applied on the audio input device when the audio is + // acquired. Unspecified when unknown. + absl::optional applied_input_volume; + bool applied_input_volume_changed; + // Recommended input volume to apply on the audio input device the next time + // that audio is acquired. Unspecified when no input volume can be + // recommended. + absl::optional recommended_input_volume; } capture_ RTC_GUARDED_BY(mutex_capture_); struct ApmCaptureNonLockedState { @@ -505,6 +582,11 @@ class AudioProcessingImpl : public AudioProcessing { RmsLevel capture_output_rms_ RTC_GUARDED_BY(mutex_capture_); int capture_rms_interval_counter_ RTC_GUARDED_BY(mutex_capture_) = 0; + InputVolumeStatsReporter applied_input_volume_stats_reporter_ + RTC_GUARDED_BY(mutex_capture_); + InputVolumeStatsReporter recommended_input_volume_stats_reporter_ + RTC_GUARDED_BY(mutex_capture_); + // Lock protection not needed. std::unique_ptr< SwapQueue, RenderQueueItemVerifier>> diff --git a/webrtc/modules/audio_processing/capture_levels_adjuster/BUILD.gn b/webrtc/modules/audio_processing/capture_levels_adjuster/BUILD.gn new file mode 100644 index 0000000..e7ff848 --- /dev/null +++ b/webrtc/modules/audio_processing/capture_levels_adjuster/BUILD.gn @@ -0,0 +1,45 @@ +# Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("capture_levels_adjuster") { + visibility = [ "*" ] + + sources = [ + "audio_samples_scaler.cc", + "audio_samples_scaler.h", + "capture_levels_adjuster.cc", + "capture_levels_adjuster.h", + ] + + defines = [] + + deps = [ + "..:audio_buffer", + "../../../api:array_view", + "../../../rtc_base:checks", + "../../../rtc_base:safe_minmax", + ] +} + +rtc_library("capture_levels_adjuster_unittests") { + testonly = true + + sources = [ + "audio_samples_scaler_unittest.cc", + "capture_levels_adjuster_unittest.cc", + ] + deps = [ + ":capture_levels_adjuster", + "..:audioproc_test_utils", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:stringutils", + "../../../test:test_support", + ] +} diff --git a/webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc b/webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc new file mode 100644 index 0000000..cb2336b --- /dev/null +++ b/webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h" + +#include + +#include "api/array_view.h" +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +AudioSamplesScaler::AudioSamplesScaler(float initial_gain) + : previous_gain_(initial_gain), target_gain_(initial_gain) {} + +void AudioSamplesScaler::Process(AudioBuffer& audio_buffer) { + if (static_cast(audio_buffer.num_frames()) != samples_per_channel_) { + // Update the members depending on audio-buffer length if needed. + RTC_DCHECK_GT(audio_buffer.num_frames(), 0); + samples_per_channel_ = static_cast(audio_buffer.num_frames()); + one_by_samples_per_channel_ = 1.f / samples_per_channel_; + } + + if (target_gain_ == 1.f && previous_gain_ == target_gain_) { + // If only a gain of 1 is to be applied, do an early return without applying + // any gain. + return; + } + + float gain = previous_gain_; + if (previous_gain_ == target_gain_) { + // Apply a non-changing gain. + for (size_t channel = 0; channel < audio_buffer.num_channels(); ++channel) { + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + sample *= gain; + } + } + } else { + const float increment = + (target_gain_ - previous_gain_) * one_by_samples_per_channel_; + + if (increment > 0.f) { + // Apply an increasing gain. + for (size_t channel = 0; channel < audio_buffer.num_channels(); + ++channel) { + gain = previous_gain_; + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + gain = std::min(gain + increment, target_gain_); + sample *= gain; + } + } + } else { + // Apply a decreasing gain. + for (size_t channel = 0; channel < audio_buffer.num_channels(); + ++channel) { + gain = previous_gain_; + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + gain = std::max(gain + increment, target_gain_); + sample *= gain; + } + } + } + } + previous_gain_ = target_gain_; + + // Saturate the samples to be in the S16 range. + for (size_t channel = 0; channel < audio_buffer.num_channels(); ++channel) { + rtc::ArrayView channel_view(audio_buffer.channels()[channel], + samples_per_channel_); + for (float& sample : channel_view) { + constexpr float kMinFloatS16Value = -32768.f; + constexpr float kMaxFloatS16Value = 32767.f; + sample = rtc::SafeClamp(sample, kMinFloatS16Value, kMaxFloatS16Value); + } + } +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h b/webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h new file mode 100644 index 0000000..2ae8533 --- /dev/null +++ b/webrtc/modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_AUDIO_SAMPLES_SCALER_H_ +#define MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_AUDIO_SAMPLES_SCALER_H_ + +#include + +#include "modules/audio_processing/audio_buffer.h" + +namespace webrtc { + +// Handles and applies a gain to the samples in an audio buffer. +// The gain is applied for each sample and any changes in the gain take effect +// gradually (in a linear manner) over one frame. +class AudioSamplesScaler { + public: + // C-tor. The supplied `initial_gain` is used immediately at the first call to + // Process(), i.e., in contrast to the gain supplied by SetGain(...) there is + // no gradual change to the `initial_gain`. + explicit AudioSamplesScaler(float initial_gain); + AudioSamplesScaler(const AudioSamplesScaler&) = delete; + AudioSamplesScaler& operator=(const AudioSamplesScaler&) = delete; + + // Applies the specified gain to the audio in `audio_buffer`. + void Process(AudioBuffer& audio_buffer); + + // Sets the gain to apply to each sample. + void SetGain(float gain) { target_gain_ = gain; } + + private: + float previous_gain_ = 1.f; + float target_gain_ = 1.f; + int samples_per_channel_ = -1; + float one_by_samples_per_channel_ = -1.f; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_AUDIO_SAMPLES_SCALER_H_ diff --git a/webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc b/webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc new file mode 100644 index 0000000..dfda582 --- /dev/null +++ b/webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h" + +#include "modules/audio_processing/audio_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" + +namespace webrtc { + +namespace { + +constexpr int kMinAnalogMicGainLevel = 0; +constexpr int kMaxAnalogMicGainLevel = 255; + +float ComputeLevelBasedGain(int emulated_analog_mic_gain_level) { + static_assert( + kMinAnalogMicGainLevel == 0, + "The minimum gain level must be 0 for the maths below to work."); + static_assert(kMaxAnalogMicGainLevel > 0, + "The minimum gain level must be larger than 0 for the maths " + "below to work."); + constexpr float kGainToLevelMultiplier = 1.f / kMaxAnalogMicGainLevel; + + RTC_DCHECK_GE(emulated_analog_mic_gain_level, kMinAnalogMicGainLevel); + RTC_DCHECK_LE(emulated_analog_mic_gain_level, kMaxAnalogMicGainLevel); + return kGainToLevelMultiplier * emulated_analog_mic_gain_level; +} + +float ComputePreGain(float pre_gain, + int emulated_analog_mic_gain_level, + bool emulated_analog_mic_gain_enabled) { + return emulated_analog_mic_gain_enabled + ? pre_gain * ComputeLevelBasedGain(emulated_analog_mic_gain_level) + : pre_gain; +} + +} // namespace + +CaptureLevelsAdjuster::CaptureLevelsAdjuster( + bool emulated_analog_mic_gain_enabled, + int emulated_analog_mic_gain_level, + float pre_gain, + float post_gain) + : emulated_analog_mic_gain_enabled_(emulated_analog_mic_gain_enabled), + emulated_analog_mic_gain_level_(emulated_analog_mic_gain_level), + pre_gain_(pre_gain), + pre_adjustment_gain_(ComputePreGain(pre_gain_, + emulated_analog_mic_gain_level_, + emulated_analog_mic_gain_enabled_)), + pre_scaler_(pre_adjustment_gain_), + post_scaler_(post_gain) {} + +void CaptureLevelsAdjuster::ApplyPreLevelAdjustment(AudioBuffer& audio_buffer) { + pre_scaler_.Process(audio_buffer); +} + +void CaptureLevelsAdjuster::ApplyPostLevelAdjustment( + AudioBuffer& audio_buffer) { + post_scaler_.Process(audio_buffer); +} + +void CaptureLevelsAdjuster::SetPreGain(float pre_gain) { + pre_gain_ = pre_gain; + UpdatePreAdjustmentGain(); +} + +void CaptureLevelsAdjuster::SetPostGain(float post_gain) { + post_scaler_.SetGain(post_gain); +} + +void CaptureLevelsAdjuster::SetAnalogMicGainLevel(int level) { + RTC_DCHECK_GE(level, kMinAnalogMicGainLevel); + RTC_DCHECK_LE(level, kMaxAnalogMicGainLevel); + int clamped_level = + rtc::SafeClamp(level, kMinAnalogMicGainLevel, kMaxAnalogMicGainLevel); + + emulated_analog_mic_gain_level_ = clamped_level; + UpdatePreAdjustmentGain(); +} + +void CaptureLevelsAdjuster::UpdatePreAdjustmentGain() { + pre_adjustment_gain_ = + ComputePreGain(pre_gain_, emulated_analog_mic_gain_level_, + emulated_analog_mic_gain_enabled_); + pre_scaler_.SetGain(pre_adjustment_gain_); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h b/webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h new file mode 100644 index 0000000..38b68ad --- /dev/null +++ b/webrtc/modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_CAPTURE_LEVELS_ADJUSTER_H_ +#define MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_CAPTURE_LEVELS_ADJUSTER_H_ + +#include + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/capture_levels_adjuster/audio_samples_scaler.h" + +namespace webrtc { + +// Adjusts the level of the capture signal before and after all capture-side +// processing is done using a combination of explicitly specified gains +// and an emulated analog gain functionality where a specified analog level +// results in an additional gain. The pre-adjustment is achieved by combining +// the gain value `pre_gain` and the level `emulated_analog_mic_gain_level` to +// form a combined gain of `pre_gain`*`emulated_analog_mic_gain_level`/255 which +// is multiplied to each sample. The intention of the +// `emulated_analog_mic_gain_level` is to be controlled by the analog AGC +// functionality and to produce an emulated analog mic gain equal to +// `emulated_analog_mic_gain_level`/255. The post level adjustment is achieved +// by multiplying each sample with the value of `post_gain`. Any changes in the +// gains take are done smoothly over one frame and the scaled samples are +// clamped to fit into the allowed S16 sample range. +class CaptureLevelsAdjuster { + public: + // C-tor. The values for the level and the gains must fulfill + // 0 <= emulated_analog_mic_gain_level <= 255. + // 0.f <= pre_gain. + // 0.f <= post_gain. + CaptureLevelsAdjuster(bool emulated_analog_mic_gain_enabled, + int emulated_analog_mic_gain_level, + float pre_gain, + float post_gain); + CaptureLevelsAdjuster(const CaptureLevelsAdjuster&) = delete; + CaptureLevelsAdjuster& operator=(const CaptureLevelsAdjuster&) = delete; + + // Adjusts the level of the signal. This should be called before any of the + // other processing is performed. + void ApplyPreLevelAdjustment(AudioBuffer& audio_buffer); + + // Adjusts the level of the signal. This should be called after all of the + // other processing have been performed. + void ApplyPostLevelAdjustment(AudioBuffer& audio_buffer); + + // Sets the gain to apply to each sample before any of the other processing is + // performed. + void SetPreGain(float pre_gain); + + // Returns the total pre-adjustment gain applied, comprising both the pre_gain + // as well as the gain from the emulated analog mic, to each sample before any + // of the other processing is performed. + float GetPreAdjustmentGain() const { return pre_adjustment_gain_; } + + // Sets the gain to apply to each sample after all of the other processing + // have been performed. + void SetPostGain(float post_gain); + + // Sets the analog gain level to use for the emulated analog gain. + // `level` must be in the range [0...255]. + void SetAnalogMicGainLevel(int level); + + // Returns the current analog gain level used for the emulated analog gain. + int GetAnalogMicGainLevel() const { return emulated_analog_mic_gain_level_; } + + private: + // Updates the value of `pre_adjustment_gain_` based on the supplied values + // for `pre_gain` and `emulated_analog_mic_gain_level_`. + void UpdatePreAdjustmentGain(); + + const bool emulated_analog_mic_gain_enabled_; + int emulated_analog_mic_gain_level_; + float pre_gain_; + float pre_adjustment_gain_; + AudioSamplesScaler pre_scaler_; + AudioSamplesScaler post_scaler_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_CAPTURE_LEVELS_ADJUSTER_CAPTURE_LEVELS_ADJUSTER_H_ diff --git a/webrtc/modules/audio_processing/common.h b/webrtc/modules/audio_processing/common.h deleted file mode 100644 index d8532c5..0000000 --- a/webrtc/modules/audio_processing/common.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_COMMON_H_ -#define MODULES_AUDIO_PROCESSING_COMMON_H_ - -#include "modules/audio_processing/include/audio_processing.h" -#include "rtc_base/checks.h" - -namespace webrtc { - -static inline size_t ChannelsFromLayout(AudioProcessing::ChannelLayout layout) { - switch (layout) { - case AudioProcessing::kMono: - case AudioProcessing::kMonoAndKeyboard: - return 1; - case AudioProcessing::kStereo: - case AudioProcessing::kStereoAndKeyboard: - return 2; - } - RTC_NOTREACHED(); - return 0; -} - -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_COMMON_H_ diff --git a/webrtc/modules/audio_processing/debug.proto b/webrtc/modules/audio_processing/debug.proto index 07cce23..cc5efbc 100644 --- a/webrtc/modules/audio_processing/debug.proto +++ b/webrtc/modules/audio_processing/debug.proto @@ -35,7 +35,7 @@ message Stream { optional int32 delay = 3; optional sint32 drift = 4; - optional int32 level = 5; + optional int32 applied_input_volume = 5; optional bool keypress = 6; // float deinterleaved data, where each repeated element points to a single @@ -92,6 +92,7 @@ message RuntimeSetting { optional int32 playout_volume_change = 4; optional PlayoutAudioDeviceInfo playout_audio_device_change = 5; optional bool capture_output_used = 6; + optional float capture_post_gain = 7; } message Event { diff --git a/webrtc/modules/audio_processing/echo_control_mobile_impl.cc b/webrtc/modules/audio_processing/echo_control_mobile_impl.cc index 8116608..fa5cb8f 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_impl.cc +++ b/webrtc/modules/audio_processing/echo_control_mobile_impl.cc @@ -18,7 +18,6 @@ #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/include/audio_processing.h" #include "rtc_base/checks.h" -#include "rtc_base/constructor_magic.h" namespace webrtc { @@ -36,7 +35,7 @@ int16_t MapSetting(EchoControlMobileImpl::RoutingMode mode) { case EchoControlMobileImpl::kLoudSpeakerphone: return 4; } - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return -1; } @@ -85,6 +84,9 @@ class EchoControlMobileImpl::Canceller { WebRtcAecm_Free(state_); } + Canceller(const Canceller&) = delete; + Canceller& operator=(const Canceller&) = delete; + void* state() { RTC_DCHECK(state_); return state_; @@ -98,7 +100,6 @@ class EchoControlMobileImpl::Canceller { private: void* state_; - RTC_DISALLOW_COPY_AND_ASSIGN(Canceller); }; EchoControlMobileImpl::EchoControlMobileImpl() diff --git a/webrtc/modules/audio_processing/echo_control_mobile_impl.h b/webrtc/modules/audio_processing/echo_control_mobile_impl.h index 23f3c06..f7f2626 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_impl.h +++ b/webrtc/modules/audio_processing/echo_control_mobile_impl.h @@ -42,7 +42,7 @@ class EchoControlMobileImpl { kLoudSpeakerphone }; - // Sets echo control appropriate for the audio routing |mode| on the device. + // Sets echo control appropriate for the audio routing `mode` on the device. // It can and should be updated during a call if the audio routing changes. int set_routing_mode(RoutingMode mode); RoutingMode routing_mode() const; diff --git a/webrtc/modules/audio_processing/gain_control_impl.cc b/webrtc/modules/audio_processing/gain_control_impl.cc index b5454c0..5f2b487 100644 --- a/webrtc/modules/audio_processing/gain_control_impl.cc +++ b/webrtc/modules/audio_processing/gain_control_impl.cc @@ -35,16 +35,12 @@ int16_t MapSetting(GainControl::Mode mode) { case GainControl::kFixedDigital: return kAgcModeFixedDigital; } - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return -1; } -// Checks whether the legacy digital gain application should be used. -bool UseLegacyDigitalGainApplier() { - return field_trial::IsEnabled("WebRTC-UseLegacyDigitalGainApplier"); -} - -// Floating point variant of WebRtcAgc_Process. +// Applies the sub-frame `gains` to all the bands in `out` and clamps the output +// in the signed 16 bit range. void ApplyDigitalGain(const int32_t gains[11], size_t num_bands, float* const* out) { @@ -97,7 +93,6 @@ int GainControlImpl::instance_counter_ = 0; GainControlImpl::GainControlImpl() : data_dumper_(new ApmDataDumper(instance_counter_)), - use_legacy_gain_applier_(UseLegacyDigitalGainApplier()), mode_(kAdaptiveAnalog), minimum_capture_level_(0), maximum_capture_level_(255), @@ -236,26 +231,9 @@ int GainControlImpl::ProcessCaptureAudio(AudioBuffer* audio, } } - if (use_legacy_gain_applier_) { - for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) { - int16_t split_band_data[AudioBuffer::kMaxNumBands] - [AudioBuffer::kMaxSplitFrameLength]; - int16_t* split_bands[AudioBuffer::kMaxNumBands] = { - split_band_data[0], split_band_data[1], split_band_data[2]}; - audio->ExportSplitChannelData(ch, split_bands); - - int err_process = WebRtcAgc_Process( - mono_agcs_[ch]->state, mono_agcs_[index_to_apply]->gains, split_bands, - audio->num_bands(), split_bands); - RTC_DCHECK_EQ(err_process, 0); - - audio->ImportSplitChannelData(ch, split_bands); - } - } else { - for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) { - ApplyDigitalGain(mono_agcs_[index_to_apply]->gains, audio->num_bands(), - audio->split_bands(ch)); - } + for (size_t ch = 0; ch < mono_agcs_.size(); ++ch) { + ApplyDigitalGain(mono_agcs_[index_to_apply]->gains, audio->num_bands(), + audio->split_bands(ch)); } RTC_DCHECK_LT(0ul, *num_proc_channels_); @@ -277,7 +255,6 @@ int GainControlImpl::ProcessCaptureAudio(AudioBuffer* audio, return AudioProcessing::kNoError; } - // TODO(ajm): ensure this is called under kAdaptiveAnalog. int GainControlImpl::set_stream_analog_level(int level) { data_dumper_->DumpRaw("gain_control_set_stream_analog_level", 1, &level); @@ -309,7 +286,6 @@ int GainControlImpl::set_mode(Mode mode) { return AudioProcessing::kNoError; } - int GainControlImpl::set_analog_level_limits(int minimum, int maximum) { if (minimum < 0 || maximum > 65535 || maximum < minimum) { return AudioProcessing::kBadParameterError; @@ -324,7 +300,6 @@ int GainControlImpl::set_analog_level_limits(int minimum, int maximum) { return AudioProcessing::kNoError; } - int GainControlImpl::set_target_level_dbfs(int level) { if (level > 31 || level < 0) { return AudioProcessing::kBadParameterError; diff --git a/webrtc/modules/audio_processing/gain_control_impl.h b/webrtc/modules/audio_processing/gain_control_impl.h index b65d697..8aea8f2 100644 --- a/webrtc/modules/audio_processing/gain_control_impl.h +++ b/webrtc/modules/audio_processing/gain_control_impl.h @@ -68,7 +68,6 @@ class GainControlImpl : public GainControl { std::unique_ptr data_dumper_; - const bool use_legacy_gain_applier_; Mode mode_; int minimum_capture_level_; int maximum_capture_level_; diff --git a/webrtc/modules/audio_processing/gain_controller2.cc b/webrtc/modules/audio_processing/gain_controller2.cc index 8e71d40..dd35212 100644 --- a/webrtc/modules/audio_processing/gain_controller2.cc +++ b/webrtc/modules/audio_processing/gain_controller2.cc @@ -10,129 +10,274 @@ #include "modules/audio_processing/gain_controller2.h" +#include +#include + #include "common_audio/include/audio_util.h" +#include "modules/audio_processing/agc2/agc2_common.h" +#include "modules/audio_processing/agc2/cpu_features.h" #include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/include/audio_frame_view.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { +namespace { -int GainController2::instance_count_ = 0; +using Agc2Config = AudioProcessing::Config::GainController2; +using InputVolumeControllerConfig = InputVolumeController::Config; -GainController2::GainController2() - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), - gain_applier_(/*hard_clip_samples=*/false, - /*initial_gain_factor=*/0.f), - limiter_(static_cast(48000), data_dumper_.get(), "Agc2") { - if (config_.adaptive_digital.enabled) { - adaptive_agc_.reset(new AdaptiveAgc(data_dumper_.get())); +constexpr int kLogLimiterStatsPeriodMs = 30'000; +constexpr int kFrameLengthMs = 10; +constexpr int kLogLimiterStatsPeriodNumFrames = + kLogLimiterStatsPeriodMs / kFrameLengthMs; + +// Detects the available CPU features and applies any kill-switches. +AvailableCpuFeatures GetAllowedCpuFeatures() { + AvailableCpuFeatures features = GetAvailableCpuFeatures(); + if (field_trial::IsEnabled("WebRTC-Agc2SimdSse2KillSwitch")) { + features.sse2 = false; + } + if (field_trial::IsEnabled("WebRTC-Agc2SimdAvx2KillSwitch")) { + features.avx2 = false; + } + if (field_trial::IsEnabled("WebRTC-Agc2SimdNeonKillSwitch")) { + features.neon = false; + } + return features; +} + +// Peak and RMS audio levels in dBFS. +struct AudioLevels { + float peak_dbfs; + float rms_dbfs; +}; + +// Speech level info. +struct SpeechLevel { + bool is_confident; + float rms_dbfs; +}; + +// Computes the audio levels for the first channel in `frame`. +AudioLevels ComputeAudioLevels(AudioFrameView frame, + ApmDataDumper& data_dumper) { + float peak = 0.0f; + float rms = 0.0f; + for (const auto& x : frame.channel(0)) { + peak = std::max(std::fabs(x), peak); + rms += x * x; + } + AudioLevels levels{ + FloatS16ToDbfs(peak), + FloatS16ToDbfs(std::sqrt(rms / frame.samples_per_channel()))}; + data_dumper.DumpRaw("agc2_input_rms_dbfs", levels.rms_dbfs); + data_dumper.DumpRaw("agc2_input_peak_dbfs", levels.peak_dbfs); + return levels; +} + +} // namespace + +std::atomic GainController2::instance_count_(0); + +GainController2::GainController2( + const Agc2Config& config, + const InputVolumeControllerConfig& input_volume_controller_config, + int sample_rate_hz, + int num_channels, + bool use_internal_vad) + : cpu_features_(GetAllowedCpuFeatures()), + data_dumper_(instance_count_.fetch_add(1) + 1), + fixed_gain_applier_( + /*hard_clip_samples=*/false, + /*initial_gain_factor=*/DbToRatio(config.fixed_digital.gain_db)), + limiter_(sample_rate_hz, &data_dumper_, /*histogram_name_prefix=*/"Agc2"), + calls_since_last_limiter_log_(0) { + RTC_DCHECK(Validate(config)); + data_dumper_.InitiateNewSetOfRecordings(); + + if (config.input_volume_controller.enabled || + config.adaptive_digital.enabled) { + // Create dependencies. + speech_level_estimator_ = std::make_unique( + &data_dumper_, config.adaptive_digital, kAdjacentSpeechFramesThreshold); + if (use_internal_vad) + vad_ = std::make_unique( + kVadResetPeriodMs, cpu_features_, sample_rate_hz); + } + + if (config.input_volume_controller.enabled) { + // Create controller. + input_volume_controller_ = std::make_unique( + num_channels, input_volume_controller_config); + // TODO(bugs.webrtc.org/7494): Call `Initialize` in ctor and remove method. + input_volume_controller_->Initialize(); + } + + if (config.adaptive_digital.enabled) { + // Create dependencies. + noise_level_estimator_ = CreateNoiseFloorEstimator(&data_dumper_); + saturation_protector_ = CreateSaturationProtector( + kSaturationProtectorInitialHeadroomDb, kAdjacentSpeechFramesThreshold, + &data_dumper_); + // Create controller. + adaptive_digital_controller_ = + std::make_unique( + &data_dumper_, config.adaptive_digital, + kAdjacentSpeechFramesThreshold); } } GainController2::~GainController2() = default; -void GainController2::Initialize(int sample_rate_hz) { - RTC_DCHECK(sample_rate_hz == AudioProcessing::kSampleRate8kHz || - sample_rate_hz == AudioProcessing::kSampleRate16kHz || - sample_rate_hz == AudioProcessing::kSampleRate32kHz || - sample_rate_hz == AudioProcessing::kSampleRate48kHz); - limiter_.SetSampleRate(sample_rate_hz); - data_dumper_->InitiateNewSetOfRecordings(); - data_dumper_->DumpRaw("sample_rate_hz", sample_rate_hz); -} - -void GainController2::Process(AudioBuffer* audio) { - AudioFrameView float_frame(audio->channels(), audio->num_channels(), - audio->num_frames()); - // Apply fixed gain first, then the adaptive one. - gain_applier_.ApplyGain(float_frame); - if (adaptive_agc_) { - adaptive_agc_->Process(float_frame, limiter_.LastAudioLevel()); +// TODO(webrtc:7494): Pass the flag also to the other components. +void GainController2::SetCaptureOutputUsed(bool capture_output_used) { + if (input_volume_controller_) { + input_volume_controller_->HandleCaptureOutputUsedChange( + capture_output_used); } - limiter_.Process(float_frame); } -void GainController2::NotifyAnalogLevel(int level) { - if (analog_level_ != level && adaptive_agc_) { - adaptive_agc_->Reset(); - } - analog_level_ = level; -} - -void GainController2::ApplyConfig( - const AudioProcessing::Config::GainController2& config) { - RTC_DCHECK(Validate(config)) - << " the invalid config was " << ToString(config); - - config_ = config; - if (config.fixed_digital.gain_db != config_.fixed_digital.gain_db) { +void GainController2::SetFixedGainDb(float gain_db) { + const float gain_factor = DbToRatio(gain_db); + if (fixed_gain_applier_.GetGainFactor() != gain_factor) { // Reset the limiter to quickly react on abrupt level changes caused by // large changes of the fixed gain. limiter_.Reset(); } - gain_applier_.SetGainFactor(DbToRatio(config_.fixed_digital.gain_db)); - if (config_.adaptive_digital.enabled) { - adaptive_agc_.reset(new AdaptiveAgc(data_dumper_.get(), config_)); - } else { - adaptive_agc_.reset(); + fixed_gain_applier_.SetGainFactor(gain_factor); +} + +void GainController2::Analyze(int applied_input_volume, + const AudioBuffer& audio_buffer) { + recommended_input_volume_ = absl::nullopt; + + RTC_DCHECK_GE(applied_input_volume, 0); + RTC_DCHECK_LE(applied_input_volume, 255); + + if (input_volume_controller_) { + input_volume_controller_->AnalyzeInputAudio(applied_input_volume, + audio_buffer); + } +} + +void GainController2::Process(absl::optional speech_probability, + bool input_volume_changed, + AudioBuffer* audio) { + recommended_input_volume_ = absl::nullopt; + + data_dumper_.DumpRaw("agc2_applied_input_volume_changed", + input_volume_changed); + if (input_volume_changed) { + // Handle input volume changes. + if (speech_level_estimator_) + speech_level_estimator_->Reset(); + if (saturation_protector_) + saturation_protector_->Reset(); + } + + AudioFrameView float_frame(audio->channels(), audio->num_channels(), + audio->num_frames()); + // Compute speech probability. + if (vad_) { + // When the VAD component runs, `speech_probability` should not be specified + // because APM should not run the same VAD twice (as an APM sub-module and + // internally in AGC2). + RTC_DCHECK(!speech_probability.has_value()); + speech_probability = vad_->Analyze(float_frame); + } + if (speech_probability.has_value()) { + RTC_DCHECK_GE(*speech_probability, 0.0f); + RTC_DCHECK_LE(*speech_probability, 1.0f); + } + // The speech probability may not be defined at this step (e.g., when the + // fixed digital controller alone is enabled). + if (speech_probability.has_value()) + data_dumper_.DumpRaw("agc2_speech_probability", *speech_probability); + + // Compute audio, noise and speech levels. + AudioLevels audio_levels = ComputeAudioLevels(float_frame, data_dumper_); + absl::optional noise_rms_dbfs; + if (noise_level_estimator_) { + // TODO(bugs.webrtc.org/7494): Pass `audio_levels` to remove duplicated + // computation in `noise_level_estimator_`. + noise_rms_dbfs = noise_level_estimator_->Analyze(float_frame); + } + absl::optional speech_level; + if (speech_level_estimator_) { + RTC_DCHECK(speech_probability.has_value()); + speech_level_estimator_->Update( + audio_levels.rms_dbfs, audio_levels.peak_dbfs, *speech_probability); + speech_level = + SpeechLevel{.is_confident = speech_level_estimator_->is_confident(), + .rms_dbfs = speech_level_estimator_->level_dbfs()}; + } + + // Update the recommended input volume. + if (input_volume_controller_) { + RTC_DCHECK(speech_level.has_value()); + RTC_DCHECK(speech_probability.has_value()); + if (speech_probability.has_value()) { + recommended_input_volume_ = + input_volume_controller_->RecommendInputVolume( + *speech_probability, + speech_level->is_confident + ? absl::optional(speech_level->rms_dbfs) + : absl::nullopt); + } + } + + if (adaptive_digital_controller_) { + RTC_DCHECK(saturation_protector_); + RTC_DCHECK(speech_probability.has_value()); + RTC_DCHECK(speech_level.has_value()); + saturation_protector_->Analyze(*speech_probability, audio_levels.peak_dbfs, + speech_level->rms_dbfs); + float headroom_db = saturation_protector_->HeadroomDb(); + data_dumper_.DumpRaw("agc2_headroom_db", headroom_db); + float limiter_envelope_dbfs = FloatS16ToDbfs(limiter_.LastAudioLevel()); + data_dumper_.DumpRaw("agc2_limiter_envelope_dbfs", limiter_envelope_dbfs); + RTC_DCHECK(noise_rms_dbfs.has_value()); + adaptive_digital_controller_->Process( + /*info=*/{.speech_probability = *speech_probability, + .speech_level_dbfs = speech_level->rms_dbfs, + .speech_level_reliable = speech_level->is_confident, + .noise_rms_dbfs = *noise_rms_dbfs, + .headroom_db = headroom_db, + .limiter_envelope_dbfs = limiter_envelope_dbfs}, + float_frame); + } + + // TODO(bugs.webrtc.org/7494): Pass `audio_levels` to remove duplicated + // computation in `limiter_`. + fixed_gain_applier_.ApplyGain(float_frame); + + limiter_.Process(float_frame); + + // Periodically log limiter stats. + if (++calls_since_last_limiter_log_ == kLogLimiterStatsPeriodNumFrames) { + calls_since_last_limiter_log_ = 0; + InterpolatedGainCurve::Stats stats = limiter_.GetGainCurveStats(); + RTC_LOG(LS_INFO) << "[AGC2] limiter stats" + << " | identity: " << stats.look_ups_identity_region + << " | knee: " << stats.look_ups_knee_region + << " | limiter: " << stats.look_ups_limiter_region + << " | saturation: " << stats.look_ups_saturation_region; } } bool GainController2::Validate( const AudioProcessing::Config::GainController2& config) { - return config.fixed_digital.gain_db >= 0.f && - config.fixed_digital.gain_db < 50.f && - config.adaptive_digital.extra_saturation_margin_db >= 0.f && - config.adaptive_digital.extra_saturation_margin_db <= 100.f; -} - -std::string GainController2::ToString( - const AudioProcessing::Config::GainController2& config) { - rtc::StringBuilder ss; - std::string adaptive_digital_level_estimator; - using LevelEstimatorType = - AudioProcessing::Config::GainController2::LevelEstimator; - switch (config.adaptive_digital.level_estimator) { - case LevelEstimatorType::kRms: - adaptive_digital_level_estimator = "RMS"; - break; - case LevelEstimatorType::kPeak: - adaptive_digital_level_estimator = "peak"; - break; - } - // clang-format off - // clang formatting doesn't respect custom nested style. - ss << "{" - "enabled: " << (config.enabled ? "true" : "false") << ", " - "fixed_digital: {gain_db: " << config.fixed_digital.gain_db << "}, " - "adaptive_digital: {" - "enabled: " - << (config.adaptive_digital.enabled ? "true" : "false") << ", " - "level_estimator: {" - "type: " << adaptive_digital_level_estimator << ", " - "adjacent_speech_frames_threshold: " - << config.adaptive_digital - .level_estimator_adjacent_speech_frames_threshold << ", " - "initial_saturation_margin_db: " - << config.adaptive_digital.initial_saturation_margin_db << ", " - "extra_saturation_margin_db: " - << config.adaptive_digital.extra_saturation_margin_db << "}, " - "gain_applier: {" - "adjacent_speech_frames_threshold: " - << config.adaptive_digital - .gain_applier_adjacent_speech_frames_threshold << ", " - "max_gain_change_db_per_second: " - << config.adaptive_digital.max_gain_change_db_per_second << ", " - "max_output_noise_level_dbfs: " - << config.adaptive_digital.max_output_noise_level_dbfs << "}" - "}" - "}"; - // clang-format on - return ss.Release(); + const auto& fixed = config.fixed_digital; + const auto& adaptive = config.adaptive_digital; + return fixed.gain_db >= 0.0f && fixed.gain_db < 50.0f && + adaptive.headroom_db >= 0.0f && adaptive.max_gain_db > 0.0f && + adaptive.initial_gain_db >= 0.0f && + adaptive.max_gain_change_db_per_second > 0.0f && + adaptive.max_output_noise_level_dbfs <= 0.0f; } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/gain_controller2.h b/webrtc/modules/audio_processing/gain_controller2.h index 7ed310e..43b5828 100644 --- a/webrtc/modules/audio_processing/gain_controller2.h +++ b/webrtc/modules/audio_processing/gain_controller2.h @@ -11,46 +11,98 @@ #ifndef MODULES_AUDIO_PROCESSING_GAIN_CONTROLLER2_H_ #define MODULES_AUDIO_PROCESSING_GAIN_CONTROLLER2_H_ +#include #include #include -#include "modules/audio_processing/agc2/adaptive_agc.h" +#include "modules/audio_processing/agc2/adaptive_digital_gain_controller.h" +#include "modules/audio_processing/agc2/cpu_features.h" #include "modules/audio_processing/agc2/gain_applier.h" +#include "modules/audio_processing/agc2/input_volume_controller.h" #include "modules/audio_processing/agc2/limiter.h" +#include "modules/audio_processing/agc2/noise_level_estimator.h" +#include "modules/audio_processing/agc2/saturation_protector.h" +#include "modules/audio_processing/agc2/speech_level_estimator.h" +#include "modules/audio_processing/agc2/vad_wrapper.h" #include "modules/audio_processing/include/audio_processing.h" -#include "rtc_base/constructor_magic.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" namespace webrtc { -class ApmDataDumper; class AudioBuffer; // Gain Controller 2 aims to automatically adjust levels by acting on the // microphone gain and/or applying digital gain. class GainController2 { public: - GainController2(); + // Ctor. If `use_internal_vad` is true, an internal voice activity + // detector is used for digital adaptive gain. + GainController2( + const AudioProcessing::Config::GainController2& config, + const InputVolumeController::Config& input_volume_controller_config, + int sample_rate_hz, + int num_channels, + bool use_internal_vad); + GainController2(const GainController2&) = delete; + GainController2& operator=(const GainController2&) = delete; ~GainController2(); - void Initialize(int sample_rate_hz); - void Process(AudioBuffer* audio); - void NotifyAnalogLevel(int level); + // Sets the fixed digital gain. + void SetFixedGainDb(float gain_db); + + // Updates the input volume controller about whether the capture output is + // used or not. + void SetCaptureOutputUsed(bool capture_output_used); + + // Analyzes `audio_buffer` before `Process()` is called so that the analysis + // can be performed before digital processing operations take place (e.g., + // echo cancellation). The analysis consists of input clipping detection and + // prediction (if enabled). The value of `applied_input_volume` is limited to + // [0, 255]. + void Analyze(int applied_input_volume, const AudioBuffer& audio_buffer); + + // Updates the recommended input volume, applies the adaptive digital and the + // fixed digital gains and runs a limiter on `audio`. + // When the internal VAD is not used, `speech_probability` should be specified + // and in the [0, 1] range. Otherwise ignores `speech_probability` and + // computes the speech probability via `vad_`. + // Handles input volume changes; if the caller cannot determine whether an + // input volume change occurred, set `input_volume_changed` to false. + void Process(absl::optional speech_probability, + bool input_volume_changed, + AudioBuffer* audio); - void ApplyConfig(const AudioProcessing::Config::GainController2& config); static bool Validate(const AudioProcessing::Config::GainController2& config); - static std::string ToString( - const AudioProcessing::Config::GainController2& config); + + AvailableCpuFeatures GetCpuFeatures() const { return cpu_features_; } + + absl::optional recommended_input_volume() const { + return recommended_input_volume_; + } private: - static int instance_count_; - std::unique_ptr data_dumper_; - AudioProcessing::Config::GainController2 config_; - GainApplier gain_applier_; - std::unique_ptr adaptive_agc_; - Limiter limiter_; - int analog_level_ = -1; + static std::atomic instance_count_; + const AvailableCpuFeatures cpu_features_; + ApmDataDumper data_dumper_; - RTC_DISALLOW_COPY_AND_ASSIGN(GainController2); + GainApplier fixed_gain_applier_; + std::unique_ptr noise_level_estimator_; + std::unique_ptr vad_; + std::unique_ptr speech_level_estimator_; + std::unique_ptr input_volume_controller_; + // TODO(bugs.webrtc.org/7494): Rename to `CrestFactorEstimator`. + std::unique_ptr saturation_protector_; + std::unique_ptr adaptive_digital_controller_; + Limiter limiter_; + + int calls_since_last_limiter_log_; + + // TODO(bugs.webrtc.org/7494): Remove intermediate storing at this level once + // APM refactoring is completed. + // Recommended input volume from `InputVolumecontroller`. Non-empty after + // `Process()` if input volume controller is enabled and + // `InputVolumeController::Process()` has returned a non-empty value. + absl::optional recommended_input_volume_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/high_pass_filter.cc b/webrtc/modules/audio_processing/high_pass_filter.cc index bff7209..3b4740f 100644 --- a/webrtc/modules/audio_processing/high_pass_filter.cc +++ b/webrtc/modules/audio_processing/high_pass_filter.cc @@ -44,9 +44,9 @@ const CascadedBiQuadFilter::BiQuadCoefficients& ChooseCoefficients( case 48000: return kHighPassFilterCoefficients48kHz; default: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); return kHighPassFilterCoefficients16kHz; } diff --git a/webrtc/modules/audio_processing/include/aec_dump.cc b/webrtc/modules/audio_processing/include/aec_dump.cc index 67809d0..8f788cb 100644 --- a/webrtc/modules/audio_processing/include/aec_dump.cc +++ b/webrtc/modules/audio_processing/include/aec_dump.cc @@ -17,7 +17,7 @@ InternalAPMConfig::InternalAPMConfig(InternalAPMConfig&&) = default; InternalAPMConfig& InternalAPMConfig::operator=(const InternalAPMConfig&) = default; -bool InternalAPMConfig::operator==(const InternalAPMConfig& other) { +bool InternalAPMConfig::operator==(const InternalAPMConfig& other) const { return aec_enabled == other.aec_enabled && aec_delay_agnostic_enabled == other.aec_delay_agnostic_enabled && aec_drift_compensation_enabled == diff --git a/webrtc/modules/audio_processing/include/aec_dump.h b/webrtc/modules/audio_processing/include/aec_dump.h index ed5acb0..6f2eb64 100644 --- a/webrtc/modules/audio_processing/include/aec_dump.h +++ b/webrtc/modules/audio_processing/include/aec_dump.h @@ -15,9 +15,10 @@ #include +#include "absl/base/attributes.h" +#include "absl/types/optional.h" #include "modules/audio_processing/include/audio_frame_view.h" #include "modules/audio_processing/include/audio_processing.h" -#include "rtc_base/deprecation.h" namespace webrtc { @@ -31,7 +32,7 @@ struct InternalAPMConfig { InternalAPMConfig& operator=(const InternalAPMConfig&); InternalAPMConfig& operator=(InternalAPMConfig&&) = delete; - bool operator==(const InternalAPMConfig& other); + bool operator==(const InternalAPMConfig& other) const; bool aec_enabled = false; bool aec_delay_agnostic_enabled = false; @@ -67,7 +68,7 @@ class AecDump { struct AudioProcessingState { int delay; int drift; - int level; + absl::optional applied_input_volume; bool keypress; }; @@ -76,7 +77,8 @@ class AecDump { // Logs Event::Type INIT message. virtual void WriteInitMessage(const ProcessingConfig& api_format, int64_t time_now_ms) = 0; - RTC_DEPRECATED void WriteInitMessage(const ProcessingConfig& api_format) { + ABSL_DEPRECATED("") + void WriteInitMessage(const ProcessingConfig& api_format) { WriteInitMessage(api_format, 0); } diff --git a/webrtc/modules/audio_processing/include/audio_frame_proxies.cc b/webrtc/modules/audio_processing/include/audio_frame_proxies.cc index b960e72..7cc4fb7 100644 --- a/webrtc/modules/audio_processing/include/audio_frame_proxies.cc +++ b/webrtc/modules/audio_processing/include/audio_frame_proxies.cc @@ -20,10 +20,8 @@ int ProcessAudioFrame(AudioProcessing* ap, AudioFrame* frame) { return AudioProcessing::Error::kNullPointerError; } - StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_, - /*has_keyboard=*/false); - StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_, - /*has_keyboard=*/false); + StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_); + StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_); RTC_DCHECK_EQ(frame->samples_per_channel(), input_config.num_frames()); int result = ap->ProcessStream(frame->data(), input_config, output_config, @@ -57,10 +55,8 @@ int ProcessReverseAudioFrame(AudioProcessing* ap, AudioFrame* frame) { return AudioProcessing::Error::kBadNumberChannelsError; } - StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_, - /*has_keyboard=*/false); - StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_, - /*has_keyboard=*/false); + StreamConfig input_config(frame->sample_rate_hz_, frame->num_channels_); + StreamConfig output_config(frame->sample_rate_hz_, frame->num_channels_); int result = ap->ProcessReverseStream(frame->data(), input_config, output_config, frame->mutable_data()); diff --git a/webrtc/modules/audio_processing/include/audio_frame_proxies.h b/webrtc/modules/audio_processing/include/audio_frame_proxies.h index 2d0f5b5..5dd111c 100644 --- a/webrtc/modules/audio_processing/include/audio_frame_proxies.h +++ b/webrtc/modules/audio_processing/include/audio_frame_proxies.h @@ -16,21 +16,21 @@ namespace webrtc { class AudioFrame; class AudioProcessing; -// Processes a 10 ms |frame| of the primary audio stream using the provided +// Processes a 10 ms `frame` of the primary audio stream using the provided // AudioProcessing object. On the client-side, this is the near-end (or -// captured) audio. The |sample_rate_hz_|, |num_channels_|, and -// |samples_per_channel_| members of |frame| must be valid. If changed from the +// captured) audio. The `sample_rate_hz_`, `num_channels_`, and +// `samples_per_channel_` members of `frame` must be valid. If changed from the // previous call to this function, it will trigger an initialization of the // provided AudioProcessing object. // The function returns any error codes passed from the AudioProcessing // ProcessStream method. int ProcessAudioFrame(AudioProcessing* ap, AudioFrame* frame); -// Processes a 10 ms |frame| of the reverse direction audio stream using the +// Processes a 10 ms `frame` of the reverse direction audio stream using the // provided AudioProcessing object. The frame may be modified. On the // client-side, this is the far-end (or to be rendered) audio. The -// |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| members of -// |frame| must be valid. If changed from the previous call to this function, it +// `sample_rate_hz_`, `num_channels_`, and `samples_per_channel_` members of +// `frame` must be valid. If changed from the previous call to this function, it // will trigger an initialization of the provided AudioProcessing object. // The function returns any error codes passed from the AudioProcessing // ProcessReverseStream method. diff --git a/webrtc/modules/audio_processing/include/audio_frame_view.h b/webrtc/modules/audio_processing/include/audio_frame_view.h index ab5779a..164784a 100644 --- a/webrtc/modules/audio_processing/include/audio_frame_view.h +++ b/webrtc/modules/audio_processing/include/audio_frame_view.h @@ -19,15 +19,16 @@ namespace webrtc { template class AudioFrameView { public: - // |num_channels| and |channel_size| describe the T** - // |audio_samples|. |audio_samples| is assumed to point to a + // `num_channels` and `channel_size` describe the T** + // `audio_samples`. `audio_samples` is assumed to point to a // two-dimensional |num_channels * channel_size| array of floats. - AudioFrameView(T* const* audio_samples, - size_t num_channels, - size_t channel_size) + AudioFrameView(T* const* audio_samples, int num_channels, int channel_size) : audio_samples_(audio_samples), num_channels_(num_channels), - channel_size_(channel_size) {} + channel_size_(channel_size) { + RTC_DCHECK_GE(num_channels_, 0); + RTC_DCHECK_GE(channel_size_, 0); + } // Implicit cast to allow converting Frame to // Frame. @@ -39,17 +40,17 @@ class AudioFrameView { AudioFrameView() = delete; - size_t num_channels() const { return num_channels_; } + int num_channels() const { return num_channels_; } - size_t samples_per_channel() const { return channel_size_; } + int samples_per_channel() const { return channel_size_; } - rtc::ArrayView channel(size_t idx) { + rtc::ArrayView channel(int idx) { RTC_DCHECK_LE(0, idx); RTC_DCHECK_LE(idx, num_channels_); return rtc::ArrayView(audio_samples_[idx], channel_size_); } - rtc::ArrayView channel(size_t idx) const { + rtc::ArrayView channel(int idx) const { RTC_DCHECK_LE(0, idx); RTC_DCHECK_LE(idx, num_channels_); return rtc::ArrayView(audio_samples_[idx], channel_size_); @@ -59,8 +60,8 @@ class AudioFrameView { private: T* const* audio_samples_; - size_t num_channels_; - size_t channel_size_; + int num_channels_; + int channel_size_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/include/audio_processing.cc b/webrtc/modules/audio_processing/include/audio_processing.cc index 8854415..13ddcc5 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.cc +++ b/webrtc/modules/audio_processing/include/audio_processing.cc @@ -16,6 +16,9 @@ namespace webrtc { namespace { +using Agc1Config = AudioProcessing::Config::GainController1; +using Agc2Config = AudioProcessing::Config::GainController2; + std::string NoiseSuppressionLevelToString( const AudioProcessing::Config::NoiseSuppression::Level& level) { switch (level) { @@ -28,36 +31,19 @@ std::string NoiseSuppressionLevelToString( case AudioProcessing::Config::NoiseSuppression::Level::kVeryHigh: return "VeryHigh"; } + RTC_CHECK_NOTREACHED(); } -std::string GainController1ModeToString( - const AudioProcessing::Config::GainController1::Mode& mode) { +std::string GainController1ModeToString(const Agc1Config::Mode& mode) { switch (mode) { - case AudioProcessing::Config::GainController1::Mode::kAdaptiveAnalog: + case Agc1Config::Mode::kAdaptiveAnalog: return "AdaptiveAnalog"; - case AudioProcessing::Config::GainController1::Mode::kAdaptiveDigital: + case Agc1Config::Mode::kAdaptiveDigital: return "AdaptiveDigital"; - case AudioProcessing::Config::GainController1::Mode::kFixedDigital: + case Agc1Config::Mode::kFixedDigital: return "FixedDigital"; } -} - -std::string GainController2LevelEstimatorToString( - const AudioProcessing::Config::GainController2::LevelEstimator& level) { - switch (level) { - case AudioProcessing::Config::GainController2::LevelEstimator::kRms: - return "Rms"; - case AudioProcessing::Config::GainController2::LevelEstimator::kPeak: - return "Peak"; - } -} - -int GetDefaultMaxInternalRate() { -#ifdef WEBRTC_ARCH_ARM_FAMILY - return 32000; -#else - return 48000; -#endif + RTC_CHECK_NOTREACHED(); } } // namespace @@ -67,25 +53,92 @@ constexpr int AudioProcessing::kNativeSampleRatesHz[]; void CustomProcessing::SetRuntimeSetting( AudioProcessing::RuntimeSetting setting) {} -AudioProcessing::Config::Pipeline::Pipeline() - : maximum_internal_processing_rate(GetDefaultMaxInternalRate()) {} +bool Agc1Config::operator==(const Agc1Config& rhs) const { + const auto& analog_lhs = analog_gain_controller; + const auto& analog_rhs = rhs.analog_gain_controller; + return enabled == rhs.enabled && mode == rhs.mode && + target_level_dbfs == rhs.target_level_dbfs && + compression_gain_db == rhs.compression_gain_db && + enable_limiter == rhs.enable_limiter && + analog_lhs.enabled == analog_rhs.enabled && + analog_lhs.startup_min_volume == analog_rhs.startup_min_volume && + analog_lhs.clipped_level_min == analog_rhs.clipped_level_min && + analog_lhs.enable_digital_adaptive == + analog_rhs.enable_digital_adaptive && + analog_lhs.clipped_level_step == analog_rhs.clipped_level_step && + analog_lhs.clipped_ratio_threshold == + analog_rhs.clipped_ratio_threshold && + analog_lhs.clipped_wait_frames == analog_rhs.clipped_wait_frames && + analog_lhs.clipping_predictor.mode == + analog_rhs.clipping_predictor.mode && + analog_lhs.clipping_predictor.window_length == + analog_rhs.clipping_predictor.window_length && + analog_lhs.clipping_predictor.reference_window_length == + analog_rhs.clipping_predictor.reference_window_length && + analog_lhs.clipping_predictor.reference_window_delay == + analog_rhs.clipping_predictor.reference_window_delay && + analog_lhs.clipping_predictor.clipping_threshold == + analog_rhs.clipping_predictor.clipping_threshold && + analog_lhs.clipping_predictor.crest_factor_margin == + analog_rhs.clipping_predictor.crest_factor_margin && + analog_lhs.clipping_predictor.use_predicted_step == + analog_rhs.clipping_predictor.use_predicted_step; +} + +bool Agc2Config::AdaptiveDigital::operator==( + const Agc2Config::AdaptiveDigital& rhs) const { + return enabled == rhs.enabled && headroom_db == rhs.headroom_db && + max_gain_db == rhs.max_gain_db && + initial_gain_db == rhs.initial_gain_db && + max_gain_change_db_per_second == rhs.max_gain_change_db_per_second && + max_output_noise_level_dbfs == rhs.max_output_noise_level_dbfs; +} + +bool Agc2Config::InputVolumeController::operator==( + const Agc2Config::InputVolumeController& rhs) const { + return enabled == rhs.enabled; +} + +bool Agc2Config::operator==(const Agc2Config& rhs) const { + return enabled == rhs.enabled && + fixed_digital.gain_db == rhs.fixed_digital.gain_db && + adaptive_digital == rhs.adaptive_digital && + input_volume_controller == rhs.input_volume_controller; +} + +bool AudioProcessing::Config::CaptureLevelAdjustment::operator==( + const AudioProcessing::Config::CaptureLevelAdjustment& rhs) const { + return enabled == rhs.enabled && pre_gain_factor == rhs.pre_gain_factor && + post_gain_factor == rhs.post_gain_factor && + analog_mic_gain_emulation == rhs.analog_mic_gain_emulation; +} + +bool AudioProcessing::Config::CaptureLevelAdjustment::AnalogMicGainEmulation:: +operator==(const AudioProcessing::Config::CaptureLevelAdjustment:: + AnalogMicGainEmulation& rhs) const { + return enabled == rhs.enabled && initial_level == rhs.initial_level; +} std::string AudioProcessing::Config::ToString() const { - char buf[1024]; + char buf[2048]; rtc::SimpleStringBuilder builder(buf); builder << "AudioProcessing::Config{ " - "pipeline: {" + "pipeline: { " "maximum_internal_processing_rate: " << pipeline.maximum_internal_processing_rate << ", multi_channel_render: " << pipeline.multi_channel_render - << ", " - ", multi_channel_capture: " - << pipeline.multi_channel_capture - << "}, " - "pre_amplifier: { enabled: " - << pre_amplifier.enabled + << ", multi_channel_capture: " << pipeline.multi_channel_capture + << " }, pre_amplifier: { enabled: " << pre_amplifier.enabled << ", fixed_gain_factor: " << pre_amplifier.fixed_gain_factor - << " }, high_pass_filter: { enabled: " << high_pass_filter.enabled + << " },capture_level_adjustment: { enabled: " + << capture_level_adjustment.enabled + << ", pre_gain_factor: " << capture_level_adjustment.pre_gain_factor + << ", post_gain_factor: " << capture_level_adjustment.post_gain_factor + << ", analog_mic_gain_emulation: { enabled: " + << capture_level_adjustment.analog_mic_gain_emulation.enabled + << ", initial_level: " + << capture_level_adjustment.analog_mic_gain_emulation.initial_level + << " }}, high_pass_filter: { enabled: " << high_pass_filter.enabled << " }, echo_canceller: { enabled: " << echo_canceller.enabled << ", mobile_mode: " << echo_canceller.mobile_mode << ", enforce_high_pass_filtering: " @@ -95,29 +148,62 @@ std::string AudioProcessing::Config::ToString() const { << NoiseSuppressionLevelToString(noise_suppression.level) << " }, transient_suppression: { enabled: " << transient_suppression.enabled - << " }, voice_detection: { enabled: " << voice_detection.enabled << " }, gain_controller1: { enabled: " << gain_controller1.enabled << ", mode: " << GainController1ModeToString(gain_controller1.mode) << ", target_level_dbfs: " << gain_controller1.target_level_dbfs << ", compression_gain_db: " << gain_controller1.compression_gain_db << ", enable_limiter: " << gain_controller1.enable_limiter - << ", analog_level_minimum: " << gain_controller1.analog_level_minimum - << ", analog_level_maximum: " << gain_controller1.analog_level_maximum - << " }, gain_controller2: { enabled: " << gain_controller2.enabled + << ", analog_gain_controller { enabled: " + << gain_controller1.analog_gain_controller.enabled + << ", startup_min_volume: " + << gain_controller1.analog_gain_controller.startup_min_volume + << ", clipped_level_min: " + << gain_controller1.analog_gain_controller.clipped_level_min + << ", enable_digital_adaptive: " + << gain_controller1.analog_gain_controller.enable_digital_adaptive + << ", clipped_level_step: " + << gain_controller1.analog_gain_controller.clipped_level_step + << ", clipped_ratio_threshold: " + << gain_controller1.analog_gain_controller.clipped_ratio_threshold + << ", clipped_wait_frames: " + << gain_controller1.analog_gain_controller.clipped_wait_frames + << ", clipping_predictor: { enabled: " + << gain_controller1.analog_gain_controller.clipping_predictor.enabled + << ", mode: " + << gain_controller1.analog_gain_controller.clipping_predictor.mode + << ", window_length: " + << gain_controller1.analog_gain_controller.clipping_predictor + .window_length + << ", reference_window_length: " + << gain_controller1.analog_gain_controller.clipping_predictor + .reference_window_length + << ", reference_window_delay: " + << gain_controller1.analog_gain_controller.clipping_predictor + .reference_window_delay + << ", clipping_threshold: " + << gain_controller1.analog_gain_controller.clipping_predictor + .clipping_threshold + << ", crest_factor_margin: " + << gain_controller1.analog_gain_controller.clipping_predictor + .crest_factor_margin + << ", use_predicted_step: " + << gain_controller1.analog_gain_controller.clipping_predictor + .use_predicted_step + << " }}}, gain_controller2: { enabled: " << gain_controller2.enabled << ", fixed_digital: { gain_db: " << gain_controller2.fixed_digital.gain_db << " }, adaptive_digital: { enabled: " - << gain_controller2.adaptive_digital.enabled << ", level_estimator: " - << GainController2LevelEstimatorToString( - gain_controller2.adaptive_digital.level_estimator) - << ", use_saturation_protector: " - << gain_controller2.adaptive_digital.use_saturation_protector - << ", extra_saturation_margin_db: " - << gain_controller2.adaptive_digital.extra_saturation_margin_db - << " } }, residual_echo_detector: { enabled: " - << residual_echo_detector.enabled - << " }, level_estimation: { enabled: " << level_estimation.enabled - << " } }"; + << gain_controller2.adaptive_digital.enabled + << ", headroom_db: " << gain_controller2.adaptive_digital.headroom_db + << ", max_gain_db: " << gain_controller2.adaptive_digital.max_gain_db + << ", initial_gain_db: " + << gain_controller2.adaptive_digital.initial_gain_db + << ", max_gain_change_db_per_second: " + << gain_controller2.adaptive_digital.max_gain_change_db_per_second + << ", max_output_noise_level_dbfs: " + << gain_controller2.adaptive_digital.max_output_noise_level_dbfs + << " }, input_volume_control : { enabled " + << gain_controller2.input_volume_controller.enabled << "}}"; return builder.str(); } diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index d09e2ba..f613a38 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -23,16 +23,14 @@ #include +#include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/array_view.h" #include "api/audio/echo_canceller3_config.h" #include "api/audio/echo_control.h" #include "api/scoped_refptr.h" #include "modules/audio_processing/include/audio_processing_statistics.h" -#include "modules/audio_processing/include/config.h" #include "rtc_base/arraysize.h" -#include "rtc_base/constructor_magic.h" -#include "rtc_base/deprecation.h" #include "rtc_base/ref_count.h" #include "rtc_base/system/file_wrapper.h" #include "rtc_base/system/rtc_export.h" @@ -53,74 +51,13 @@ class EchoDetector; class CustomAudioAnalyzer; class CustomProcessing; -// Use to enable experimental gain control (AGC). At startup the experimental -// AGC moves the microphone volume up to |startup_min_volume| if the current -// microphone volume is set too low. The value is clamped to its operating range -// [12, 255]. Here, 255 maps to 100%. -// -// Must be provided through AudioProcessingBuilder().Create(config). -#if defined(WEBRTC_CHROMIUM_BUILD) -static const int kAgcStartupMinVolume = 85; -#else -static const int kAgcStartupMinVolume = 0; -#endif // defined(WEBRTC_CHROMIUM_BUILD) -static constexpr int kClippedLevelMin = 70; - -// To be deprecated: Please instead use the flag in the -// AudioProcessing::Config::AnalogGainController. -// TODO(webrtc:5298): Remove. -struct ExperimentalAgc { - ExperimentalAgc() = default; - explicit ExperimentalAgc(bool enabled) : enabled(enabled) {} - ExperimentalAgc(bool enabled, - bool enabled_agc2_level_estimator, - bool digital_adaptive_disabled) - : enabled(enabled), - enabled_agc2_level_estimator(enabled_agc2_level_estimator), - digital_adaptive_disabled(digital_adaptive_disabled) {} - // Deprecated constructor: will be removed. - ExperimentalAgc(bool enabled, - bool enabled_agc2_level_estimator, - bool digital_adaptive_disabled, - bool analyze_before_aec) - : enabled(enabled), - enabled_agc2_level_estimator(enabled_agc2_level_estimator), - digital_adaptive_disabled(digital_adaptive_disabled) {} - ExperimentalAgc(bool enabled, int startup_min_volume) - : enabled(enabled), startup_min_volume(startup_min_volume) {} - ExperimentalAgc(bool enabled, int startup_min_volume, int clipped_level_min) - : enabled(enabled), - startup_min_volume(startup_min_volume), - clipped_level_min(clipped_level_min) {} - static const ConfigOptionID identifier = ConfigOptionID::kExperimentalAgc; - bool enabled = true; - int startup_min_volume = kAgcStartupMinVolume; - // Lowest microphone level that will be applied in response to clipping. - int clipped_level_min = kClippedLevelMin; - bool enabled_agc2_level_estimator = false; - bool digital_adaptive_disabled = false; -}; - -// To be deprecated: Please instead use the flag in the -// AudioProcessing::Config::TransientSuppression. -// -// Use to enable experimental noise suppression. It can be set in the -// constructor. -// TODO(webrtc:5298): Remove. -struct ExperimentalNs { - ExperimentalNs() : enabled(false) {} - explicit ExperimentalNs(bool enabled) : enabled(enabled) {} - static const ConfigOptionID identifier = ConfigOptionID::kExperimentalNs; - bool enabled; -}; - // The Audio Processing Module (APM) provides a collection of voice processing // components designed for real-time communications software. // // APM operates on two audio streams on a frame-by-frame basis. Frames of the // primary stream, on which all processing is applied, are passed to -// |ProcessStream()|. Frames of the reverse direction stream are passed to -// |ProcessReverseStream()|. On the client-side, this will typically be the +// `ProcessStream()`. Frames of the reverse direction stream are passed to +// `ProcessReverseStream()`. On the client-side, this will typically be the // near-end (capture) and far-end (render) streams, respectively. APM should be // placed in the signal chain as close to the audio hardware abstraction layer // (HAL) as possible. @@ -143,12 +80,13 @@ struct ExperimentalNs { // 2. Parameter getters are never called concurrently with the corresponding // setter. // -// APM accepts only linear PCM audio data in chunks of 10 ms. The int16 -// interfaces use interleaved data, while the float interfaces use deinterleaved -// data. +// APM accepts only linear PCM audio data in chunks of ~10 ms (see +// AudioProcessing::GetFrameSize() for details) and sample rates ranging from +// 8000 Hz to 384000 Hz. The int16 interfaces use interleaved data, while the +// float interfaces use deinterleaved data. // // Usage example, omitting error checking: -// AudioProcessing* apm = AudioProcessingBuilder().Create(); +// rtc::scoped_refptr apm = AudioProcessingBuilder().Create(); // // AudioProcessing::Config config; // config.echo_canceller.enabled = true; @@ -164,13 +102,8 @@ struct ExperimentalNs { // // config.high_pass_filter.enabled = true; // -// config.voice_detection.enabled = true; -// // apm->ApplyConfig(config) // -// apm->noise_reduction()->set_level(kHighSuppression); -// apm->noise_reduction()->Enable(true); -// // // Start a voice call... // // // ... Render frame arrives bound for the audio HAL ... @@ -187,12 +120,12 @@ struct ExperimentalNs { // analog_level = apm->recommended_stream_analog_level(); // has_voice = apm->stream_has_voice(); // -// // Repeate render and capture processing for the duration of the call... +// // Repeat render and capture processing for the duration of the call... // // Start a new call... // apm->Initialize(); // // // Close the application... -// delete apm; +// apm.reset(); // class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { public: @@ -211,30 +144,60 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // submodule resets, affecting the audio quality. Use the RuntimeSetting // construct for runtime configuration. struct RTC_EXPORT Config { - // Sets the properties of the audio processing pipeline. struct RTC_EXPORT Pipeline { - Pipeline(); + // Ways to downmix a multi-channel track to mono. + enum class DownmixMethod { + kAverageChannels, // Average across channels. + kUseFirstChannel // Use the first channel. + }; // Maximum allowed processing rate used internally. May only be set to - // 32000 or 48000 and any differing values will be treated as 48000. The - // default rate is currently selected based on the CPU architecture, but - // that logic may change. - int maximum_internal_processing_rate; + // 32000 or 48000 and any differing values will be treated as 48000. + int maximum_internal_processing_rate = 48000; // Allow multi-channel processing of render audio. bool multi_channel_render = false; // Allow multi-channel processing of capture audio when AEC3 is active // or a custom AEC is injected.. bool multi_channel_capture = false; + // Indicates how to downmix multi-channel capture audio to mono (when + // needed). + DownmixMethod capture_downmix_method = DownmixMethod::kAverageChannels; } pipeline; // Enabled the pre-amplifier. It amplifies the capture signal // before any other processing is done. + // TODO(webrtc:5298): Deprecate and use the pre-gain functionality in + // capture_level_adjustment instead. struct PreAmplifier { bool enabled = false; - float fixed_gain_factor = 1.f; + float fixed_gain_factor = 1.0f; } pre_amplifier; + // Functionality for general level adjustment in the capture pipeline. This + // should not be used together with the legacy PreAmplifier functionality. + struct CaptureLevelAdjustment { + bool operator==(const CaptureLevelAdjustment& rhs) const; + bool operator!=(const CaptureLevelAdjustment& rhs) const { + return !(*this == rhs); + } + bool enabled = false; + // The `pre_gain_factor` scales the signal before any processing is done. + float pre_gain_factor = 1.0f; + // The `post_gain_factor` scales the signal after all processing is done. + float post_gain_factor = 1.0f; + struct AnalogMicGainEmulation { + bool operator==(const AnalogMicGainEmulation& rhs) const; + bool operator!=(const AnalogMicGainEmulation& rhs) const { + return !(*this == rhs); + } + bool enabled = false; + // Initial analog gain level to use for the emulated analog gain. Must + // be in the range [0...255]. + int initial_level = 255; + } analog_mic_gain_emulation; + } capture_level_adjustment; + struct HighPassFilter { bool enabled = false; bool apply_in_full_band = true; @@ -262,18 +225,18 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { bool enabled = false; } transient_suppression; - // Enables reporting of |voice_detected| in webrtc::AudioProcessingStats. - struct VoiceDetection { - bool enabled = false; - } voice_detection; - // Enables automatic gain control (AGC) functionality. // The automatic gain control (AGC) component brings the signal to an // appropriate range. This is done by applying a digital gain directly and, // in the analog mode, prescribing an analog gain to be applied at the audio // HAL. // Recommended to be enabled on the client-side. - struct GainController1 { + struct RTC_EXPORT GainController1 { + bool operator==(const GainController1& rhs) const; + bool operator!=(const GainController1& rhs) const { + return !(*this == rhs); + } + bool enabled = false; enum Mode { // Adaptive mode intended for use if an analog volume control is @@ -314,73 +277,112 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // target level. Otherwise, the signal will be compressed but not limited // above the target level. bool enable_limiter = true; - // Sets the minimum and maximum analog levels of the audio capture device. - // Must be set if an analog mode is used. Limited to [0, 65535]. - int analog_level_minimum = 0; - int analog_level_maximum = 255; // Enables the analog gain controller functionality. struct AnalogGainController { bool enabled = true; - int startup_min_volume = kAgcStartupMinVolume; + // TODO(bugs.webrtc.org/7494): Deprecated. Stop using and remove. + int startup_min_volume = 0; // Lowest analog microphone level that will be applied in response to // clipping. - int clipped_level_min = kClippedLevelMin; - bool enable_agc2_level_estimator = false; + int clipped_level_min = 70; + // If true, an adaptive digital gain is applied. bool enable_digital_adaptive = true; + // Amount the microphone level is lowered with every clipping event. + // Limited to (0, 255]. + int clipped_level_step = 15; + // Proportion of clipped samples required to declare a clipping event. + // Limited to (0.f, 1.f). + float clipped_ratio_threshold = 0.1f; + // Time in frames to wait after a clipping event before checking again. + // Limited to values higher than 0. + int clipped_wait_frames = 300; + + // Enables clipping prediction functionality. + struct ClippingPredictor { + bool enabled = false; + enum Mode { + // Clipping event prediction mode with fixed step estimation. + kClippingEventPrediction, + // Clipped peak estimation mode with adaptive step estimation. + kAdaptiveStepClippingPeakPrediction, + // Clipped peak estimation mode with fixed step estimation. + kFixedStepClippingPeakPrediction, + }; + Mode mode = kClippingEventPrediction; + // Number of frames in the sliding analysis window. + int window_length = 5; + // Number of frames in the sliding reference window. + int reference_window_length = 5; + // Reference window delay (unit: number of frames). + int reference_window_delay = 5; + // Clipping prediction threshold (dBFS). + float clipping_threshold = -1.0f; + // Crest factor drop threshold (dB). + float crest_factor_margin = 3.0f; + // If true, the recommended clipped level step is used to modify the + // analog gain. Otherwise, the predictor runs without affecting the + // analog gain. + bool use_predicted_step = true; + } clipping_predictor; } analog_gain_controller; } gain_controller1; - // Enables the next generation AGC functionality. This feature replaces the - // standard methods of gain control in the previous AGC. Enabling this - // submodule enables an adaptive digital AGC followed by a limiter. By - // setting |fixed_gain_db|, the limiter can be turned into a compressor that - // first applies a fixed gain. The adaptive digital AGC can be turned off by - // setting |adaptive_digital_mode=false|. - struct GainController2 { - enum LevelEstimator { kRms, kPeak }; + // Parameters for AGC2, an Automatic Gain Control (AGC) sub-module which + // replaces the AGC sub-module parametrized by `gain_controller1`. + // AGC2 brings the captured audio signal to the desired level by combining + // three different controllers (namely, input volume controller, adapative + // digital controller and fixed digital controller) and a limiter. + // TODO(bugs.webrtc.org:7494): Name `GainController` when AGC1 removed. + struct RTC_EXPORT GainController2 { + bool operator==(const GainController2& rhs) const; + bool operator!=(const GainController2& rhs) const { + return !(*this == rhs); + } + + // AGC2 must be created if and only if `enabled` is true. bool enabled = false; - struct { - float gain_db = 0.f; - } fixed_digital; - struct { + + // Parameters for the input volume controller, which adjusts the input + // volume applied when the audio is captured (e.g., microphone volume on + // a soundcard, input volume on HAL). + struct InputVolumeController { + bool operator==(const InputVolumeController& rhs) const; + bool operator!=(const InputVolumeController& rhs) const { + return !(*this == rhs); + } bool enabled = false; - float vad_probability_attack = 1.f; - LevelEstimator level_estimator = kRms; - int level_estimator_adjacent_speech_frames_threshold = 1; - // TODO(crbug.com/webrtc/7494): Remove `use_saturation_protector`. - bool use_saturation_protector = true; - float initial_saturation_margin_db = 20.f; - float extra_saturation_margin_db = 2.f; - int gain_applier_adjacent_speech_frames_threshold = 1; - float max_gain_change_db_per_second = 3.f; - float max_output_noise_level_dbfs = -50.f; + } input_volume_controller; + + // Parameters for the adaptive digital controller, which adjusts and + // applies a digital gain after echo cancellation and after noise + // suppression. + struct RTC_EXPORT AdaptiveDigital { + bool operator==(const AdaptiveDigital& rhs) const; + bool operator!=(const AdaptiveDigital& rhs) const { + return !(*this == rhs); + } + bool enabled = false; + float headroom_db = 6.0f; + float max_gain_db = 30.0f; + float initial_gain_db = 8.0f; + float max_gain_change_db_per_second = 3.0f; + float max_output_noise_level_dbfs = -50.0f; } adaptive_digital; + + // Parameters for the fixed digital controller, which applies a fixed + // digital gain after the adaptive digital controller and before the + // limiter. + struct FixedDigital { + // By setting `gain_db` to a value greater than zero, the limiter can be + // turned into a compressor that first applies a fixed gain. + float gain_db = 0.0f; + } fixed_digital; } gain_controller2; - struct ResidualEchoDetector { - bool enabled = true; - } residual_echo_detector; - - // Enables reporting of |output_rms_dbfs| in webrtc::AudioProcessingStats. - struct LevelEstimation { - bool enabled = false; - } level_estimation; - std::string ToString() const; }; - // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone. - enum ChannelLayout { - kMono, - // Left, right. - kStereo, - // Mono, keyboard, and mic. - kMonoAndKeyboard, - // Left, right, keyboard, and mic. - kStereoAndKeyboard - }; - // Specifies the properties of a setting to be passed to AudioProcessing at // runtime. class RuntimeSetting { @@ -393,6 +395,7 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { kPlayoutVolumeChange, kCustomRenderProcessingRuntimeSetting, kPlayoutAudioDeviceChange, + kCapturePostGain, kCaptureOutputUsed }; @@ -402,14 +405,17 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { int max_volume; // Maximum play-out volume. }; - RuntimeSetting() : type_(Type::kNotSpecified), value_(0.f) {} + RuntimeSetting() : type_(Type::kNotSpecified), value_(0.0f) {} ~RuntimeSetting() = default; static RuntimeSetting CreateCapturePreGain(float gain) { - RTC_DCHECK_GE(gain, 1.f) << "Attenuation is not allowed."; return {Type::kCapturePreGain, gain}; } + static RuntimeSetting CreateCapturePostGain(float gain) { + return {Type::kCapturePostGain, gain}; + } + // Corresponds to Config::GainController1::compression_gain_db, but for // runtime configuration. static RuntimeSetting CreateCompressionGainDb(int gain_db) { @@ -421,8 +427,8 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // Corresponds to Config::GainController2::fixed_digital::gain_db, but for // runtime configuration. static RuntimeSetting CreateCaptureFixedPostGain(float gain_db) { - RTC_DCHECK_GE(gain_db, 0.f); - RTC_DCHECK_LE(gain_db, 90.f); + RTC_DCHECK_GE(gain_db, 0.0f); + RTC_DCHECK_LE(gain_db, 90.0f); return {Type::kCaptureFixedPostGain, gain_db}; } @@ -434,7 +440,7 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { } // Creates a runtime setting to notify play-out (aka render) volume changes. - // |volume| is the unnormalized volume, the maximum of which + // `volume` is the unnormalized volume, the maximum of which static RuntimeSetting CreatePlayoutVolumeChange(int volume) { return {Type::kPlayoutVolumeChange, volume}; } @@ -443,8 +449,9 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { return {Type::kCustomRenderProcessingRuntimeSetting, payload}; } - static RuntimeSetting CreateCaptureOutputUsedSetting(bool payload) { - return {Type::kCaptureOutputUsed, payload}; + static RuntimeSetting CreateCaptureOutputUsedSetting( + bool capture_output_used) { + return {Type::kCaptureOutputUsed, capture_output_used}; } Type type() const { return type_; } @@ -494,32 +501,22 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // // It is also not necessary to call if the audio parameters (sample // rate and number of channels) have changed. Passing updated parameters - // directly to |ProcessStream()| and |ProcessReverseStream()| is permissible. + // directly to `ProcessStream()` and `ProcessReverseStream()` is permissible. // If the parameters are known at init-time though, they may be provided. // TODO(webrtc:5298): Change to return void. virtual int Initialize() = 0; // The int16 interfaces require: - // - only |NativeRate|s be used + // - only `NativeRate`s be used // - that the input, output and reverse rates must match - // - that |processing_config.output_stream()| matches - // |processing_config.input_stream()|. + // - that `processing_config.output_stream()` matches + // `processing_config.input_stream()`. // // The float interfaces accept arbitrary rates and support differing input and // output layouts, but the output must have either one channel or the same // number of channels as the input. virtual int Initialize(const ProcessingConfig& processing_config) = 0; - // Initialize with unpacked parameters. See Initialize() above for details. - // - // TODO(mgraczyk): Remove once clients are updated to use the new interface. - virtual int Initialize(int capture_input_sample_rate_hz, - int capture_output_sample_rate_hz, - int render_sample_rate_hz, - ChannelLayout capture_input_layout, - ChannelLayout capture_output_layout, - ChannelLayout render_input_layout) = 0; - // TODO(peah): This method is a temporary solution used to take control // over the parameters in the audio processing module and is likely to change. virtual void ApplyConfig(const Config& config) = 0; @@ -536,14 +533,19 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // Set to true when the output of AudioProcessing will be muted or in some // other way not used. Ideally, the captured audio would still be processed, // but some components may change behavior based on this information. - // Default false. + // Default false. This method takes a lock. To achieve this in a lock-less + // manner the PostRuntimeSetting can instead be used. virtual void set_output_will_be_muted(bool muted) = 0; - // Enqueue a runtime setting. + // Enqueues a runtime setting. virtual void SetRuntimeSetting(RuntimeSetting setting) = 0; - // Accepts and produces a 10 ms frame interleaved 16 bit integer audio as - // specified in |input_config| and |output_config|. |src| and |dest| may use + // Enqueues a runtime setting. Returns a bool indicating whether the + // enqueueing was successfull. + virtual bool PostRuntimeSetting(RuntimeSetting setting) = 0; + + // Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio as + // specified in `input_config` and `output_config`. `src` and `dest` may use // the same memory, if desired. virtual int ProcessStream(const int16_t* const src, const StreamConfig& input_config, @@ -551,58 +553,59 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { int16_t* const dest) = 0; // Accepts deinterleaved float audio with the range [-1, 1]. Each element of - // |src| points to a channel buffer, arranged according to |input_stream|. At - // output, the channels will be arranged according to |output_stream| in - // |dest|. + // `src` points to a channel buffer, arranged according to `input_stream`. At + // output, the channels will be arranged according to `output_stream` in + // `dest`. // - // The output must have one channel or as many channels as the input. |src| - // and |dest| may use the same memory, if desired. + // The output must have one channel or as many channels as the input. `src` + // and `dest` may use the same memory, if desired. virtual int ProcessStream(const float* const* src, const StreamConfig& input_config, const StreamConfig& output_config, float* const* dest) = 0; - // Accepts and produces a 10 ms frame of interleaved 16 bit integer audio for - // the reverse direction audio stream as specified in |input_config| and - // |output_config|. |src| and |dest| may use the same memory, if desired. + // Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio for + // the reverse direction audio stream as specified in `input_config` and + // `output_config`. `src` and `dest` may use the same memory, if desired. virtual int ProcessReverseStream(const int16_t* const src, const StreamConfig& input_config, const StreamConfig& output_config, int16_t* const dest) = 0; // Accepts deinterleaved float audio with the range [-1, 1]. Each element of - // |data| points to a channel buffer, arranged according to |reverse_config|. + // `data` points to a channel buffer, arranged according to `reverse_config`. virtual int ProcessReverseStream(const float* const* src, const StreamConfig& input_config, const StreamConfig& output_config, float* const* dest) = 0; // Accepts deinterleaved float audio with the range [-1, 1]. Each element - // of |data| points to a channel buffer, arranged according to - // |reverse_config|. + // of `data` points to a channel buffer, arranged according to + // `reverse_config`. virtual int AnalyzeReverseStream(const float* const* data, const StreamConfig& reverse_config) = 0; - // Returns the most recently produced 10 ms of the linear AEC output at a rate - // of 16 kHz. If there is more than one capture channel, a mono representation - // of the input is returned. Returns true/false to indicate whether an output - // returned. + // Returns the most recently produced ~10 ms of the linear AEC output at a + // rate of 16 kHz. If there is more than one capture channel, a mono + // representation of the input is returned. Returns true/false to indicate + // whether an output returned. virtual bool GetLinearAecOutput( rtc::ArrayView> linear_output) const = 0; // This must be called prior to ProcessStream() if and only if adaptive analog // gain control is enabled, to pass the current analog level from the audio - // HAL. Must be within the range provided in Config::GainController1. + // HAL. Must be within the range [0, 255]. virtual void set_stream_analog_level(int level) = 0; - // When an analog mode is set, this should be called after ProcessStream() - // to obtain the recommended new analog level for the audio HAL. It is the - // user's responsibility to apply this level. + // When an analog mode is set, this should be called after + // `set_stream_analog_level()` and `ProcessStream()` to obtain the recommended + // new analog level for the audio HAL. It is the user's responsibility to + // apply this level. virtual int recommended_stream_analog_level() const = 0; // This must be called if and only if echo processing is enabled. // - // Sets the |delay| in ms between ProcessReverseStream() receiving a far-end + // Sets the `delay` in ms between ProcessReverseStream() receiving a far-end // frame and ProcessStream() receiving a near-end frame containing the // corresponding echo. On the client-side this can be expressed as // delay = (t_render - t_analyze) + (t_process - t_capture) @@ -622,14 +625,14 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // Creates and attaches an webrtc::AecDump for recording debugging // information. - // The |worker_queue| may not be null and must outlive the created + // The `worker_queue` may not be null and must outlive the created // AecDump instance. |max_log_size_bytes == -1| means the log size - // will be unlimited. |handle| may not be null. The AecDump takes - // responsibility for |handle| and closes it in the destructor. A + // will be unlimited. `handle` may not be null. The AecDump takes + // responsibility for `handle` and closes it in the destructor. A // return value of true indicates that the file has been // sucessfully opened, while a value of false indicates that // opening the file failed. - virtual bool CreateAndAttachAecDump(const std::string& file_name, + virtual bool CreateAndAttachAecDump(absl::string_view file_name, int64_t max_log_size_bytes, rtc::TaskQueue* worker_queue) = 0; virtual bool CreateAndAttachAecDump(FILE* handle, @@ -653,7 +656,7 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { // Get audio processing statistics. virtual AudioProcessingStats GetStatistics() = 0; - // TODO(webrtc:5298) Deprecated variant. The |has_remote_tracks| argument + // TODO(webrtc:5298) Deprecated variant. The `has_remote_tracks` argument // should be set if there are active remote tracks (this would usually be true // during a call). If there are no remote tracks some of the stats will not be // set by AudioProcessing, because they only make sense if there is at least @@ -703,77 +706,101 @@ class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface { static constexpr int kMaxNativeSampleRateHz = kNativeSampleRatesHz[kNumNativeSampleRates - 1]; - static const int kChunkSizeMs = 10; + // APM processes audio in chunks of about 10 ms. See GetFrameSize() for + // details. + static constexpr int kChunkSizeMs = 10; + + // Returns floor(sample_rate_hz/100): the number of samples per channel used + // as input and output to the audio processing module in calls to + // ProcessStream, ProcessReverseStream, AnalyzeReverseStream, and + // GetLinearAecOutput. + // + // This is exactly 10 ms for sample rates divisible by 100. For example: + // - 48000 Hz (480 samples per channel), + // - 44100 Hz (441 samples per channel), + // - 16000 Hz (160 samples per channel). + // + // Sample rates not divisible by 100 are received/produced in frames of + // approximately 10 ms. For example: + // - 22050 Hz (220 samples per channel, or ~9.98 ms per frame), + // - 11025 Hz (110 samples per channel, or ~9.98 ms per frame). + // These nondivisible sample rates yield lower audio quality compared to + // multiples of 100. Internal resampling to 10 ms frames causes a simulated + // clock drift effect which impacts the performance of (for example) echo + // cancellation. + static int GetFrameSize(int sample_rate_hz) { return sample_rate_hz / 100; } }; class RTC_EXPORT AudioProcessingBuilder { public: AudioProcessingBuilder(); + AudioProcessingBuilder(const AudioProcessingBuilder&) = delete; + AudioProcessingBuilder& operator=(const AudioProcessingBuilder&) = delete; ~AudioProcessingBuilder(); - // The AudioProcessingBuilder takes ownership of the echo_control_factory. + + // Sets the APM configuration. + AudioProcessingBuilder& SetConfig(const AudioProcessing::Config& config) { + config_ = config; + return *this; + } + + // Sets the echo controller factory to inject when APM is created. AudioProcessingBuilder& SetEchoControlFactory( std::unique_ptr echo_control_factory) { echo_control_factory_ = std::move(echo_control_factory); return *this; } - // The AudioProcessingBuilder takes ownership of the capture_post_processing. + + // Sets the capture post-processing sub-module to inject when APM is created. AudioProcessingBuilder& SetCapturePostProcessing( std::unique_ptr capture_post_processing) { capture_post_processing_ = std::move(capture_post_processing); return *this; } - // The AudioProcessingBuilder takes ownership of the render_pre_processing. + + // Sets the render pre-processing sub-module to inject when APM is created. AudioProcessingBuilder& SetRenderPreProcessing( std::unique_ptr render_pre_processing) { render_pre_processing_ = std::move(render_pre_processing); return *this; } - // The AudioProcessingBuilder takes ownership of the echo_detector. + + // Sets the echo detector to inject when APM is created. AudioProcessingBuilder& SetEchoDetector( rtc::scoped_refptr echo_detector) { echo_detector_ = std::move(echo_detector); return *this; } - // The AudioProcessingBuilder takes ownership of the capture_analyzer. + + // Sets the capture analyzer sub-module to inject when APM is created. AudioProcessingBuilder& SetCaptureAnalyzer( std::unique_ptr capture_analyzer) { capture_analyzer_ = std::move(capture_analyzer); return *this; } - // This creates an APM instance using the previously set components. Calling - // the Create function resets the AudioProcessingBuilder to its initial state. - AudioProcessing* Create(); - AudioProcessing* Create(const webrtc::Config& config); + + // Creates an APM instance with the specified config or the default one if + // unspecified. Injects the specified components transferring the ownership + // to the newly created APM instance - i.e., except for the config, the + // builder is reset to its initial state. + rtc::scoped_refptr Create(); private: + AudioProcessing::Config config_; std::unique_ptr echo_control_factory_; std::unique_ptr capture_post_processing_; std::unique_ptr render_pre_processing_; rtc::scoped_refptr echo_detector_; std::unique_ptr capture_analyzer_; - RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder); }; class StreamConfig { public: // sample_rate_hz: The sampling rate of the stream. - // - // num_channels: The number of audio channels in the stream, excluding the - // keyboard channel if it is present. When passing a - // StreamConfig with an array of arrays T*[N], - // - // N == {num_channels + 1 if has_keyboard - // {num_channels if !has_keyboard - // - // has_keyboard: True if the stream has a keyboard channel. When has_keyboard - // is true, the last channel in any corresponding list of - // channels is the keyboard channel. - StreamConfig(int sample_rate_hz = 0, - size_t num_channels = 0, - bool has_keyboard = false) + // num_channels: The number of audio channels in the stream. + StreamConfig(int sample_rate_hz = 0, size_t num_channels = 0) : sample_rate_hz_(sample_rate_hz), num_channels_(num_channels), - has_keyboard_(has_keyboard), num_frames_(calculate_frames(sample_rate_hz)) {} void set_sample_rate_hz(int value) { @@ -781,35 +808,29 @@ class StreamConfig { num_frames_ = calculate_frames(value); } void set_num_channels(size_t value) { num_channels_ = value; } - void set_has_keyboard(bool value) { has_keyboard_ = value; } int sample_rate_hz() const { return sample_rate_hz_; } - // The number of channels in the stream, not including the keyboard channel if - // present. + // The number of channels in the stream. size_t num_channels() const { return num_channels_; } - bool has_keyboard() const { return has_keyboard_; } size_t num_frames() const { return num_frames_; } size_t num_samples() const { return num_channels_ * num_frames_; } bool operator==(const StreamConfig& other) const { return sample_rate_hz_ == other.sample_rate_hz_ && - num_channels_ == other.num_channels_ && - has_keyboard_ == other.has_keyboard_; + num_channels_ == other.num_channels_; } bool operator!=(const StreamConfig& other) const { return !(*this == other); } private: static size_t calculate_frames(int sample_rate_hz) { - return static_cast(AudioProcessing::kChunkSizeMs * sample_rate_hz / - 1000); + return static_cast(AudioProcessing::GetFrameSize(sample_rate_hz)); } int sample_rate_hz_; size_t num_channels_; - bool has_keyboard_; size_t num_frames_; }; @@ -899,17 +920,13 @@ class EchoDetector : public rtc::RefCountInterface { int render_sample_rate_hz, int num_render_channels) = 0; - // Analysis (not changing) of the render signal. + // Analysis (not changing) of the first channel of the render signal. virtual void AnalyzeRenderAudio(rtc::ArrayView render_audio) = 0; // Analysis (not changing) of the capture signal. virtual void AnalyzeCaptureAudio( rtc::ArrayView capture_audio) = 0; - // Pack an AudioBuffer into a vector. - static void PackRenderAudioBuffer(AudioBuffer* audio, - std::vector* packed_buffer); - struct Metrics { absl::optional echo_likelihood; absl::optional echo_likelihood_recent_max; diff --git a/webrtc/modules/audio_processing/include/audio_processing_statistics.h b/webrtc/modules/audio_processing/include/audio_processing_statistics.h index 87babee..3b43319 100644 --- a/webrtc/modules/audio_processing/include/audio_processing_statistics.h +++ b/webrtc/modules/audio_processing/include/audio_processing_statistics.h @@ -24,14 +24,8 @@ struct RTC_EXPORT AudioProcessingStats { AudioProcessingStats(const AudioProcessingStats& other); ~AudioProcessingStats(); - // The root mean square (RMS) level in dBFS (decibels from digital - // full-scale) of the last capture frame, after processing. It is - // constrained to [-127, 0]. - // The computation follows: https://tools.ietf.org/html/rfc6465 - // with the intent that it can provide the RTP audio level indication. - // Only reported if level estimation is enabled in AudioProcessing::Config. - absl::optional output_rms_dbfs; - + // Deprecated. + // TODO(bugs.webrtc.org/11226): Remove. // True if voice is detected in the last capture frame, after processing. // It is conservative in flagging audio as speech, with low likelihood of // incorrectly flagging a frame as voice. @@ -50,9 +44,9 @@ struct RTC_EXPORT AudioProcessingStats { // The delay metrics consists of the delay median and standard deviation. It // also consists of the fraction of delay estimates that can make the echo // cancellation perform poorly. The values are aggregated until the first - // call to |GetStatistics()| and afterwards aggregated and updated every + // call to `GetStatistics()` and afterwards aggregated and updated every // second. Note that if there are several clients pulling metrics from - // |GetStatistics()| during a session the first call from any of them will + // `GetStatistics()` during a session the first call from any of them will // change to one second aggregation window for all. absl::optional delay_median_ms; absl::optional delay_standard_deviation_ms; @@ -64,7 +58,7 @@ struct RTC_EXPORT AudioProcessingStats { // The instantaneous delay estimate produced in the AEC. The unit is in // milliseconds and the value is the instantaneous value at the time of the - // call to |GetStatistics()|. + // call to `GetStatistics()`. absl::optional delay_ms; }; diff --git a/webrtc/modules/audio_processing/level_estimator.cc b/webrtc/modules/audio_processing/level_estimator.cc deleted file mode 100644 index e707288..0000000 --- a/webrtc/modules/audio_processing/level_estimator.cc +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/level_estimator.h" - -#include "api/array_view.h" - -namespace webrtc { - -LevelEstimator::LevelEstimator() { - rms_.Reset(); -} - -LevelEstimator::~LevelEstimator() = default; - -void LevelEstimator::ProcessStream(const AudioBuffer& audio) { - for (size_t i = 0; i < audio.num_channels(); i++) { - rms_.Analyze(rtc::ArrayView(audio.channels_const()[i], - audio.num_frames())); - } -} -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/level_estimator.h b/webrtc/modules/audio_processing/level_estimator.h deleted file mode 100644 index 1d8a071..0000000 --- a/webrtc/modules/audio_processing/level_estimator.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_LEVEL_ESTIMATOR_H_ -#define MODULES_AUDIO_PROCESSING_LEVEL_ESTIMATOR_H_ - -#include "modules/audio_processing/audio_buffer.h" -#include "modules/audio_processing/rms_level.h" - -namespace webrtc { - -// An estimation component used to retrieve level metrics. -class LevelEstimator { - public: - LevelEstimator(); - ~LevelEstimator(); - - LevelEstimator(LevelEstimator&) = delete; - LevelEstimator& operator=(LevelEstimator&) = delete; - - void ProcessStream(const AudioBuffer& audio); - - // Returns the root mean square (RMS) level in dBFs (decibels from digital - // full-scale), or alternately dBov. It is computed over all primary stream - // frames since the last call to RMS(). The returned value is positive but - // should be interpreted as negative. It is constrained to [0, 127]. - // - // The computation follows: https://tools.ietf.org/html/rfc6465 - // with the intent that it can provide the RTP audio level indication. - // - // Frames passed to ProcessStream() with an |_energy| of zero are considered - // to have been muted. The RMS of the frame will be interpreted as -127. - int RMS() { return rms_.Average(); } - - private: - RmsLevel rms_; -}; -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_LEVEL_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/logging/apm_data_dumper.cc b/webrtc/modules/audio_processing/logging/apm_data_dumper.cc index 917df60..65d2167 100644 --- a/webrtc/modules/audio_processing/logging/apm_data_dumper.cc +++ b/webrtc/modules/audio_processing/logging/apm_data_dumper.cc @@ -10,6 +10,7 @@ #include "modules/audio_processing/logging/apm_data_dumper.h" +#include "absl/strings/string_view.h" #include "rtc_base/strings/string_builder.h" // Check to verify that the define is properly set. @@ -29,17 +30,16 @@ constexpr char kPathDelimiter = '\\'; constexpr char kPathDelimiter = '/'; #endif -std::string FormFileName(const char* output_dir, - const char* name, +std::string FormFileName(absl::string_view output_dir, + absl::string_view name, int instance_index, int reinit_index, - const std::string& suffix) { + absl::string_view suffix) { char buf[1024]; rtc::SimpleStringBuilder ss(buf); - const size_t output_dir_size = strlen(output_dir); - if (output_dir_size > 0) { + if (!output_dir.empty()) { ss << output_dir; - if (output_dir[output_dir_size - 1] != kPathDelimiter) { + if (output_dir.back() != kPathDelimiter) { ss << kPathDelimiter; } } @@ -61,9 +61,10 @@ ApmDataDumper::~ApmDataDumper() = default; #if WEBRTC_APM_DEBUG_DUMP == 1 bool ApmDataDumper::recording_activated_ = false; +absl::optional ApmDataDumper::dump_set_to_use_; char ApmDataDumper::output_dir_[] = ""; -FILE* ApmDataDumper::GetRawFile(const char* name) { +FILE* ApmDataDumper::GetRawFile(absl::string_view name) { std::string filename = FormFileName(output_dir_, name, instance_index_, recording_set_index_, ".dat"); auto& f = raw_files_[filename]; @@ -74,7 +75,7 @@ FILE* ApmDataDumper::GetRawFile(const char* name) { return f.get(); } -WavWriter* ApmDataDumper::GetWavFile(const char* name, +WavWriter* ApmDataDumper::GetWavFile(absl::string_view name, int sample_rate_hz, int num_channels, WavFile::SampleFormat format) { diff --git a/webrtc/modules/audio_processing/logging/apm_data_dumper.h b/webrtc/modules/audio_processing/logging/apm_data_dumper.h index 1824fdd..76f8b34 100644 --- a/webrtc/modules/audio_processing/logging/apm_data_dumper.h +++ b/webrtc/modules/audio_processing/logging/apm_data_dumper.h @@ -13,18 +13,20 @@ #include #include -#include -#include #if WEBRTC_APM_DEBUG_DUMP == 1 #include +#include #include #endif +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "api/array_view.h" #if WEBRTC_APM_DEBUG_DUMP == 1 #include "common_audio/wav_file.h" #include "rtc_base/checks.h" +#include "rtc_base/string_utils.h" #endif // Check to verify that the define is properly set. @@ -64,11 +66,32 @@ class ApmDataDumper { #endif } + // Returns whether dumping functionality is enabled/available. + static bool IsAvailable() { +#if WEBRTC_APM_DEBUG_DUMP == 1 + return true; +#else + return false; +#endif + } + + // Default dump set. + static constexpr size_t kDefaultDumpSet = 0; + + // Specifies what dump set to use. All dump commands with a different dump set + // than the one specified will be discarded. If not specificed, all dump sets + // will be used. + static void SetDumpSetToUse(int dump_set_to_use) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + dump_set_to_use_ = dump_set_to_use; +#endif + } + // Set an optional output directory. - static void SetOutputDirectory(const std::string& output_dir) { + static void SetOutputDirectory(absl::string_view output_dir) { #if WEBRTC_APM_DEBUG_DUMP == 1 RTC_CHECK_LT(output_dir.size(), kOutputDirMaxLength); - strncpy(output_dir_, output_dir.c_str(), output_dir.size()); + rtc::strcpyn(output_dir_, kOutputDirMaxLength, output_dir); #endif } @@ -82,8 +105,13 @@ class ApmDataDumper { // Methods for performing dumping of data of various types into // various formats. - void DumpRaw(const char* name, double v) { + void DumpRaw(absl::string_view name, + double v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(&v, sizeof(v), 1, file); @@ -91,8 +119,14 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, size_t v_length, const double* v) { + void DumpRaw(absl::string_view name, + size_t v_length, + const double* v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -100,16 +134,26 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, rtc::ArrayView v) { + void DumpRaw(absl::string_view name, + rtc::ArrayView v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpRaw(name, v.size(), v.data()); } #endif } - void DumpRaw(const char* name, float v) { + void DumpRaw(absl::string_view name, + float v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(&v, sizeof(v), 1, file); @@ -117,8 +161,14 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, size_t v_length, const float* v) { + void DumpRaw(absl::string_view name, + size_t v_length, + const float* v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -126,24 +176,38 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, rtc::ArrayView v) { + void DumpRaw(absl::string_view name, + rtc::ArrayView v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpRaw(name, v.size(), v.data()); } #endif } - void DumpRaw(const char* name, bool v) { + void DumpRaw(absl::string_view name, bool v, int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpRaw(name, static_cast(v)); } #endif } - void DumpRaw(const char* name, size_t v_length, const bool* v) { + void DumpRaw(absl::string_view name, + size_t v_length, + const bool* v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); for (size_t k = 0; k < v_length; ++k) { @@ -154,16 +218,26 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, rtc::ArrayView v) { + void DumpRaw(absl::string_view name, + rtc::ArrayView v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpRaw(name, v.size(), v.data()); } #endif } - void DumpRaw(const char* name, int16_t v) { + void DumpRaw(absl::string_view name, + int16_t v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(&v, sizeof(v), 1, file); @@ -171,8 +245,14 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, size_t v_length, const int16_t* v) { + void DumpRaw(absl::string_view name, + size_t v_length, + const int16_t* v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -180,16 +260,26 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, rtc::ArrayView v) { + void DumpRaw(absl::string_view name, + rtc::ArrayView v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpRaw(name, v.size(), v.data()); } #endif } - void DumpRaw(const char* name, int32_t v) { + void DumpRaw(absl::string_view name, + int32_t v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(&v, sizeof(v), 1, file); @@ -197,8 +287,14 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, size_t v_length, const int32_t* v) { + void DumpRaw(absl::string_view name, + size_t v_length, + const int32_t* v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -206,8 +302,13 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, size_t v) { + void DumpRaw(absl::string_view name, + size_t v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(&v, sizeof(v), 1, file); @@ -215,8 +316,14 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, size_t v_length, const size_t* v) { + void DumpRaw(absl::string_view name, + size_t v_length, + const size_t* v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -224,26 +331,40 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, rtc::ArrayView v) { + void DumpRaw(absl::string_view name, + rtc::ArrayView v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpRaw(name, v.size(), v.data()); } #endif } - void DumpRaw(const char* name, rtc::ArrayView v) { + void DumpRaw(absl::string_view name, + rtc::ArrayView v, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + DumpRaw(name, v.size(), v.data()); #endif } - void DumpWav(const char* name, + void DumpWav(absl::string_view name, size_t v_length, const float* v, int sample_rate_hz, - int num_channels) { + int num_channels, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { WavWriter* file = GetWavFile(name, sample_rate_hz, num_channels, WavFile::SampleFormat::kFloat); @@ -252,11 +373,15 @@ class ApmDataDumper { #endif } - void DumpWav(const char* name, + void DumpWav(absl::string_view name, rtc::ArrayView v, int sample_rate_hz, - int num_channels) { + int num_channels, + int dump_set = kDefaultDumpSet) { #if WEBRTC_APM_DEBUG_DUMP == 1 + if (dump_set_to_use_ && *dump_set_to_use_ != dump_set) + return; + if (recording_activated_) { DumpWav(name, v.size(), v.data(), sample_rate_hz, num_channels); } @@ -266,6 +391,7 @@ class ApmDataDumper { private: #if WEBRTC_APM_DEBUG_DUMP == 1 static bool recording_activated_; + static absl::optional dump_set_to_use_; static constexpr size_t kOutputDirMaxLength = 1024; static char output_dir_[kOutputDirMaxLength]; const int instance_index_; @@ -274,8 +400,8 @@ class ApmDataDumper { raw_files_; std::unordered_map> wav_files_; - FILE* GetRawFile(const char* name); - WavWriter* GetWavFile(const char* name, + FILE* GetRawFile(absl::string_view name); + WavWriter* GetWavFile(absl::string_view name, int sample_rate_hz, int num_channels, WavFile::SampleFormat format); diff --git a/webrtc/modules/audio_processing/meson.build b/webrtc/modules/audio_processing/meson.build index 1309bc5..dc8d43b 100644 --- a/webrtc/modules/audio_processing/meson.build +++ b/webrtc/modules/audio_processing/meson.build @@ -17,6 +17,7 @@ webrtc_audio_processing_sources = [ 'aec3/clockdrift_detector.cc', 'aec3/coarse_filter_update_gain.cc', 'aec3/comfort_noise_generator.cc', + 'aec3/config_selector.cc', 'aec3/decimator.cc', 'aec3/dominant_nearend_detector.cc', 'aec3/downsampled_render_buffer.cc', @@ -35,6 +36,7 @@ webrtc_audio_processing_sources = [ 'aec3/matched_filter.cc', 'aec3/matched_filter_lag_aggregator.cc', 'aec3/moving_average.cc', + 'aec3/multi_channel_content_detector.cc', 'aec3/refined_filter_update_gain.cc', 'aec3/render_buffer.cc', 'aec3/render_delay_buffer.cc', @@ -66,37 +68,42 @@ webrtc_audio_processing_sources = [ 'agc/legacy/digital_agc.cc', 'agc/loudness_histogram.cc', 'agc/utility.cc', - 'agc2/adaptive_agc.cc', - 'agc2/adaptive_digital_gain_applier.cc', - 'agc2/adaptive_mode_level_estimator_agc.cc', - 'agc2/adaptive_mode_level_estimator.cc', + 'agc2/adaptive_digital_gain_controller.cc', 'agc2/agc2_testing_common.cc', 'agc2/biquad_filter.cc', + 'agc2/clipping_predictor.cc', + 'agc2/clipping_predictor_level_buffer.cc', 'agc2/compute_interpolated_gain_curve.cc', - 'agc2/down_sampler.cc', + 'agc2/cpu_features.cc', 'agc2/fixed_digital_level_estimator.cc', 'agc2/gain_applier.cc', + 'agc2/input_volume_controller.cc', + 'agc2/input_volume_stats_reporter.cc', 'agc2/interpolated_gain_curve.cc', 'agc2/limiter.cc', 'agc2/limiter_db_gain_curve.cc', 'agc2/noise_level_estimator.cc', - 'agc2/noise_spectrum_estimator.cc', 'agc2/rnn_vad/auto_correlation.cc', - 'agc2/rnn_vad/common.cc', 'agc2/rnn_vad/features_extraction.cc', 'agc2/rnn_vad/lp_residual.cc', 'agc2/rnn_vad/pitch_search.cc', 'agc2/rnn_vad/pitch_search_internal.cc', 'agc2/rnn_vad/rnn.cc', + 'agc2/rnn_vad/rnn_fc.cc', + 'agc2/rnn_vad/rnn_gru.cc', 'agc2/rnn_vad/spectral_features.cc', 'agc2/rnn_vad/spectral_features_internal.cc', 'agc2/saturation_protector.cc', - 'agc2/signal_classifier.cc', - 'agc2/vad_with_level.cc', + 'agc2/saturation_protector_buffer.cc', + 'agc2/speech_level_estimator.cc', + 'agc2/speech_probability_buffer.cc', + 'agc2/vad_wrapper.cc', 'agc2/vector_float_frame.cc', 'audio_buffer.cc', 'audio_processing_builder_impl.cc', 'audio_processing_impl.cc', + 'capture_levels_adjuster/audio_samples_scaler.cc', + 'capture_levels_adjuster/capture_levels_adjuster.cc', 'echo_control_mobile_impl.cc', 'echo_detector/circular_buffer.cc', 'echo_detector/mean_variance_estimator.cc', @@ -110,7 +117,6 @@ webrtc_audio_processing_sources = [ 'include/audio_processing.cc', 'include/audio_processing_statistics.cc', 'include/config.cc', - 'level_estimator.cc', 'logging/apm_data_dumper.cc', 'ns/fast_math.cc', 'ns/histograms.cc', @@ -134,9 +140,9 @@ webrtc_audio_processing_sources = [ 'transient/moving_moments.cc', 'transient/transient_detector.cc', 'transient/transient_suppressor_impl.cc', + 'transient/voice_probability_delay_unit.cc', 'transient/wpd_node.cc', 'transient/wpd_tree.cc', - 'typing_detection.cc', 'utility/cascaded_biquad_filter.cc', 'utility/delay_estimator.cc', 'utility/delay_estimator_wrapper.cc', @@ -149,7 +155,6 @@ webrtc_audio_processing_sources = [ 'vad/vad_audio_proc.cc', 'vad/vad_circular_buffer.cc', 'vad/voice_activity_detector.cc', - 'voice_detection.cc', ] webrtc_audio_processing_include_headers = [ @@ -168,6 +173,7 @@ if have_x86 'aec3/fft_data_avx2.cc', 'aec3/matched_filter_avx2.cc', 'aec3/vector_math_avx2.cc', + 'agc2/rnn_vad/vector_math_avx2.cc', ], dependencies: common_deps, include_directories: webrtc_inc, diff --git a/webrtc/modules/audio_processing/ns/BUILD.gn b/webrtc/modules/audio_processing/ns/BUILD.gn index f0842c5..d818e23 100644 --- a/webrtc/modules/audio_processing/ns/BUILD.gn +++ b/webrtc/modules/audio_processing/ns/BUILD.gn @@ -57,7 +57,6 @@ rtc_static_library("ns") { "../../../common_audio/third_party/ooura:fft_size_128", "../../../common_audio/third_party/ooura:fft_size_256", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", "../../../rtc_base:safe_minmax", "../../../rtc_base/system:arch", "../../../system_wrappers", @@ -80,12 +79,11 @@ if (rtc_include_tests) { "..:apm_logging", "..:audio_buffer", "..:audio_processing", - "..:audio_processing_unittests", "..:high_pass_filter", "../../../api:array_view", "../../../rtc_base:checks", - "../../../rtc_base:rtc_base_approved", "../../../rtc_base:safe_minmax", + "../../../rtc_base:stringutils", "../../../rtc_base/system:arch", "../../../system_wrappers", "../../../test:test_support", @@ -98,5 +96,9 @@ if (rtc_include_tests) { if (rtc_enable_protobuf) { sources += [] } + + if (!build_with_chromium) { + deps += [ "..:audio_processing_unittests" ] + } } } diff --git a/webrtc/modules/audio_processing/ns/noise_suppressor.cc b/webrtc/modules/audio_processing/ns/noise_suppressor.cc index 89e1fe0..7c524da 100644 --- a/webrtc/modules/audio_processing/ns/noise_suppressor.cc +++ b/webrtc/modules/audio_processing/ns/noise_suppressor.cc @@ -13,6 +13,7 @@ #include #include #include + #include #include "modules/audio_processing/ns/fast_math.h" @@ -448,6 +449,12 @@ void NoiseSuppressor::Process(AudioBuffer* audio) { } } + // Only do the below processing if the output of the audio processing module + // is used. + if (!capture_output_used_) { + return; + } + // Aggregate the Wiener filters for all channels. std::array filter_data; rtc::ArrayView filter = filter_data; diff --git a/webrtc/modules/audio_processing/ns/noise_suppressor.h b/webrtc/modules/audio_processing/ns/noise_suppressor.h index d962886..1e321cf 100644 --- a/webrtc/modules/audio_processing/ns/noise_suppressor.h +++ b/webrtc/modules/audio_processing/ns/noise_suppressor.h @@ -41,12 +41,21 @@ class NoiseSuppressor { // Applies noise suppression. void Process(AudioBuffer* audio); + // Specifies whether the capture output will be used. The purpose of this is + // to allow the noise suppressor to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + void SetCaptureOutputUsage(bool capture_output_used) { + capture_output_used_ = capture_output_used; + } + private: const size_t num_bands_; const size_t num_channels_; const SuppressionParams suppression_params_; int32_t num_analyzed_frames_ = -1; NrFft fft_; + bool capture_output_used_ = true; struct ChannelState { ChannelState(const SuppressionParams& suppression_params, size_t num_bands); diff --git a/webrtc/modules/audio_processing/ns/prior_signal_model_estimator.cc b/webrtc/modules/audio_processing/ns/prior_signal_model_estimator.cc index c814658..f77dcd6 100644 --- a/webrtc/modules/audio_processing/ns/prior_signal_model_estimator.cc +++ b/webrtc/modules/audio_processing/ns/prior_signal_model_estimator.cc @@ -11,6 +11,7 @@ #include "modules/audio_processing/ns/prior_signal_model_estimator.h" #include + #include #include "modules/audio_processing/ns/fast_math.h" diff --git a/webrtc/modules/audio_processing/ns/quantile_noise_estimator.h b/webrtc/modules/audio_processing/ns/quantile_noise_estimator.h index 67d1512..55b0bfa 100644 --- a/webrtc/modules/audio_processing/ns/quantile_noise_estimator.h +++ b/webrtc/modules/audio_processing/ns/quantile_noise_estimator.h @@ -12,6 +12,7 @@ #define MODULES_AUDIO_PROCESSING_NS_QUANTILE_NOISE_ESTIMATOR_H_ #include + #include #include "api/array_view.h" diff --git a/webrtc/modules/audio_processing/ns/speech_probability_estimator.cc b/webrtc/modules/audio_processing/ns/speech_probability_estimator.cc index fce9bc8..65f17f4 100644 --- a/webrtc/modules/audio_processing/ns/speech_probability_estimator.cc +++ b/webrtc/modules/audio_processing/ns/speech_probability_estimator.cc @@ -11,6 +11,7 @@ #include "modules/audio_processing/ns/speech_probability_estimator.h" #include + #include #include "modules/audio_processing/ns/fast_math.h" diff --git a/webrtc/modules/audio_processing/ns/suppression_params.cc b/webrtc/modules/audio_processing/ns/suppression_params.cc index 9a6bd5a..7bf1834 100644 --- a/webrtc/modules/audio_processing/ns/suppression_params.cc +++ b/webrtc/modules/audio_processing/ns/suppression_params.cc @@ -42,7 +42,7 @@ SuppressionParams::SuppressionParams( use_attenuation_adjustment = true; break; default: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); } } diff --git a/webrtc/modules/audio_processing/ns/wiener_filter.cc b/webrtc/modules/audio_processing/ns/wiener_filter.cc index e14b797..1eb50a7 100644 --- a/webrtc/modules/audio_processing/ns/wiener_filter.cc +++ b/webrtc/modules/audio_processing/ns/wiener_filter.cc @@ -13,6 +13,7 @@ #include #include #include + #include #include "modules/audio_processing/ns/fast_math.h" diff --git a/webrtc/modules/audio_processing/optionally_built_submodule_creators.cc b/webrtc/modules/audio_processing/optionally_built_submodule_creators.cc index 62a1632..cea5c83 100644 --- a/webrtc/modules/audio_processing/optionally_built_submodule_creators.cc +++ b/webrtc/modules/audio_processing/optionally_built_submodule_creators.cc @@ -17,14 +17,19 @@ namespace webrtc { std::unique_ptr CreateTransientSuppressor( - const ApmSubmoduleCreationOverrides& overrides) { + const ApmSubmoduleCreationOverrides& overrides, + TransientSuppressor::VadMode vad_mode, + int sample_rate_hz, + int detection_rate_hz, + int num_channels) { #ifdef WEBRTC_EXCLUDE_TRANSIENT_SUPPRESSOR return nullptr; #else if (overrides.transient_suppression) { return nullptr; } - return std::make_unique(); + return std::make_unique( + vad_mode, sample_rate_hz, detection_rate_hz, num_channels); #endif } diff --git a/webrtc/modules/audio_processing/optionally_built_submodule_creators.h b/webrtc/modules/audio_processing/optionally_built_submodule_creators.h index c96e66f..1be2743 100644 --- a/webrtc/modules/audio_processing/optionally_built_submodule_creators.h +++ b/webrtc/modules/audio_processing/optionally_built_submodule_creators.h @@ -20,7 +20,7 @@ namespace webrtc { // These overrides are only to be used for testing purposes. // Each flag emulates a preprocessor macro to exclude a submodule of APM from // the build, e.g. WEBRTC_EXCLUDE_TRANSIENT_SUPPRESSOR. If the corresponding -// flag |transient_suppression| is enabled, then the creators will return +// flag `transient_suppression` is enabled, then the creators will return // nullptr instead of a submodule instance, as if the macro had been defined. struct ApmSubmoduleCreationOverrides { bool transient_suppression = false; @@ -29,9 +29,13 @@ struct ApmSubmoduleCreationOverrides { // Creates a transient suppressor. // Will instead return nullptr if one of the following is true: // * WEBRTC_EXCLUDE_TRANSIENT_SUPPRESSOR is defined -// * The corresponding override in |overrides| is enabled. +// * The corresponding override in `overrides` is enabled. std::unique_ptr CreateTransientSuppressor( - const ApmSubmoduleCreationOverrides& overrides); + const ApmSubmoduleCreationOverrides& overrides, + TransientSuppressor::VadMode vad_mode, + int sample_rate_hz, + int detection_rate_hz, + int num_channels); } // namespace webrtc diff --git a/webrtc/modules/audio_processing/residual_echo_detector.cc b/webrtc/modules/audio_processing/residual_echo_detector.cc index 6188883..2a564fc 100644 --- a/webrtc/modules/audio_processing/residual_echo_detector.cc +++ b/webrtc/modules/audio_processing/residual_echo_detector.cc @@ -14,9 +14,7 @@ #include #include "absl/types/optional.h" -#include "modules/audio_processing/audio_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" -#include "rtc_base/atomic_ops.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "system_wrappers/include/metrics.h" @@ -42,11 +40,10 @@ constexpr size_t kAggregationBufferSize = 10 * 100; namespace webrtc { -int ResidualEchoDetector::instance_count_ = 0; +std::atomic ResidualEchoDetector::instance_count_(0); ResidualEchoDetector::ResidualEchoDetector() - : data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)), render_buffer_(kRenderBufferSize), render_power_(kLookbackFrames), render_power_mean_(kLookbackFrames), @@ -199,13 +196,6 @@ void ResidualEchoDetector::Initialize(int /*capture_sample_rate_hz*/, reliability_ = 0.f; } -void EchoDetector::PackRenderAudioBuffer(AudioBuffer* audio, - std::vector* packed_buffer) { - packed_buffer->clear(); - packed_buffer->insert(packed_buffer->end(), audio->channels()[0], - audio->channels()[0] + audio->num_frames()); -} - EchoDetector::Metrics ResidualEchoDetector::GetMetrics() const { EchoDetector::Metrics metrics; metrics.echo_likelihood = echo_likelihood_; diff --git a/webrtc/modules/audio_processing/residual_echo_detector.h b/webrtc/modules/audio_processing/residual_echo_detector.h index 5d18ecb..ac554b1 100644 --- a/webrtc/modules/audio_processing/residual_echo_detector.h +++ b/webrtc/modules/audio_processing/residual_echo_detector.h @@ -11,6 +11,7 @@ #ifndef MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ #define MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_ +#include #include #include "api/array_view.h" @@ -49,14 +50,14 @@ class ResidualEchoDetector : public EchoDetector { EchoDetector::Metrics GetMetrics() const override; private: - static int instance_count_; + static std::atomic instance_count_; std::unique_ptr data_dumper_; - // Keep track if the |Process| function has been previously called. + // Keep track if the `Process` function has been previously called. bool first_process_call_ = true; // Buffer for storing the power of incoming farend buffers. This is needed for // cases where calls to BufferFarend and Process are jittery. CircularBuffer render_buffer_; - // Count how long ago it was that the size of |render_buffer_| was zero. This + // Count how long ago it was that the size of `render_buffer_` was zero. This // value is also reset to zero when clock drift is detected and a value from // the renderbuffer is discarded, even though the buffer is not actually zero // at that point. This is done to avoid repeatedly removing elements in this diff --git a/webrtc/modules/audio_processing/rms_level.cc b/webrtc/modules/audio_processing/rms_level.cc index 6992a15..b0a45cb 100644 --- a/webrtc/modules/audio_processing/rms_level.cc +++ b/webrtc/modules/audio_processing/rms_level.cc @@ -101,8 +101,18 @@ void RmsLevel::AnalyzeMuted(size_t length) { } int RmsLevel::Average() { - int rms = (sample_count_ == 0) ? RmsLevel::kMinLevelDb - : ComputeRms(sum_square_ / sample_count_); + const bool have_samples = (sample_count_ != 0); + int rms = have_samples ? ComputeRms(sum_square_ / sample_count_) + : RmsLevel::kMinLevelDb; + + // To ensure that kMinLevelDb represents digital silence (muted audio + // sources) we'll check here if the sum_square is actually 0. If it's not + // we'll bump up the return value to `kInaudibleButNotMuted`. + // https://datatracker.ietf.org/doc/html/rfc6464 + if (have_samples && rms == RmsLevel::kMinLevelDb && sum_square_ != 0.0f) { + rms = kInaudibleButNotMuted; + } + Reset(); return rms; } diff --git a/webrtc/modules/audio_processing/rms_level.h b/webrtc/modules/audio_processing/rms_level.h index e1a6d56..fbece19 100644 --- a/webrtc/modules/audio_processing/rms_level.h +++ b/webrtc/modules/audio_processing/rms_level.h @@ -34,7 +34,7 @@ class RmsLevel { int peak; }; - enum : int { kMinLevelDb = 127 }; + enum : int { kMinLevelDb = 127, kInaudibleButNotMuted = 126 }; RmsLevel(); ~RmsLevel(); @@ -47,7 +47,7 @@ class RmsLevel { void Analyze(rtc::ArrayView data); void Analyze(rtc::ArrayView data); - // If all samples with the given |length| have a magnitude of zero, this is + // If all samples with the given `length` have a magnitude of zero, this is // a shortcut to avoid some computation. void AnalyzeMuted(size_t length); @@ -62,7 +62,7 @@ class RmsLevel { Levels AverageAndPeak(); private: - // Compares |block_size| with |block_size_|. If they are different, calls + // Compares `block_size` with `block_size_`. If they are different, calls // Reset() and stores the new size. void CheckBlockSize(size_t block_size); diff --git a/webrtc/modules/audio_processing/three_band_filter_bank.cc b/webrtc/modules/audio_processing/three_band_filter_bank.cc index 2a7d272..bd1c504 100644 --- a/webrtc/modules/audio_processing/three_band_filter_bank.cc +++ b/webrtc/modules/audio_processing/three_band_filter_bank.cc @@ -39,16 +39,16 @@ namespace webrtc { namespace { -// Factors to take into account when choosing |kFilterSize|: -// 1. Higher |kFilterSize|, means faster transition, which ensures less +// Factors to take into account when choosing `kFilterSize`: +// 1. Higher `kFilterSize`, means faster transition, which ensures less // aliasing. This is especially important when there is non-linear // processing between the splitting and merging. // 2. The delay that this filter bank introduces is -// |kNumBands| * |kSparsity| * |kFilterSize| / 2, so it increases linearly -// with |kFilterSize|. -// 3. The computation complexity also increases linearly with |kFilterSize|. +// `kNumBands` * `kSparsity` * `kFilterSize` / 2, so it increases linearly +// with `kFilterSize`. +// 3. The computation complexity also increases linearly with `kFilterSize`. -// The Matlab code to generate these |kFilterCoeffs| is: +// The Matlab code to generate these `kFilterCoeffs` is: // // N = kNumBands * kSparsity * kFilterSize - 1; // h = fir1(N, 1 / (2 * kNumBands), kaiser(N + 1, 3.5)); @@ -59,7 +59,7 @@ namespace { // Because the total bandwidth of the lower and higher band is double the middle // one (because of the spectrum parity), the low-pass prototype is half the -// bandwidth of 1 / (2 * |kNumBands|) and is then shifted with cosine modulation +// bandwidth of 1 / (2 * `kNumBands`) and is then shifted with cosine modulation // to the right places. // A Kaiser window is used because of its flexibility and the alpha is set to // 3.5, since that sets a stop band attenuation of 40dB ensuring a fast @@ -100,8 +100,8 @@ const float kDctModulation[ThreeBandFilterBank::kNumNonZeroFilters][kDctSize] = {1.f, -2.f, 1.f}, {1.73205077f, 0.f, -1.73205077f}}; -// Filters the input signal |in| with the filter |filter| using a shift by -// |in_shift|, taking into account the previous state. +// Filters the input signal `in` with the filter `filter` using a shift by +// `in_shift`, taking into account the previous state. void FilterCore( rtc::ArrayView filter, rtc::ArrayView in, @@ -164,10 +164,10 @@ ThreeBandFilterBank::ThreeBandFilterBank() { ThreeBandFilterBank::~ThreeBandFilterBank() = default; // The analysis can be separated in these steps: -// 1. Serial to parallel downsampling by a factor of |kNumBands|. -// 2. Filtering of |kSparsity| different delayed signals with polyphase +// 1. Serial to parallel downsampling by a factor of `kNumBands`. +// 2. Filtering of `kSparsity` different delayed signals with polyphase // decomposition of the low-pass prototype filter and upsampled by a factor -// of |kSparsity|. +// of `kSparsity`. // 3. Modulating with cosines and accumulating to get the desired band. void ThreeBandFilterBank::Analysis( rtc::ArrayView in, @@ -211,8 +211,9 @@ void ThreeBandFilterBank::Analysis( // Band and modulate the output. for (int band = 0; band < ThreeBandFilterBank::kNumBands; ++band) { + float* out_band = out[band].data(); for (int n = 0; n < kSplitBandSize; ++n) { - out[band][n] += dct_modulation[band] * out_subsampled[n]; + out_band[n] += dct_modulation[band] * out_subsampled[n]; } } } @@ -222,9 +223,9 @@ void ThreeBandFilterBank::Analysis( // The synthesis can be separated in these steps: // 1. Modulating with cosines. // 2. Filtering each one with a polyphase decomposition of the low-pass -// prototype filter upsampled by a factor of |kSparsity| and accumulating -// |kSparsity| signals with different delays. -// 3. Parallel to serial upsampling by a factor of |kNumBands|. +// prototype filter upsampled by a factor of `kSparsity` and accumulating +// `kSparsity` signals with different delays. +// 3. Parallel to serial upsampling by a factor of `kNumBands`. void ThreeBandFilterBank::Synthesis( rtc::ArrayView, ThreeBandFilterBank::kNumBands> in, @@ -254,8 +255,9 @@ void ThreeBandFilterBank::Synthesis( std::fill(in_subsampled.begin(), in_subsampled.end(), 0.f); for (int band = 0; band < ThreeBandFilterBank::kNumBands; ++band) { RTC_DCHECK_EQ(in[band].size(), kSplitBandSize); + const float* in_band = in[band].data(); for (int n = 0; n < kSplitBandSize; ++n) { - in_subsampled[n] += dct_modulation[band] * in[band][n]; + in_subsampled[n] += dct_modulation[band] * in_band[n]; } } diff --git a/webrtc/modules/audio_processing/three_band_filter_bank.h b/webrtc/modules/audio_processing/three_band_filter_bank.h index e6346de..db66cab 100644 --- a/webrtc/modules/audio_processing/three_band_filter_bank.h +++ b/webrtc/modules/audio_processing/three_band_filter_bank.h @@ -55,13 +55,13 @@ class ThreeBandFilterBank final { ThreeBandFilterBank(); ~ThreeBandFilterBank(); - // Splits |in| of size kFullBandSize into 3 downsampled frequency bands in - // |out|, each of size 160. + // Splits `in` of size kFullBandSize into 3 downsampled frequency bands in + // `out`, each of size 160. void Analysis(rtc::ArrayView in, rtc::ArrayView, kNumBands> out); - // Merges the 3 downsampled frequency bands in |in|, each of size 160, into - // |out|, which is of size kFullBandSize. + // Merges the 3 downsampled frequency bands in `in`, each of size 160, into + // `out`, which is of size kFullBandSize. void Synthesis(rtc::ArrayView, kNumBands> in, rtc::ArrayView out); diff --git a/webrtc/modules/audio_processing/transient/BUILD.gn b/webrtc/modules/audio_processing/transient/BUILD.gn index 13e319f..41aeab0 100644 --- a/webrtc/modules/audio_processing/transient/BUILD.gn +++ b/webrtc/modules/audio_processing/transient/BUILD.gn @@ -14,10 +14,10 @@ rtc_source_set("transient_suppressor_api") { rtc_library("transient_suppressor_impl") { visibility = [ - "..:optionally_built_submodule_creators", + ":click_annotate", ":transient_suppression_test", ":transient_suppression_unittests", - ":click_annotate", + "..:optionally_built_submodule_creators", ] sources = [ "common.h", @@ -37,6 +37,7 @@ rtc_library("transient_suppressor_impl") { ] deps = [ ":transient_suppressor_api", + ":voice_probability_delay_unit", "../../../common_audio:common_audio", "../../../common_audio:common_audio_c", "../../../common_audio:fir_filter", @@ -48,43 +49,56 @@ rtc_library("transient_suppressor_impl") { ] } -if (rtc_include_tests) { - rtc_executable("click_annotate") { - testonly = true - sources = [ - "click_annotate.cc", - "file_utils.cc", - "file_utils.h", - ] - deps = [ - ":transient_suppressor_impl", - "..:audio_processing", - "../../../rtc_base/system:file_wrapper", - "../../../system_wrappers", - ] - } +rtc_library("voice_probability_delay_unit") { + sources = [ + "voice_probability_delay_unit.cc", + "voice_probability_delay_unit.h", + ] + deps = [ "../../../rtc_base:checks" ] +} - rtc_executable("transient_suppression_test") { - testonly = true - sources = [ - "file_utils.cc", - "file_utils.h", - "transient_suppression_test.cc", - ] - deps = [ - ":transient_suppressor_impl", - "..:audio_processing", - "../../../common_audio", - "../../../rtc_base:rtc_base_approved", - "../../../rtc_base/system:file_wrapper", - "../../../system_wrappers", - "../../../test:fileutils", - "../../../test:test_support", - "../agc:level_estimation", - "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", - "//third_party/abseil-cpp/absl/flags:parse", - ] +if (rtc_include_tests) { + if (!build_with_chromium) { + rtc_executable("click_annotate") { + testonly = true + sources = [ + "click_annotate.cc", + "file_utils.cc", + "file_utils.h", + ] + deps = [ + ":transient_suppressor_impl", + "..:audio_processing", + "../../../rtc_base/system:file_wrapper", + "../../../system_wrappers", + ] + } + + rtc_executable("transient_suppression_test") { + testonly = true + sources = [ + "file_utils.cc", + "file_utils.h", + "transient_suppression_test.cc", + "voice_probability_delay_unit_unittest.cc", + ] + deps = [ + ":transient_suppressor_api", + ":transient_suppressor_impl", + ":voice_probability_delay_unit", + "..:audio_processing", + "../../../common_audio", + "../../../rtc_base/system:file_wrapper", + "../../../system_wrappers", + "../../../test:fileutils", + "../../../test:test_support", + "../agc:level_estimation", + "//testing/gtest", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + "//third_party/abseil-cpp/absl/types:optional", + ] + } } rtc_library("transient_suppression_unittests") { @@ -97,16 +111,23 @@ if (rtc_include_tests) { "moving_moments_unittest.cc", "transient_detector_unittest.cc", "transient_suppressor_unittest.cc", + "voice_probability_delay_unit_unittest.cc", "wpd_node_unittest.cc", "wpd_tree_unittest.cc", ] deps = [ + ":transient_suppressor_api", ":transient_suppressor_impl", + ":voice_probability_delay_unit", "../../../rtc_base:stringutils", "../../../rtc_base/system:file_wrapper", "../../../test:fileutils", "../../../test:test_support", "//testing/gtest", ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] } } diff --git a/webrtc/modules/audio_processing/transient/click_annotate.cc b/webrtc/modules/audio_processing/transient/click_annotate.cc deleted file mode 100644 index 21641f8..0000000 --- a/webrtc/modules/audio_processing/transient/click_annotate.cc +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include -#include -#include -#include - -#include "modules/audio_processing/transient/file_utils.h" -#include "modules/audio_processing/transient/transient_detector.h" -#include "rtc_base/system/file_wrapper.h" - -using webrtc::FileWrapper; -using webrtc::TransientDetector; - -// Application to generate a RTP timing file. -// Opens the PCM file and divides the signal in frames. -// Creates a send times array, one for each step. -// Each block that contains a transient, has an infinite send time. -// The resultant array is written to a DAT file -// Returns -1 on error or |lost_packets| otherwise. -int main(int argc, char* argv[]) { - if (argc != 5) { - printf("\n%s - Application to generate a RTP timing file.\n\n", argv[0]); - printf("%s PCMfile DATfile chunkSize sampleRate\n\n", argv[0]); - printf("Opens the PCMfile with sampleRate in Hertz.\n"); - printf("Creates a send times array, one for each chunkSize "); - printf("milliseconds step.\n"); - printf("Each block that contains a transient, has an infinite send time. "); - printf("The resultant array is written to a DATfile.\n\n"); - return 0; - } - - FileWrapper pcm_file = FileWrapper::OpenReadOnly(argv[1]); - if (!pcm_file.is_open()) { - printf("\nThe %s could not be opened.\n\n", argv[1]); - return -1; - } - - FileWrapper dat_file = FileWrapper::OpenWriteOnly(argv[2]); - if (!dat_file.is_open()) { - printf("\nThe %s could not be opened.\n\n", argv[2]); - return -1; - } - - int chunk_size_ms = atoi(argv[3]); - if (chunk_size_ms <= 0) { - printf("\nThe chunkSize must be a positive integer\n\n"); - return -1; - } - - int sample_rate_hz = atoi(argv[4]); - if (sample_rate_hz <= 0) { - printf("\nThe sampleRate must be a positive integer\n\n"); - return -1; - } - - TransientDetector detector(sample_rate_hz); - int lost_packets = 0; - size_t audio_buffer_length = chunk_size_ms * sample_rate_hz / 1000; - std::unique_ptr audio_buffer(new float[audio_buffer_length]); - std::vector send_times; - - // Read first buffer from the PCM test file. - size_t file_samples_read = ReadInt16FromFileToFloatBuffer( - &pcm_file, audio_buffer_length, audio_buffer.get()); - for (int time = 0; file_samples_read > 0; time += chunk_size_ms) { - // Pad the rest of the buffer with zeros. - for (size_t i = file_samples_read; i < audio_buffer_length; ++i) { - audio_buffer[i] = 0.0; - } - float value = - detector.Detect(audio_buffer.get(), audio_buffer_length, NULL, 0); - if (value < 0.5f) { - value = time; - } else { - value = FLT_MAX; - ++lost_packets; - } - send_times.push_back(value); - - // Read next buffer from the PCM test file. - file_samples_read = ReadInt16FromFileToFloatBuffer( - &pcm_file, audio_buffer_length, audio_buffer.get()); - } - - size_t floats_written = - WriteFloatBufferToFile(&dat_file, send_times.size(), &send_times[0]); - - if (floats_written == 0) { - printf("\nThe send times could not be written to DAT file\n\n"); - return -1; - } - - pcm_file.Close(); - dat_file.Close(); - - return lost_packets; -} diff --git a/webrtc/modules/audio_processing/transient/dyadic_decimator.h b/webrtc/modules/audio_processing/transient/dyadic_decimator.h index fcb56b7..52467e8 100644 --- a/webrtc/modules/audio_processing/transient/dyadic_decimator.h +++ b/webrtc/modules/audio_processing/transient/dyadic_decimator.h @@ -18,7 +18,7 @@ namespace webrtc { // Returns the proper length of the output buffer that you should use for the -// given |in_length| and decimation |odd_sequence|. +// given `in_length` and decimation `odd_sequence`. // Return -1 on error. inline size_t GetOutLengthToDyadicDecimate(size_t in_length, bool odd_sequence) { @@ -34,10 +34,10 @@ inline size_t GetOutLengthToDyadicDecimate(size_t in_length, // Performs a dyadic decimation: removes every odd/even member of a sequence // halving its overall length. // Arguments: -// in: array of |in_length|. +// in: array of `in_length`. // odd_sequence: If false, the odd members will be removed (1, 3, 5, ...); // if true, the even members will be removed (0, 2, 4, ...). -// out: array of |out_length|. |out_length| must be large enough to +// out: array of `out_length`. `out_length` must be large enough to // hold the decimated output. The necessary length can be provided by // GetOutLengthToDyadicDecimate(). // Must be previously allocated. diff --git a/webrtc/modules/audio_processing/transient/file_utils.h b/webrtc/modules/audio_processing/transient/file_utils.h index 1c9370d..b748337 100644 --- a/webrtc/modules/audio_processing/transient/file_utils.h +++ b/webrtc/modules/audio_processing/transient/file_utils.h @@ -12,7 +12,6 @@ #define MODULES_AUDIO_PROCESSING_TRANSIENT_FILE_UTILS_H_ #include -#include #include "rtc_base/system/file_wrapper.h" @@ -51,63 +50,63 @@ int ConvertFloatToByteArray(float value, uint8_t out_bytes[4]); // Returns 0 if correct, -1 on error. int ConvertDoubleToByteArray(double value, uint8_t out_bytes[8]); -// Reads |length| 16-bit integers from |file| to |buffer|. -// |file| must be previously opened. +// Reads `length` 16-bit integers from `file` to `buffer`. +// `file` must be previously opened. // Returns the number of 16-bit integers read or -1 on error. size_t ReadInt16BufferFromFile(FileWrapper* file, size_t length, int16_t* buffer); -// Reads |length| 16-bit integers from |file| and stores those values -// (converting them) in |buffer|. -// |file| must be previously opened. +// Reads `length` 16-bit integers from `file` and stores those values +// (converting them) in `buffer`. +// `file` must be previously opened. // Returns the number of 16-bit integers read or -1 on error. size_t ReadInt16FromFileToFloatBuffer(FileWrapper* file, size_t length, float* buffer); -// Reads |length| 16-bit integers from |file| and stores those values -// (converting them) in |buffer|. -// |file| must be previously opened. +// Reads `length` 16-bit integers from `file` and stores those values +// (converting them) in `buffer`. +// `file` must be previously opened. // Returns the number of 16-bit integers read or -1 on error. size_t ReadInt16FromFileToDoubleBuffer(FileWrapper* file, size_t length, double* buffer); -// Reads |length| floats in binary representation (4 bytes) from |file| to -// |buffer|. -// |file| must be previously opened. +// Reads `length` floats in binary representation (4 bytes) from `file` to +// `buffer`. +// `file` must be previously opened. // Returns the number of floats read or -1 on error. size_t ReadFloatBufferFromFile(FileWrapper* file, size_t length, float* buffer); -// Reads |length| doubles in binary representation (8 bytes) from |file| to -// |buffer|. -// |file| must be previously opened. +// Reads `length` doubles in binary representation (8 bytes) from `file` to +// `buffer`. +// `file` must be previously opened. // Returns the number of doubles read or -1 on error. size_t ReadDoubleBufferFromFile(FileWrapper* file, size_t length, double* buffer); -// Writes |length| 16-bit integers from |buffer| in binary representation (2 -// bytes) to |file|. It flushes |file|, so after this call there are no +// Writes `length` 16-bit integers from `buffer` in binary representation (2 +// bytes) to `file`. It flushes `file`, so after this call there are no // writings pending. -// |file| must be previously opened. +// `file` must be previously opened. // Returns the number of doubles written or -1 on error. size_t WriteInt16BufferToFile(FileWrapper* file, size_t length, const int16_t* buffer); -// Writes |length| floats from |buffer| in binary representation (4 bytes) to -// |file|. It flushes |file|, so after this call there are no writtings pending. -// |file| must be previously opened. +// Writes `length` floats from `buffer` in binary representation (4 bytes) to +// `file`. It flushes `file`, so after this call there are no writtings pending. +// `file` must be previously opened. // Returns the number of doubles written or -1 on error. size_t WriteFloatBufferToFile(FileWrapper* file, size_t length, const float* buffer); -// Writes |length| doubles from |buffer| in binary representation (8 bytes) to -// |file|. It flushes |file|, so after this call there are no writings pending. -// |file| must be previously opened. +// Writes `length` doubles from `buffer` in binary representation (8 bytes) to +// `file`. It flushes `file`, so after this call there are no writings pending. +// `file` must be previously opened. // Returns the number of doubles written or -1 on error. size_t WriteDoubleBufferToFile(FileWrapper* file, size_t length, diff --git a/webrtc/modules/audio_processing/transient/moving_moments.h b/webrtc/modules/audio_processing/transient/moving_moments.h index 6dc0520..70451dc 100644 --- a/webrtc/modules/audio_processing/transient/moving_moments.h +++ b/webrtc/modules/audio_processing/transient/moving_moments.h @@ -26,13 +26,13 @@ namespace webrtc { // the last values of the moments. When needed. class MovingMoments { public: - // Creates a Moving Moments object, that uses the last |length| values + // Creates a Moving Moments object, that uses the last `length` values // (including the new value introduced in every new calculation). explicit MovingMoments(size_t length); ~MovingMoments(); - // Calculates the new values using |in|. Results will be in the out buffers. - // |first| and |second| must be allocated with at least |in_length|. + // Calculates the new values using `in`. Results will be in the out buffers. + // `first` and `second` must be allocated with at least `in_length`. void CalculateMoments(const float* in, size_t in_length, float* first, @@ -40,7 +40,7 @@ class MovingMoments { private: size_t length_; - // A queue holding the |length_| latest input values. + // A queue holding the `length_` latest input values. std::queue queue_; // Sum of the values of the queue. float sum_; diff --git a/webrtc/modules/audio_processing/transient/transient_detector.cc b/webrtc/modules/audio_processing/transient/transient_detector.cc index f03a2ea..5c35505 100644 --- a/webrtc/modules/audio_processing/transient/transient_detector.cc +++ b/webrtc/modules/audio_processing/transient/transient_detector.cc @@ -43,8 +43,8 @@ TransientDetector::TransientDetector(int sample_rate_hz) sample_rate_hz == ts::kSampleRate48kHz); int samples_per_transient = sample_rate_hz * kTransientLengthMs / 1000; // Adjustment to avoid data loss while downsampling, making - // |samples_per_chunk_| and |samples_per_transient| always divisible by - // |kLeaves|. + // `samples_per_chunk_` and `samples_per_transient` always divisible by + // `kLeaves`. samples_per_chunk_ -= samples_per_chunk_ % kLeaves; samples_per_transient -= samples_per_transient % kLeaves; @@ -137,7 +137,7 @@ float TransientDetector::Detect(const float* data, // In the current implementation we return the max of the current result and // the previous results, so the high results have a width equals to - // |transient_length|. + // `transient_length`. return *std::max_element(previous_results_.begin(), previous_results_.end()); } diff --git a/webrtc/modules/audio_processing/transient/transient_detector.h b/webrtc/modules/audio_processing/transient/transient_detector.h index 5ede2e8..a3dbb7f 100644 --- a/webrtc/modules/audio_processing/transient/transient_detector.h +++ b/webrtc/modules/audio_processing/transient/transient_detector.h @@ -37,8 +37,8 @@ class TransientDetector { ~TransientDetector(); - // Calculates the log-likelihood of the existence of a transient in |data|. - // |data_length| has to be equal to |samples_per_chunk_|. + // Calculates the log-likelihood of the existence of a transient in `data`. + // `data_length` has to be equal to `samples_per_chunk_`. // Returns a value between 0 and 1, as a non linear representation of this // likelihood. // Returns a negative value on error. @@ -71,7 +71,7 @@ class TransientDetector { float last_second_moment_[kLeaves]; // We keep track of the previous results from the previous chunks, so it can - // be used to effectively give results according to the |transient_length|. + // be used to effectively give results according to the `transient_length`. std::deque previous_results_; // Number of chunks that are going to return only zeros at the beginning of diff --git a/webrtc/modules/audio_processing/transient/transient_suppressor.h b/webrtc/modules/audio_processing/transient/transient_suppressor.h index bb262b0..ecb3c3b 100644 --- a/webrtc/modules/audio_processing/transient/transient_suppressor.h +++ b/webrtc/modules/audio_processing/transient/transient_suppressor.h @@ -11,9 +11,7 @@ #ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_SUPPRESSOR_H_ #define MODULES_AUDIO_PROCESSING_TRANSIENT_TRANSIENT_SUPPRESSOR_H_ -#include -#include -#include +#include namespace webrtc { @@ -21,38 +19,55 @@ namespace webrtc { // restoration algorithm that attenuates unexpected spikes in the spectrum. class TransientSuppressor { public: + // Type of VAD used by the caller to compute the `voice_probability` argument + // `Suppress()`. + enum class VadMode { + // By default, `TransientSuppressor` assumes that `voice_probability` is + // computed by `AgcManagerDirect`. + kDefault = 0, + // Use this mode when `TransientSuppressor` must assume that + // `voice_probability` is computed by the RNN VAD. + kRnnVad, + // Use this mode to let `TransientSuppressor::Suppressor()` ignore + // `voice_probability` and behave as if voice information is unavailable + // (regardless of the passed value). + kNoVad, + }; + virtual ~TransientSuppressor() {} - virtual int Initialize(int sample_rate_hz, - int detector_rate_hz, - int num_channels) = 0; + virtual void Initialize(int sample_rate_hz, + int detector_rate_hz, + int num_channels) = 0; - // Processes a |data| chunk, and returns it with keystrokes suppressed from + // Processes a `data` chunk, and returns it with keystrokes suppressed from // it. The float format is assumed to be int16 ranged. If there are more than - // one channel, the chunks are concatenated one after the other in |data|. - // |data_length| must be equal to |data_length_|. - // |num_channels| must be equal to |num_channels_|. - // A sub-band, ideally the higher, can be used as |detection_data|. If it is - // NULL, |data| is used for the detection too. The |detection_data| is always + // one channel, the chunks are concatenated one after the other in `data`. + // `data_length` must be equal to `data_length_`. + // `num_channels` must be equal to `num_channels_`. + // A sub-band, ideally the higher, can be used as `detection_data`. If it is + // NULL, `data` is used for the detection too. The `detection_data` is always // assumed mono. // If a reference signal (e.g. keyboard microphone) is available, it can be - // passed in as |reference_data|. It is assumed mono and must have the same - // length as |data|. NULL is accepted if unavailable. + // passed in as `reference_data`. It is assumed mono and must have the same + // length as `data`. NULL is accepted if unavailable. // This suppressor performs better if voice information is available. - // |voice_probability| is the probability of voice being present in this chunk - // of audio. If voice information is not available, |voice_probability| must + // `voice_probability` is the probability of voice being present in this chunk + // of audio. If voice information is not available, `voice_probability` must // always be set to 1. - // |key_pressed| determines if a key was pressed on this audio chunk. - // Returns 0 on success and -1 otherwise. - virtual int Suppress(float* data, - size_t data_length, - int num_channels, - const float* detection_data, - size_t detection_length, - const float* reference_data, - size_t reference_length, - float voice_probability, - bool key_pressed) = 0; + // `key_pressed` determines if a key was pressed on this audio chunk. + // Returns a delayed version of `voice_probability` according to the + // algorithmic delay introduced by this method. In this way, the modified + // `data` and the returned voice probability will be temporally aligned. + virtual float Suppress(float* data, + size_t data_length, + int num_channels, + const float* detection_data, + size_t detection_length, + const float* reference_data, + size_t reference_length, + float voice_probability, + bool key_pressed) = 0; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/transient/transient_suppressor_impl.cc b/webrtc/modules/audio_processing/transient/transient_suppressor_impl.cc index d515d30..9042846 100644 --- a/webrtc/modules/audio_processing/transient/transient_suppressor_impl.cc +++ b/webrtc/modules/audio_processing/transient/transient_suppressor_impl.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include "common_audio/include/audio_util.h" #include "common_audio/signal_processing/include/signal_processing_library.h" @@ -32,7 +33,6 @@ namespace webrtc { static const float kMeanIIRCoefficient = 0.5f; -static const float kVoiceThreshold = 0.02f; // TODO(aluebs): Check if these values work also for 48kHz. static const size_t kMinVoiceBin = 3; @@ -44,10 +44,27 @@ float ComplexMagnitude(float a, float b) { return std::abs(a) + std::abs(b); } +std::string GetVadModeLabel(TransientSuppressor::VadMode vad_mode) { + switch (vad_mode) { + case TransientSuppressor::VadMode::kDefault: + return "default"; + case TransientSuppressor::VadMode::kRnnVad: + return "RNN VAD"; + case TransientSuppressor::VadMode::kNoVad: + return "no VAD"; + } +} + } // namespace -TransientSuppressorImpl::TransientSuppressorImpl() - : data_length_(0), +TransientSuppressorImpl::TransientSuppressorImpl(VadMode vad_mode, + int sample_rate_hz, + int detector_rate_hz, + int num_channels) + : vad_mode_(vad_mode), + voice_probability_delay_unit_(/*delay_num_samples=*/0, sample_rate_hz), + analyzed_audio_is_silent_(false), + data_length_(0), detection_length_(0), analysis_length_(0), buffer_delay_(0), @@ -62,13 +79,26 @@ TransientSuppressorImpl::TransientSuppressorImpl() use_hard_restoration_(false), chunks_since_voice_change_(0), seed_(182), - using_reference_(false) {} + using_reference_(false) { + RTC_LOG(LS_INFO) << "VAD mode: " << GetVadModeLabel(vad_mode_); + Initialize(sample_rate_hz, detector_rate_hz, num_channels); +} TransientSuppressorImpl::~TransientSuppressorImpl() {} -int TransientSuppressorImpl::Initialize(int sample_rate_hz, - int detection_rate_hz, - int num_channels) { +void TransientSuppressorImpl::Initialize(int sample_rate_hz, + int detection_rate_hz, + int num_channels) { + RTC_DCHECK(sample_rate_hz == ts::kSampleRate8kHz || + sample_rate_hz == ts::kSampleRate16kHz || + sample_rate_hz == ts::kSampleRate32kHz || + sample_rate_hz == ts::kSampleRate48kHz); + RTC_DCHECK(detection_rate_hz == ts::kSampleRate8kHz || + detection_rate_hz == ts::kSampleRate16kHz || + detection_rate_hz == ts::kSampleRate32kHz || + detection_rate_hz == ts::kSampleRate48kHz); + RTC_DCHECK_GT(num_channels, 0); + switch (sample_rate_hz) { case ts::kSampleRate8kHz: analysis_length_ = 128u; @@ -87,26 +117,18 @@ int TransientSuppressorImpl::Initialize(int sample_rate_hz, window_ = kBlocks480w1024; break; default: - return -1; - } - if (detection_rate_hz != ts::kSampleRate8kHz && - detection_rate_hz != ts::kSampleRate16kHz && - detection_rate_hz != ts::kSampleRate32kHz && - detection_rate_hz != ts::kSampleRate48kHz) { - return -1; - } - if (num_channels <= 0) { - return -1; + RTC_DCHECK_NOTREACHED(); + return; } detector_.reset(new TransientDetector(detection_rate_hz)); data_length_ = sample_rate_hz * ts::kChunkSizeMs / 1000; - if (data_length_ > analysis_length_) { - RTC_NOTREACHED(); - return -1; - } + RTC_DCHECK_LE(data_length_, analysis_length_); buffer_delay_ = analysis_length_ - data_length_; + voice_probability_delay_unit_.Initialize(/*delay_num_samples=*/buffer_delay_, + sample_rate_hz); + complex_analysis_length_ = analysis_length_ / 2 + 1; RTC_DCHECK_GE(complex_analysis_length_, kMaxVoiceBin); num_channels_ = num_channels; @@ -155,28 +177,28 @@ int TransientSuppressorImpl::Initialize(int sample_rate_hz, chunks_since_voice_change_ = 0; seed_ = 182; using_reference_ = false; - return 0; } -int TransientSuppressorImpl::Suppress(float* data, - size_t data_length, - int num_channels, - const float* detection_data, - size_t detection_length, - const float* reference_data, - size_t reference_length, - float voice_probability, - bool key_pressed) { +float TransientSuppressorImpl::Suppress(float* data, + size_t data_length, + int num_channels, + const float* detection_data, + size_t detection_length, + const float* reference_data, + size_t reference_length, + float voice_probability, + bool key_pressed) { if (!data || data_length != data_length_ || num_channels != num_channels_ || detection_length != detection_length_ || voice_probability < 0 || voice_probability > 1) { - return -1; + // The audio is not modified, so the voice probability is returned as is + // (delay not applied). + return voice_probability; } UpdateKeypress(key_pressed); UpdateBuffers(data); - int result = 0; if (detection_enabled_) { UpdateRestoration(voice_probability); @@ -189,12 +211,14 @@ int TransientSuppressorImpl::Suppress(float* data, float detector_result = detector_->Detect(detection_data, detection_length, reference_data, reference_length); if (detector_result < 0) { - return -1; + // The audio is not modified, so the voice probability is returned as is + // (delay not applied). + return voice_probability; } using_reference_ = detector_->using_reference(); - // |detector_smoothed_| follows the |detector_result| when this last one is + // `detector_smoothed_` follows the `detector_result` when this last one is // increasing, but has an exponential decaying tail to be able to suppress // the ringing of keyclicks. float smooth_factor = using_reference_ ? 0.6 : 0.1; @@ -219,11 +243,13 @@ int TransientSuppressorImpl::Suppress(float* data, : &in_buffer_[i * analysis_length_], data_length_ * sizeof(*data)); } - return result; + + // The audio has been modified, return the delayed voice probability. + return voice_probability_delay_unit_.Delay(voice_probability); } // This should only be called when detection is enabled. UpdateBuffers() must -// have been called. At return, |out_buffer_| will be filled with the +// have been called. At return, `out_buffer_` will be filled with the // processed output. void TransientSuppressorImpl::Suppress(float* in_ptr, float* spectral_mean, @@ -304,16 +330,34 @@ void TransientSuppressorImpl::UpdateKeypress(bool key_pressed) { } void TransientSuppressorImpl::UpdateRestoration(float voice_probability) { - const int kHardRestorationOffsetDelay = 3; - const int kHardRestorationOnsetDelay = 80; - - bool not_voiced = voice_probability < kVoiceThreshold; + bool not_voiced; + switch (vad_mode_) { + case TransientSuppressor::VadMode::kDefault: { + constexpr float kVoiceThreshold = 0.02f; + not_voiced = voice_probability < kVoiceThreshold; + break; + } + case TransientSuppressor::VadMode::kRnnVad: { + constexpr float kVoiceThreshold = 0.7f; + not_voiced = voice_probability < kVoiceThreshold; + break; + } + case TransientSuppressor::VadMode::kNoVad: + // Always assume that voice is detected. + not_voiced = false; + break; + } if (not_voiced == use_hard_restoration_) { chunks_since_voice_change_ = 0; } else { ++chunks_since_voice_change_; + // Number of 10 ms frames to wait to transition to and from hard + // restoration. + constexpr int kHardRestorationOffsetDelay = 3; + constexpr int kHardRestorationOnsetDelay = 80; + if ((use_hard_restoration_ && chunks_since_voice_change_ > kHardRestorationOffsetDelay) || (!use_hard_restoration_ && @@ -325,7 +369,7 @@ void TransientSuppressorImpl::UpdateRestoration(float voice_probability) { } // Shift buffers to make way for new data. Must be called after -// |detection_enabled_| is updated by UpdateKeypress(). +// `detection_enabled_` is updated by UpdateKeypress(). void TransientSuppressorImpl::UpdateBuffers(float* data) { // TODO(aluebs): Change to ring buffer. memmove(in_buffer_.get(), &in_buffer_[data_length_], @@ -350,9 +394,9 @@ void TransientSuppressorImpl::UpdateBuffers(float* data) { } // Restores the unvoiced signal if a click is present. -// Attenuates by a certain factor every peak in the |fft_buffer_| that exceeds -// the spectral mean. The attenuation depends on |detector_smoothed_|. -// If a restoration takes place, the |magnitudes_| are updated to the new value. +// Attenuates by a certain factor every peak in the `fft_buffer_` that exceeds +// the spectral mean. The attenuation depends on `detector_smoothed_`. +// If a restoration takes place, the `magnitudes_` are updated to the new value. void TransientSuppressorImpl::HardRestoration(float* spectral_mean) { const float detector_result = 1.f - std::pow(1.f - detector_smoothed_, using_reference_ ? 200.f : 50.f); @@ -376,10 +420,10 @@ void TransientSuppressorImpl::HardRestoration(float* spectral_mean) { } // Restores the voiced signal if a click is present. -// Attenuates by a certain factor every peak in the |fft_buffer_| that exceeds +// Attenuates by a certain factor every peak in the `fft_buffer_` that exceeds // the spectral mean and that is lower than some function of the current block -// frequency mean. The attenuation depends on |detector_smoothed_|. -// If a restoration takes place, the |magnitudes_| are updated to the new value. +// frequency mean. The attenuation depends on `detector_smoothed_`. +// If a restoration takes place, the `magnitudes_` are updated to the new value. void TransientSuppressorImpl::SoftRestoration(float* spectral_mean) { // Get the spectral magnitude mean of the current block. float block_frequency_mean = 0; diff --git a/webrtc/modules/audio_processing/transient/transient_suppressor_impl.h b/webrtc/modules/audio_processing/transient/transient_suppressor_impl.h index 4737af5..4005a16 100644 --- a/webrtc/modules/audio_processing/transient/transient_suppressor_impl.h +++ b/webrtc/modules/audio_processing/transient/transient_suppressor_impl.h @@ -17,6 +17,7 @@ #include #include "modules/audio_processing/transient/transient_suppressor.h" +#include "modules/audio_processing/transient/voice_probability_delay_unit.h" #include "rtc_base/gtest_prod_util.h" namespace webrtc { @@ -27,42 +28,28 @@ class TransientDetector; // restoration algorithm that attenuates unexpected spikes in the spectrum. class TransientSuppressorImpl : public TransientSuppressor { public: - TransientSuppressorImpl(); + TransientSuppressorImpl(VadMode vad_mode, + int sample_rate_hz, + int detector_rate_hz, + int num_channels); ~TransientSuppressorImpl() override; - int Initialize(int sample_rate_hz, - int detector_rate_hz, - int num_channels) override; + void Initialize(int sample_rate_hz, + int detector_rate_hz, + int num_channels) override; - // Processes a |data| chunk, and returns it with keystrokes suppressed from - // it. The float format is assumed to be int16 ranged. If there are more than - // one channel, the chunks are concatenated one after the other in |data|. - // |data_length| must be equal to |data_length_|. - // |num_channels| must be equal to |num_channels_|. - // A sub-band, ideally the higher, can be used as |detection_data|. If it is - // NULL, |data| is used for the detection too. The |detection_data| is always - // assumed mono. - // If a reference signal (e.g. keyboard microphone) is available, it can be - // passed in as |reference_data|. It is assumed mono and must have the same - // length as |data|. NULL is accepted if unavailable. - // This suppressor performs better if voice information is available. - // |voice_probability| is the probability of voice being present in this chunk - // of audio. If voice information is not available, |voice_probability| must - // always be set to 1. - // |key_pressed| determines if a key was pressed on this audio chunk. - // Returns 0 on success and -1 otherwise. - int Suppress(float* data, - size_t data_length, - int num_channels, - const float* detection_data, - size_t detection_length, - const float* reference_data, - size_t reference_length, - float voice_probability, - bool key_pressed) override; + float Suppress(float* data, + size_t data_length, + int num_channels, + const float* detection_data, + size_t detection_length, + const float* reference_data, + size_t reference_length, + float voice_probability, + bool key_pressed) override; private: - FRIEND_TEST_ALL_PREFIXES(TransientSuppressorImplTest, + FRIEND_TEST_ALL_PREFIXES(TransientSuppressorVadModeParametrization, TypingDetectionLogicWorksAsExpectedForMono); void Suppress(float* in_ptr, float* spectral_mean, float* out_ptr); @@ -74,8 +61,13 @@ class TransientSuppressorImpl : public TransientSuppressor { void HardRestoration(float* spectral_mean); void SoftRestoration(float* spectral_mean); + const VadMode vad_mode_; + VoiceProbabilityDelayUnit voice_probability_delay_unit_; + std::unique_ptr detector_; + bool analyzed_audio_is_silent_; + size_t data_length_; size_t detection_length_; size_t analysis_length_; diff --git a/webrtc/modules/audio_processing/transient/voice_probability_delay_unit.cc b/webrtc/modules/audio_processing/transient/voice_probability_delay_unit.cc new file mode 100644 index 0000000..27b2b42 --- /dev/null +++ b/webrtc/modules/audio_processing/transient/voice_probability_delay_unit.cc @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/transient/voice_probability_delay_unit.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +VoiceProbabilityDelayUnit::VoiceProbabilityDelayUnit(int delay_num_samples, + int sample_rate_hz) { + Initialize(delay_num_samples, sample_rate_hz); +} + +void VoiceProbabilityDelayUnit::Initialize(int delay_num_samples, + int sample_rate_hz) { + RTC_DCHECK_GE(delay_num_samples, 0); + RTC_DCHECK_LE(delay_num_samples, sample_rate_hz / 50) + << "The implementation does not support delays greater than 20 ms."; + int frame_size = rtc::CheckedDivExact(sample_rate_hz, 100); // 10 ms. + if (delay_num_samples <= frame_size) { + weights_[0] = 0.0f; + weights_[1] = static_cast(delay_num_samples) / frame_size; + weights_[2] = + static_cast(frame_size - delay_num_samples) / frame_size; + } else { + delay_num_samples -= frame_size; + weights_[0] = static_cast(delay_num_samples) / frame_size; + weights_[1] = + static_cast(frame_size - delay_num_samples) / frame_size; + weights_[2] = 0.0f; + } + + // Resets the delay unit. + last_probabilities_.fill(0.0f); +} + +float VoiceProbabilityDelayUnit::Delay(float voice_probability) { + float weighted_probability = weights_[0] * last_probabilities_[0] + + weights_[1] * last_probabilities_[1] + + weights_[2] * voice_probability; + last_probabilities_[0] = last_probabilities_[1]; + last_probabilities_[1] = voice_probability; + return weighted_probability; +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/transient/voice_probability_delay_unit.h b/webrtc/modules/audio_processing/transient/voice_probability_delay_unit.h new file mode 100644 index 0000000..0596166 --- /dev/null +++ b/webrtc/modules/audio_processing/transient/voice_probability_delay_unit.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_VOICE_PROBABILITY_DELAY_UNIT_H_ +#define MODULES_AUDIO_PROCESSING_TRANSIENT_VOICE_PROBABILITY_DELAY_UNIT_H_ + +#include + +namespace webrtc { + +// Iteratively produces a sequence of delayed voice probability values given a +// fixed delay between 0 and 20 ms and given a sequence of voice probability +// values observed every 10 ms. Supports fractional delays, that are delays +// which are not a multiple integer of 10 ms. Applies interpolation with +// fractional delays; otherwise, returns a previously observed value according +// to the given fixed delay. +class VoiceProbabilityDelayUnit { + public: + // Ctor. `delay_num_samples` is the delay in number of samples and it must be + // non-negative and less than 20 ms. + VoiceProbabilityDelayUnit(int delay_num_samples, int sample_rate_hz); + + // Handles delay and sample rate changes and resets the delay unit. + void Initialize(int delay_num_samples, int sample_rate_hz); + + // Observes `voice_probability` and returns a delayed voice probability. + float Delay(float voice_probability); + + private: + std::array weights_; + std::array last_probabilities_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_TRANSIENT_VOICE_PROBABILITY_DELAY_UNIT_H_ diff --git a/webrtc/modules/audio_processing/transient/wpd_node.h b/webrtc/modules/audio_processing/transient/wpd_node.h index 6a52fb7..645bc5f 100644 --- a/webrtc/modules/audio_processing/transient/wpd_node.h +++ b/webrtc/modules/audio_processing/transient/wpd_node.h @@ -11,6 +11,7 @@ #ifndef MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_NODE_H_ #define MODULES_AUDIO_PROCESSING_TRANSIENT_WPD_NODE_H_ +#include #include namespace webrtc { @@ -25,7 +26,7 @@ class WPDNode { WPDNode(size_t length, const float* coefficients, size_t coefficients_length); ~WPDNode(); - // Updates the node data. |parent_data| / 2 must be equals to |length_|. + // Updates the node data. `parent_data` / 2 must be equals to `length_`. // Returns 0 if correct, and -1 otherwise. int Update(const float* parent_data, size_t parent_data_length); diff --git a/webrtc/modules/audio_processing/transient/wpd_tree.h b/webrtc/modules/audio_processing/transient/wpd_tree.h index c54220f..13cb8d9 100644 --- a/webrtc/modules/audio_processing/transient/wpd_tree.h +++ b/webrtc/modules/audio_processing/transient/wpd_tree.h @@ -65,7 +65,7 @@ class WPDTree { // If level or index are out of bounds the function will return NULL. WPDNode* NodeAt(int level, int index); - // Updates all the nodes of the tree with the new data. |data_length| must be + // Updates all the nodes of the tree with the new data. `data_length` must be // teh same that was used for the creation of the tree. // Returns 0 if correct, and -1 otherwise. int Update(const float* data, size_t data_length); diff --git a/webrtc/modules/audio_processing/typing_detection.cc b/webrtc/modules/audio_processing/typing_detection.cc deleted file mode 100644 index e725b26..0000000 --- a/webrtc/modules/audio_processing/typing_detection.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/typing_detection.h" - -namespace webrtc { - -TypingDetection::TypingDetection() - : time_active_(0), - time_since_last_typing_(0), - penalty_counter_(0), - counter_since_last_detection_update_(0), - detection_to_report_(false), - new_detection_to_report_(false), - time_window_(10), - cost_per_typing_(100), - reporting_threshold_(300), - penalty_decay_(1), - type_event_delay_(2), - report_detection_update_period_(1) {} - -TypingDetection::~TypingDetection() {} - -bool TypingDetection::Process(bool key_pressed, bool vad_activity) { - if (vad_activity) - time_active_++; - else - time_active_ = 0; - - // Keep track if time since last typing event - if (key_pressed) - time_since_last_typing_ = 0; - else - ++time_since_last_typing_; - - if (time_since_last_typing_ < type_event_delay_ && vad_activity && - time_active_ < time_window_) { - penalty_counter_ += cost_per_typing_; - if (penalty_counter_ > reporting_threshold_) - new_detection_to_report_ = true; - } - - if (penalty_counter_ > 0) - penalty_counter_ -= penalty_decay_; - - if (++counter_since_last_detection_update_ == - report_detection_update_period_) { - detection_to_report_ = new_detection_to_report_; - new_detection_to_report_ = false; - counter_since_last_detection_update_ = 0; - } - - return detection_to_report_; -} - -int TypingDetection::TimeSinceLastDetectionInSeconds() { - // Round to whole seconds. - return (time_since_last_typing_ + 50) / 100; -} - -void TypingDetection::SetParameters(int time_window, - int cost_per_typing, - int reporting_threshold, - int penalty_decay, - int type_event_delay, - int report_detection_update_period) { - if (time_window) - time_window_ = time_window; - - if (cost_per_typing) - cost_per_typing_ = cost_per_typing; - - if (reporting_threshold) - reporting_threshold_ = reporting_threshold; - - if (penalty_decay) - penalty_decay_ = penalty_decay; - - if (type_event_delay) - type_event_delay_ = type_event_delay; - - if (report_detection_update_period) - report_detection_update_period_ = report_detection_update_period; -} - -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/typing_detection.h b/webrtc/modules/audio_processing/typing_detection.h deleted file mode 100644 index d8fb359..0000000 --- a/webrtc/modules/audio_processing/typing_detection.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_ -#define MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_ - -#include "rtc_base/system/rtc_export.h" - -namespace webrtc { - -class RTC_EXPORT TypingDetection { - public: - TypingDetection(); - virtual ~TypingDetection(); - - // Run the detection algortihm. Shall be called every 10 ms. Returns true if - // typing is detected, or false if not, based on the update period as set with - // SetParameters(). See |report_detection_update_period_| description below. - bool Process(bool key_pressed, bool vad_activity); - - // Gets the time in seconds since the last detection. - int TimeSinceLastDetectionInSeconds(); - - // Sets the algorithm parameters. A parameter value of 0 leaves it unchanged. - // See the correspondning member variables below for descriptions. - void SetParameters(int time_window, - int cost_per_typing, - int reporting_threshold, - int penalty_decay, - int type_event_delay, - int report_detection_update_period); - - private: - int time_active_; - int time_since_last_typing_; - int penalty_counter_; - - // Counter since last time the detection status reported by Process() was - // updated. See also |report_detection_update_period_|. - int counter_since_last_detection_update_; - - // The detection status to report. Updated every - // |report_detection_update_period_| call to Process(). - bool detection_to_report_; - - // What |detection_to_report_| should be set to next time it is updated. - bool new_detection_to_report_; - - // Settable threshold values. - - // Number of 10 ms slots accepted to count as a hit. - int time_window_; - - // Penalty added for a typing + activity coincide. - int cost_per_typing_; - - // Threshold for |penalty_counter_|. - int reporting_threshold_; - - // How much we reduce |penalty_counter_| every 10 ms. - int penalty_decay_; - - // How old typing events we allow. - int type_event_delay_; - - // Settable update period. - - // Number of 10 ms slots between each update of the detection status returned - // by Process(). This inertia added to the algorithm is usually desirable and - // provided so that consumers of the class don't have to implement that - // themselves if they don't wish. - // If set to 1, each call to Process() will return the detection status for - // that 10 ms slot. - // If set to N (where N > 1), the detection status returned from Process() - // will remain the same until Process() has been called N times. Then, if none - // of the last N calls to Process() has detected typing for each respective - // 10 ms slot, Process() will return false. If at least one of the last N - // calls has detected typing, Process() will return true. And that returned - // status will then remain the same until the next N calls have been done. - int report_detection_update_period_; -}; - -} // namespace webrtc - -#endif // #ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_ diff --git a/webrtc/modules/audio_processing/utility/BUILD.gn b/webrtc/modules/audio_processing/utility/BUILD.gn index 437b544..4851e77 100644 --- a/webrtc/modules/audio_processing/utility/BUILD.gn +++ b/webrtc/modules/audio_processing/utility/BUILD.gn @@ -50,7 +50,6 @@ if (rtc_include_tests) { sources = [ "cascaded_biquad_filter_unittest.cc" ] deps = [ ":cascaded_biquad_filter", - "../../../rtc_base:rtc_base_approved", "../../../test:test_support", "//testing/gtest", ] @@ -62,7 +61,6 @@ if (rtc_include_tests) { sources = [ "delay_estimator_unittest.cc" ] deps = [ ":legacy_delay_estimator", - "../../../rtc_base:rtc_base_approved", "../../../test:test_support", "//testing/gtest", ] diff --git a/webrtc/modules/audio_processing/utility/cascaded_biquad_filter.cc b/webrtc/modules/audio_processing/utility/cascaded_biquad_filter.cc index 08b9464..0d236ce 100644 --- a/webrtc/modules/audio_processing/utility/cascaded_biquad_filter.cc +++ b/webrtc/modules/audio_processing/utility/cascaded_biquad_filter.cc @@ -99,19 +99,28 @@ void CascadedBiQuadFilter::ApplyBiQuad(rtc::ArrayView x, rtc::ArrayView y, CascadedBiQuadFilter::BiQuad* biquad) { RTC_DCHECK_EQ(x.size(), y.size()); - const auto* c_b = biquad->coefficients.b; - const auto* c_a = biquad->coefficients.a; - auto* m_x = biquad->x; - auto* m_y = biquad->y; + const float c_a_0 = biquad->coefficients.a[0]; + const float c_a_1 = biquad->coefficients.a[1]; + const float c_b_0 = biquad->coefficients.b[0]; + const float c_b_1 = biquad->coefficients.b[1]; + const float c_b_2 = biquad->coefficients.b[2]; + float m_x_0 = biquad->x[0]; + float m_x_1 = biquad->x[1]; + float m_y_0 = biquad->y[0]; + float m_y_1 = biquad->y[1]; for (size_t k = 0; k < x.size(); ++k) { const float tmp = x[k]; - y[k] = c_b[0] * tmp + c_b[1] * m_x[0] + c_b[2] * m_x[1] - c_a[0] * m_y[0] - - c_a[1] * m_y[1]; - m_x[1] = m_x[0]; - m_x[0] = tmp; - m_y[1] = m_y[0]; - m_y[0] = y[k]; + y[k] = c_b_0 * tmp + c_b_1 * m_x_0 + c_b_2 * m_x_1 - c_a_0 * m_y_0 - + c_a_1 * m_y_1; + m_x_1 = m_x_0; + m_x_0 = tmp; + m_y_1 = m_y_0; + m_y_0 = y[k]; } + biquad->x[0] = m_x_0; + biquad->x[1] = m_x_1; + biquad->y[0] = m_y_0; + biquad->y[1] = m_y_1; } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/utility/delay_estimator.cc b/webrtc/modules/audio_processing/utility/delay_estimator.cc index 73c70b0..6868392 100644 --- a/webrtc/modules/audio_processing/utility/delay_estimator.cc +++ b/webrtc/modules/audio_processing/utility/delay_estimator.cc @@ -55,7 +55,7 @@ static int BitCount(uint32_t u32) { return ((int)tmp); } -// Compares the |binary_vector| with all rows of the |binary_matrix| and counts +// Compares the `binary_vector` with all rows of the `binary_matrix` and counts // per row the number of times they have the same value. // // Inputs: @@ -74,7 +74,7 @@ static void BitCountComparison(uint32_t binary_vector, int32_t* bit_counts) { int n = 0; - // Compare |binary_vector| with all rows of the |binary_matrix| + // Compare `binary_vector` with all rows of the `binary_matrix` for (; n < matrix_size; n++) { bit_counts[n] = (int32_t)BitCount(binary_vector ^ binary_matrix[n]); } @@ -83,9 +83,9 @@ static void BitCountComparison(uint32_t binary_vector, // Collects necessary statistics for the HistogramBasedValidation(). This // function has to be called prior to calling HistogramBasedValidation(). The // statistics updated and used by the HistogramBasedValidation() are: -// 1. the number of |candidate_hits|, which states for how long we have had the -// same |candidate_delay| -// 2. the |histogram| of candidate delays over time. This histogram is +// 1. the number of `candidate_hits`, which states for how long we have had the +// same `candidate_delay` +// 2. the `histogram` of candidate delays over time. This histogram is // weighted with respect to a reliability measure and time-varying to cope // with possible delay shifts. // For further description see commented code. @@ -93,7 +93,7 @@ static void BitCountComparison(uint32_t binary_vector, // Inputs: // - candidate_delay : The delay to validate. // - valley_depth_q14 : The cost function has a valley/minimum at the -// |candidate_delay| location. |valley_depth_q14| is the +// `candidate_delay` location. `valley_depth_q14` is the // cost function difference between the minimum and // maximum locations. The value is in the Q14 domain. // - valley_level_q14 : Is the cost function value at the minimum, in Q14. @@ -109,37 +109,37 @@ static void UpdateRobustValidationStatistics(BinaryDelayEstimator* self, int i = 0; RTC_DCHECK_EQ(self->history_size, self->farend->history_size); - // Reset |candidate_hits| if we have a new candidate. + // Reset `candidate_hits` if we have a new candidate. if (candidate_delay != self->last_candidate_delay) { self->candidate_hits = 0; self->last_candidate_delay = candidate_delay; } self->candidate_hits++; - // The |histogram| is updated differently across the bins. - // 1. The |candidate_delay| histogram bin is increased with the - // |valley_depth|, which is a simple measure of how reliable the - // |candidate_delay| is. The histogram is not increased above - // |kHistogramMax|. + // The `histogram` is updated differently across the bins. + // 1. The `candidate_delay` histogram bin is increased with the + // `valley_depth`, which is a simple measure of how reliable the + // `candidate_delay` is. The histogram is not increased above + // `kHistogramMax`. self->histogram[candidate_delay] += valley_depth; if (self->histogram[candidate_delay] > kHistogramMax) { self->histogram[candidate_delay] = kHistogramMax; } - // 2. The histogram bins in the neighborhood of |candidate_delay| are + // 2. The histogram bins in the neighborhood of `candidate_delay` are // unaffected. The neighborhood is defined as x + {-2, -1, 0, 1}. - // 3. The histogram bins in the neighborhood of |last_delay| are decreased - // with |decrease_in_last_set|. This value equals the difference between - // the cost function values at the locations |candidate_delay| and - // |last_delay| until we reach |max_hits_for_slow_change| consecutive hits - // at the |candidate_delay|. If we exceed this amount of hits the - // |candidate_delay| is a "potential" candidate and we start decreasing - // these histogram bins more rapidly with |valley_depth|. + // 3. The histogram bins in the neighborhood of `last_delay` are decreased + // with `decrease_in_last_set`. This value equals the difference between + // the cost function values at the locations `candidate_delay` and + // `last_delay` until we reach `max_hits_for_slow_change` consecutive hits + // at the `candidate_delay`. If we exceed this amount of hits the + // `candidate_delay` is a "potential" candidate and we start decreasing + // these histogram bins more rapidly with `valley_depth`. if (self->candidate_hits < max_hits_for_slow_change) { decrease_in_last_set = (self->mean_bit_counts[self->compare_delay] - valley_level_q14) * kQ14Scaling; } - // 4. All other bins are decreased with |valley_depth|. + // 4. All other bins are decreased with `valley_depth`. // TODO(bjornv): Investigate how to make this loop more efficient. Split up // the loop? Remove parts that doesn't add too much. for (i = 0; i < self->history_size; ++i) { @@ -157,15 +157,15 @@ static void UpdateRobustValidationStatistics(BinaryDelayEstimator* self, } } -// Validates the |candidate_delay|, estimated in WebRtc_ProcessBinarySpectrum(), +// Validates the `candidate_delay`, estimated in WebRtc_ProcessBinarySpectrum(), // based on a mix of counting concurring hits with a modified histogram // of recent delay estimates. In brief a candidate is valid (returns 1) if it // is the most likely according to the histogram. There are a couple of // exceptions that are worth mentioning: -// 1. If the |candidate_delay| < |last_delay| it can be that we are in a +// 1. If the `candidate_delay` < `last_delay` it can be that we are in a // non-causal state, breaking a possible echo control algorithm. Hence, we // open up for a quicker change by allowing the change even if the -// |candidate_delay| is not the most likely one according to the histogram. +// `candidate_delay` is not the most likely one according to the histogram. // 2. There's a minimum number of hits (kMinRequiredHits) and the histogram // value has to reached a minimum (kMinHistogramThreshold) to be valid. // 3. The action is also depending on the filter length used for echo control. @@ -177,7 +177,7 @@ static void UpdateRobustValidationStatistics(BinaryDelayEstimator* self, // - candidate_delay : The delay to validate. // // Return value: -// - is_histogram_valid : 1 - The |candidate_delay| is valid. +// - is_histogram_valid : 1 - The `candidate_delay` is valid. // 0 - Otherwise. static int HistogramBasedValidation(const BinaryDelayEstimator* self, int candidate_delay) { @@ -186,22 +186,22 @@ static int HistogramBasedValidation(const BinaryDelayEstimator* self, const int delay_difference = candidate_delay - self->last_delay; int is_histogram_valid = 0; - // The histogram based validation of |candidate_delay| is done by comparing - // the |histogram| at bin |candidate_delay| with a |histogram_threshold|. - // This |histogram_threshold| equals a |fraction| of the |histogram| at bin - // |last_delay|. The |fraction| is a piecewise linear function of the - // |delay_difference| between the |candidate_delay| and the |last_delay| + // The histogram based validation of `candidate_delay` is done by comparing + // the `histogram` at bin `candidate_delay` with a `histogram_threshold`. + // This `histogram_threshold` equals a `fraction` of the `histogram` at bin + // `last_delay`. The `fraction` is a piecewise linear function of the + // `delay_difference` between the `candidate_delay` and the `last_delay` // allowing for a quicker move if // i) a potential echo control filter can not handle these large differences. - // ii) keeping |last_delay| instead of updating to |candidate_delay| could + // ii) keeping `last_delay` instead of updating to `candidate_delay` could // force an echo control into a non-causal state. // We further require the histogram to have reached a minimum value of - // |kMinHistogramThreshold|. In addition, we also require the number of - // |candidate_hits| to be more than |kMinRequiredHits| to remove spurious + // `kMinHistogramThreshold`. In addition, we also require the number of + // `candidate_hits` to be more than `kMinRequiredHits` to remove spurious // values. - // Calculate a comparison histogram value (|histogram_threshold|) that is - // depending on the distance between the |candidate_delay| and |last_delay|. + // Calculate a comparison histogram value (`histogram_threshold`) that is + // depending on the distance between the `candidate_delay` and `last_delay`. // TODO(bjornv): How much can we gain by turning the fraction calculation // into tables? if (delay_difference > self->allowed_offset) { @@ -226,9 +226,9 @@ static int HistogramBasedValidation(const BinaryDelayEstimator* self, return is_histogram_valid; } -// Performs a robust validation of the |candidate_delay| estimated in +// Performs a robust validation of the `candidate_delay` estimated in // WebRtc_ProcessBinarySpectrum(). The algorithm takes the -// |is_instantaneous_valid| and the |is_histogram_valid| and combines them +// `is_instantaneous_valid` and the `is_histogram_valid` and combines them // into a robust validation. The HistogramBasedValidation() has to be called // prior to this call. // For further description on how the combination is done, see commented code. @@ -250,18 +250,18 @@ static int RobustValidation(const BinaryDelayEstimator* self, int is_robust = 0; // The final robust validation is based on the two algorithms; 1) the - // |is_instantaneous_valid| and 2) the histogram based with result stored in - // |is_histogram_valid|. - // i) Before we actually have a valid estimate (|last_delay| == -2), we say + // `is_instantaneous_valid` and 2) the histogram based with result stored in + // `is_histogram_valid`. + // i) Before we actually have a valid estimate (`last_delay` == -2), we say // a candidate is valid if either algorithm states so - // (|is_instantaneous_valid| OR |is_histogram_valid|). + // (`is_instantaneous_valid` OR `is_histogram_valid`). is_robust = (self->last_delay < 0) && (is_instantaneous_valid || is_histogram_valid); // ii) Otherwise, we need both algorithms to be certain - // (|is_instantaneous_valid| AND |is_histogram_valid|) + // (`is_instantaneous_valid` AND `is_histogram_valid`) is_robust |= is_instantaneous_valid && is_histogram_valid; // iii) With one exception, i.e., the histogram based algorithm can overrule - // the instantaneous one if |is_histogram_valid| = 1 and the histogram + // the instantaneous one if `is_histogram_valid` = 1 and the histogram // is significantly strong. is_robust |= is_histogram_valid && (self->histogram[candidate_delay] > self->last_delay_histogram); @@ -373,13 +373,13 @@ void WebRtc_SoftResetBinaryDelayEstimatorFarend( void WebRtc_AddBinaryFarSpectrum(BinaryDelayEstimatorFarend* handle, uint32_t binary_far_spectrum) { RTC_DCHECK(handle); - // Shift binary spectrum history and insert current |binary_far_spectrum|. + // Shift binary spectrum history and insert current `binary_far_spectrum`. memmove(&(handle->binary_far_history[1]), &(handle->binary_far_history[0]), (handle->history_size - 1) * sizeof(uint32_t)); handle->binary_far_history[0] = binary_far_spectrum; // Shift history of far-end binary spectrum bit counts and insert bit count - // of current |binary_far_spectrum|. + // of current `binary_far_spectrum`. memmove(&(handle->far_bit_counts[1]), &(handle->far_bit_counts[0]), (handle->history_size - 1) * sizeof(int)); handle->far_bit_counts[0] = BitCount(binary_far_spectrum); @@ -402,7 +402,7 @@ void WebRtc_FreeBinaryDelayEstimator(BinaryDelayEstimator* self) { free(self->histogram); self->histogram = NULL; - // BinaryDelayEstimator does not have ownership of |farend|, hence we do not + // BinaryDelayEstimator does not have ownership of `farend`, hence we do not // free the memory here. That should be handled separately by the user. self->farend = NULL; @@ -454,8 +454,8 @@ int WebRtc_AllocateHistoryBufferMemory(BinaryDelayEstimator* self, // Only update far-end buffers if we need. history_size = WebRtc_AllocateFarendBufferMemory(far, history_size); } - // The extra array element in |mean_bit_counts| and |histogram| is a dummy - // element only used while |last_delay| == -2, i.e., before we have a valid + // The extra array element in `mean_bit_counts` and `histogram` is a dummy + // element only used while `last_delay` == -2, i.e., before we have a valid // estimate. self->mean_bit_counts = static_cast( realloc(self->mean_bit_counts, @@ -539,36 +539,36 @@ int WebRtc_ProcessBinarySpectrum(BinaryDelayEstimator* self, } if (self->near_history_size > 1) { // If we apply lookahead, shift near-end binary spectrum history. Insert - // current |binary_near_spectrum| and pull out the delayed one. + // current `binary_near_spectrum` and pull out the delayed one. memmove(&(self->binary_near_history[1]), &(self->binary_near_history[0]), (self->near_history_size - 1) * sizeof(uint32_t)); self->binary_near_history[0] = binary_near_spectrum; binary_near_spectrum = self->binary_near_history[self->lookahead]; } - // Compare with delayed spectra and store the |bit_counts| for each delay. + // Compare with delayed spectra and store the `bit_counts` for each delay. BitCountComparison(binary_near_spectrum, self->farend->binary_far_history, self->history_size, self->bit_counts); - // Update |mean_bit_counts|, which is the smoothed version of |bit_counts|. + // Update `mean_bit_counts`, which is the smoothed version of `bit_counts`. for (i = 0; i < self->history_size; i++) { - // |bit_counts| is constrained to [0, 32], meaning we can smooth with a + // `bit_counts` is constrained to [0, 32], meaning we can smooth with a // factor up to 2^26. We use Q9. int32_t bit_count = (self->bit_counts[i] << 9); // Q9. - // Update |mean_bit_counts| only when far-end signal has something to - // contribute. If |far_bit_counts| is zero the far-end signal is weak and + // Update `mean_bit_counts` only when far-end signal has something to + // contribute. If `far_bit_counts` is zero the far-end signal is weak and // we likely have a poor echo condition, hence don't update. if (self->farend->far_bit_counts[i] > 0) { - // Make number of right shifts piecewise linear w.r.t. |far_bit_counts|. + // Make number of right shifts piecewise linear w.r.t. `far_bit_counts`. int shifts = kShiftsAtZero; shifts -= (kShiftsLinearSlope * self->farend->far_bit_counts[i]) >> 4; WebRtc_MeanEstimatorFix(bit_count, shifts, &(self->mean_bit_counts[i])); } } - // Find |candidate_delay|, |value_best_candidate| and |value_worst_candidate| - // of |mean_bit_counts|. + // Find `candidate_delay`, `value_best_candidate` and `value_worst_candidate` + // of `mean_bit_counts`. for (i = 0; i < self->history_size; i++) { if (self->mean_bit_counts[i] < value_best_candidate) { value_best_candidate = self->mean_bit_counts[i]; @@ -580,25 +580,25 @@ int WebRtc_ProcessBinarySpectrum(BinaryDelayEstimator* self, } valley_depth = value_worst_candidate - value_best_candidate; - // The |value_best_candidate| is a good indicator on the probability of - // |candidate_delay| being an accurate delay (a small |value_best_candidate| + // The `value_best_candidate` is a good indicator on the probability of + // `candidate_delay` being an accurate delay (a small `value_best_candidate` // means a good binary match). In the following sections we make a decision - // whether to update |last_delay| or not. + // whether to update `last_delay` or not. // 1) If the difference bit counts between the best and the worst delay // candidates is too small we consider the situation to be unreliable and - // don't update |last_delay|. - // 2) If the situation is reliable we update |last_delay| if the value of the + // don't update `last_delay`. + // 2) If the situation is reliable we update `last_delay` if the value of the // best candidate delay has a value less than - // i) an adaptive threshold |minimum_probability|, or - // ii) this corresponding value |last_delay_probability|, but updated at + // i) an adaptive threshold `minimum_probability`, or + // ii) this corresponding value `last_delay_probability`, but updated at // this time instant. - // Update |minimum_probability|. + // Update `minimum_probability`. if ((self->minimum_probability > kProbabilityLowerLimit) && (valley_depth > kProbabilityMinSpread)) { // The "hard" threshold can't be lower than 17 (in Q9). // The valley in the curve also has to be distinct, i.e., the - // difference between |value_worst_candidate| and |value_best_candidate| has + // difference between `value_worst_candidate` and `value_best_candidate` has // to be large enough. int32_t threshold = value_best_candidate + kProbabilityOffset; if (threshold < kProbabilityLowerLimit) { @@ -608,17 +608,17 @@ int WebRtc_ProcessBinarySpectrum(BinaryDelayEstimator* self, self->minimum_probability = threshold; } } - // Update |last_delay_probability|. + // Update `last_delay_probability`. // We use a Markov type model, i.e., a slowly increasing level over time. self->last_delay_probability++; - // Validate |candidate_delay|. We have a reliable instantaneous delay + // Validate `candidate_delay`. We have a reliable instantaneous delay // estimate if - // 1) The valley is distinct enough (|valley_depth| > |kProbabilityOffset|) + // 1) The valley is distinct enough (`valley_depth` > `kProbabilityOffset`) // and // 2) The depth of the valley is deep enough - // (|value_best_candidate| < |minimum_probability|) + // (`value_best_candidate` < `minimum_probability`) // and deeper than the best estimate so far - // (|value_best_candidate| < |last_delay_probability|) + // (`value_best_candidate` < `last_delay_probability`) valid_candidate = ((valley_depth > kProbabilityOffset) && ((value_best_candidate < self->minimum_probability) || (value_best_candidate < self->last_delay_probability))); @@ -650,7 +650,7 @@ int WebRtc_ProcessBinarySpectrum(BinaryDelayEstimator* self, (self->histogram[candidate_delay] > kLastHistogramMax ? kLastHistogramMax : self->histogram[candidate_delay]); - // Adjust the histogram if we made a change to |last_delay|, though it was + // Adjust the histogram if we made a change to `last_delay`, though it was // not the most likely one according to the histogram. if (self->histogram[candidate_delay] < self->histogram[self->compare_delay]) { @@ -680,7 +680,7 @@ float WebRtc_binary_last_delay_quality(BinaryDelayEstimator* self) { // Simply a linear function of the histogram height at delay estimate. quality = self->histogram[self->compare_delay] / kHistogramMax; } else { - // Note that |last_delay_probability| states how deep the minimum of the + // Note that `last_delay_probability` states how deep the minimum of the // cost function is, so it is rather an error probability. quality = (float)(kMaxBitCountsQ9 - self->last_delay_probability) / kMaxBitCountsQ9; diff --git a/webrtc/modules/audio_processing/utility/delay_estimator.h b/webrtc/modules/audio_processing/utility/delay_estimator.h index df281bc..b6fc36a 100644 --- a/webrtc/modules/audio_processing/utility/delay_estimator.h +++ b/webrtc/modules/audio_processing/utility/delay_estimator.h @@ -81,7 +81,7 @@ void WebRtc_FreeBinaryDelayEstimatorFarend(BinaryDelayEstimatorFarend* self); // // Return value: // - BinaryDelayEstimatorFarend* -// : Created |handle|. If the memory can't be allocated +// : Created `handle`. If the memory can't be allocated // or if any of the input parameters are invalid NULL // is returned. // @@ -159,7 +159,7 @@ BinaryDelayEstimator* WebRtc_CreateBinaryDelayEstimator( BinaryDelayEstimatorFarend* farend, int max_lookahead); -// Re-allocates |history_size| dependent buffers. The far-end buffers will be +// Re-allocates `history_size` dependent buffers. The far-end buffers will be // updated at the same time if needed. // // Input: @@ -237,7 +237,7 @@ int WebRtc_binary_last_delay(BinaryDelayEstimator* self); // delay value. float WebRtc_binary_last_delay_quality(BinaryDelayEstimator* self); -// Updates the |mean_value| recursively with a step size of 2^-|factor|. This +// Updates the `mean_value` recursively with a step size of 2^-`factor`. This // function is used internally in the Binary Delay Estimator as well as the // Fixed point wrapper. // diff --git a/webrtc/modules/audio_processing/utility/delay_estimator_internal.h b/webrtc/modules/audio_processing/utility/delay_estimator_internal.h index fce95d8..891e200 100644 --- a/webrtc/modules/audio_processing/utility/delay_estimator_internal.h +++ b/webrtc/modules/audio_processing/utility/delay_estimator_internal.h @@ -25,7 +25,7 @@ typedef union { typedef struct { // Pointers to mean values of spectrum. SpectrumType* mean_far_spectrum; - // |mean_far_spectrum| initialization indicator. + // `mean_far_spectrum` initialization indicator. int far_spectrum_initialized; int spectrum_size; @@ -37,7 +37,7 @@ typedef struct { typedef struct { // Pointers to mean values of spectrum. SpectrumType* mean_near_spectrum; - // |mean_near_spectrum| initialization indicator. + // `mean_near_spectrum` initialization indicator. int near_spectrum_initialized; int spectrum_size; diff --git a/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc b/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc index 8eac2f6..3b1409c 100644 --- a/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc +++ b/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.cc @@ -19,10 +19,10 @@ namespace webrtc { -// Only bit |kBandFirst| through bit |kBandLast| are processed and -// |kBandFirst| - |kBandLast| must be < 32. -enum { kBandFirst = 12 }; -enum { kBandLast = 43 }; +// Only bit `kBandFirst` through bit `kBandLast` are processed and +// `kBandFirst` - `kBandLast` must be < 32. +constexpr int kBandFirst = 12; +constexpr int kBandLast = 43; static __inline uint32_t SetBit(uint32_t in, int pos) { uint32_t mask = (1 << pos); @@ -48,8 +48,8 @@ static void MeanEstimatorFloat(float new_value, *mean_value += (new_value - *mean_value) * scale; } -// Computes the binary spectrum by comparing the input |spectrum| with a -// |threshold_spectrum|. Float and fixed point versions. +// Computes the binary spectrum by comparing the input `spectrum` with a +// `threshold_spectrum`. Float and fixed point versions. // // Inputs: // - spectrum : Spectrum of which the binary spectrum should be @@ -69,11 +69,11 @@ static uint32_t BinarySpectrumFix(const uint16_t* spectrum, RTC_DCHECK_LT(q_domain, 16); if (!(*threshold_initialized)) { - // Set the |threshold_spectrum| to half the input |spectrum| as starting + // Set the `threshold_spectrum` to half the input `spectrum` as starting // value. This speeds up the convergence. for (i = kBandFirst; i <= kBandLast; i++) { if (spectrum[i] > 0) { - // Convert input spectrum from Q(|q_domain|) to Q15. + // Convert input spectrum from Q(`q_domain`) to Q15. int32_t spectrum_q15 = ((int32_t)spectrum[i]) << (15 - q_domain); threshold_spectrum[i].int32_ = (spectrum_q15 >> 1); *threshold_initialized = 1; @@ -81,11 +81,11 @@ static uint32_t BinarySpectrumFix(const uint16_t* spectrum, } } for (i = kBandFirst; i <= kBandLast; i++) { - // Convert input spectrum from Q(|q_domain|) to Q15. + // Convert input spectrum from Q(`q_domain`) to Q15. int32_t spectrum_q15 = ((int32_t)spectrum[i]) << (15 - q_domain); - // Update the |threshold_spectrum|. + // Update the `threshold_spectrum`. WebRtc_MeanEstimatorFix(spectrum_q15, 6, &(threshold_spectrum[i].int32_)); - // Convert |spectrum| at current frequency bin to a binary value. + // Convert `spectrum` at current frequency bin to a binary value. if (spectrum_q15 > threshold_spectrum[i].int32_) { out = SetBit(out, i - kBandFirst); } @@ -102,7 +102,7 @@ static uint32_t BinarySpectrumFloat(const float* spectrum, const float kScale = 1 / 64.0; if (!(*threshold_initialized)) { - // Set the |threshold_spectrum| to half the input |spectrum| as starting + // Set the `threshold_spectrum` to half the input `spectrum` as starting // value. This speeds up the convergence. for (i = kBandFirst; i <= kBandLast; i++) { if (spectrum[i] > 0.0f) { @@ -113,9 +113,9 @@ static uint32_t BinarySpectrumFloat(const float* spectrum, } for (i = kBandFirst; i <= kBandLast; i++) { - // Update the |threshold_spectrum|. + // Update the `threshold_spectrum`. MeanEstimatorFloat(spectrum[i], kScale, &(threshold_spectrum[i].float_)); - // Convert |spectrum| at current frequency bin to a binary value. + // Convert `spectrum` at current frequency bin to a binary value. if (spectrum[i] > threshold_spectrum[i].float_) { out = SetBit(out, i - kBandFirst); } @@ -219,7 +219,7 @@ int WebRtc_AddFarSpectrumFix(void* handle, return -1; } if (far_q > 15) { - // If |far_q| is larger than 15 we cannot guarantee no wrap around. + // If `far_q` is larger than 15 we cannot guarantee no wrap around. return -1; } @@ -433,7 +433,7 @@ int WebRtc_DelayEstimatorProcessFix(void* handle, return -1; } if (near_q > 15) { - // If |near_q| is larger than 15 we cannot guarantee no wrap around. + // If `near_q` is larger than 15 we cannot guarantee no wrap around. return -1; } diff --git a/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h b/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h index dbcafaf..a90cbe3 100644 --- a/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h +++ b/webrtc/modules/audio_processing/utility/delay_estimator_wrapper.h @@ -35,7 +35,7 @@ void WebRtc_FreeDelayEstimatorFarend(void* handle); // determined together with WebRtc_set_lookahead(). // // Return value: -// - void* : Created |handle|. If the memory can't be allocated or +// - void* : Created `handle`. If the memory can't be allocated or // if any of the input parameters are invalid NULL is // returned. void* WebRtc_CreateDelayEstimatorFarend(int spectrum_size, int history_size); @@ -85,13 +85,13 @@ void WebRtc_FreeDelayEstimator(void* handle); // WebRtc_CreateDelayEstimatorFarend(). // // Note that WebRtc_CreateDelayEstimator does not take -// ownership of |farend_handle|, which has to be torn +// ownership of `farend_handle`, which has to be torn // down properly after this instance. // // - max_lookahead : Maximum amount of non-causal lookahead allowed. The // actual amount of lookahead used can be controlled by -// WebRtc_set_lookahead(...). The default |lookahead| is -// set to |max_lookahead| at create time. Use +// WebRtc_set_lookahead(...). The default `lookahead` is +// set to `max_lookahead` at create time. Use // WebRtc_set_lookahead(...) before start if a different // value is desired. // @@ -106,12 +106,12 @@ void WebRtc_FreeDelayEstimator(void* handle); // estimated. // // Note that the effective range of delay estimates is -// [-|lookahead|,... ,|history_size|-|lookahead|) -// where |history_size| is set through +// [-`lookahead`,... ,`history_size`-`lookahead`) +// where `history_size` is set through // WebRtc_set_history_size(). // // Return value: -// - void* : Created |handle|. If the memory can't be allocated or +// - void* : Created `handle`. If the memory can't be allocated or // if any of the input parameters are invalid NULL is // returned. void* WebRtc_CreateDelayEstimator(void* farend_handle, int max_lookahead); @@ -129,12 +129,12 @@ int WebRtc_InitDelayEstimator(void* handle); // - actual_shifts : The actual number of shifts performed. int WebRtc_SoftResetDelayEstimator(void* handle, int delay_shift); -// Sets the effective |history_size| used. Valid values from 2. We simply need -// at least two delays to compare to perform an estimate. If |history_size| is +// Sets the effective `history_size` used. Valid values from 2. We simply need +// at least two delays to compare to perform an estimate. If `history_size` is // changed, buffers are reallocated filling in with zeros if necessary. -// Note that changing the |history_size| affects both buffers in far-end and +// Note that changing the `history_size` affects both buffers in far-end and // near-end. Hence it is important to change all DelayEstimators that use the -// same reference far-end, to the same |history_size| value. +// same reference far-end, to the same `history_size` value. // Inputs: // - handle : Pointer to the delay estimation instance. // - history_size : Effective history size to be used. @@ -148,8 +148,8 @@ int WebRtc_set_history_size(void* handle, int history_size); // - handle : Pointer to the delay estimation instance. int WebRtc_history_size(const void* handle); -// Sets the amount of |lookahead| to use. Valid values are [0, max_lookahead] -// where |max_lookahead| was set at create time through +// Sets the amount of `lookahead` to use. Valid values are [0, max_lookahead] +// where `max_lookahead` was set at create time through // WebRtc_CreateDelayEstimator(...). // // Input: @@ -157,8 +157,8 @@ int WebRtc_history_size(const void* handle); // - lookahead : The amount of lookahead to be used. // // Return value: -// - new_lookahead : The actual amount of lookahead set, unless |handle| is -// a NULL pointer or |lookahead| is invalid, for which an +// - new_lookahead : The actual amount of lookahead set, unless `handle` is +// a NULL pointer or `lookahead` is invalid, for which an // error is returned. int WebRtc_set_lookahead(void* handle, int lookahead); @@ -167,12 +167,12 @@ int WebRtc_set_lookahead(void* handle, int lookahead); // - handle : Pointer to the delay estimation instance. int WebRtc_lookahead(void* handle); -// Sets the |allowed_offset| used in the robust validation scheme. If the +// Sets the `allowed_offset` used in the robust validation scheme. If the // delay estimator is used in an echo control component, this parameter is -// related to the filter length. In principle |allowed_offset| should be set to +// related to the filter length. In principle `allowed_offset` should be set to // the echo control filter length minus the expected echo duration, i.e., the // delay offset the echo control can handle without quality regression. The -// default value, used if not set manually, is zero. Note that |allowed_offset| +// default value, used if not set manually, is zero. Note that `allowed_offset` // has to be non-negative. // Inputs: // - handle : Pointer to the delay estimation instance. @@ -180,7 +180,7 @@ int WebRtc_lookahead(void* handle); // the echo control filter can handle. int WebRtc_set_allowed_offset(void* handle, int allowed_offset); -// Returns the |allowed_offset| in number of partitions. +// Returns the `allowed_offset` in number of partitions. int WebRtc_get_allowed_offset(const void* handle); // Enables/Disables a robust validation functionality in the delay estimation. diff --git a/webrtc/modules/audio_processing/utility/pffft_wrapper.h b/webrtc/modules/audio_processing/utility/pffft_wrapper.h index 160f0da..983c2fd 100644 --- a/webrtc/modules/audio_processing/utility/pffft_wrapper.h +++ b/webrtc/modules/audio_processing/utility/pffft_wrapper.h @@ -51,7 +51,7 @@ class Pffft { // TODO(https://crbug.com/webrtc/9577): Consider adding a factory and making // the ctor private. // static std::unique_ptr Create(size_t fft_size, - // FftType fft_type); Ctor. |fft_size| must be a supported size (see + // FftType fft_type); Ctor. `fft_size` must be a supported size (see // Pffft::IsValidFftSize()). If not supported, the code will crash. Pffft(size_t fft_size, FftType fft_type); Pffft(const Pffft&) = delete; @@ -73,9 +73,9 @@ class Pffft { // Computes the backward fast Fourier transform. void BackwardTransform(const FloatBuffer& in, FloatBuffer* out, bool ordered); - // Multiplies the frequency components of |fft_x| and |fft_y| and accumulates - // them into |out|. The arrays must have been obtained with - // ForwardTransform(..., /*ordered=*/false) - i.e., |fft_x| and |fft_y| must + // Multiplies the frequency components of `fft_x` and `fft_y` and accumulates + // them into `out`. The arrays must have been obtained with + // ForwardTransform(..., /*ordered=*/false) - i.e., `fft_x` and `fft_y` must // not be ordered. void FrequencyDomainConvolve(const FloatBuffer& fft_x, const FloatBuffer& fft_y, diff --git a/webrtc/modules/audio_processing/vad/gmm.h b/webrtc/modules/audio_processing/vad/gmm.h index 93eb675..d9d68ec 100644 --- a/webrtc/modules/audio_processing/vad/gmm.h +++ b/webrtc/modules/audio_processing/vad/gmm.h @@ -20,13 +20,13 @@ namespace webrtc { // Where a 'mixture' is a Gaussian density. struct GmmParameters { - // weight[n] = log(w[n]) - |dimension|/2 * log(2*pi) - 1/2 * log(det(cov[n])); + // weight[n] = log(w[n]) - `dimension`/2 * log(2*pi) - 1/2 * log(det(cov[n])); // where cov[n] is the covariance matrix of mixture n; const double* weight; - // pointer to the first element of a |num_mixtures|x|dimension| matrix + // pointer to the first element of a `num_mixtures`x`dimension` matrix // where kth row is the mean of the kth mixture. const double* mean; - // pointer to the first element of a |num_mixtures|x|dimension|x|dimension| + // pointer to the first element of a `num_mixtures`x`dimension`x`dimension` // 3D-matrix, where the kth 2D-matrix is the inverse of the covariance // matrix of the kth mixture. const double* covar_inverse; @@ -36,8 +36,8 @@ struct GmmParameters { int num_mixtures; }; -// Evaluate the given GMM, according to |gmm_parameters|, at the given point -// |x|. If the dimensionality of the given GMM is larger that the maximum +// Evaluate the given GMM, according to `gmm_parameters`, at the given point +// `x`. If the dimensionality of the given GMM is larger that the maximum // acceptable dimension by the following function -1 is returned. double EvaluateGmm(const double* x, const GmmParameters& gmm_parameters); diff --git a/webrtc/modules/audio_processing/vad/pitch_based_vad.h b/webrtc/modules/audio_processing/vad/pitch_based_vad.h index e005e23..fa3abc2 100644 --- a/webrtc/modules/audio_processing/vad/pitch_based_vad.h +++ b/webrtc/modules/audio_processing/vad/pitch_based_vad.h @@ -34,7 +34,7 @@ class PitchBasedVad { // p_combined: an array which contains the combined activity probabilities // computed prior to the call of this function. The method, // then, computes the voicing probabilities and combine them - // with the given values. The result are returned in |p|. + // with the given values. The result are returned in `p`. int VoicingProbability(const AudioFeatures& features, double* p_combined); private: diff --git a/webrtc/modules/audio_processing/vad/pitch_internal.h b/webrtc/modules/audio_processing/vad/pitch_internal.h index 938745d..e382c1f 100644 --- a/webrtc/modules/audio_processing/vad/pitch_internal.h +++ b/webrtc/modules/audio_processing/vad/pitch_internal.h @@ -14,7 +14,7 @@ namespace webrtc { // TODO(turajs): Write a description of this function. Also be consistent with -// usage of |sampling_rate_hz| vs |kSamplingFreqHz|. +// usage of `sampling_rate_hz` vs `kSamplingFreqHz`. void GetSubframesPitchParameters(int sampling_rate_hz, double* gains, double* lags, diff --git a/webrtc/modules/audio_processing/vad/standalone_vad.h b/webrtc/modules/audio_processing/vad/standalone_vad.h index 3dff416..b084633 100644 --- a/webrtc/modules/audio_processing/vad/standalone_vad.h +++ b/webrtc/modules/audio_processing/vad/standalone_vad.h @@ -26,12 +26,12 @@ class StandaloneVad { // Outputs // p: a buffer where probabilities are written to. - // length_p: number of elements of |p|. + // length_p: number of elements of `p`. // // return value: // -1: if no audio is stored or VAD returns error. // 0: in success. - // In case of error the content of |activity| is unchanged. + // In case of error the content of `activity` is unchanged. // // Note that due to a high false-positive (VAD decision is active while the // processed audio is just background noise) rate, stand-alone VAD is used as diff --git a/webrtc/modules/audio_processing/vad/vad_audio_proc.cc b/webrtc/modules/audio_processing/vad/vad_audio_proc.cc index 97cf651..aaf8214 100644 --- a/webrtc/modules/audio_processing/vad/vad_audio_proc.cc +++ b/webrtc/modules/audio_processing/vad/vad_audio_proc.cc @@ -132,7 +132,7 @@ void VadAudioProc::SubframeCorrelation(double* corr, kNumSubframeSamples + kNumPastSignalSamples, kLpcOrder); } -// Compute |kNum10msSubframes| sets of LPC coefficients, one per 10 ms input. +// Compute `kNum10msSubframes` sets of LPC coefficients, one per 10 ms input. // The analysis window is 15 ms long and it is centered on the first half of // each 10ms sub-frame. This is equivalent to computing LPC coefficients for the // first half of each 10 ms subframe. @@ -169,7 +169,7 @@ static float QuadraticInterpolation(float prev_val, return fractional_index; } -// 1 / A(z), where A(z) is defined by |lpc| is a model of the spectral envelope +// 1 / A(z), where A(z) is defined by `lpc` is a model of the spectral envelope // of the input signal. The local maximum of the spectral envelope corresponds // with the local minimum of A(z). It saves complexity, as we save one // inversion. Furthermore, we find the first local maximum of magnitude squared, diff --git a/webrtc/modules/audio_processing/vad/vad_audio_proc.h b/webrtc/modules/audio_processing/vad/vad_audio_proc.h index 4a71ce3..905c687 100644 --- a/webrtc/modules/audio_processing/vad/vad_audio_proc.h +++ b/webrtc/modules/audio_processing/vad/vad_audio_proc.h @@ -35,7 +35,7 @@ class VadAudioProc { size_t length, AudioFeatures* audio_features); - static const size_t kDftSize = 512; + static constexpr size_t kDftSize = 512; private: void PitchAnalysis(double* pitch_gains, double* pitch_lags_hz, size_t length); @@ -51,28 +51,22 @@ class VadAudioProc { // For every 30 ms we compute 3 spectral peak there for 3 LPC analysis. // LPC is computed over 15 ms of windowed audio. For every 10 ms sub-frame // we need 5 ms of past signal to create the input of LPC analysis. - enum : size_t { - kNumPastSignalSamples = static_cast(kSampleRateHz / 200) - }; + static constexpr size_t kNumPastSignalSamples = size_t{kSampleRateHz / 200}; // TODO(turajs): maybe defining this at a higher level (maybe enum) so that // all the code recognize it as "no-error." - enum : int { kNoError = 0 }; + static constexpr int kNoError = 0; - enum : size_t { kNum10msSubframes = 3 }; - enum : size_t { - kNumSubframeSamples = static_cast(kSampleRateHz / 100) - }; - enum : size_t { - // Samples in 30 ms @ given sampling rate. - kNumSamplesToProcess = kNum10msSubframes * kNumSubframeSamples - }; - enum : size_t { - kBufferLength = kNumPastSignalSamples + kNumSamplesToProcess - }; - enum : size_t { kIpLength = kDftSize >> 1 }; - enum : size_t { kWLength = kDftSize >> 1 }; - enum : size_t { kLpcOrder = 16 }; + static constexpr size_t kNum10msSubframes = 3; + static constexpr size_t kNumSubframeSamples = size_t{kSampleRateHz / 100}; + // Samples in 30 ms @ given sampling rate. + static constexpr size_t kNumSamplesToProcess = + kNum10msSubframes * kNumSubframeSamples; + static constexpr size_t kBufferLength = + kNumPastSignalSamples + kNumSamplesToProcess; + static constexpr size_t kIpLength = kDftSize >> 1; + static constexpr size_t kWLength = kDftSize >> 1; + static constexpr size_t kLpcOrder = 16; size_t ip_[kIpLength]; float w_fft_[kWLength]; diff --git a/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h b/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h index 915524f..93589af 100644 --- a/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h +++ b/webrtc/modules/audio_processing/vad/vad_audio_proc_internal.h @@ -11,6 +11,8 @@ #ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_INTERNAL_H_ #define MODULES_AUDIO_PROCESSING_VAD_VAD_AUDIO_PROC_INTERNAL_H_ +#include + namespace webrtc { // These values should match MATLAB counterparts for unit-tests to pass. diff --git a/webrtc/modules/audio_processing/vad/vad_circular_buffer.h b/webrtc/modules/audio_processing/vad/vad_circular_buffer.h index 46b03d4..c1806f9 100644 --- a/webrtc/modules/audio_processing/vad/vad_circular_buffer.h +++ b/webrtc/modules/audio_processing/vad/vad_circular_buffer.h @@ -38,8 +38,8 @@ class VadCircularBuffer { // The mean value of the elements in the buffer. The return value is zero if // buffer is empty, i.e. no value is inserted. double Mean(); - // Remove transients. If the values exceed |val_threshold| for a period - // shorter then or equal to |width_threshold|, then that period is considered + // Remove transients. If the values exceed `val_threshold` for a period + // shorter then or equal to `width_threshold`, then that period is considered // transient and set to zero. int RemoveTransient(int width_threshold, double val_threshold); @@ -49,7 +49,7 @@ class VadCircularBuffer { // insertion. |index = 1| is the one before the most recent insertion, and // so on. int Get(int index, double* value) const; - // Set a given position to |value|. |index| is interpreted as above. + // Set a given position to `value`. `index` is interpreted as above. int Set(int index, double value); // Return the number of valid elements in the buffer. int BufferLevel(); diff --git a/webrtc/modules/audio_processing/vad/voice_activity_detector.cc b/webrtc/modules/audio_processing/vad/voice_activity_detector.cc index f0d34c6..02023d6 100644 --- a/webrtc/modules/audio_processing/vad/voice_activity_detector.cc +++ b/webrtc/modules/audio_processing/vad/voice_activity_detector.cc @@ -32,12 +32,13 @@ VoiceActivityDetector::VoiceActivityDetector() VoiceActivityDetector::~VoiceActivityDetector() = default; // Because ISAC has a different chunk length, it updates -// |chunkwise_voice_probabilities_| and |chunkwise_rms_| when there is new data. +// `chunkwise_voice_probabilities_` and `chunkwise_rms_` when there is new data. // Otherwise it clears them. void VoiceActivityDetector::ProcessChunk(const int16_t* audio, size_t length, int sample_rate_hz) { RTC_DCHECK_EQ(length, sample_rate_hz / 100); + // TODO(bugs.webrtc.org/7494): Remove resampling and force 16 kHz audio. // Resample to the required rate. const int16_t* resampled_ptr = audio; if (sample_rate_hz != kSampleRateHz) { @@ -49,7 +50,7 @@ void VoiceActivityDetector::ProcessChunk(const int16_t* audio, } RTC_DCHECK_EQ(length, kLength10Ms); - // Each chunk needs to be passed into |standalone_vad_|, because internally it + // Each chunk needs to be passed into `standalone_vad_`, because internally it // buffers the audio and processes it all at once when GetActivity() is // called. RTC_CHECK_EQ(standalone_vad_->AddAudio(resampled_ptr, length), 0); diff --git a/webrtc/modules/audio_processing/vad/voice_activity_detector.h b/webrtc/modules/audio_processing/vad/voice_activity_detector.h index a19883d..92b9a8c 100644 --- a/webrtc/modules/audio_processing/vad/voice_activity_detector.h +++ b/webrtc/modules/audio_processing/vad/voice_activity_detector.h @@ -33,6 +33,8 @@ class VoiceActivityDetector { ~VoiceActivityDetector(); // Processes each audio chunk and estimates the voice probability. + // TODO(bugs.webrtc.org/7494): Switch to rtc::ArrayView and remove + // `sample_rate_hz`. void ProcessChunk(const int16_t* audio, size_t length, int sample_rate_hz); // Returns a vector of voice probabilities for each chunk. It can be empty for diff --git a/webrtc/modules/audio_processing/voice_detection.cc b/webrtc/modules/audio_processing/voice_detection.cc deleted file mode 100644 index e6c92ae..0000000 --- a/webrtc/modules/audio_processing/voice_detection.cc +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/audio_processing/voice_detection.h" - -#include "common_audio/vad/include/webrtc_vad.h" -#include "modules/audio_processing/audio_buffer.h" -#include "rtc_base/checks.h" - -namespace webrtc { -class VoiceDetection::Vad { - public: - Vad() { - state_ = WebRtcVad_Create(); - RTC_CHECK(state_); - int error = WebRtcVad_Init(state_); - RTC_DCHECK_EQ(0, error); - } - ~Vad() { WebRtcVad_Free(state_); } - - Vad(Vad&) = delete; - Vad& operator=(Vad&) = delete; - - VadInst* state() { return state_; } - - private: - VadInst* state_ = nullptr; -}; - -VoiceDetection::VoiceDetection(int sample_rate_hz, Likelihood likelihood) - : sample_rate_hz_(sample_rate_hz), - frame_size_samples_(static_cast(sample_rate_hz_ / 100)), - likelihood_(likelihood), - vad_(new Vad()) { - int mode = 2; - switch (likelihood) { - case VoiceDetection::kVeryLowLikelihood: - mode = 3; - break; - case VoiceDetection::kLowLikelihood: - mode = 2; - break; - case VoiceDetection::kModerateLikelihood: - mode = 1; - break; - case VoiceDetection::kHighLikelihood: - mode = 0; - break; - default: - RTC_NOTREACHED(); - break; - } - int error = WebRtcVad_set_mode(vad_->state(), mode); - RTC_DCHECK_EQ(0, error); -} - -VoiceDetection::~VoiceDetection() {} - -bool VoiceDetection::ProcessCaptureAudio(AudioBuffer* audio) { - RTC_DCHECK_GE(AudioBuffer::kMaxSplitFrameLength, - audio->num_frames_per_band()); - std::array mixed_low_pass_data; - rtc::ArrayView mixed_low_pass(mixed_low_pass_data.data(), - audio->num_frames_per_band()); - if (audio->num_channels() == 1) { - FloatS16ToS16(audio->split_bands_const(0)[kBand0To8kHz], - audio->num_frames_per_band(), mixed_low_pass_data.data()); - } else { - const int num_channels = static_cast(audio->num_channels()); - for (size_t i = 0; i < audio->num_frames_per_band(); ++i) { - int32_t value = - FloatS16ToS16(audio->split_channels_const(kBand0To8kHz)[0][i]); - for (int j = 1; j < num_channels; ++j) { - value += FloatS16ToS16(audio->split_channels_const(kBand0To8kHz)[j][i]); - } - mixed_low_pass_data[i] = value / num_channels; - } - } - - int vad_ret = WebRtcVad_Process(vad_->state(), sample_rate_hz_, - mixed_low_pass.data(), frame_size_samples_); - RTC_DCHECK(vad_ret == 0 || vad_ret == 1); - return vad_ret == 0 ? false : true; -} -} // namespace webrtc diff --git a/webrtc/modules/audio_processing/voice_detection.h b/webrtc/modules/audio_processing/voice_detection.h deleted file mode 100644 index 79d44e6..0000000 --- a/webrtc/modules/audio_processing/voice_detection.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_PROCESSING_VOICE_DETECTION_H_ -#define MODULES_AUDIO_PROCESSING_VOICE_DETECTION_H_ - -#include - -#include - -#include "modules/audio_processing/include/audio_processing.h" - -namespace webrtc { - -class AudioBuffer; - -// The voice activity detection (VAD) component analyzes the stream to -// determine if voice is present. -class VoiceDetection { - public: - // Specifies the likelihood that a frame will be declared to contain voice. - // A higher value makes it more likely that speech will not be clipped, at - // the expense of more noise being detected as voice. - enum Likelihood { - kVeryLowLikelihood, - kLowLikelihood, - kModerateLikelihood, - kHighLikelihood - }; - - VoiceDetection(int sample_rate_hz, Likelihood likelihood); - ~VoiceDetection(); - - VoiceDetection(VoiceDetection&) = delete; - VoiceDetection& operator=(VoiceDetection&) = delete; - - // Returns true if voice is detected in the current frame. - bool ProcessCaptureAudio(AudioBuffer* audio); - - Likelihood likelihood() const { return likelihood_; } - - private: - class Vad; - - int sample_rate_hz_; - size_t frame_size_samples_; - Likelihood likelihood_; - std::unique_ptr vad_; -}; -} // namespace webrtc - -#endif // MODULES_AUDIO_PROCESSING_VOICE_DETECTION_H_ diff --git a/webrtc/rtc_base/BUILD.gn b/webrtc/rtc_base/BUILD.gn index 3383580..6abae80 100644 --- a/webrtc/rtc_base/BUILD.gn +++ b/webrtc/rtc_base/BUILD.gn @@ -6,8 +6,6 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. -import("//build/config/crypto.gni") -import("//build/config/ui.gni") import("../webrtc.gni") if (is_android) { @@ -15,10 +13,6 @@ if (is_android) { import("//build/config/android/rules.gni") } -config("rtc_base_chromium_config") { - defines = [ "NO_MAIN_THREAD_WRAPPING" ] -} - if (!rtc_build_ssl) { config("external_ssl_library") { assert(rtc_ssl_root != "", @@ -35,6 +29,23 @@ rtc_source_set("protobuf_utils") { } } +rtc_source_set("bitstream_reader") { + sources = [ + "bitstream_reader.cc", + "bitstream_reader.h", + ] + deps = [ + ":checks", + ":safe_conversions", + "../api:array_view", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/numeric:bits", + "//third_party/abseil-cpp/absl/strings", + ] +} + rtc_source_set("compile_assert_c") { sources = [ "compile_assert_c.h" ] } @@ -48,124 +59,294 @@ rtc_source_set("untyped_function") { deps = [ "system:assume" ] } -rtc_source_set("robo_caller") { +rtc_source_set("callback_list") { sources = [ - "robo_caller.cc", - "robo_caller.h", + "callback_list.cc", + "callback_list.h", ] deps = [ + ":checks", ":untyped_function", "../api:function_view", "system:assume", "system:inline", + "system:rtc_export", ] } -# The subset of rtc_base approved for use outside of libjingle. -# TODO(bugs.webrtc.org/9838): Create small and focused build targets and remove -# the old concept of rtc_base and rtc_base_approved. -rtc_library("rtc_base_approved") { +rtc_source_set("buffer") { visibility = [ "*" ] + sources = [ "buffer.h" ] deps = [ ":checks", - ":rtc_task_queue", - ":safe_compare", ":type_traits", + ":zero_memory", "../api:array_view", - "../api:scoped_refptr", - "synchronization:mutex", - "system:arch", - "system:rtc_export", - "system:unused", - "third_party/base64", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - public_deps = [] # no-presubmit-check TODO(webrtc:8603) + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} +rtc_source_set("byte_order") { + visibility = [ "*" ] + sources = [ "byte_order.h" ] + deps = [ "system:arch" ] +} + +rtc_source_set("mod_ops") { + visibility = [ "*" ] + sources = [ "numerics/mod_ops.h" ] + deps = [ ":checks" ] +} + +rtc_source_set("moving_max_counter") { + visibility = [ "*" ] + sources = [ "numerics/moving_max_counter.h" ] + deps = [ ":checks" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_source_set("one_time_event") { + visibility = [ "*" ] + sources = [ "one_time_event.h" ] + deps = [ "synchronization:mutex" ] +} + +rtc_source_set("strong_alias") { + visibility = [ "*" ] + sources = [ "strong_alias.h" ] +} + +rtc_source_set("swap_queue") { + visibility = [ "*" ] + sources = [ "swap_queue.h" ] + deps = [ ":checks" ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] +} + +rtc_source_set("macromagic") { + sources = [ + "arraysize.h", + "thread_annotations.h", + ] + deps = [ "system:arch" ] +} + +rtc_library("bit_buffer") { + visibility = [ "*" ] sources = [ - "bind.h", "bit_buffer.cc", "bit_buffer.h", - "buffer.h", - "buffer_queue.cc", - "buffer_queue.h", + ] + deps = [ + ":checks", + "../api/units:data_size", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/numeric:bits", + "//third_party/abseil-cpp/absl/strings:strings", + ] +} + +rtc_library("byte_buffer") { + visibility = [ "*" ] + sources = [ "byte_buffer.cc", "byte_buffer.h", - "byte_order.h", + ] + deps = [ + ":buffer", + ":byte_order", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("buffer_queue") { + visibility = [ "*" ] + sources = [ + "buffer_queue.cc", + "buffer_queue.h", + ] + deps = [ + ":buffer", + ":macromagic", + "../api:sequence_checker", + "system:no_unique_address", + ] +} + +rtc_library("copy_on_write_buffer") { + visibility = [ "*" ] + sources = [ "copy_on_write_buffer.cc", "copy_on_write_buffer.h", + ] + deps = [ + ":buffer", + ":checks", + ":refcount", + ":type_traits", + "../api:scoped_refptr", + "system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("event_tracer") { + visibility = [ "*" ] + sources = [ "event_tracer.cc", "event_tracer.h", - "location.cc", - "location.h", - "message_buffer_reader.h", - "numerics/histogram_percentile_counter.cc", - "numerics/histogram_percentile_counter.h", - "numerics/mod_ops.h", - "numerics/moving_max_counter.h", - "numerics/sample_counter.cc", - "numerics/sample_counter.h", - "one_time_event.h", - "race_checker.cc", - "race_checker.h", - "random.cc", - "random.h", - "rate_statistics.cc", - "rate_statistics.h", - "rate_tracker.cc", - "rate_tracker.h", - "swap_queue.h", - "timestamp_aligner.cc", - "timestamp_aligner.h", "trace_event.h", - "zero_memory.cc", - "zero_memory.h", ] - - if (is_win) { - sources += [ - "win/windows_version.cc", - "win/windows_version.h", - ] - data_deps = [ "//build/win:runtime_libs" ] - } - - if (is_nacl) { - public_deps += # no-presubmit-check TODO(webrtc:8603) - [ "//native_client_sdk/src/libraries/nacl_io" ] - } - - if (is_android) { - libs = [ "log" ] - } - - public_deps += [ # no-presubmit-check TODO(webrtc:8603) - ":atomicops", - ":criticalsection", + deps = [ + ":checks", ":logging", ":macromagic", ":platform_thread", ":platform_thread_types", - ":refcount", ":rtc_event", - ":safe_conversions", - ":stringutils", - ":thread_checker", ":timeutils", - "synchronization:sequence_checker", + "../api:sequence_checker", + "synchronization:mutex", + "system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("histogram_percentile_counter") { + visibility = [ "*" ] + sources = [ + "numerics/histogram_percentile_counter.cc", + "numerics/histogram_percentile_counter.h", + ] + deps = [ ":checks" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("race_checker") { + visibility = [ "*" ] + sources = [ + "race_checker.cc", + "race_checker.h", + ] + deps = [ + ":checks", + ":macromagic", + ":platform_thread_types", ] } -rtc_source_set("macromagic") { - # TODO(bugs.webrtc.org/9606): This should not be public. +rtc_library("random") { visibility = [ "*" ] sources = [ - "arraysize.h", - "constructor_magic.h", - "format_macros.h", - "thread_annotations.h", + "random.cc", + "random.h", + ] + deps = [ + ":checks", + ":safe_conversions", + ] +} + +rtc_library("bitrate_tracker") { + visibility = [ "*" ] + sources = [ + "bitrate_tracker.cc", + "bitrate_tracker.h", + ] + deps = [ + ":rate_statistics", + "../api/units:data_rate", + "../api/units:data_size", + "../api/units:time_delta", + "../api/units:timestamp", + "system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("frequency_tracker") { + visibility = [ "*" ] + sources = [ + "frequency_tracker.cc", + "frequency_tracker.h", + ] + deps = [ + ":rate_statistics", + "../api/units:frequency", + "../api/units:time_delta", + "../api/units:timestamp", + "system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("rate_statistics") { + # TODO(bugs.webrtc.org/13756): Restrict visibility to private when all usage + # of the RateStatistics is migrated to BitrateTracker and FrequencyTracker. + visibility = [ "*" ] + sources = [ + "rate_statistics.cc", + "rate_statistics.h", + ] + deps = [ + ":checks", + ":logging", + ":safe_conversions", + "system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("rate_tracker") { + visibility = [ "*" ] + sources = [ + "rate_tracker.cc", + "rate_tracker.h", + ] + deps = [ + ":checks", + ":timeutils", + ] +} + +rtc_library("sample_counter") { + visibility = [ "*" ] + sources = [ + "numerics/sample_counter.cc", + "numerics/sample_counter.h", + ] + deps = [ + ":checks", + ":safe_conversions", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("timestamp_aligner") { + visibility = [ "*" ] + sources = [ + "timestamp_aligner.cc", + "timestamp_aligner.h", + ] + deps = [ + ":checks", + ":logging", + ":timeutils", + "system:rtc_export", + ] +} + +rtc_library("zero_memory") { + visibility = [ "*" ] + sources = [ + "zero_memory.cc", + "zero_memory.h", + ] + deps = [ + ":checks", + "../api:array_view", ] - deps = [ "system:arch" ] } rtc_library("platform_thread_types") { @@ -173,7 +354,10 @@ rtc_library("platform_thread_types") { "platform_thread_types.cc", "platform_thread_types.h", ] - deps = [ ":macromagic" ] + deps = [ + ":checks", + ":macromagic", + ] } rtc_source_set("refcount") { @@ -183,7 +367,10 @@ rtc_source_set("refcount") { "ref_counted_object.h", "ref_counter.h", ] - deps = [ ":macromagic" ] + deps = [ + ":macromagic", + "../api:scoped_refptr", + ] } rtc_library("criticalsection") { @@ -192,7 +379,6 @@ rtc_library("criticalsection") { "deprecated/recursive_critical_section.h", ] deps = [ - ":atomicops", ":checks", ":macromagic", ":platform_thread_types", @@ -202,28 +388,23 @@ rtc_library("criticalsection") { } rtc_library("platform_thread") { - visibility = [ - ":rtc_base_approved", - ":rtc_task_queue_libevent", - ":rtc_task_queue_stdlib", - ":rtc_task_queue_win", - "synchronization:mutex", - "synchronization:sequence_checker", - ] sources = [ "platform_thread.cc", "platform_thread.h", ] deps = [ - ":atomicops", ":checks", ":macromagic", ":platform_thread_types", ":rtc_event", - ":thread_checker", ":timeutils", + "../api:sequence_checker", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("rtc_event") { @@ -234,6 +415,7 @@ rtc_library("rtc_event") { ] deps = [ ":checks", + "../api/units:time_delta", "system:rtc_export", # Only Chromium's rtc::Event use RTC_EXPORT. "//base", # Dependency on chromium's waitable_event. ] @@ -244,6 +426,8 @@ rtc_library("rtc_event") { ] deps = [ ":checks", + ":timeutils", + "../api/units:time_delta", "synchronization:yield_policy", "system:warn_current_thread_is_deadlocked", ] @@ -251,22 +435,27 @@ rtc_library("rtc_event") { } } +config("chromium_logging_config") { + defines = [ "LOGGING_INSIDE_WEBRTC" ] +} + rtc_library("logging") { visibility = [ "*" ] libs = [] deps = [ ":checks", - ":criticalsection", ":macromagic", ":platform_thread_types", ":stringutils", ":timeutils", + "../api/units:timestamp", "synchronization:mutex", ] absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/meta:type_traits", "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] if (build_with_chromium) { @@ -276,11 +465,14 @@ rtc_library("logging") { "../../webrtc_overrides/rtc_base/logging.cc", "../../webrtc_overrides/rtc_base/logging.h", ] + + # This macro needs to be both present in all WebRTC targets (see its + # definition in //BUILD.gn but also propagated to all the targets + # depending on the Chromium component defined in + # //third_party/webrtc_overrides:webrtc_component. This public_config + # allows GN to propagate the macro accordingly. + public_configs = [ ":chromium_logging_config" ] } else { - configs += [ - "..:no_exit_time_destructors", - "..:no_global_constructors", - ] sources = [ "logging.cc", "logging.h", @@ -291,28 +483,12 @@ rtc_library("logging") { frameworks = [ "Foundation.framework" ] } - # logging.h needs the deprecation header while downstream projects are - # removing code that depends on logging implementation details. - deps += [ ":deprecation" ] - if (is_android) { libs += [ "log" ] } } } -rtc_source_set("thread_checker") { - sources = [ "thread_checker.h" ] - deps = [ - ":deprecation", - "synchronization:sequence_checker", - ] -} - -rtc_source_set("atomicops") { - sources = [ "atomic_ops.h" ] -} - rtc_library("checks") { # TODO(bugs.webrtc.org/9607): This should not be public. visibility = [ "*" ] @@ -323,9 +499,14 @@ rtc_library("checks") { ] deps = [ ":safe_compare", + "../api:scoped_refptr", "system:inline", "system:rtc_export", ] + if (build_with_chromium) { + sources += [ "../../webrtc_overrides/rtc_base/checks_overrides.cc" ] + deps += [ "//base" ] + } absl_deps = [ "//third_party/abseil-cpp/absl/meta:type_traits", "//third_party/abseil-cpp/absl/strings", @@ -341,7 +522,8 @@ rtc_library("rate_limiter") { "rate_limiter.h", ] deps = [ - ":rtc_base_approved", + ":macromagic", + ":rate_statistics", "../system_wrappers", "synchronization:mutex", ] @@ -392,6 +574,8 @@ rtc_source_set("safe_conversions") { rtc_library("timeutils") { visibility = [ "*" ] sources = [ + "system_time.cc", + "system_time.h", "time_utils.cc", "time_utils.h", ] @@ -401,9 +585,15 @@ rtc_library("timeutils") { ":stringutils", "system:rtc_export", ] + + if (rtc_exclude_system_time) { + defines = [ "WEBRTC_EXCLUDE_SYSTEM_TIME" ] + } + libs = [] if (is_win) { libs += [ "winmm.lib" ] + deps += [ ":win32" ] } } @@ -447,10 +637,6 @@ rtc_source_set("type_traits") { sources = [ "type_traits.h" ] } -rtc_source_set("deprecation") { - sources = [ "deprecation.h" ] -} - rtc_library("rtc_task_queue") { visibility = [ "*" ] sources = [ @@ -461,9 +647,11 @@ rtc_library("rtc_task_queue") { ":macromagic", "../api/task_queue", "system:rtc_export", - "task_utils:to_queued_task", ] - absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/memory", + ] } rtc_source_set("rtc_operations_chain") { @@ -476,8 +664,11 @@ rtc_source_set("rtc_operations_chain") { ":checks", ":macromagic", ":refcount", + "../api:make_ref_counted", + "../api:refcountedbase", "../api:scoped_refptr", - "synchronization:sequence_checker", + "../api:sequence_checker", + "system:no_unique_address", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } @@ -491,7 +682,6 @@ if (rtc_enable_libevent) { ] deps = [ ":checks", - ":criticalsection", ":logging", ":macromagic", ":platform_thread", @@ -499,14 +689,16 @@ if (rtc_enable_libevent) { ":safe_conversions", ":timeutils", "../api/task_queue", + "../api/units:time_delta", "synchronization:mutex", ] absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/strings", ] if (rtc_build_libevent) { - deps += [ "//base/third_party/libevent" ] + deps += [ "//third_party/libevent" ] } } } @@ -521,11 +713,16 @@ if (is_mac || is_ios) { deps = [ ":checks", ":logging", + "../api:location", "../api/task_queue", + "../api/units:time_delta", "synchronization:mutex", "system:gcd_helpers", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/strings", + ] } } @@ -538,7 +735,6 @@ if (is_win) { ] deps = [ ":checks", - ":criticalsection", ":logging", ":macromagic", ":platform_thread", @@ -546,9 +742,15 @@ if (is_win) { ":safe_conversions", ":timeutils", "../api/task_queue", + "../api/units:time_delta", + "../api/units:timestamp", "synchronization:mutex", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] } } @@ -559,7 +761,7 @@ rtc_library("rtc_task_queue_stdlib") { ] deps = [ ":checks", - ":criticalsection", + ":divide_round", ":logging", ":macromagic", ":platform_thread", @@ -567,9 +769,28 @@ rtc_library("rtc_task_queue_stdlib") { ":safe_conversions", ":timeutils", "../api/task_queue", + "../api/units:time_delta", "synchronization:mutex", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/strings", + ] +} + +if (rtc_include_tests) { + rtc_library("task_queue_stdlib_unittest") { + testonly = true + + sources = [ "task_queue_stdlib_unittest.cc" ] + deps = [ + ":gunit_helpers", + ":rtc_task_queue_stdlib", + "../api/task_queue:task_queue_test", + "../test:test_main", + "../test:test_support", + ] + } } rtc_library("weak_ptr") { @@ -580,7 +801,8 @@ rtc_library("weak_ptr") { deps = [ ":refcount", "../api:scoped_refptr", - "synchronization:sequence_checker", + "../api:sequence_checker", + "system:no_unique_address", ] } @@ -593,14 +815,15 @@ rtc_library("rtc_numerics") { "numerics/math_utils.h", "numerics/moving_average.cc", "numerics/moving_average.h", - "numerics/moving_median_filter.h", + "numerics/moving_percentile_filter.h", "numerics/percentile_filter.h", "numerics/running_statistics.h", + "numerics/sequence_number_unwrapper.h", "numerics/sequence_number_util.h", ] deps = [ ":checks", - ":rtc_base_approved", + ":mod_ops", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } @@ -624,10 +847,6 @@ rtc_library("rtc_stats_counters") { config("rtc_json_suppressions") { if (!is_win || is_clang) { cflags_cc = [ - # TODO(bugs.webrtc.org/10770): Update jsoncpp API usage and remove - # -Wno-deprecated-declarations. - "-Wno-deprecated-declarations", - # TODO(bugs.webrtc.org/10814): Remove -Wno-undef as soon as it get # removed upstream. "-Wno-undef", @@ -636,8 +855,8 @@ config("rtc_json_suppressions") { } rtc_library("rtc_json") { + testonly = true public_configs = [ ":rtc_json_suppressions" ] - poisonous = [ "rtc_json" ] defines = [] sources = [ "strings/json.cc", @@ -646,155 +865,301 @@ rtc_library("rtc_json") { deps = [ ":stringutils" ] all_dependent_configs = [ "//third_party/jsoncpp:jsoncpp_config" ] if (rtc_build_json) { - public_deps = # no-presubmit-check TODO(webrtc:8603) - [ "//third_party/jsoncpp" ] + deps += [ "//third_party/jsoncpp" ] } else { include_dirs = [ "$rtc_jsoncpp_root" ] + } + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} - # When defined changes the include path for json.h to where it is - # expected to be when building json outside of the standalone build. - defines += [ "WEBRTC_EXTERNAL_JSON" ] +rtc_library("net_helpers") { + visibility = [ "*" ] + sources = [ + "net_helpers.cc", + "net_helpers.h", + ] + deps = [ "system:rtc_export" ] + if (is_android) { + deps += [ ":ifaddrs_android" ] + } + if (is_win) { + deps += [ + ":win32", + "win:windows_version", + ] + } + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("net_test_helpers") { + # TODO(mbonadei): Enable once net_helpers don't depend + # on this target anymore. + # testonly = true + sources = [ + "net_test_helpers.cc", + "net_test_helpers.h", + ] + deps = [ "system:rtc_export" ] + if (is_android) { + deps += [ ":ifaddrs_android" ] + } + if (is_win) { + deps += [ + ":win32", + "win:windows_version", + ] } } -rtc_source_set("net_helpers") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "net_helpers.cc", - # "net_helpers.h", - # ] -} - -rtc_source_set("async_resolver_interface") { +rtc_library("async_resolver_interface") { visibility = [ "*" ] - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "async_resolver_interface.cc", - # "async_resolver_interface.h", - # ] + sources = [ + "async_resolver_interface.cc", + "async_resolver_interface.h", + ] + deps = [ + ":checks", + ":socket_address", + "system:rtc_export", + "third_party/sigslot", + ] } -rtc_source_set("ip_address") { +rtc_library("async_dns_resolver") { + sources = [ + "async_dns_resolver.cc", + "async_dns_resolver.h", + ] + deps = [ + ":logging", + ":macromagic", + ":platform_thread", + ":refcount", + "../api:async_dns_resolver", + "../api:make_ref_counted", + "../api:sequence_checker", + "../api/task_queue:pending_task_safety_flag", + "system:rtc_export", + ] +} + +rtc_library("async_dns_resolver_unittests") { + testonly = true + sources = [ "async_dns_resolver_unittest.cc" ] + deps = [ + ":async_dns_resolver", + ":gunit_helpers", + "../test:run_loop", + "../test:test_support", + ] +} + +rtc_library("ip_address") { visibility = [ "*" ] - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "ip_address.cc", - # "ip_address.h", - # ] + sources = [ + "ip_address.cc", + "ip_address.h", + ] + deps = [ + ":byte_order", + ":net_helpers", + ":stringutils", + "system:rtc_export", + ] + if (is_win) { + deps += [ ":win32" ] + } + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } -rtc_source_set("socket_address") { +rtc_library("socket_address") { visibility = [ "*" ] - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "socket_address.cc", - # "socket_address.h", - # ] + sources = [ + "socket_address.cc", + "socket_address.h", + ] + deps = [ + ":byte_order", + ":checks", + ":ip_address", + ":logging", + ":net_helpers", + ":safe_conversions", + ":stringutils", + "system:rtc_export", + ] + if (is_win) { + deps += [ ":win32" ] + } + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } -rtc_source_set("null_socket_server") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "null_socket_server.cc", - # "null_socket_server.h", - # ] +rtc_library("null_socket_server") { + sources = [ + "null_socket_server.cc", + "null_socket_server.h", + ] + deps = [ + ":checks", + ":rtc_event", + ":socket", + ":socket_server", + "../api/units:time_delta", + "system:rtc_export", + ] } rtc_source_set("socket_server") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "socket_server.h", - # ] + visibility = [ "*" ] + sources = [ "socket_server.h" ] + deps = [ + ":rtc_event", + ":socket_factory", + "../api/units:time_delta", + ] } -rtc_source_set("threading") { +rtc_library("threading") { visibility = [ "*" ] - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "asyncresolver.cc", - # "asyncresolver.h", - # "defaultsocketserver.cc", - # "defaultsocketserver.h", - # "message_handler.cc", - # "message_handler.h", - # "network_monitor.cc", - # "network_monitor.h", - # "network_monitor_factory.cc", - # "network_monitor_factory.h", - # "physical_socket_server.cc", - # "physical_socket_server.h", - # "signal_thread.cc", - # "signal_thread.h", - # "thread.cc", - # "thread.h", - # ] + + sources = [ + "async_resolver.cc", + "async_resolver.h", + "internal/default_socket_server.cc", + "internal/default_socket_server.h", + "network_monitor.cc", + "network_monitor.h", + "network_monitor_factory.cc", + "network_monitor_factory.h", + "physical_socket_server.cc", + "physical_socket_server.h", + "thread.cc", + "thread.h", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/cleanup", + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] + deps = [ + ":async_dns_resolver", + ":async_resolver_interface", + ":byte_order", + ":checks", + ":criticalsection", + ":event_tracer", + ":ip_address", + ":logging", + ":macromagic", + ":network_constants", + ":null_socket_server", + ":platform_thread", + ":platform_thread_types", + ":refcount", + ":rtc_event", + ":rtc_task_queue", + ":socket", + ":socket_address", + ":socket_server", + ":timeutils", + "../api:async_dns_resolver", + "../api:function_view", + "../api:location", + "../api:refcountedbase", + "../api:scoped_refptr", + "../api:sequence_checker", + "../api/task_queue", + "../api/task_queue:pending_task_safety_flag", + "../api/units:time_delta", + "../system_wrappers:field_trial", + "synchronization:mutex", + "system:no_unique_address", + "system:rtc_export", + "third_party/sigslot", + ] + if (is_android) { + deps += [ ":ifaddrs_android" ] + } + if (is_win) { + deps += [ ":win32" ] + } + if (is_mac || is_ios) { + deps += [ "system:cocoa_threading" ] + } } rtc_source_set("socket_factory") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "socket_factory.h", - # ] + sources = [ "socket_factory.h" ] + deps = [ ":socket" ] } -rtc_source_set("async_socket") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "async_socket.cc", - # "async_socket.h", - # ] +rtc_library("async_socket") { + sources = [ + "async_socket.cc", + "async_socket.h", + ] + deps = [ + ":checks", + ":socket", + ":socket_address", + "third_party/sigslot", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] } -rtc_source_set("socket") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "socket.cc", - # "socket.h", - # ] +rtc_library("socket") { + sources = [ + "socket.cc", + "socket.h", + ] + deps = [ + ":macromagic", + ":socket_address", + "third_party/sigslot", + ] + if (is_win) { + deps += [ ":win32" ] + } } rtc_source_set("network_constants") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "network_constants.h", - # ] + visibility = [ "*" ] + sources = [ + "network_constants.cc", + "network_constants.h", + ] + deps = [ ":checks" ] } if (is_android) { - rtc_source_set("ifaddrs_android") { - # TODO(bugs.webrtc.org/9987): This build target will soon contain - # the following files: - # sources = [ - # "ifaddrs_android.cc", - # "ifaddrs_android.h", - # ] + rtc_library("ifaddrs_android") { + sources = [ + "ifaddrs_android.cc", + "ifaddrs_android.h", + ] + libs = [ + "log", + "GLESv2", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/cleanup" ] } } if (is_win) { - rtc_source_set("win32") { + rtc_library("win32") { sources = [ "win32.cc", "win32.h", - "win32_window.cc", - "win32_window.h", ] deps = [ + ":byte_order", ":checks", ":macromagic", - ":rtc_base_approved", + ":stringutils", ] libs = [ @@ -807,133 +1172,343 @@ if (is_win) { } } -rtc_library("rtc_base") { - visibility = [ "*" ] - cflags = [] - cflags_cc = [] - libs = [] - defines = [] +rtc_library("ifaddrs_converter") { + sources = [] deps = [ ":checks", - ":deprecation", - ":rtc_task_queue", + ":ip_address", + ":logging", + ] + + if (is_android) { + deps += [ ":ifaddrs_android" ] + } + + if (is_ios || is_mac) { + sources += [ "mac_ifaddrs_converter.cc" ] + } + + if (is_posix || is_fuchsia) { + sources += [ + "ifaddrs_converter.cc", + "ifaddrs_converter.h", + ] + } +} + +rtc_library("rolling_accumulator") { + sources = [ "rolling_accumulator.h" ] + deps = [ + ":checks", + ":rtc_numerics", + ] +} + +if (is_win) { + rtc_library("win32_socket_init") { + sources = [ "win32_socket_init.h" ] + deps = [ ":win32" ] + } +} + +if (!build_with_chromium) { + rtc_library("log_sinks") { + visibility = [ "*" ] + sources = [ + "log_sinks.cc", + "log_sinks.h", + ] + deps = [ + ":checks", + ":file_rotating_stream", + ":logging", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } +} + +rtc_library("network") { + visibility = [ "*" ] + sources = [ + "network.cc", + "network.h", + ] + deps = [ + ":checks", + ":ifaddrs_converter", + ":ip_address", + ":logging", + ":macromagic", + ":mdns_responder_interface", + ":socket", + ":socket_factory", ":stringutils", + ":threading", "../api:array_view", - "../api:function_view", - "../api:scoped_refptr", - "../api/numerics", - "../api/task_queue", - "../system_wrappers:field_trial", - "network:sent_packet", - "synchronization:mutex", - "synchronization:sequence_checker", - "system:file_wrapper", - "system:inline", + "../api:field_trials_view", + "../api:sequence_checker", + "../api/task_queue:pending_task_safety_flag", + "../api/transport:field_trial_based_config", + "../api/units:time_delta", + "experiments:field_trial_parser", + "memory:always_valid_pointer", "system:rtc_export", - "task_utils:pending_task_safety_flag", - "task_utils:repeating_task", - "task_utils:to_queued_task", - "third_party/base64", "third_party/sigslot", ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", - "//third_party/abseil-cpp/absl/container:flat_hash_map", + "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings", + ] + + if (is_win) { + deps += [ ":win32" ] + } +} + +rtc_library("socket_address_pair") { + sources = [ + "socket_address_pair.cc", + "socket_address_pair.h", + ] + deps = [ ":socket_address" ] +} + +rtc_library("net_helper") { + visibility = [ "*" ] + sources = [ + "net_helper.cc", + "net_helper.h", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + deps = [ "system:rtc_export" ] +} + +rtc_library("socket_adapters") { + visibility = [ "*" ] + sources = [ + "socket_adapters.cc", + "socket_adapters.h", + ] + deps = [ + ":async_socket", + ":buffer", + ":byte_buffer", + ":checks", + ":crypt_string", + ":http_common", + ":logging", + ":stringutils", + ":zero_memory", + "../api:array_view", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("network_route") { + sources = [ + "network_route.cc", + "network_route.h", + ] + deps = [ + ":network_constants", + ":stringutils", + "system:inline", + ] +} + +rtc_library("async_tcp_socket") { + sources = [ + "async_tcp_socket.cc", + "async_tcp_socket.h", + ] + deps = [ + ":async_packet_socket", + ":buffer", + ":byte_order", + ":checks", + ":logging", + ":socket", + ":socket_address", + ":timeutils", + "../api:array_view", + "network:sent_packet", + ] +} + +rtc_library("async_udp_socket") { + visibility = [ "*" ] + sources = [ + "async_udp_socket.cc", + "async_udp_socket.h", + ] + deps = [ + ":async_packet_socket", + ":checks", + ":logging", + ":macromagic", + ":socket", + ":socket_address", + ":socket_factory", + ":timeutils", + "../api:sequence_checker", + "../system_wrappers:field_trial", + "network:sent_packet", + "system:no_unique_address", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("async_packet_socket") { + visibility = [ "*" ] + sources = [ + "async_packet_socket.cc", + "async_packet_socket.h", + ] + deps = [ + ":callback_list", + ":dscp", + ":socket", + ":timeutils", + "../api:sequence_checker", + "network:sent_packet", + "system:no_unique_address", + "system:rtc_export", + "third_party/sigslot", + ] +} + +rtc_library("mdns_responder_interface") { + sources = [ "mdns_responder_interface.h" ] + deps = [ ":ip_address" ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("dscp") { + sources = [ "dscp.h" ] +} + +rtc_library("proxy_info") { + visibility = [ "*" ] + sources = [ + "proxy_info.cc", + "proxy_info.h", + ] + deps = [ + ":crypt_string", + ":socket_address", + ] +} + +rtc_library("file_rotating_stream") { + sources = [ + "file_rotating_stream.cc", + "file_rotating_stream.h", + ] + deps = [ + ":checks", + ":logging", + ":stringutils", + "system:file_wrapper", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] - public_deps = [ ":rtc_base_approved" ] # no-presubmit-check TODO(webrtc:8603) - public_configs = [] +} +rtc_library("data_rate_limiter") { sources = [ - "async_invoker.cc", - "async_invoker.h", - "async_invoker_inl.h", - "async_packet_socket.cc", - "async_packet_socket.h", - "async_resolver_interface.cc", - "async_resolver_interface.h", - "async_socket.cc", - "async_socket.h", - "async_tcp_socket.cc", - "async_tcp_socket.h", - "async_udp_socket.cc", - "async_udp_socket.h", - "crc32.cc", - "crc32.h", - "crypt_string.cc", - "crypt_string.h", "data_rate_limiter.cc", "data_rate_limiter.h", - "deprecated/signal_thread.cc", - "deprecated/signal_thread.h", - "dscp.h", - "file_rotating_stream.cc", - "file_rotating_stream.h", + ] + deps = [ "system:rtc_export" ] +} + +rtc_library("unique_id_generator") { + sources = [ + "unique_id_generator.cc", + "unique_id_generator.h", + ] + deps = [ + ":ssl", + ":stringutils", + "../api:array_view", + "../api:sequence_checker", + "synchronization:mutex", + "system:no_unique_address", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("crc32") { + sources = [ + "crc32.cc", + "crc32.h", + ] + deps = [ ":macromagic" ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("stream") { + visibility = [ "*" ] + sources = [ + "stream.cc", + "stream.h", + ] + deps = [ + ":buffer", + ":checks", + ":threading", + "../api:array_view", + "system:rtc_export", + "third_party/sigslot", + ] +} + +rtc_library("rtc_certificate_generator") { + visibility = [ "*" ] + sources = [ + "rtc_certificate_generator.cc", + "rtc_certificate_generator.h", + ] + deps = [ + ":checks", + ":ssl", + ":threading", + "../api:scoped_refptr", + "system:rtc_export", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("ssl") { + visibility = [ "*" ] + sources = [ "helpers.cc", "helpers.h", - "http_common.cc", - "http_common.h", - "ip_address.cc", - "ip_address.h", - "keep_ref_until_done.h", - "mdns_responder_interface.h", "message_digest.cc", "message_digest.h", - "message_handler.cc", - "message_handler.h", - "net_helper.cc", - "net_helper.h", - "net_helpers.cc", - "net_helpers.h", - "network.cc", - "network.h", - "network_constants.cc", - "network_constants.h", - "network_monitor.cc", - "network_monitor.h", - "network_monitor_factory.cc", - "network_monitor_factory.h", - "network_route.cc", - "network_route.h", - "null_socket_server.cc", - "null_socket_server.h", "openssl.h", "openssl_adapter.cc", "openssl_adapter.h", - "openssl_certificate.cc", - "openssl_certificate.h", "openssl_digest.cc", "openssl_digest.h", - "openssl_identity.cc", - "openssl_identity.h", + "openssl_key_pair.cc", + "openssl_key_pair.h", "openssl_session_cache.cc", "openssl_session_cache.h", "openssl_stream_adapter.cc", "openssl_stream_adapter.h", "openssl_utility.cc", "openssl_utility.h", - "physical_socket_server.cc", - "physical_socket_server.h", - "proxy_info.cc", - "proxy_info.h", "rtc_certificate.cc", "rtc_certificate.h", - "rtc_certificate_generator.cc", - "rtc_certificate_generator.h", - "signal_thread.h", - "sigslot_repeater.h", - "socket.cc", - "socket.h", - "socket_adapters.cc", - "socket_adapters.h", - "socket_address.cc", - "socket_address.h", - "socket_address_pair.cc", - "socket_address_pair.h", - "socket_factory.h", - "socket_server.h", "ssl_adapter.cc", "ssl_adapter.h", "ssl_certificate.cc", @@ -944,98 +1519,99 @@ rtc_library("rtc_base") { "ssl_identity.h", "ssl_stream_adapter.cc", "ssl_stream_adapter.h", - "stream.cc", - "stream.h", - "thread.cc", - "thread.h", - "thread_message.h", - "unique_id_generator.cc", - "unique_id_generator.h", ] - if (build_with_chromium) { - include_dirs = [ "../../boringssl/src/include" ] - public_configs += [ ":rtc_base_chromium_config" ] - } else { + deps = [ + ":async_socket", + ":buffer", + ":checks", + ":copy_on_write_buffer", + ":logging", + ":macromagic", + ":safe_conversions", + ":socket", + ":socket_address", + ":stream", + ":stringutils", + ":threading", + ":timeutils", + "../api:array_view", + "../api:refcountedbase", + "../api:scoped_refptr", + "../api/task_queue:pending_task_safety_flag", + "../api/units:time_delta", + "../system_wrappers:field_trial", + "synchronization:mutex", + "system:rtc_export", + "task_utils:repeating_task", + "third_party/base64", + "third_party/sigslot", + ] + + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + # If we are building the SSL library ourselves, we know it's BoringSSL. + if (rtc_build_ssl) { sources += [ - "callback.h", - "log_sinks.cc", - "log_sinks.h", - "rolling_accumulator.h", - "ssl_roots.h", + "boringssl_certificate.cc", + "boringssl_certificate.h", + "boringssl_identity.cc", + "boringssl_identity.h", ] - deps += [ ":rtc_numerics" ] - - if (is_win) { - sources += [ "win32_socket_init.h" ] - if (current_os != "winuwp") { - sources += [ - "win32_socket_server.cc", - "win32_socket_server.h", - ] - } - } - } # !build_with_chromium - - if (rtc_build_ssl) { deps += [ "//third_party/boringssl" ] } else { + sources += [ + "openssl_certificate.cc", + "openssl_certificate.h", + "openssl_identity.cc", + "openssl_identity.h", + ] + configs += [ ":external_ssl_library" ] } - if (is_android) { - sources += [ - "ifaddrs_android.cc", - "ifaddrs_android.h", - ] - - libs += [ - "log", - "GLESv2", - ] - } - - if (is_ios || is_mac) { - sources += [ "mac_ifaddrs_converter.cc" ] - deps += [ "system:cocoa_threading" ] - } - - if (is_linux || is_chromeos) { - libs += [ - "dl", - "rt", - ] - } - - if (is_ios) { - frameworks = [ - "CFNetwork.framework", - "Foundation.framework", - "Security.framework", - "SystemConfiguration.framework", - "UIKit.framework", - ] + if (build_with_chromium) { + include_dirs = [ "../../boringssl/src/include" ] + } else { + sources += [ "ssl_roots.h" ] } if (is_win) { deps += [ ":win32" ] } +} - if (is_posix || is_fuchsia) { - sources += [ - "ifaddrs_converter.cc", - "ifaddrs_converter.h", - ] - } +rtc_library("crypt_string") { + sources = [ + "crypt_string.cc", + "crypt_string.h", + ] +} - if (is_nacl) { - public_deps += # no-presubmit-check TODO(webrtc:8603) - [ "//native_client_sdk/src/libraries/nacl_io" ] +rtc_library("http_common") { + sources = [ + "http_common.cc", + "http_common.h", + ] + deps = [ + ":crypt_string", + ":logging", + ":socket_address", + ":ssl", + ":stringutils", + ":zero_memory", + "third_party/base64", + ] - defines += [ "timezone=_timezone" ] - sources -= [ "ifaddrs_converter.cc" ] - } + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_source_set("gtest_prod") { @@ -1050,9 +1626,9 @@ rtc_library("gunit_helpers") { ] deps = [ ":logging", - ":rtc_base", ":rtc_base_tests_utils", ":stringutils", + ":threading", "../test:test_support", ] absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] @@ -1065,23 +1641,22 @@ rtc_library("testclient") { "test_client.h", ] deps = [ - ":criticalsection", + ":async_udp_socket", ":gunit_helpers", - ":rtc_base", ":rtc_base_tests_utils", + ":threading", ":timeutils", "synchronization:mutex", ] } -rtc_library("robo_caller_unittests") { +rtc_library("callback_list_unittests") { testonly = true - sources = [ "robo_caller_unittest.cc" ] + sources = [ "callback_list_unittest.cc" ] deps = [ + ":callback_list", ":gunit_helpers", - ":robo_caller", - ":rtc_base", "../api:function_view", "../test:test_support", ] @@ -1127,10 +1702,37 @@ rtc_library("rtc_base_tests_utils") { "virtual_socket_server.h", ] deps = [ + ":async_packet_socket", + ":async_socket", + ":async_tcp_socket", + ":async_udp_socket", + ":byte_buffer", ":checks", - ":rtc_base", + ":ip_address", + ":logging", + ":macromagic", + ":mdns_responder_interface", + ":network", + ":rtc_event", + ":socket", + ":socket_adapters", + ":socket_address", + ":socket_address_pair", + ":socket_factory", + ":socket_server", + ":ssl", + ":stream", + ":stringutils", + ":threading", + ":timeutils", + "../api:make_ref_counted", + "../api:refcountedbase", + "../api:scoped_refptr", + "../api/task_queue", "../api/units:time_delta", "../api/units:timestamp", + "../test:scoped_key_value_config", + "memory:always_valid_pointer", "memory:fifo_buffer", "synchronization:mutex", "third_party/sigslot", @@ -1138,7 +1740,12 @@ rtc_library("rtc_base_tests_utils") { absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", ] + if (is_fuchsia) { + deps += [ "//third_party/fuchsia-sdk/sdk/pkg/zx" ] + } } rtc_library("task_queue_for_test") { @@ -1150,14 +1757,17 @@ rtc_library("task_queue_for_test") { ] deps = [ ":checks", - ":rtc_base_approved", + ":macromagic", ":rtc_event", ":rtc_task_queue", + "../api:function_view", "../api/task_queue", "../api/task_queue:default_task_queue_factory", - "task_utils:to_queued_task", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/cleanup", + "//third_party/abseil-cpp/absl/strings", + ] } if (rtc_include_tests) { @@ -1166,7 +1776,6 @@ if (rtc_include_tests) { sources = [ "sigslot_unittest.cc" ] deps = [ ":gunit_helpers", - ":rtc_base", ":rtc_base_tests_utils", "../test:test_support", "synchronization:mutex", @@ -1183,280 +1792,380 @@ if (rtc_include_tests) { ] } - rtc_library("rtc_base_nonparallel_tests") { - testonly = true - - sources = [ - "cpu_time_unittest.cc", - "file_rotating_stream_unittest.cc", - "null_socket_server_unittest.cc", - "physical_socket_server_unittest.cc", - "socket_address_unittest.cc", - "socket_unittest.cc", - "socket_unittest.h", - ] - deps = [ - ":checks", - ":gunit_helpers", - ":rtc_base", - ":rtc_base_tests_utils", - ":testclient", - "../system_wrappers", - "../test:fileutils", - "../test:test_main", - "../test:test_support", - "third_party/sigslot", - "//testing/gtest", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] - if (is_win) { - sources += [ "win32_socket_server_unittest.cc" ] - } - } - - rtc_library("rtc_base_approved_unittests") { - testonly = true - sources = [ - "atomic_ops_unittest.cc", - "base64_unittest.cc", - "bind_unittest.cc", - "bit_buffer_unittest.cc", - "bounded_inline_vector_unittest.cc", - "buffer_queue_unittest.cc", - "buffer_unittest.cc", - "byte_buffer_unittest.cc", - "byte_order_unittest.cc", - "checks_unittest.cc", - "copy_on_write_buffer_unittest.cc", - "deprecated/recursive_critical_section_unittest.cc", - "event_tracer_unittest.cc", - "event_unittest.cc", - "logging_unittest.cc", - "numerics/divide_round_unittest.cc", - "numerics/histogram_percentile_counter_unittest.cc", - "numerics/mod_ops_unittest.cc", - "numerics/moving_max_counter_unittest.cc", - "numerics/safe_compare_unittest.cc", - "numerics/safe_minmax_unittest.cc", - "numerics/sample_counter_unittest.cc", - "one_time_event_unittest.cc", - "platform_thread_unittest.cc", - "random_unittest.cc", - "rate_limiter_unittest.cc", - "rate_statistics_unittest.cc", - "rate_tracker_unittest.cc", - "ref_counted_object_unittest.cc", - "sanitizer_unittest.cc", - "string_encode_unittest.cc", - "string_to_number_unittest.cc", - "string_utils_unittest.cc", - "strings/string_builder_unittest.cc", - "strings/string_format_unittest.cc", - "swap_queue_unittest.cc", - "thread_annotations_unittest.cc", - "thread_checker_unittest.cc", - "time_utils_unittest.cc", - "timestamp_aligner_unittest.cc", - "virtual_socket_unittest.cc", - "zero_memory_unittest.cc", - ] - if (is_win) { - sources += [ "win/windows_version_unittest.cc" ] - } - deps = [ - ":bounded_inline_vector", - ":checks", - ":divide_round", - ":gunit_helpers", - ":rate_limiter", - ":rtc_base", - ":rtc_base_approved", - ":rtc_base_tests_utils", - ":rtc_numerics", - ":rtc_task_queue", - ":safe_compare", - ":safe_minmax", - ":sanitizer", - ":stringutils", - ":testclient", - "../api:array_view", - "../api:scoped_refptr", - "../api/numerics", - "../api/units:time_delta", - "../system_wrappers", - "../test:fileutils", - "../test:test_main", - "../test:test_support", - "memory:unittests", - "synchronization:mutex", - "task_utils:to_queued_task", - "third_party/base64", - "third_party/sigslot", - ] - absl_deps = [ - "//third_party/abseil-cpp/absl/base:core_headers", - "//third_party/abseil-cpp/absl/memory", - ] - } - - rtc_library("rtc_task_queue_unittests") { - testonly = true - - sources = [ "task_queue_unittest.cc" ] - deps = [ - ":gunit_helpers", - ":rtc_base_approved", - ":rtc_base_tests_utils", - ":rtc_task_queue", - ":task_queue_for_test", - "../test:test_main", - "../test:test_support", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] - } - rtc_library("rtc_operations_chain_unittests") { testonly = true sources = [ "operations_chain_unittest.cc" ] deps = [ ":gunit_helpers", - ":rtc_base", - ":rtc_base_approved", ":rtc_event", ":rtc_operations_chain", + ":threading", "../test:test_support", ] } - rtc_library("weak_ptr_unittests") { - testonly = true + if (!build_with_chromium) { + rtc_library("rtc_base_nonparallel_tests") { + testonly = true - sources = [ "weak_ptr_unittest.cc" ] - deps = [ - ":gunit_helpers", - ":rtc_base_approved", - ":rtc_base_tests_utils", - ":rtc_event", - ":task_queue_for_test", - ":weak_ptr", - "../test:test_main", - "../test:test_support", - ] - } - - rtc_library("rtc_numerics_unittests") { - testonly = true - - sources = [ - "numerics/event_based_exponential_moving_average_unittest.cc", - "numerics/exp_filter_unittest.cc", - "numerics/moving_average_unittest.cc", - "numerics/moving_median_filter_unittest.cc", - "numerics/percentile_filter_unittest.cc", - "numerics/running_statistics_unittest.cc", - "numerics/sequence_number_util_unittest.cc", - ] - deps = [ - ":rtc_base_approved", - ":rtc_numerics", - "../test:test_main", - "../test:test_support", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] - } - - rtc_library("rtc_json_unittests") { - testonly = true - - sources = [ "strings/json_unittest.cc" ] - deps = [ - ":gunit_helpers", - ":rtc_base_tests_utils", - ":rtc_json", - "../test:test_main", - "../test:test_support", - ] - } - - rtc_library("rtc_base_unittests") { - testonly = true - defines = [] - - sources = [ - "callback_unittest.cc", - "crc32_unittest.cc", - "data_rate_limiter_unittest.cc", - "deprecated/signal_thread_unittest.cc", - "fake_clock_unittest.cc", - "helpers_unittest.cc", - "ip_address_unittest.cc", - "memory_usage_unittest.cc", - "message_digest_unittest.cc", - "nat_unittest.cc", - "network_route_unittest.cc", - "network_unittest.cc", - "proxy_unittest.cc", - "rolling_accumulator_unittest.cc", - "rtc_certificate_generator_unittest.cc", - "rtc_certificate_unittest.cc", - "sigslot_tester_unittest.cc", - "test_client_unittest.cc", - "thread_unittest.cc", - "unique_id_generator_unittest.cc", - ] - deps = [ - ":checks", - ":gunit_helpers", - ":rtc_base_tests_utils", - ":stringutils", - ":testclient", - "../api:array_view", - "../api/task_queue", - "../api/task_queue:task_queue_test", - "../test:field_trial", - "../test:fileutils", - "../test:rtc_expect_death", - "../test:test_main", - "../test:test_support", - "memory:fifo_buffer", - "synchronization:mutex", - "synchronization:synchronization_unittests", - "task_utils:pending_task_safety_flag", - "task_utils:to_queued_task", - "third_party/sigslot", - ] - if (is_win) { - sources += [ - "win32_unittest.cc", - "win32_window_unittest.cc", + sources = [ + "cpu_time_unittest.cc", + "file_rotating_stream_unittest.cc", + "null_socket_server_unittest.cc", + "physical_socket_server_unittest.cc", + "socket_address_unittest.cc", + "socket_unittest.cc", + "socket_unittest.h", ] - deps += [ ":win32" ] - } - if (is_posix || is_fuchsia) { - sources += [ - "openssl_adapter_unittest.cc", - "openssl_session_cache_unittest.cc", - "openssl_utility_unittest.cc", - "ssl_adapter_unittest.cc", - "ssl_identity_unittest.cc", - "ssl_stream_adapter_unittest.cc", + deps = [ + ":async_packet_socket", + ":async_udp_socket", + ":buffer", + ":checks", + ":file_rotating_stream", + ":gunit_helpers", + ":ip_address", + ":logging", + ":macromagic", + ":net_helpers", + ":net_test_helpers", + ":null_socket_server", + ":platform_thread", + ":rtc_base_tests_utils", + ":socket", + ":socket_address", + ":socket_server", + ":testclient", + ":threading", + ":timeutils", + "../api/units:time_delta", + "../system_wrappers", + "../test:field_trial", + "../test:fileutils", + "../test:test_main", + "../test:test_support", + "third_party/sigslot", + "//testing/gtest", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", ] } - absl_deps = [ - "//third_party/abseil-cpp/absl/algorithm:container", - "//third_party/abseil-cpp/absl/memory", - "//third_party/abseil-cpp/absl/strings", - "//third_party/abseil-cpp/absl/types:optional", - ] - public_deps = [ ":rtc_base" ] # no-presubmit-check TODO(webrtc:8603) - if (build_with_chromium) { - include_dirs = [ "../../boringssl/src/include" ] + + rtc_library("rtc_base_approved_unittests") { + testonly = true + sources = [ + "base64_unittest.cc", + "bit_buffer_unittest.cc", + "bitrate_tracker_unittest.cc", + "bitstream_reader_unittest.cc", + "bounded_inline_vector_unittest.cc", + "buffer_queue_unittest.cc", + "buffer_unittest.cc", + "byte_buffer_unittest.cc", + "byte_order_unittest.cc", + "checks_unittest.cc", + "copy_on_write_buffer_unittest.cc", + "deprecated/recursive_critical_section_unittest.cc", + "event_tracer_unittest.cc", + "event_unittest.cc", + "frequency_tracker_unittest.cc", + "logging_unittest.cc", + "numerics/divide_round_unittest.cc", + "numerics/histogram_percentile_counter_unittest.cc", + "numerics/mod_ops_unittest.cc", + "numerics/moving_max_counter_unittest.cc", + "numerics/safe_compare_unittest.cc", + "numerics/safe_minmax_unittest.cc", + "numerics/sample_counter_unittest.cc", + "one_time_event_unittest.cc", + "platform_thread_unittest.cc", + "random_unittest.cc", + "rate_limiter_unittest.cc", + "rate_statistics_unittest.cc", + "rate_tracker_unittest.cc", + "ref_counted_object_unittest.cc", + "sanitizer_unittest.cc", + "string_encode_unittest.cc", + "string_to_number_unittest.cc", + "string_utils_unittest.cc", + "strings/string_builder_unittest.cc", + "strings/string_format_unittest.cc", + "strong_alias_unittest.cc", + "swap_queue_unittest.cc", + "thread_annotations_unittest.cc", + "time_utils_unittest.cc", + "timestamp_aligner_unittest.cc", + "virtual_socket_unittest.cc", + "zero_memory_unittest.cc", + ] + deps = [ + ":async_packet_socket", + ":async_udp_socket", + ":bit_buffer", + ":bitrate_tracker", + ":bitstream_reader", + ":bounded_inline_vector", + ":buffer", + ":buffer_queue", + ":byte_buffer", + ":byte_order", + ":checks", + ":copy_on_write_buffer", + ":criticalsection", + ":divide_round", + ":event_tracer", + ":frequency_tracker", + ":gunit_helpers", + ":histogram_percentile_counter", + ":ip_address", + ":logging", + ":macromagic", + ":mod_ops", + ":moving_max_counter", + ":null_socket_server", + ":one_time_event", + ":platform_thread", + ":random", + ":rate_limiter", + ":rate_statistics", + ":rate_tracker", + ":refcount", + ":rtc_base_tests_utils", + ":rtc_event", + ":rtc_numerics", + ":rtc_task_queue", + ":safe_compare", + ":safe_minmax", + ":sample_counter", + ":sanitizer", + ":socket", + ":socket_address", + ":socket_server", + ":ssl", + ":stringutils", + ":strong_alias", + ":swap_queue", + ":testclient", + ":threading", + ":timestamp_aligner", + ":timeutils", + ":zero_memory", + "../api:array_view", + "../api:make_ref_counted", + "../api:scoped_refptr", + "../api/numerics", + "../api/units:data_rate", + "../api/units:data_size", + "../api/units:frequency", + "../api/units:time_delta", + "../api/units:timestamp", + "../system_wrappers", + "../test:fileutils", + "../test:test_main", + "../test:test_support", + "containers:flat_map", + "containers:unittests", + "memory:unittests", + "synchronization:mutex", + "task_utils:repeating_task", + "third_party/base64", + "third_party/sigslot", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/numeric:bits", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + if (is_win) { + deps += [ "win:windows_version_unittest" ] + } } - if (rtc_build_ssl) { - deps += [ "//third_party/boringssl" ] - } else { - configs += [ ":external_ssl_library" ] + + rtc_library("rtc_task_queue_unittests") { + testonly = true + + sources = [ "task_queue_unittest.cc" ] + deps = [ + ":gunit_helpers", + ":rtc_base_tests_utils", + ":rtc_event", + ":rtc_task_queue", + ":task_queue_for_test", + ":timeutils", + "../api/units:time_delta", + "../test:test_main", + "../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] + } + + rtc_library("weak_ptr_unittests") { + testonly = true + + sources = [ "weak_ptr_unittest.cc" ] + deps = [ + ":gunit_helpers", + ":rtc_base_tests_utils", + ":rtc_event", + ":task_queue_for_test", + ":weak_ptr", + "../test:test_main", + "../test:test_support", + ] + } + + rtc_library("rtc_numerics_unittests") { + testonly = true + + sources = [ + "numerics/event_based_exponential_moving_average_unittest.cc", + "numerics/exp_filter_unittest.cc", + "numerics/moving_average_unittest.cc", + "numerics/moving_percentile_filter_unittest.cc", + "numerics/percentile_filter_unittest.cc", + "numerics/running_statistics_unittest.cc", + "numerics/sequence_number_unwrapper_unittest.cc", + "numerics/sequence_number_util_unittest.cc", + ] + deps = [ + ":rtc_numerics", + ":timeutils", + "../test:test_main", + "../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] + } + + rtc_library("rtc_json_unittests") { + testonly = true + + sources = [ "strings/json_unittest.cc" ] + deps = [ + ":gunit_helpers", + ":rtc_base_tests_utils", + ":rtc_json", + "../test:test_main", + "../test:test_support", + ] + } + + rtc_library("rtc_base_unittests") { + testonly = true + defines = [] + + sources = [ + "crc32_unittest.cc", + "data_rate_limiter_unittest.cc", + "fake_clock_unittest.cc", + "helpers_unittest.cc", + "ip_address_unittest.cc", + "memory_usage_unittest.cc", + "message_digest_unittest.cc", + "nat_unittest.cc", + "network_route_unittest.cc", + "network_unittest.cc", + "proxy_unittest.cc", + "rolling_accumulator_unittest.cc", + "rtc_certificate_generator_unittest.cc", + "rtc_certificate_unittest.cc", + "sigslot_tester_unittest.cc", + "test_client_unittest.cc", + "thread_unittest.cc", + "unique_id_generator_unittest.cc", + ] + deps = [ + ":async_packet_socket", + ":async_tcp_socket", + ":async_udp_socket", + ":buffer", + ":buffer_queue", + ":checks", + ":crc32", + ":data_rate_limiter", + ":gunit_helpers", + ":ifaddrs_converter", + ":ip_address", + ":logging", + ":macromagic", + ":net_helpers", + ":net_test_helpers", + ":network", + ":network_route", + ":null_socket_server", + ":refcount", + ":rolling_accumulator", + ":rtc_base_tests_utils", + ":rtc_certificate_generator", + ":rtc_event", + ":safe_conversions", + ":socket", + ":socket_adapters", + ":socket_address", + ":socket_factory", + ":socket_server", + ":ssl", + ":stream", + ":stringutils", + ":testclient", + ":threading", + ":timeutils", + ":unique_id_generator", + "../api:array_view", + "../api:field_trials_view", + "../api:make_ref_counted", + "../api/task_queue", + "../api/task_queue:pending_task_safety_flag", + "../api/task_queue:task_queue_test", + "../api/units:time_delta", + "../test:field_trial", + "../test:fileutils", + "../test:rtc_expect_death", + "../test:scoped_key_value_config", + "../test:test_main", + "../test:test_support", + "memory:fifo_buffer", + "synchronization:mutex", + "third_party/sigslot", + ] + if (rtc_enable_google_benchmarks) { + deps += [ "synchronization:synchronization_unittests" ] + } + if (is_win) { + sources += [ "win32_unittest.cc" ] + deps += [ ":win32" ] + } + if (is_posix || is_fuchsia || is_win) { + sources += [ + "openssl_adapter_unittest.cc", + "openssl_session_cache_unittest.cc", + "openssl_utility_unittest.cc", + "ssl_adapter_unittest.cc", + "ssl_identity_unittest.cc", + "ssl_stream_adapter_unittest.cc", + ] + } + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + + if (build_with_chromium) { + include_dirs = [ "../../boringssl/src/include" ] + } + if (rtc_build_ssl) { + deps += [ "//third_party/boringssl" ] + } else { + configs += [ ":external_ssl_library" ] + } } } } @@ -1471,9 +2180,7 @@ if (is_android) { "java/src/org/webrtc/Size.java", "java/src/org/webrtc/ThreadUtils.java", ] - deps = [ - "//third_party/android_deps:com_android_support_support_annotations_java", - ] + deps = [ "//third_party/androidx:androidx_annotation_annotation_java" ] } java_cpp_enum("network_monitor_enums") { sources = [ "network_monitor.h" ] diff --git a/webrtc/rtc_base/atomic_ops.h b/webrtc/rtc_base/atomic_ops.h deleted file mode 100644 index 18a24a8..0000000 --- a/webrtc/rtc_base/atomic_ops.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2011 The WebRTC Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef RTC_BASE_ATOMIC_OPS_H_ -#define RTC_BASE_ATOMIC_OPS_H_ - -#if defined(WEBRTC_WIN) -// clang-format off -// clang formating would change include order. - -// Include winsock2.h before including to maintain consistency with -// win32.h. To include win32.h directly, it must be broken out into its own -// build target. -#include -#include -// clang-format on -#endif // defined(WEBRTC_WIN) - -namespace rtc { -class AtomicOps { - public: -#if defined(WEBRTC_WIN) - // Assumes sizeof(int) == sizeof(LONG), which it is on Win32 and Win64. - static int Increment(volatile int* i) { - return ::InterlockedIncrement(reinterpret_cast(i)); - } - static int Decrement(volatile int* i) { - return ::InterlockedDecrement(reinterpret_cast(i)); - } - static int AcquireLoad(volatile const int* i) { return *i; } - static void ReleaseStore(volatile int* i, int value) { *i = value; } - static int CompareAndSwap(volatile int* i, int old_value, int new_value) { - return ::InterlockedCompareExchange(reinterpret_cast(i), - new_value, old_value); - } - // Pointer variants. - template - static T* AcquireLoadPtr(T* volatile* ptr) { - return *ptr; - } - template - static T* CompareAndSwapPtr(T* volatile* ptr, T* old_value, T* new_value) { - return static_cast(::InterlockedCompareExchangePointer( - reinterpret_cast(ptr), new_value, old_value)); - } -#else - static int Increment(volatile int* i) { return __sync_add_and_fetch(i, 1); } - static int Decrement(volatile int* i) { return __sync_sub_and_fetch(i, 1); } - static int AcquireLoad(volatile const int* i) { - return __atomic_load_n(i, __ATOMIC_ACQUIRE); - } - static void ReleaseStore(volatile int* i, int value) { - __atomic_store_n(i, value, __ATOMIC_RELEASE); - } - static int CompareAndSwap(volatile int* i, int old_value, int new_value) { - return __sync_val_compare_and_swap(i, old_value, new_value); - } - // Pointer variants. - template - static T* AcquireLoadPtr(T* volatile* ptr) { - return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); - } - template - static T* CompareAndSwapPtr(T* volatile* ptr, T* old_value, T* new_value) { - return __sync_val_compare_and_swap(ptr, old_value, new_value); - } -#endif -}; - -} // namespace rtc - -#endif // RTC_BASE_ATOMIC_OPS_H_ diff --git a/webrtc/rtc_base/buffer.h b/webrtc/rtc_base/buffer.h index d1639e2..6663c68 100644 --- a/webrtc/rtc_base/buffer.h +++ b/webrtc/rtc_base/buffer.h @@ -19,6 +19,7 @@ #include #include +#include "absl/strings/string_view.h" #include "api/array_view.h" #include "rtc_base/checks.h" #include "rtc_base/type_traits.h" @@ -105,7 +106,10 @@ class BufferT { internal::BufferCompat::value>::type* = nullptr> BufferT(U* data, size_t size, size_t capacity) : BufferT(size, capacity) { static_assert(sizeof(T) == sizeof(U), ""); - std::memcpy(data_.get(), data, size * sizeof(U)); + if (size > 0) { + RTC_DCHECK(data); + std::memcpy(data_.get(), data, size * sizeof(U)); + } } // Construct a buffer from the contents of an array. @@ -117,6 +121,13 @@ class BufferT { ~BufferT() { MaybeZeroCompleteBuffer(); } + // Implicit conversion to absl::string_view if T is compatible with char. + template + operator typename std::enable_if::value, + absl::string_view>::type() const { + return absl::string_view(data(), size()); + } + // Get a pointer to the data. Just .data() will give you a (const) T*, but if // T is a byte-sized integer, you may also use .data() for any other // byte-sized integer U. @@ -229,13 +240,13 @@ class BufferT { SetData(w.data(), w.size()); } - // Replaces the data in the buffer with at most |max_elements| of data, using - // the function |setter|, which should have the following signature: + // Replaces the data in the buffer with at most `max_elements` of data, using + // the function `setter`, which should have the following signature: // // size_t setter(ArrayView view) // - // |setter| is given an appropriately typed ArrayView of length exactly - // |max_elements| that describes the area where it should write the data; it + // `setter` is given an appropriately typed ArrayView of length exactly + // `max_elements` that describes the area where it should write the data; it // should return the number of elements actually written. (If it doesn't fill // the whole ArrayView, it should leave the unused space at the end.) template ::value>::type* = nullptr> void AppendData(const U* data, size_t size) { + if (size == 0) { + return; + } + RTC_DCHECK(data); RTC_DCHECK(IsConsistent()); const size_t new_size = size_ + size; EnsureCapacityWithHeadroom(new_size, true); @@ -290,13 +305,13 @@ class BufferT { AppendData(&item, 1); } - // Appends at most |max_elements| to the end of the buffer, using the function - // |setter|, which should have the following signature: + // Appends at most `max_elements` to the end of the buffer, using the function + // `setter`, which should have the following signature: // // size_t setter(ArrayView view) // - // |setter| is given an appropriately typed ArrayView of length exactly - // |max_elements| that describes the area where it should write the data; it + // `setter` is given an appropriately typed ArrayView of length exactly + // `max_elements` that describes the area where it should write the data; it // should return the number of elements actually written. (If it doesn't fill // the whole ArrayView, it should leave the unused space at the end.) template #include +#include "absl/strings/string_view.h" + #if defined(WEBRTC_ANDROID) #define RTC_LOG_TAG_ANDROID "rtc" #include // NOLINT @@ -36,6 +38,7 @@ #include "rtc_base/checks.h" namespace { + #if defined(__GNUC__) __attribute__((__format__(__printf__, 2, 3))) #endif @@ -59,6 +62,30 @@ void AppendFormat(std::string* s, const char* fmt, ...) { namespace rtc { namespace webrtc_checks_impl { +#if !defined(WEBRTC_CHROMIUM_BUILD) +RTC_NORETURN void WriteFatalLog(absl::string_view output) { +#if defined(WEBRTC_ANDROID) + std::string output_str(output); + __android_log_print(ANDROID_LOG_ERROR, RTC_LOG_TAG_ANDROID, "%s\n", + output_str.c_str()); +#endif + fflush(stdout); + fwrite(output.data(), output.size(), 1, stderr); + fflush(stderr); +#if defined(WEBRTC_WIN) + DebugBreak(); +#endif + abort(); +} + +RTC_NORETURN void WriteFatalLog(const char* file, + int line, + absl::string_view output) { + WriteFatalLog(output); +} + +#endif // !defined(WEBRTC_CHROMIUM_BUILD) + #if RTC_CHECK_MSG_ENABLED // Reads one argument from args, appends it to s and advances fmt. // Returns true iff an argument was sucessfully parsed. @@ -149,19 +176,7 @@ RTC_NORETURN void FatalLog(const char* file, va_end(args); - const char* output = s.c_str(); - -#if defined(WEBRTC_ANDROID) - __android_log_print(ANDROID_LOG_ERROR, RTC_LOG_TAG_ANDROID, "%s\n", output); -#endif - - fflush(stdout); - fprintf(stderr, "%s", output); - fflush(stderr); -#if defined(WEBRTC_WIN) - DebugBreak(); -#endif - abort(); + WriteFatalLog(file, line, s); } #else // RTC_CHECK_MSG_ENABLED RTC_NORETURN void FatalLog(const char* file, int line) { @@ -174,22 +189,40 @@ RTC_NORETURN void FatalLog(const char* file, int line) { "# Check failed.\n" "# ", file, line, LAST_SYSTEM_ERROR); - const char* output = s.c_str(); - -#if defined(WEBRTC_ANDROID) - __android_log_print(ANDROID_LOG_ERROR, RTC_LOG_TAG_ANDROID, "%s\n", output); -#endif - - fflush(stdout); - fprintf(stderr, "%s", output); - fflush(stderr); -#if defined(WEBRTC_WIN) - DebugBreak(); -#endif - abort(); + WriteFatalLog(file, line, s); } #endif // RTC_CHECK_MSG_ENABLED +#if RTC_DCHECK_IS_ON + +RTC_NORETURN void UnreachableCodeReached(const char* file, int line) { + std::string s; + AppendFormat(&s, + "\n\n" + "#\n" + "# Unreachable code reached: %s, line %d\n" + "# last system error: %u\n" + "# ", + file, line, LAST_SYSTEM_ERROR); + WriteFatalLog(file, line, s); +} + +#else // !RTC_DCHECK_IS_ON + +RTC_NORETURN void UnreachableCodeReached() { + std::string s; + AppendFormat(&s, + "\n\n" + "#\n" + "# Unreachable code reached (file and line unknown)\n" + "# last system error: %u\n" + "# ", + LAST_SYSTEM_ERROR); + WriteFatalLog(s); +} + +#endif // !RTC_DCHECK_IS_ON + } // namespace webrtc_checks_impl } // namespace rtc diff --git a/webrtc/rtc_base/checks.h b/webrtc/rtc_base/checks.h index 61c074a..99fee97 100644 --- a/webrtc/rtc_base/checks.h +++ b/webrtc/rtc_base/checks.h @@ -56,6 +56,7 @@ RTC_NORETURN void rtc_FatalMessage(const char* file, int line, const char* msg); #include "absl/meta/type_traits.h" #include "absl/strings/string_view.h" +#include "api/scoped_refptr.h" #include "rtc_base/numerics/safe_compare.h" #include "rtc_base/system/inline.h" #include "rtc_base/system/rtc_export.h" @@ -95,7 +96,7 @@ RTC_NORETURN void rtc_FatalMessage(const char* file, int line, const char* msg); // messages if the condition doesn't hold. Prefer them to raw RTC_CHECK and // RTC_DCHECK. // -// - FATAL() aborts unconditionally. +// - RTC_FATAL() aborts unconditionally. namespace rtc { namespace webrtc_checks_impl { @@ -121,6 +122,13 @@ enum class CheckArgType : int8_t { kCheckOp, }; +// These two functions are public so they can be overridden from +// webrtc_overrides in chromium. +RTC_NORETURN void WriteFatalLog(const char* file, + int line, + absl::string_view output); +RTC_NORETURN void WriteFatalLog(absl::string_view output); + #if RTC_CHECK_MSG_ENABLED RTC_NORETURN RTC_EXPORT void FatalLog(const char* file, int line, @@ -192,6 +200,12 @@ inline Val MakeVal(const void* x) { return {x}; } +template +inline Val MakeVal( + const rtc::scoped_refptr& p) { + return {p.get()}; +} + // The enum class types are not implicitly convertible to arithmetic types. template ::value && @@ -338,9 +352,25 @@ class FatalLogCall final { const char* message_; }; +#if RTC_DCHECK_IS_ON + +// Be helpful, and include file and line in the RTC_CHECK_NOTREACHED error +// message. +#define RTC_UNREACHABLE_FILE_AND_LINE_CALL_ARGS __FILE__, __LINE__ +RTC_NORETURN RTC_EXPORT void UnreachableCodeReached(const char* file, int line); + +#else + +// Be mindful of binary size, and don't include file and line in the +// RTC_CHECK_NOTREACHED error message. +#define RTC_UNREACHABLE_FILE_AND_LINE_CALL_ARGS +RTC_NORETURN RTC_EXPORT void UnreachableCodeReached(); + +#endif + } // namespace webrtc_checks_impl -// The actual stream used isn't important. We reference |ignored| in the code +// The actual stream used isn't important. We reference `ignored` in the code // but don't evaluate it; this is to avoid "unused variable" warnings (we do so // in a particularly convoluted way with an extra ?: because that appears to be // the simplest construct that keeps Visual Studio from complaining about @@ -352,8 +382,8 @@ class FatalLogCall final { ::rtc::webrtc_checks_impl::LogStreamer<>() // Call RTC_EAT_STREAM_PARAMETERS with an argument that fails to compile if -// values of the same types as |a| and |b| can't be compared with the given -// operation, and that would evaluate |a| and |b| if evaluated. +// values of the same types as `a` and `b` can't be compared with the given +// operation, and that would evaluate `a` and `b` if evaluated. #define RTC_EAT_STREAM_PARAMETERS_OP(op, a, b) \ RTC_EAT_STREAM_PARAMETERS(((void)::rtc::Safe##op(a, b))) @@ -361,7 +391,7 @@ class FatalLogCall final { // controlled by NDEBUG or anything else, so the check will be executed // regardless of compilation mode. // -// We make sure RTC_CHECK et al. always evaluates |condition|, as +// We make sure RTC_CHECK et al. always evaluates `condition`, as // doing RTC_CHECK(FunctionWithSideEffect()) is a common idiom. // // RTC_CHECK_OP is a helper macro for binary operators. @@ -381,22 +411,20 @@ class FatalLogCall final { ::rtc::webrtc_checks_impl::LogStreamer<>() << (val1) << (val2) #else #define RTC_CHECK(condition) \ - (condition) \ - ? static_cast(0) \ - : true ? ::rtc::webrtc_checks_impl::FatalLogCall(__FILE__, \ - __LINE__, "") & \ - ::rtc::webrtc_checks_impl::LogStreamer<>() \ - : ::rtc::webrtc_checks_impl::FatalLogCall("", 0, "") & \ - ::rtc::webrtc_checks_impl::LogStreamer<>() + (condition) ? static_cast(0) \ + : true ? ::rtc::webrtc_checks_impl::FatalLogCall(__FILE__, __LINE__, \ + "") & \ + ::rtc::webrtc_checks_impl::LogStreamer<>() \ + : ::rtc::webrtc_checks_impl::FatalLogCall("", 0, "") & \ + ::rtc::webrtc_checks_impl::LogStreamer<>() #define RTC_CHECK_OP(name, op, val1, val2) \ - ::rtc::Safe##name((val1), (val2)) \ - ? static_cast(0) \ - : true ? ::rtc::webrtc_checks_impl::FatalLogCall(__FILE__, \ - __LINE__, "") & \ - ::rtc::webrtc_checks_impl::LogStreamer<>() \ - : ::rtc::webrtc_checks_impl::FatalLogCall("", 0, "") & \ - ::rtc::webrtc_checks_impl::LogStreamer<>() + ::rtc::Safe##name((val1), (val2)) ? static_cast(0) \ + : true ? ::rtc::webrtc_checks_impl::FatalLogCall(__FILE__, __LINE__, \ + "") & \ + ::rtc::webrtc_checks_impl::LogStreamer<>() \ + : ::rtc::webrtc_checks_impl::FatalLogCall("", 0, "") & \ + ::rtc::webrtc_checks_impl::LogStreamer<>() #endif #define RTC_CHECK_EQ(val1, val2) RTC_CHECK_OP(Eq, ==, val1, val2) @@ -428,10 +456,17 @@ class FatalLogCall final { #endif #define RTC_UNREACHABLE_CODE_HIT false -#define RTC_NOTREACHED() RTC_DCHECK(RTC_UNREACHABLE_CODE_HIT) +#define RTC_DCHECK_NOTREACHED() RTC_DCHECK(RTC_UNREACHABLE_CODE_HIT) -// TODO(bugs.webrtc.org/8454): Add an RTC_ prefix or rename differently. -#define FATAL() \ +// Kills the process with an error message. Never returns. Use when you wish to +// assert that a point in the code is never reached. +#define RTC_CHECK_NOTREACHED() \ + do { \ + ::rtc::webrtc_checks_impl::UnreachableCodeReached( \ + RTC_UNREACHABLE_FILE_AND_LINE_CALL_ARGS); \ + } while (0) + +#define RTC_FATAL() \ ::rtc::webrtc_checks_impl::FatalLogCall(__FILE__, __LINE__, \ "FATAL()") & \ ::rtc::webrtc_checks_impl::LogStreamer<>() diff --git a/webrtc/rtc_base/containers/BUILD.gn b/webrtc/rtc_base/containers/BUILD.gn new file mode 100644 index 0000000..621b612 --- /dev/null +++ b/webrtc/rtc_base/containers/BUILD.gn @@ -0,0 +1,56 @@ +# Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_library("flat_containers_internal") { + sources = [ + "flat_tree.cc", + "flat_tree.h", + "identity.h", + "invoke.h", + "move_only_int.h", + ] + deps = [ + "..:checks", + "../system:no_unique_address", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] + visibility = [ ":*" ] +} + +rtc_source_set("flat_set") { + sources = [ "flat_set.h" ] + deps = [ ":flat_containers_internal" ] +} + +rtc_source_set("flat_map") { + sources = [ "flat_map.h" ] + deps = [ + ":flat_containers_internal", + "..:checks", + ] +} + +rtc_library("unittests") { + testonly = true + sources = [ + "flat_map_unittest.cc", + "flat_set_unittest.cc", + "flat_tree_unittest.cc", + ] + deps = [ + ":flat_containers_internal", + ":flat_map", + ":flat_set", + "../../test:test_support", + "//testing/gmock:gmock", + "//testing/gtest:gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] +} diff --git a/webrtc/rtc_base/containers/flat_set.h b/webrtc/rtc_base/containers/flat_set.h new file mode 100644 index 0000000..355690b --- /dev/null +++ b/webrtc/rtc_base/containers/flat_set.h @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This implementation is borrowed from Chromium. + +#ifndef RTC_BASE_CONTAINERS_FLAT_SET_H_ +#define RTC_BASE_CONTAINERS_FLAT_SET_H_ + +#include +#include + +#include "rtc_base/containers/flat_tree.h" // IWYU pragma: export +#include "rtc_base/containers/identity.h" + +namespace webrtc { + +// flat_set is a container with a std::set-like interface that stores its +// contents in a sorted container, by default a vector. +// +// Its implementation mostly tracks the corresponding standardization proposal +// https://wg21.link/P1222. +// +// +// PROS +// +// - Good memory locality. +// - Low overhead, especially for smaller sets. +// - Performance is good for more workloads than you might expect (see +// //base/containers/README.md in Chromium repository) +// - Supports C++14 set interface. +// +// CONS +// +// - Inserts and removals are O(n). +// +// IMPORTANT NOTES +// +// - Iterators are invalidated across mutations. +// - If possible, construct a flat_set in one operation by inserting into +// a container and moving that container into the flat_set constructor. +// - For multiple removals use base::EraseIf() which is O(n) rather than +// O(n * removed_items). +// +// QUICK REFERENCE +// +// Most of the core functionality is inherited from flat_tree. Please see +// flat_tree.h for more details for most of these functions. As a quick +// reference, the functions available are: +// +// Constructors (inputs need not be sorted): +// flat_set(const flat_set&); +// flat_set(flat_set&&); +// flat_set(InputIterator first, InputIterator last, +// const Compare& compare = Compare()); +// flat_set(const container_type& items, +// const Compare& compare = Compare()); +// flat_set(container_type&& items, +// const Compare& compare = Compare()); // Re-use storage. +// flat_set(std::initializer_list ilist, +// const Compare& comp = Compare()); +// +// Constructors (inputs need to be sorted): +// flat_set(sorted_unique_t, +// InputIterator first, InputIterator last, +// const Compare& compare = Compare()); +// flat_set(sorted_unique_t, +// const container_type& items, +// const Compare& compare = Compare()); +// flat_set(sorted_unique_t, +// container_type&& items, +// const Compare& compare = Compare()); // Re-use storage. +// flat_set(sorted_unique_t, +// std::initializer_list ilist, +// const Compare& comp = Compare()); +// +// Assignment functions: +// flat_set& operator=(const flat_set&); +// flat_set& operator=(flat_set&&); +// flat_set& operator=(initializer_list); +// +// Memory management functions: +// void reserve(size_t); +// size_t capacity() const; +// void shrink_to_fit(); +// +// Size management functions: +// void clear(); +// size_t size() const; +// size_t max_size() const; +// bool empty() const; +// +// Iterator functions: +// iterator begin(); +// const_iterator begin() const; +// const_iterator cbegin() const; +// iterator end(); +// const_iterator end() const; +// const_iterator cend() const; +// reverse_iterator rbegin(); +// const reverse_iterator rbegin() const; +// const_reverse_iterator crbegin() const; +// reverse_iterator rend(); +// const_reverse_iterator rend() const; +// const_reverse_iterator crend() const; +// +// Insert and accessor functions: +// pair insert(const key_type&); +// pair insert(key_type&&); +// void insert(InputIterator first, InputIterator last); +// iterator insert(const_iterator hint, const key_type&); +// iterator insert(const_iterator hint, key_type&&); +// pair emplace(Args&&...); +// iterator emplace_hint(const_iterator, Args&&...); +// +// Underlying type functions: +// container_type extract() &&; +// void replace(container_type&&); +// +// Erase functions: +// iterator erase(iterator); +// iterator erase(const_iterator); +// iterator erase(const_iterator first, const_iterator& last); +// template size_t erase(const K& key); +// +// Comparators (see std::set documentation). +// key_compare key_comp() const; +// value_compare value_comp() const; +// +// Search functions: +// template size_t count(const K&) const; +// template iterator find(const K&); +// template const_iterator find(const K&) const; +// template bool contains(const K&) const; +// template pair equal_range(K&); +// template iterator lower_bound(const K&); +// template const_iterator lower_bound(const K&) const; +// template iterator upper_bound(const K&); +// template const_iterator upper_bound(const K&) const; +// +// General functions: +// void swap(flat_set&); +// +// Non-member operators: +// bool operator==(const flat_set&, const flat_set); +// bool operator!=(const flat_set&, const flat_set); +// bool operator<(const flat_set&, const flat_set); +// bool operator>(const flat_set&, const flat_set); +// bool operator>=(const flat_set&, const flat_set); +// bool operator<=(const flat_set&, const flat_set); +// +template , + class Container = std::vector> +using flat_set = typename ::webrtc::flat_containers_internal:: + flat_tree; + +// ---------------------------------------------------------------------------- +// General operations. + +// Erases all elements that match predicate. It has O(size) complexity. +// +// flat_set numbers; +// ... +// EraseIf(numbers, [](int number) { return number % 2 == 1; }); + +// NOLINTNEXTLINE(misc-unused-using-decls) +using ::webrtc::flat_containers_internal::EraseIf; + +} // namespace webrtc + +#endif // RTC_BASE_CONTAINERS_FLAT_SET_H_ diff --git a/webrtc/rtc_base/synchronization/rw_lock_wrapper.cc b/webrtc/rtc_base/containers/flat_tree.cc similarity index 50% rename from webrtc/rtc_base/synchronization/rw_lock_wrapper.cc rename to webrtc/rtc_base/containers/flat_tree.cc index fb46419..9e86db1 100644 --- a/webrtc/rtc_base/synchronization/rw_lock_wrapper.cc +++ b/webrtc/rtc_base/containers/flat_tree.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,22 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "rtc_base/synchronization/rw_lock_wrapper.h" +// This implementation is borrowed from Chromium. -#if defined(_WIN32) -#include "rtc_base/synchronization/rw_lock_win.h" -#else -#include "rtc_base/synchronization/rw_lock_posix.h" -#endif +#include "rtc_base/containers/flat_tree.h" namespace webrtc { -RWLockWrapper* RWLockWrapper::CreateRWLock() { -#ifdef _WIN32 - return RWLockWin::Create(); -#else - return RWLockPosix::Create(); -#endif -} +sorted_unique_t sorted_unique; } // namespace webrtc diff --git a/webrtc/rtc_base/containers/flat_tree.h b/webrtc/rtc_base/containers/flat_tree.h new file mode 100644 index 0000000..480784c --- /dev/null +++ b/webrtc/rtc_base/containers/flat_tree.h @@ -0,0 +1,1099 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This implementation is borrowed from Chromium. + +#ifndef RTC_BASE_CONTAINERS_FLAT_TREE_H_ +#define RTC_BASE_CONTAINERS_FLAT_TREE_H_ + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" +#include "rtc_base/system/no_unique_address.h" + +namespace webrtc { +// Tag type that allows skipping the sort_and_unique step when constructing a +// flat_tree in case the underlying container is already sorted and has no +// duplicate elements. +struct sorted_unique_t { + constexpr sorted_unique_t() = default; +}; +extern sorted_unique_t sorted_unique; + +namespace flat_containers_internal { + +// Helper functions used in RTC_DCHECKs below to make sure that inputs tagged +// with sorted_unique are indeed sorted and unique. +template +constexpr bool is_sorted_and_unique(const Range& range, Comp comp) { + // Being unique implies that there are no adjacent elements that + // compare equal. So this checks that each element is strictly less + // than the element after it. + return absl::c_adjacent_find(range, std::not_fn(comp)) == std::end(range); +} + +// This is a convenience trait inheriting from std::true_type if Iterator is at +// least a ForwardIterator and thus supports multiple passes over a range. +template +using is_multipass = + std::is_base_of::iterator_category>; + +// Uses SFINAE to detect whether type has is_transparent member. +template +struct IsTransparentCompare : std::false_type {}; +template +struct IsTransparentCompare> + : std::true_type {}; + +// Helper inspired by C++20's std::to_array to convert a C-style array to a +// std::array. As opposed to the C++20 version this implementation does not +// provide an overload for rvalues and does not strip cv qualifers from the +// returned std::array::value_type. The returned value_type needs to be +// specified explicitly, allowing the construction of std::arrays with const +// elements. +// +// Reference: https://en.cppreference.com/w/cpp/container/array/to_array +template +constexpr std::array ToArrayImpl(const T (&data)[N], + std::index_sequence) { + return {{data[I]...}}; +} + +template +constexpr std::array ToArray(const T (&data)[N]) { + return ToArrayImpl(data, std::make_index_sequence()); +} + +// std::pair's operator= is not constexpr prior to C++20. Thus we need this +// small helper to invoke operator= on the .first and .second member explicitly. +template +constexpr void Assign(T& lhs, T&& rhs) { + lhs = std::move(rhs); +} + +template +constexpr void Assign(std::pair& lhs, std::pair&& rhs) { + Assign(lhs.first, std::move(rhs.first)); + Assign(lhs.second, std::move(rhs.second)); +} + +// constexpr swap implementation. std::swap is not constexpr prior to C++20. +template +constexpr void Swap(T& lhs, T& rhs) { + T tmp = std::move(lhs); + Assign(lhs, std::move(rhs)); + Assign(rhs, std::move(tmp)); +} + +// constexpr prev implementation. std::prev is not constexpr prior to C++17. +template +constexpr BidirIt Prev(BidirIt it) { + return --it; +} + +// constexpr next implementation. std::next is not constexpr prior to C++17. +template +constexpr InputIt Next(InputIt it) { + return ++it; +} + +// constexpr sort implementation. std::sort is not constexpr prior to C++20. +// While insertion sort has a quadratic worst case complexity, it was chosen +// because it has linear complexity for nearly sorted data, is stable, and +// simple to implement. +template +constexpr void InsertionSort(BidirIt first, BidirIt last, const Compare& comp) { + if (first == last) + return; + + for (auto it = Next(first); it != last; ++it) { + for (auto curr = it; curr != first && comp(*curr, *Prev(curr)); --curr) + Swap(*curr, *Prev(curr)); + } +} + +// Implementation ------------------------------------------------------------- + +// Implementation for the sorted associative flat_set and flat_map using a +// sorted vector as the backing store. Do not use directly. +// +// The use of "value" in this is like std::map uses, meaning it's the thing +// contained (in the case of map it's a pair). The Key is how +// things are looked up. In the case of a set, Key == Value. In the case of +// a map, the Key is a component of a Value. +// +// The helper class GetKeyFromValue provides the means to extract a key from a +// value for comparison purposes. It should implement: +// const Key& operator()(const Value&). +template +class flat_tree { + public: + // -------------------------------------------------------------------------- + // Types. + // + using key_type = Key; + using key_compare = KeyCompare; + using value_type = typename Container::value_type; + + // Wraps the templated key comparison to compare values. + struct value_compare { + constexpr bool operator()(const value_type& left, + const value_type& right) const { + GetKeyFromValue extractor; + return comp(extractor(left), extractor(right)); + } + + RTC_NO_UNIQUE_ADDRESS key_compare comp; + }; + + using pointer = typename Container::pointer; + using const_pointer = typename Container::const_pointer; + using reference = typename Container::reference; + using const_reference = typename Container::const_reference; + using size_type = typename Container::size_type; + using difference_type = typename Container::difference_type; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + using reverse_iterator = typename Container::reverse_iterator; + using const_reverse_iterator = typename Container::const_reverse_iterator; + using container_type = Container; + + // -------------------------------------------------------------------------- + // Lifetime. + // + // Constructors that take range guarantee O(N * log^2(N)) + O(N) complexity + // and take O(N * log(N)) + O(N) if extra memory is available (N is a range + // length). + // + // Assume that move constructors invalidate iterators and references. + // + // The constructors that take ranges, lists, and vectors do not require that + // the input be sorted. + // + // When passing the webrtc::sorted_unique tag as the first argument no sort + // and unique step takes places. This is useful if the underlying container + // already has the required properties. + + flat_tree() = default; + flat_tree(const flat_tree&) = default; + flat_tree(flat_tree&&) = default; + + explicit flat_tree(const key_compare& comp); + + template + flat_tree(InputIterator first, + InputIterator last, + const key_compare& comp = key_compare()); + + flat_tree(const container_type& items, + const key_compare& comp = key_compare()); + + explicit flat_tree(container_type&& items, + const key_compare& comp = key_compare()); + + flat_tree(std::initializer_list ilist, + const key_compare& comp = key_compare()); + + template + flat_tree(sorted_unique_t, + InputIterator first, + InputIterator last, + const key_compare& comp = key_compare()); + + flat_tree(sorted_unique_t, + const container_type& items, + const key_compare& comp = key_compare()); + + constexpr flat_tree(sorted_unique_t, + container_type&& items, + const key_compare& comp = key_compare()); + + flat_tree(sorted_unique_t, + std::initializer_list ilist, + const key_compare& comp = key_compare()); + + ~flat_tree() = default; + + // -------------------------------------------------------------------------- + // Assignments. + // + // Assume that move assignment invalidates iterators and references. + + flat_tree& operator=(const flat_tree&) = default; + flat_tree& operator=(flat_tree&&) = default; + // Takes the first if there are duplicates in the initializer list. + flat_tree& operator=(std::initializer_list ilist); + + // -------------------------------------------------------------------------- + // Memory management. + // + // Beware that shrink_to_fit() simply forwards the request to the + // container_type and its implementation is free to optimize otherwise and + // leave capacity() to be greater that its size. + // + // reserve() and shrink_to_fit() invalidate iterators and references. + + void reserve(size_type new_capacity); + size_type capacity() const; + void shrink_to_fit(); + + // -------------------------------------------------------------------------- + // Size management. + // + // clear() leaves the capacity() of the flat_tree unchanged. + + void clear(); + + constexpr size_type size() const; + constexpr size_type max_size() const; + constexpr bool empty() const; + + // -------------------------------------------------------------------------- + // Iterators. + // + // Iterators follow the ordering defined by the key comparator used in + // construction of the flat_tree. + + iterator begin(); + constexpr const_iterator begin() const; + const_iterator cbegin() const; + + iterator end(); + constexpr const_iterator end() const; + const_iterator cend() const; + + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + const_reverse_iterator crbegin() const; + + reverse_iterator rend(); + const_reverse_iterator rend() const; + const_reverse_iterator crend() const; + + // -------------------------------------------------------------------------- + // Insert operations. + // + // Assume that every operation invalidates iterators and references. + // Insertion of one element can take O(size). Capacity of flat_tree grows in + // an implementation-defined manner. + // + // NOTE: Prefer to build a new flat_tree from a std::vector (or similar) + // instead of calling insert() repeatedly. + + std::pair insert(const value_type& val); + std::pair insert(value_type&& val); + + iterator insert(const_iterator position_hint, const value_type& x); + iterator insert(const_iterator position_hint, value_type&& x); + + // This method inserts the values from the range [first, last) into the + // current tree. + template + void insert(InputIterator first, InputIterator last); + + template + std::pair emplace(Args&&... args); + + template + iterator emplace_hint(const_iterator position_hint, Args&&... args); + + // -------------------------------------------------------------------------- + // Underlying type operations. + // + // Assume that either operation invalidates iterators and references. + + // Extracts the container_type and returns it to the caller. Ensures that + // `this` is `empty()` afterwards. + container_type extract() &&; + + // Replaces the container_type with `body`. Expects that `body` is sorted + // and has no repeated elements with regard to value_comp(). + void replace(container_type&& body); + + // -------------------------------------------------------------------------- + // Erase operations. + // + // Assume that every operation invalidates iterators and references. + // + // erase(position), erase(first, last) can take O(size). + // erase(key) may take O(size) + O(log(size)). + // + // Prefer webrtc::EraseIf() or some other variation on erase(remove(), end()) + // idiom when deleting multiple non-consecutive elements. + + iterator erase(iterator position); + // Artificially templatized to break ambiguity if `iterator` and + // `const_iterator` are the same type. + template + iterator erase(const_iterator position); + iterator erase(const_iterator first, const_iterator last); + template + size_type erase(const K& key); + + // -------------------------------------------------------------------------- + // Comparators. + + constexpr key_compare key_comp() const; + constexpr value_compare value_comp() const; + + // -------------------------------------------------------------------------- + // Search operations. + // + // Search operations have O(log(size)) complexity. + + template + size_type count(const K& key) const; + + template + iterator find(const K& key); + + template + const_iterator find(const K& key) const; + + template + bool contains(const K& key) const; + + template + std::pair equal_range(const K& key); + + template + std::pair equal_range(const K& key) const; + + template + iterator lower_bound(const K& key); + + template + const_iterator lower_bound(const K& key) const; + + template + iterator upper_bound(const K& key); + + template + const_iterator upper_bound(const K& key) const; + + // -------------------------------------------------------------------------- + // General operations. + // + // Assume that swap invalidates iterators and references. + // + // Implementation note: currently we use operator==() and operator<() on + // std::vector, because they have the same contract we need, so we use them + // directly for brevity and in case it is more optimal than calling equal() + // and lexicograhpical_compare(). If the underlying container type is changed, + // this code may need to be modified. + + void swap(flat_tree& other) noexcept; + + friend bool operator==(const flat_tree& lhs, const flat_tree& rhs) { + return lhs.body_ == rhs.body_; + } + + friend bool operator!=(const flat_tree& lhs, const flat_tree& rhs) { + return !(lhs == rhs); + } + + friend bool operator<(const flat_tree& lhs, const flat_tree& rhs) { + return lhs.body_ < rhs.body_; + } + + friend bool operator>(const flat_tree& lhs, const flat_tree& rhs) { + return rhs < lhs; + } + + friend bool operator>=(const flat_tree& lhs, const flat_tree& rhs) { + return !(lhs < rhs); + } + + friend bool operator<=(const flat_tree& lhs, const flat_tree& rhs) { + return !(lhs > rhs); + } + + friend void swap(flat_tree& lhs, flat_tree& rhs) noexcept { lhs.swap(rhs); } + + protected: + // Emplaces a new item into the tree that is known not to be in it. This + // is for implementing map operator[]. + template + iterator unsafe_emplace(const_iterator position, Args&&... args); + + // Attempts to emplace a new element with key `key`. Only if `key` is not yet + // present, construct value_type from `args` and insert it. Returns an + // iterator to the element with key `key` and a bool indicating whether an + // insertion happened. + template + std::pair emplace_key_args(const K& key, Args&&... args); + + // Similar to `emplace_key_args`, but checks `hint` first as a possible + // insertion position. + template + std::pair emplace_hint_key_args(const_iterator hint, + const K& key, + Args&&... args); + + private: + // Helper class for e.g. lower_bound that can compare a value on the left + // to a key on the right. + struct KeyValueCompare { + // The key comparison object must outlive this class. + explicit KeyValueCompare(const key_compare& comp) : comp_(comp) {} + + template + bool operator()(const T& lhs, const U& rhs) const { + return comp_(extract_if_value_type(lhs), extract_if_value_type(rhs)); + } + + private: + const key_type& extract_if_value_type(const value_type& v) const { + GetKeyFromValue extractor; + return extractor(v); + } + + template + const K& extract_if_value_type(const K& k) const { + return k; + } + + const key_compare& comp_; + }; + + iterator const_cast_it(const_iterator c_it) { + auto distance = std::distance(cbegin(), c_it); + return std::next(begin(), distance); + } + + // This method is inspired by both std::map::insert(P&&) and + // std::map::insert_or_assign(const K&, V&&). It inserts val if an equivalent + // element is not present yet, otherwise it overwrites. It returns an iterator + // to the modified element and a flag indicating whether insertion or + // assignment happened. + template + std::pair insert_or_assign(V&& val) { + auto position = lower_bound(GetKeyFromValue()(val)); + + if (position == end() || value_comp()(val, *position)) + return {body_.emplace(position, std::forward(val)), true}; + + *position = std::forward(val); + return {position, false}; + } + + // This method is similar to insert_or_assign, with the following differences: + // - Instead of searching [begin(), end()) it only searches [first, last). + // - In case no equivalent element is found, val is appended to the end of the + // underlying body and an iterator to the next bigger element in [first, + // last) is returned. + template + std::pair append_or_assign(iterator first, + iterator last, + V&& val) { + auto position = std::lower_bound(first, last, val, value_comp()); + + if (position == last || value_comp()(val, *position)) { + // emplace_back might invalidate position, which is why distance needs to + // be cached. + const difference_type distance = std::distance(begin(), position); + body_.emplace_back(std::forward(val)); + return {std::next(begin(), distance), true}; + } + + *position = std::forward(val); + return {position, false}; + } + + // This method is similar to insert, with the following differences: + // - Instead of searching [begin(), end()) it only searches [first, last). + // - In case no equivalent element is found, val is appended to the end of the + // underlying body and an iterator to the next bigger element in [first, + // last) is returned. + template + std::pair append_unique(iterator first, + iterator last, + V&& val) { + auto position = std::lower_bound(first, last, val, value_comp()); + + if (position == last || value_comp()(val, *position)) { + // emplace_back might invalidate position, which is why distance needs to + // be cached. + const difference_type distance = std::distance(begin(), position); + body_.emplace_back(std::forward(val)); + return {std::next(begin(), distance), true}; + } + + return {position, false}; + } + + void sort_and_unique(iterator first, iterator last) { + // Preserve stability for the unique code below. + std::stable_sort(first, last, value_comp()); + + // lhs is already <= rhs due to sort, therefore !(lhs < rhs) <=> lhs == rhs. + auto equal_comp = std::not_fn(value_comp()); + erase(std::unique(first, last, equal_comp), last); + } + + void sort_and_unique() { sort_and_unique(begin(), end()); } + + // To support comparators that may not be possible to default-construct, we + // have to store an instance of Compare. Since Compare commonly is stateless, + // we use the RTC_NO_UNIQUE_ADDRESS attribute to save space. + RTC_NO_UNIQUE_ADDRESS key_compare comp_; + // Declare after `key_compare_comp_` to workaround GCC ICE. For details + // see https://crbug.com/1156268 + container_type body_; + + // If the compare is not transparent we want to construct key_type once. + template + using KeyTypeOrK = typename std:: + conditional::value, K, key_type>::type; +}; + +// ---------------------------------------------------------------------------- +// Lifetime. + +template +flat_tree::flat_tree( + const KeyCompare& comp) + : comp_(comp) {} + +template +template +flat_tree::flat_tree( + InputIterator first, + InputIterator last, + const KeyCompare& comp) + : comp_(comp), body_(first, last) { + sort_and_unique(); +} + +template +flat_tree::flat_tree( + const container_type& items, + const KeyCompare& comp) + : comp_(comp), body_(items) { + sort_and_unique(); +} + +template +flat_tree::flat_tree( + container_type&& items, + const KeyCompare& comp) + : comp_(comp), body_(std::move(items)) { + sort_and_unique(); +} + +template +flat_tree::flat_tree( + std::initializer_list ilist, + const KeyCompare& comp) + : flat_tree(std::begin(ilist), std::end(ilist), comp) {} + +template +template +flat_tree::flat_tree( + sorted_unique_t, + InputIterator first, + InputIterator last, + const KeyCompare& comp) + : comp_(comp), body_(first, last) { + RTC_DCHECK(is_sorted_and_unique(*this, value_comp())); +} + +template +flat_tree::flat_tree( + sorted_unique_t, + const container_type& items, + const KeyCompare& comp) + : comp_(comp), body_(items) { + RTC_DCHECK(is_sorted_and_unique(*this, value_comp())); +} + +template +constexpr flat_tree::flat_tree( + sorted_unique_t, + container_type&& items, + const KeyCompare& comp) + : comp_(comp), body_(std::move(items)) { + RTC_DCHECK(is_sorted_and_unique(*this, value_comp())); +} + +template +flat_tree::flat_tree( + sorted_unique_t, + std::initializer_list ilist, + const KeyCompare& comp) + : flat_tree(sorted_unique, std::begin(ilist), std::end(ilist), comp) {} + +// ---------------------------------------------------------------------------- +// Assignments. + +template +auto flat_tree::operator=( + std::initializer_list ilist) -> flat_tree& { + body_ = ilist; + sort_and_unique(); + return *this; +} + +// ---------------------------------------------------------------------------- +// Memory management. + +template +void flat_tree::reserve( + size_type new_capacity) { + body_.reserve(new_capacity); +} + +template +auto flat_tree::capacity() const + -> size_type { + return body_.capacity(); +} + +template +void flat_tree::shrink_to_fit() { + body_.shrink_to_fit(); +} + +// ---------------------------------------------------------------------------- +// Size management. + +template +void flat_tree::clear() { + body_.clear(); +} + +template +constexpr auto flat_tree::size() + const -> size_type { + return body_.size(); +} + +template +constexpr auto +flat_tree::max_size() const + -> size_type { + return body_.max_size(); +} + +template +constexpr bool flat_tree::empty() + const { + return body_.empty(); +} + +// ---------------------------------------------------------------------------- +// Iterators. + +template +auto flat_tree::begin() + -> iterator { + return body_.begin(); +} + +template +constexpr auto flat_tree::begin() + const -> const_iterator { + return std::begin(body_); +} + +template +auto flat_tree::cbegin() const + -> const_iterator { + return body_.cbegin(); +} + +template +auto flat_tree::end() -> iterator { + return body_.end(); +} + +template +constexpr auto flat_tree::end() + const -> const_iterator { + return std::end(body_); +} + +template +auto flat_tree::cend() const + -> const_iterator { + return body_.cend(); +} + +template +auto flat_tree::rbegin() + -> reverse_iterator { + return body_.rbegin(); +} + +template +auto flat_tree::rbegin() const + -> const_reverse_iterator { + return body_.rbegin(); +} + +template +auto flat_tree::crbegin() const + -> const_reverse_iterator { + return body_.crbegin(); +} + +template +auto flat_tree::rend() + -> reverse_iterator { + return body_.rend(); +} + +template +auto flat_tree::rend() const + -> const_reverse_iterator { + return body_.rend(); +} + +template +auto flat_tree::crend() const + -> const_reverse_iterator { + return body_.crend(); +} + +// ---------------------------------------------------------------------------- +// Insert operations. +// +// Currently we use position_hint the same way as eastl or boost: +// https://github.com/electronicarts/EASTL/blob/master/include/EASTL/vector_set.h#L493 + +template +auto flat_tree::insert( + const value_type& val) -> std::pair { + return emplace_key_args(GetKeyFromValue()(val), val); +} + +template +auto flat_tree::insert( + value_type&& val) -> std::pair { + return emplace_key_args(GetKeyFromValue()(val), std::move(val)); +} + +template +auto flat_tree::insert( + const_iterator position_hint, + const value_type& val) -> iterator { + return emplace_hint_key_args(position_hint, GetKeyFromValue()(val), val) + .first; +} + +template +auto flat_tree::insert( + const_iterator position_hint, + value_type&& val) -> iterator { + return emplace_hint_key_args(position_hint, GetKeyFromValue()(val), + std::move(val)) + .first; +} + +template +template +void flat_tree::insert( + InputIterator first, + InputIterator last) { + if (first == last) + return; + + // Dispatch to single element insert if the input range contains a single + // element. + if (is_multipass() && std::next(first) == last) { + insert(end(), *first); + return; + } + + // Provide a convenience lambda to obtain an iterator pointing past the last + // old element. This needs to be dymanic due to possible re-allocations. + auto middle = [this, size = size()] { return std::next(begin(), size); }; + + // For batch updates initialize the first insertion point. + difference_type pos_first_new = size(); + + // Loop over the input range while appending new values and overwriting + // existing ones, if applicable. Keep track of the first insertion point. + for (; first != last; ++first) { + std::pair result = append_unique(begin(), middle(), *first); + if (result.second) { + pos_first_new = + std::min(pos_first_new, std::distance(begin(), result.first)); + } + } + + // The new elements might be unordered and contain duplicates, so post-process + // the just inserted elements and merge them with the rest, inserting them at + // the previously found spot. + sort_and_unique(middle(), end()); + std::inplace_merge(std::next(begin(), pos_first_new), middle(), end(), + value_comp()); +} + +template +template +auto flat_tree::emplace( + Args&&... args) -> std::pair { + return insert(value_type(std::forward(args)...)); +} + +template +template +auto flat_tree::emplace_hint( + const_iterator position_hint, + Args&&... args) -> iterator { + return insert(position_hint, value_type(std::forward(args)...)); +} + +// ---------------------------------------------------------------------------- +// Underlying type operations. + +template +auto flat_tree:: + extract() && -> container_type { + return std::exchange(body_, container_type()); +} + +template +void flat_tree::replace( + container_type&& body) { + // Ensure that `body` is sorted and has no repeated elements according to + // `value_comp()`. + RTC_DCHECK(is_sorted_and_unique(body, value_comp())); + body_ = std::move(body); +} + +// ---------------------------------------------------------------------------- +// Erase operations. + +template +auto flat_tree::erase( + iterator position) -> iterator { + RTC_CHECK(position != body_.end()); + return body_.erase(position); +} + +template +template +auto flat_tree::erase( + const_iterator position) -> iterator { + RTC_CHECK(position != body_.end()); + return body_.erase(position); +} + +template +template +auto flat_tree::erase(const K& val) + -> size_type { + auto eq_range = equal_range(val); + auto res = std::distance(eq_range.first, eq_range.second); + erase(eq_range.first, eq_range.second); + return res; +} + +template +auto flat_tree::erase( + const_iterator first, + const_iterator last) -> iterator { + return body_.erase(first, last); +} + +// ---------------------------------------------------------------------------- +// Comparators. + +template +constexpr auto +flat_tree::key_comp() const + -> key_compare { + return comp_; +} + +template +constexpr auto +flat_tree::value_comp() const + -> value_compare { + return value_compare{comp_}; +} + +// ---------------------------------------------------------------------------- +// Search operations. + +template +template +auto flat_tree::count( + const K& key) const -> size_type { + auto eq_range = equal_range(key); + return std::distance(eq_range.first, eq_range.second); +} + +template +template +auto flat_tree::find(const K& key) + -> iterator { + return const_cast_it(std::as_const(*this).find(key)); +} + +template +template +auto flat_tree::find( + const K& key) const -> const_iterator { + auto eq_range = equal_range(key); + return (eq_range.first == eq_range.second) ? end() : eq_range.first; +} + +template +template +bool flat_tree::contains( + const K& key) const { + auto lower = lower_bound(key); + return lower != end() && !comp_(key, GetKeyFromValue()(*lower)); +} + +template +template +auto flat_tree::equal_range( + const K& key) -> std::pair { + auto res = std::as_const(*this).equal_range(key); + return {const_cast_it(res.first), const_cast_it(res.second)}; +} + +template +template +auto flat_tree::equal_range( + const K& key) const -> std::pair { + auto lower = lower_bound(key); + + KeyValueCompare comp(comp_); + if (lower == end() || comp(key, *lower)) + return {lower, lower}; + + return {lower, std::next(lower)}; +} + +template +template +auto flat_tree::lower_bound( + const K& key) -> iterator { + return const_cast_it(std::as_const(*this).lower_bound(key)); +} + +template +template +auto flat_tree::lower_bound( + const K& key) const -> const_iterator { + static_assert(std::is_convertible&, const K&>::value, + "Requested type cannot be bound to the container's key_type " + "which is required for a non-transparent compare."); + + const KeyTypeOrK& key_ref = key; + + KeyValueCompare comp(comp_); + return absl::c_lower_bound(*this, key_ref, comp); +} + +template +template +auto flat_tree::upper_bound( + const K& key) -> iterator { + return const_cast_it(std::as_const(*this).upper_bound(key)); +} + +template +template +auto flat_tree::upper_bound( + const K& key) const -> const_iterator { + static_assert(std::is_convertible&, const K&>::value, + "Requested type cannot be bound to the container's key_type " + "which is required for a non-transparent compare."); + + const KeyTypeOrK& key_ref = key; + + KeyValueCompare comp(comp_); + return absl::c_upper_bound(*this, key_ref, comp); +} + +// ---------------------------------------------------------------------------- +// General operations. + +template +void flat_tree::swap( + flat_tree& other) noexcept { + std::swap(*this, other); +} + +template +template +auto flat_tree::unsafe_emplace( + const_iterator position, + Args&&... args) -> iterator { + return body_.emplace(position, std::forward(args)...); +} + +template +template +auto flat_tree::emplace_key_args( + const K& key, + Args&&... args) -> std::pair { + auto lower = lower_bound(key); + if (lower == end() || comp_(key, GetKeyFromValue()(*lower))) + return {unsafe_emplace(lower, std::forward(args)...), true}; + return {lower, false}; +} + +template +template +auto flat_tree:: + emplace_hint_key_args(const_iterator hint, const K& key, Args&&... args) + -> std::pair { + KeyValueCompare comp(comp_); + if ((hint == begin() || comp(*std::prev(hint), key))) { + if (hint == end() || comp(key, *hint)) { + // *(hint - 1) < key < *hint => key did not exist and hint is correct. + return {unsafe_emplace(hint, std::forward(args)...), true}; + } + if (!comp(*hint, key)) { + // key == *hint => no-op, return correct hint. + return {const_cast_it(hint), false}; + } + } + // hint was not helpful, dispatch to hintless version. + return emplace_key_args(key, std::forward(args)...); +} + +// ---------------------------------------------------------------------------- +// Free functions. + +// Erases all elements that match predicate. It has O(size) complexity. +template +size_t EraseIf( + webrtc::flat_containers_internal:: + flat_tree& container, + Predicate pred) { + auto it = std::remove_if(container.begin(), container.end(), + std::forward(pred)); + size_t removed = std::distance(it, container.end()); + container.erase(it, container.end()); + return removed; +} + +} // namespace flat_containers_internal +} // namespace webrtc + +#endif // RTC_BASE_CONTAINERS_FLAT_TREE_H_ diff --git a/webrtc/rtc_base/containers/identity.h b/webrtc/rtc_base/containers/identity.h new file mode 100644 index 0000000..2959293 --- /dev/null +++ b/webrtc/rtc_base/containers/identity.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This implementation is borrowed from Chromium. + +#ifndef RTC_BASE_CONTAINERS_IDENTITY_H_ +#define RTC_BASE_CONTAINERS_IDENTITY_H_ + +#include + +namespace webrtc { + +// Implementation of C++20's std::identity. +// +// Reference: +// - https://en.cppreference.com/w/cpp/utility/functional/identity +// - https://wg21.link/func.identity +struct identity { + template + constexpr T&& operator()(T&& t) const noexcept { + return std::forward(t); + } + + using is_transparent = void; +}; + +} // namespace webrtc + +#endif // RTC_BASE_CONTAINERS_IDENTITY_H_ diff --git a/webrtc/rtc_base/deprecation.h b/webrtc/rtc_base/deprecation.h deleted file mode 100644 index f285ab0..0000000 --- a/webrtc/rtc_base/deprecation.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef RTC_BASE_DEPRECATION_H_ -#define RTC_BASE_DEPRECATION_H_ - -// Annotate the declarations of deprecated functions with this to cause a -// compiler warning when they're used. Like so: -// -// RTC_DEPRECATED std::pony PonyPlz(const std::pony_spec& ps); -// -// NOTE 1: The annotation goes on the declaration in the .h file, not the -// definition in the .cc file! -// -// NOTE 2: In order to keep unit testing the deprecated function without -// getting warnings, do something like this: -// -// std::pony DEPRECATED_PonyPlz(const std::pony_spec& ps); -// RTC_DEPRECATED inline std::pony PonyPlz(const std::pony_spec& ps) { -// return DEPRECATED_PonyPlz(ps); -// } -// -// In other words, rename the existing function, and provide an inline wrapper -// using the original name that calls it. That way, callers who are willing to -// call it using the DEPRECATED_-prefixed name don't get the warning. -// -// TODO(kwiberg): Remove this when we can use [[deprecated]] from C++14. -#if defined(_MSC_VER) -// Note: Deprecation warnings seem to fail to trigger on Windows -// (https://bugs.chromium.org/p/webrtc/issues/detail?id=5368). -#define RTC_DEPRECATED __declspec(deprecated) -#elif defined(__GNUC__) -#define RTC_DEPRECATED __attribute__((__deprecated__)) -#else -#define RTC_DEPRECATED -#endif - -#endif // RTC_BASE_DEPRECATION_H_ diff --git a/webrtc/rtc_base/event.cc b/webrtc/rtc_base/event.cc index 67c8746..c2f6f8a 100644 --- a/webrtc/rtc_base/event.cc +++ b/webrtc/rtc_base/event.cc @@ -25,9 +25,12 @@ #include "rtc_base/checks.h" #include "rtc_base/synchronization/yield_policy.h" #include "rtc_base/system/warn_current_thread_is_deadlocked.h" +#include "rtc_base/time_utils.h" namespace rtc { +using ::webrtc::TimeDelta; + Event::Event() : Event(false, false) {} #if defined(WEBRTC_WIN) @@ -51,9 +54,12 @@ void Event::Reset() { ResetEvent(event_handle_); } -bool Event::Wait(const int give_up_after_ms, int /*warn_after_ms*/) { +bool Event::Wait(TimeDelta give_up_after, TimeDelta /*warn_after*/) { ScopedYieldPolicy::YieldExecution(); - const DWORD ms = give_up_after_ms == kForever ? INFINITE : give_up_after_ms; + const DWORD ms = + give_up_after.IsPlusInfinity() + ? INFINITE + : give_up_after.RoundUpTo(webrtc::TimeDelta::Millis(1)).ms(); return (WaitForSingleObject(event_handle_, ms) == WAIT_OBJECT_0); } @@ -108,7 +114,7 @@ void Event::Reset() { namespace { -timespec GetTimespec(const int milliseconds_from_now) { +timespec GetTimespec(TimeDelta duration_from_now) { timespec ts; // Get the current time. @@ -118,17 +124,19 @@ timespec GetTimespec(const int milliseconds_from_now) { timeval tv; gettimeofday(&tv, nullptr); ts.tv_sec = tv.tv_sec; - ts.tv_nsec = tv.tv_usec * 1000; + ts.tv_nsec = tv.tv_usec * kNumNanosecsPerMicrosec; #endif // Add the specified number of milliseconds to it. - ts.tv_sec += (milliseconds_from_now / 1000); - ts.tv_nsec += (milliseconds_from_now % 1000) * 1000000; + int64_t microsecs_from_now = duration_from_now.us(); + ts.tv_sec += microsecs_from_now / kNumMicrosecsPerSec; + ts.tv_nsec += + (microsecs_from_now % kNumMicrosecsPerSec) * kNumNanosecsPerMicrosec; // Normalize. - if (ts.tv_nsec >= 1000000000) { + if (ts.tv_nsec >= kNumNanosecsPerSec) { ts.tv_sec++; - ts.tv_nsec -= 1000000000; + ts.tv_nsec -= kNumNanosecsPerSec; } return ts; @@ -136,22 +144,21 @@ timespec GetTimespec(const int milliseconds_from_now) { } // namespace -bool Event::Wait(const int give_up_after_ms, const int warn_after_ms) { +bool Event::Wait(TimeDelta give_up_after, TimeDelta warn_after) { // Instant when we'll log a warning message (because we've been waiting so // long it might be a bug), but not yet give up waiting. nullopt if we // shouldn't log a warning. const absl::optional warn_ts = - warn_after_ms == kForever || - (give_up_after_ms != kForever && warn_after_ms > give_up_after_ms) + warn_after >= give_up_after ? absl::nullopt - : absl::make_optional(GetTimespec(warn_after_ms)); + : absl::make_optional(GetTimespec(warn_after)); // Instant when we'll stop waiting and return an error. nullopt if we should // never give up. const absl::optional give_up_ts = - give_up_after_ms == kForever + give_up_after.IsPlusInfinity() ? absl::nullopt - : absl::make_optional(GetTimespec(give_up_after_ms)); + : absl::make_optional(GetTimespec(give_up_after)); ScopedYieldPolicy::YieldExecution(); pthread_mutex_lock(&event_mutex_); diff --git a/webrtc/rtc_base/event.h b/webrtc/rtc_base/event.h index 584ad5d..12f6a7d 100644 --- a/webrtc/rtc_base/event.h +++ b/webrtc/rtc_base/event.h @@ -11,6 +11,8 @@ #ifndef RTC_BASE_EVENT_H_ #define RTC_BASE_EVENT_H_ +#include "api/units/time_delta.h" + #if defined(WEBRTC_WIN) #include #elif defined(WEBRTC_POSIX) @@ -19,11 +21,43 @@ #error "Must define either WEBRTC_WIN or WEBRTC_POSIX." #endif +#include "rtc_base/synchronization/yield_policy.h" + namespace rtc { +// RTC_DISALLOW_WAIT() utility +// +// Sets a stack-scoped flag that disallows use of `rtc::Event::Wait` by means +// of raising a DCHECK when a call to `rtc::Event::Wait()` is made.. +// This is useful to guard synchronization-free scopes against regressions. +// +// Example of what this would catch (`ScopeToProtect` calls `Foo`): +// +// void Foo(TaskQueue* tq) { +// Event event; +// tq->PostTask([&event]() { +// event.Set(); +// }); +// event.Wait(Event::kForever); // <- Will trigger a DCHECK. +// } +// +// void ScopeToProtect() { +// TaskQueue* tq = GetSomeTaskQueue(); +// RTC_DISALLOW_WAIT(); // Policy takes effect. +// Foo(tq); +// } +// +#if RTC_DCHECK_IS_ON +#define RTC_DISALLOW_WAIT() ScopedDisallowWait disallow_wait_##__LINE__ +#else +#define RTC_DISALLOW_WAIT() +#endif + class Event { public: - static const int kForever = -1; + // TODO(bugs.webrtc.org/14366): Consider removing this redundant alias. + static constexpr webrtc::TimeDelta kForever = + webrtc::TimeDelta::PlusInfinity(); Event(); Event(bool manual_reset, bool initially_signaled); @@ -35,19 +69,22 @@ class Event { void Reset(); // Waits for the event to become signaled, but logs a warning if it takes more - // than `warn_after_ms` milliseconds, and gives up completely if it takes more - // than `give_up_after_ms` milliseconds. (If `warn_after_ms >= - // give_up_after_ms`, no warning will be logged.) Either or both may be - // `kForever`, which means wait indefinitely. + // than `warn_after`, and gives up completely if it takes more than + // `give_up_after`. (If `warn_after >= give_up_after`, no warning will be + // logged.) Either or both may be `kForever`, which means wait indefinitely. + // + // Care is taken so that the underlying OS wait call isn't requested to sleep + // shorter than `give_up_after`. // // Returns true if the event was signaled, false if there was a timeout or // some other error. - bool Wait(int give_up_after_ms, int warn_after_ms); + bool Wait(webrtc::TimeDelta give_up_after, webrtc::TimeDelta warn_after); // Waits with the given timeout and a reasonable default warning timeout. - bool Wait(int give_up_after_ms) { - return Wait(give_up_after_ms, - give_up_after_ms == kForever ? 3000 : kForever); + bool Wait(webrtc::TimeDelta give_up_after) { + return Wait(give_up_after, give_up_after.IsPlusInfinity() + ? webrtc::TimeDelta::Seconds(3) + : kForever); } private: @@ -81,6 +118,20 @@ class ScopedAllowBaseSyncPrimitivesForTesting { ~ScopedAllowBaseSyncPrimitivesForTesting() {} }; +#if RTC_DCHECK_IS_ON +class ScopedDisallowWait { + public: + ScopedDisallowWait() = default; + + private: + class DisallowYieldHandler : public YieldInterface { + public: + void YieldExecution() override { RTC_DCHECK_NOTREACHED(); } + } handler_; + rtc::ScopedYieldPolicy policy{&handler_}; +}; +#endif + } // namespace rtc #endif // RTC_BASE_EVENT_H_ diff --git a/webrtc/rtc_base/event_tracer.cc b/webrtc/rtc_base/event_tracer.cc index 3af8183..992a2b5 100644 --- a/webrtc/rtc_base/event_tracer.cc +++ b/webrtc/rtc_base/event_tracer.cc @@ -14,10 +14,12 @@ #include #include +#include #include #include -#include "rtc_base/atomic_ops.h" +#include "absl/strings/string_view.h" +#include "api/sequence_checker.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" #include "rtc_base/logging.h" @@ -25,7 +27,6 @@ #include "rtc_base/platform_thread_types.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" -#include "rtc_base/thread_checker.h" #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" @@ -79,19 +80,12 @@ namespace rtc { namespace tracing { namespace { -static void EventTracingThreadFunc(void* params); - // Atomic-int fast path for avoiding logging when disabled. -static volatile int g_event_logging_active = 0; +static std::atomic g_event_logging_active(0); // TODO(pbos): Log metadata for all threads, etc. class EventLogger final { public: - EventLogger() - : logging_thread_(EventTracingThreadFunc, - this, - "EventTracingThread", - kLowPriority) {} ~EventLogger() { RTC_DCHECK(thread_checker_.IsCurrent()); } void AddTraceEvent(const char* name, @@ -129,11 +123,12 @@ class EventLogger final { // https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview void Log() { RTC_DCHECK(output_file_); - static const int kLoggingIntervalMs = 100; + static constexpr webrtc::TimeDelta kLoggingInterval = + webrtc::TimeDelta::Millis(100); fprintf(output_file_, "{ \"traceEvents\": [\n"); bool has_logged_event = false; while (true) { - bool shutting_down = shutdown_event_.Wait(kLoggingIntervalMs); + bool shutting_down = shutdown_event_.Wait(kLoggingInterval); std::vector events; { webrtc::MutexLock lock(&mutex_); @@ -205,11 +200,12 @@ class EventLogger final { } // Enable event logging (fast-path). This should be disabled since starting // shouldn't be done twice. - RTC_CHECK_EQ(0, - rtc::AtomicOps::CompareAndSwap(&g_event_logging_active, 0, 1)); + int zero = 0; + RTC_CHECK(g_event_logging_active.compare_exchange_strong(zero, 1)); // Finally start, everything should be set up now. - logging_thread_.Start(); + logging_thread_ = + PlatformThread::SpawnJoinable([this] { Log(); }, "EventTracingThread"); TRACE_EVENT_INSTANT0("webrtc", "EventLogger::Start"); } @@ -217,13 +213,14 @@ class EventLogger final { RTC_DCHECK(thread_checker_.IsCurrent()); TRACE_EVENT_INSTANT0("webrtc", "EventLogger::Stop"); // Try to stop. Abort if we're not currently logging. - if (rtc::AtomicOps::CompareAndSwap(&g_event_logging_active, 1, 0) == 0) + int one = 1; + if (g_event_logging_active.compare_exchange_strong(one, 0)) return; // Wake up logging thread to finish writing. shutdown_event_.Set(); // Join the logging thread. - logging_thread_.Stop(); + logging_thread_.Finalize(); } private: @@ -321,16 +318,12 @@ class EventLogger final { std::vector trace_events_ RTC_GUARDED_BY(mutex_); rtc::PlatformThread logging_thread_; rtc::Event shutdown_event_; - rtc::ThreadChecker thread_checker_; + webrtc::SequenceChecker thread_checker_; FILE* output_file_ = nullptr; bool output_file_owned_ = false; }; -static void EventTracingThreadFunc(void* params) { - static_cast(params)->Log(); -} - -static EventLogger* volatile g_event_logger = nullptr; +static std::atomic g_event_logger(nullptr); static const char* const kDisabledTracePrefix = TRACE_DISABLED_BY_DEFAULT(""); const unsigned char* InternalGetCategoryEnabled(const char* name) { const char* prefix_ptr = &kDisabledTracePrefix[0]; @@ -344,6 +337,10 @@ const unsigned char* InternalGetCategoryEnabled(const char* name) { : name); } +const unsigned char* InternalEnableAllCategories(const char* name) { + return reinterpret_cast(name); +} + void InternalAddTraceEvent(char phase, const unsigned char* category_enabled, const char* name, @@ -354,56 +351,59 @@ void InternalAddTraceEvent(char phase, const unsigned long long* arg_values, unsigned char flags) { // Fast path for when event tracing is inactive. - if (rtc::AtomicOps::AcquireLoad(&g_event_logging_active) == 0) + if (g_event_logging_active.load() == 0) return; - g_event_logger->AddTraceEvent(name, category_enabled, phase, num_args, - arg_names, arg_types, arg_values, - rtc::TimeMicros(), 1, rtc::CurrentThreadId()); + g_event_logger.load()->AddTraceEvent( + name, category_enabled, phase, num_args, arg_names, arg_types, arg_values, + rtc::TimeMicros(), 1, rtc::CurrentThreadId()); } } // namespace -void SetupInternalTracer() { - RTC_CHECK(rtc::AtomicOps::CompareAndSwapPtr( - &g_event_logger, static_cast(nullptr), - new EventLogger()) == nullptr); - webrtc::SetupEventTracer(InternalGetCategoryEnabled, InternalAddTraceEvent); +void SetupInternalTracer(bool enable_all_categories) { + EventLogger* null_logger = nullptr; + RTC_CHECK( + g_event_logger.compare_exchange_strong(null_logger, new EventLogger())); + webrtc::SetupEventTracer(enable_all_categories ? InternalEnableAllCategories + : InternalGetCategoryEnabled, + InternalAddTraceEvent); } void StartInternalCaptureToFile(FILE* file) { - if (g_event_logger) { - g_event_logger->Start(file, false); + EventLogger* event_logger = g_event_logger.load(); + if (event_logger) { + event_logger->Start(file, false); } } -bool StartInternalCapture(const char* filename) { - if (!g_event_logger) +bool StartInternalCapture(absl::string_view filename) { + EventLogger* event_logger = g_event_logger.load(); + if (!event_logger) return false; - FILE* file = fopen(filename, "w"); + FILE* file = fopen(std::string(filename).c_str(), "w"); if (!file) { RTC_LOG(LS_ERROR) << "Failed to open trace file '" << filename << "' for writing."; return false; } - g_event_logger->Start(file, true); + event_logger->Start(file, true); return true; } void StopInternalCapture() { - if (g_event_logger) { - g_event_logger->Stop(); + EventLogger* event_logger = g_event_logger.load(); + if (event_logger) { + event_logger->Stop(); } } void ShutdownInternalTracer() { StopInternalCapture(); - EventLogger* old_logger = rtc::AtomicOps::AcquireLoadPtr(&g_event_logger); + EventLogger* old_logger = g_event_logger.load(std::memory_order_acquire); RTC_DCHECK(old_logger); - RTC_CHECK(rtc::AtomicOps::CompareAndSwapPtr( - &g_event_logger, old_logger, - static_cast(nullptr)) == old_logger); + RTC_CHECK(g_event_logger.compare_exchange_strong(old_logger, nullptr)); delete old_logger; webrtc::SetupEventTracer(nullptr, nullptr); } diff --git a/webrtc/rtc_base/event_tracer.h b/webrtc/rtc_base/event_tracer.h index 4bbda57..dc2eaed 100644 --- a/webrtc/rtc_base/event_tracer.h +++ b/webrtc/rtc_base/event_tracer.h @@ -28,6 +28,9 @@ #include +#include "absl/strings/string_view.h" +#include "rtc_base/system/rtc_export.h" + namespace webrtc { typedef const unsigned char* (*GetCategoryEnabledPtr)(const char* name); @@ -70,12 +73,12 @@ class EventTracer { namespace rtc { namespace tracing { // Set up internal event tracer. -void SetupInternalTracer(); -bool StartInternalCapture(const char* filename); -void StartInternalCaptureToFile(FILE* file); -void StopInternalCapture(); +RTC_EXPORT void SetupInternalTracer(bool enable_all_categories = true); +RTC_EXPORT bool StartInternalCapture(absl::string_view filename); +RTC_EXPORT void StartInternalCaptureToFile(FILE* file); +RTC_EXPORT void StopInternalCapture(); // Make sure we run this, this will tear down the internal tracing. -void ShutdownInternalTracer(); +RTC_EXPORT void ShutdownInternalTracer(); } // namespace tracing } // namespace rtc diff --git a/webrtc/rtc_base/experiments/field_trial_parser.cc b/webrtc/rtc_base/experiments/field_trial_parser.cc index b88d0f9..78d5489 100644 --- a/webrtc/rtc_base/experiments/field_trial_parser.cc +++ b/webrtc/rtc_base/experiments/field_trial_parser.cc @@ -16,21 +16,15 @@ #include #include +#include "absl/strings/string_view.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" namespace webrtc { -namespace { -int FindOrEnd(std::string str, size_t start, char delimiter) { - size_t pos = str.find(delimiter, start); - pos = (pos == std::string::npos) ? str.length() : pos; - return static_cast(pos); -} -} // namespace - -FieldTrialParameterInterface::FieldTrialParameterInterface(std::string key) +FieldTrialParameterInterface::FieldTrialParameterInterface( + absl::string_view key) : key_(key) {} FieldTrialParameterInterface::~FieldTrialParameterInterface() { RTC_DCHECK(used_) << "Field trial parameter with key: '" << key_ @@ -39,8 +33,8 @@ FieldTrialParameterInterface::~FieldTrialParameterInterface() { void ParseFieldTrial( std::initializer_list fields, - std::string trial_string) { - std::map field_map; + absl::string_view trial_string) { + std::map field_map; FieldTrialParameterInterface* keyless_field = nullptr; for (FieldTrialParameterInterface* field : fields) { field->MarkAsUsed(); @@ -60,18 +54,29 @@ void ParseFieldTrial( field_map[field->key_] = field; } } + bool logged_unknown_key = false; - size_t i = 0; - while (i < trial_string.length()) { - int val_end = FindOrEnd(trial_string, i, ','); - int colon_pos = FindOrEnd(trial_string, i, ':'); - int key_end = std::min(val_end, colon_pos); - int val_begin = key_end + 1; - std::string key = trial_string.substr(i, key_end - i); + absl::string_view tail = trial_string; + while (!tail.empty()) { + size_t key_end = tail.find_first_of(",:"); + absl::string_view key = tail.substr(0, key_end); absl::optional opt_value; - if (val_end >= val_begin) - opt_value = trial_string.substr(val_begin, val_end - val_begin); - i = val_end + 1; + if (key_end == absl::string_view::npos) { + tail = ""; + } else if (tail[key_end] == ':') { + tail = tail.substr(key_end + 1); + size_t value_end = tail.find(','); + opt_value.emplace(tail.substr(0, value_end)); + if (value_end == absl::string_view::npos) { + tail = ""; + } else { + tail = tail.substr(value_end + 1); + } + } else { + RTC_DCHECK_EQ(tail[key_end], ','); + tail = tail.substr(key_end + 1); + } + auto field = field_map.find(key); if (field != field_map.end()) { if (!field->second->Parse(std::move(opt_value))) { @@ -79,19 +84,25 @@ void ParseFieldTrial( << "' in trial: \"" << trial_string << "\""; } } else if (!opt_value && keyless_field && !key.empty()) { - if (!keyless_field->Parse(key)) { + if (!keyless_field->Parse(std::string(key))) { RTC_LOG(LS_WARNING) << "Failed to read empty key field with value '" << key << "' in trial: \"" << trial_string << "\""; } - } else { - RTC_LOG(LS_INFO) << "No field with key: '" << key - << "' (found in trial: \"" << trial_string << "\")"; - std::string valid_keys; - for (const auto& f : field_map) { - valid_keys += f.first; - valid_keys += ", "; + } else if (key.empty() || key[0] != '_') { + // "_" is be used to prefix keys that are part of the string for + // debugging purposes but not neccessarily used. + // e.g. WebRTC-Experiment/param: value, _DebuggingString + if (!logged_unknown_key) { + RTC_LOG(LS_INFO) << "No field with key: '" << key + << "' (found in trial: \"" << trial_string << "\")"; + std::string valid_keys; + for (const auto& f : field_map) { + valid_keys.append(f.first.data(), f.first.size()); + valid_keys += ", "; + } + RTC_LOG(LS_INFO) << "Valid keys are: " << valid_keys; + logged_unknown_key = true; } - RTC_LOG(LS_INFO) << "Valid keys are: " << valid_keys; } } @@ -101,7 +112,7 @@ void ParseFieldTrial( } template <> -absl::optional ParseTypedParameter(std::string str) { +absl::optional ParseTypedParameter(absl::string_view str) { if (str == "true" || str == "1") { return true; } else if (str == "false" || str == "0") { @@ -111,10 +122,10 @@ absl::optional ParseTypedParameter(std::string str) { } template <> -absl::optional ParseTypedParameter(std::string str) { +absl::optional ParseTypedParameter(absl::string_view str) { double value; char unit[2]{0, 0}; - if (sscanf(str.c_str(), "%lf%1s", &value, unit) >= 1) { + if (sscanf(std::string(str).c_str(), "%lf%1s", &value, unit) >= 1) { if (unit[0] == '%') return value / 100; return value; @@ -124,9 +135,9 @@ absl::optional ParseTypedParameter(std::string str) { } template <> -absl::optional ParseTypedParameter(std::string str) { +absl::optional ParseTypedParameter(absl::string_view str) { int64_t value; - if (sscanf(str.c_str(), "%" SCNd64, &value) == 1) { + if (sscanf(std::string(str).c_str(), "%" SCNd64, &value) == 1) { if (rtc::IsValueInRangeForNumericType(value)) { return static_cast(value); } @@ -135,9 +146,9 @@ absl::optional ParseTypedParameter(std::string str) { } template <> -absl::optional ParseTypedParameter(std::string str) { +absl::optional ParseTypedParameter(absl::string_view str) { int64_t value; - if (sscanf(str.c_str(), "%" SCNd64, &value) == 1) { + if (sscanf(std::string(str).c_str(), "%" SCNd64, &value) == 1) { if (rtc::IsValueInRangeForNumericType(value)) { return static_cast(value); } @@ -146,34 +157,36 @@ absl::optional ParseTypedParameter(std::string str) { } template <> -absl::optional ParseTypedParameter(std::string str) { - return std::move(str); +absl::optional ParseTypedParameter( + absl::string_view str) { + return std::string(str); } template <> absl::optional> ParseTypedParameter>( - std::string str) { + absl::string_view str) { return ParseOptionalParameter(str); } template <> absl::optional> ParseTypedParameter>( - std::string str) { + absl::string_view str) { return ParseOptionalParameter(str); } template <> absl::optional> -ParseTypedParameter>(std::string str) { +ParseTypedParameter>(absl::string_view str) { return ParseOptionalParameter(str); } template <> absl::optional> -ParseTypedParameter>(std::string str) { +ParseTypedParameter>(absl::string_view str) { return ParseOptionalParameter(str); } -FieldTrialFlag::FieldTrialFlag(std::string key) : FieldTrialFlag(key, false) {} +FieldTrialFlag::FieldTrialFlag(absl::string_view key) + : FieldTrialFlag(key, false) {} -FieldTrialFlag::FieldTrialFlag(std::string key, bool default_value) +FieldTrialFlag::FieldTrialFlag(absl::string_view key, bool default_value) : FieldTrialParameterInterface(key), value_(default_value) {} bool FieldTrialFlag::Get() const { @@ -198,7 +211,7 @@ bool FieldTrialFlag::Parse(absl::optional str_value) { } AbstractFieldTrialEnum::AbstractFieldTrialEnum( - std::string key, + absl::string_view key, int default_value, std::map mapping) : FieldTrialParameterInterface(key), diff --git a/webrtc/rtc_base/experiments/field_trial_parser.h b/webrtc/rtc_base/experiments/field_trial_parser.h index 42535ed..822895e 100644 --- a/webrtc/rtc_base/experiments/field_trial_parser.h +++ b/webrtc/rtc_base/experiments/field_trial_parser.h @@ -18,6 +18,7 @@ #include #include +#include "absl/strings/string_view.h" #include "absl/types/optional.h" // Field trial parser functionality. Provides funcitonality to parse field trial @@ -45,10 +46,10 @@ class FieldTrialParameterInterface { FieldTrialParameterInterface(const FieldTrialParameterInterface&) = default; FieldTrialParameterInterface& operator=(const FieldTrialParameterInterface&) = default; - explicit FieldTrialParameterInterface(std::string key); + explicit FieldTrialParameterInterface(absl::string_view key); friend void ParseFieldTrial( std::initializer_list fields, - std::string raw_string); + absl::string_view trial_string); void MarkAsUsed() { used_ = true; } virtual bool Parse(absl::optional str_value) = 0; @@ -65,19 +66,19 @@ class FieldTrialParameterInterface { // with extracted values if available. void ParseFieldTrial( std::initializer_list fields, - std::string raw_string); + absl::string_view trial_string); // Specialize this in code file for custom types. Should return absl::nullopt if // the given string cannot be properly parsed. template -absl::optional ParseTypedParameter(std::string); +absl::optional ParseTypedParameter(absl::string_view); // This class uses the ParseTypedParameter function to implement a parameter // implementation with an enforced default value. template class FieldTrialParameter : public FieldTrialParameterInterface { public: - FieldTrialParameter(std::string key, T default_value) + FieldTrialParameter(absl::string_view key, T default_value) : FieldTrialParameterInterface(key), value_(default_value) {} T Get() const { return value_; } operator T() const { return Get(); } @@ -107,7 +108,7 @@ class FieldTrialParameter : public FieldTrialParameterInterface { template class FieldTrialConstrained : public FieldTrialParameterInterface { public: - FieldTrialConstrained(std::string key, + FieldTrialConstrained(absl::string_view key, T default_value, absl::optional lower_limit, absl::optional upper_limit) @@ -140,7 +141,7 @@ class FieldTrialConstrained : public FieldTrialParameterInterface { class AbstractFieldTrialEnum : public FieldTrialParameterInterface { public: - AbstractFieldTrialEnum(std::string key, + AbstractFieldTrialEnum(absl::string_view key, int default_value, std::map mapping); ~AbstractFieldTrialEnum() override; @@ -161,7 +162,7 @@ class AbstractFieldTrialEnum : public FieldTrialParameterInterface { template class FieldTrialEnum : public AbstractFieldTrialEnum { public: - FieldTrialEnum(std::string key, + FieldTrialEnum(absl::string_view key, T default_value, std::map mapping) : AbstractFieldTrialEnum(key, @@ -184,9 +185,9 @@ class FieldTrialEnum : public AbstractFieldTrialEnum { template class FieldTrialOptional : public FieldTrialParameterInterface { public: - explicit FieldTrialOptional(std::string key) + explicit FieldTrialOptional(absl::string_view key) : FieldTrialParameterInterface(key) {} - FieldTrialOptional(std::string key, absl::optional default_value) + FieldTrialOptional(absl::string_view key, absl::optional default_value) : FieldTrialParameterInterface(key), value_(default_value) {} absl::optional GetOptional() const { return value_; } const T& Value() const { return value_.value(); } @@ -216,10 +217,10 @@ class FieldTrialOptional : public FieldTrialParameterInterface { // explicit value is provided, the flag evaluates to true. class FieldTrialFlag : public FieldTrialParameterInterface { public: - explicit FieldTrialFlag(std::string key); - FieldTrialFlag(std::string key, bool default_value); + explicit FieldTrialFlag(absl::string_view key); + FieldTrialFlag(absl::string_view key, bool default_value); bool Get() const; - operator bool() const; + explicit operator bool() const; protected: bool Parse(absl::optional str_value) override; @@ -229,7 +230,8 @@ class FieldTrialFlag : public FieldTrialParameterInterface { }; template -absl::optional> ParseOptionalParameter(std::string str) { +absl::optional> ParseOptionalParameter( + absl::string_view str) { if (str.empty()) return absl::optional(); auto parsed = ParseTypedParameter(str); @@ -239,28 +241,29 @@ absl::optional> ParseOptionalParameter(std::string str) { } template <> -absl::optional ParseTypedParameter(std::string str); +absl::optional ParseTypedParameter(absl::string_view str); template <> -absl::optional ParseTypedParameter(std::string str); +absl::optional ParseTypedParameter(absl::string_view str); template <> -absl::optional ParseTypedParameter(std::string str); +absl::optional ParseTypedParameter(absl::string_view str); template <> -absl::optional ParseTypedParameter(std::string str); +absl::optional ParseTypedParameter(absl::string_view str); template <> -absl::optional ParseTypedParameter(std::string str); +absl::optional ParseTypedParameter( + absl::string_view str); template <> absl::optional> ParseTypedParameter>( - std::string str); + absl::string_view str); template <> absl::optional> ParseTypedParameter>( - std::string str); + absl::string_view str); template <> absl::optional> -ParseTypedParameter>(std::string str); +ParseTypedParameter>(absl::string_view str); template <> absl::optional> -ParseTypedParameter>(std::string str); +ParseTypedParameter>(absl::string_view str); // Accepts true, false, else parsed with sscanf %i, true if != 0. extern template class FieldTrialParameter; diff --git a/webrtc/rtc_base/logging.cc b/webrtc/rtc_base/logging.cc index fbc6242..61a3c66 100644 --- a/webrtc/rtc_base/logging.cc +++ b/webrtc/rtc_base/logging.cc @@ -15,7 +15,6 @@ #if RTC_LOG_ENABLED() #if defined(WEBRTC_WIN) -#include #include #if _MSC_VER < 1900 #define snprintf _snprintf @@ -43,6 +42,8 @@ static const int kMaxLogLineSize = 1024 - 60; #include #include "absl/base/attributes.h" +#include "absl/strings/string_view.h" +#include "api/units/timestamp.h" #include "rtc_base/checks.h" #include "rtc_base/platform_thread_types.h" #include "rtc_base/string_encode.h" @@ -54,15 +55,18 @@ static const int kMaxLogLineSize = 1024 - 60; namespace rtc { namespace { + // By default, release builds don't log, debug builds at info level #if !defined(NDEBUG) -static LoggingSeverity g_min_sev = LS_INFO; -static LoggingSeverity g_dbg_sev = LS_INFO; +constexpr LoggingSeverity kDefaultLoggingSeverity = LS_INFO; #else -static LoggingSeverity g_min_sev = LS_NONE; -static LoggingSeverity g_dbg_sev = LS_NONE; +constexpr LoggingSeverity kDefaultLoggingSeverity = LS_NONE; #endif +// Note: `g_min_sev` and `g_dbg_sev` can be changed while running. +LoggingSeverity g_min_sev = kDefaultLoggingSeverity; +LoggingSeverity g_dbg_sev = kDefaultLoggingSeverity; + // Return the filename portion of the string (that following the last slash). const char* FilenameFromPath(const char* file) { const char* end1 = ::strrchr(file, '/'); @@ -74,11 +78,38 @@ const char* FilenameFromPath(const char* file) { } // Global lock for log subsystem, only needed to serialize access to streams_. -// TODO(bugs.webrtc.org/11665): this is not currently constant initialized and -// trivially destructible. -webrtc::Mutex g_log_mutex_; +webrtc::Mutex& GetLoggingLock() { + static webrtc::Mutex& mutex = *new webrtc::Mutex(); + return mutex; +} + } // namespace +std::string LogLineRef::DefaultLogLine() const { + rtc::StringBuilder log_output; + if (timestamp_ != webrtc::Timestamp::MinusInfinity()) { + // TODO(kwiberg): Switch to absl::StrFormat, if binary size is ok. + char timestamp[50]; // Maximum string length of an int64_t is 20. + int len = + snprintf(timestamp, sizeof(timestamp), "[%03" PRId64 ":%03" PRId64 "]", + timestamp_.ms() / 1000, timestamp_.ms() % 1000); + RTC_DCHECK_LT(len, sizeof(timestamp)); + log_output << timestamp; + } + if (thread_id_.has_value()) { + log_output << "[" << *thread_id_ << "] "; + } + if (!filename_.empty()) { +#if defined(WEBRTC_ANDROID) + log_output << "(line " << line_ << "): "; +#else + log_output << "(" << filename_ << ":" << line_ << "): "; +#endif + } + log_output << message_; + return log_output.Release(); +} + ///////////////////////////////////////////////////////////////////////////// // LogMessage ///////////////////////////////////////////////////////////////////////////// @@ -89,12 +120,13 @@ bool LogMessage::log_to_stderr_ = true; // Note: we explicitly do not clean this up, because of the uncertain ordering // of destructors at program exit. Let the person who sets the stream trigger // cleanup by setting to null, or let it leak (safe at program exit). -ABSL_CONST_INIT LogSink* LogMessage::streams_ RTC_GUARDED_BY(g_log_mutex_) = +ABSL_CONST_INIT LogSink* LogMessage::streams_ RTC_GUARDED_BY(GetLoggingLock()) = nullptr; ABSL_CONST_INIT std::atomic LogMessage::streams_empty_ = {true}; -// Boolean options default to false (0) -bool LogMessage::thread_, LogMessage::timestamp_; +// Boolean options default to false. +ABSL_CONST_INIT bool LogMessage::log_thread_ = false; +ABSL_CONST_INIT bool LogMessage::log_timestamp_ = false; LogMessage::LogMessage(const char* file, int line, LoggingSeverity sev) : LogMessage(file, line, sev, ERRCTX_NONE, 0) {} @@ -103,35 +135,28 @@ LogMessage::LogMessage(const char* file, int line, LoggingSeverity sev, LogErrorContext err_ctx, - int err) - : severity_(sev) { - if (timestamp_) { + int err) { + log_line_.set_severity(sev); + if (log_timestamp_) { + int64_t log_start_time = LogStartTime(); // Use SystemTimeMillis so that even if tests use fake clocks, the timestamp // in log messages represents the real system time. - int64_t time = TimeDiff(SystemTimeMillis(), LogStartTime()); + int64_t time = TimeDiff(SystemTimeMillis(), log_start_time); // Also ensure WallClockStartTime is initialized, so that it matches // LogStartTime. WallClockStartTime(); - // TODO(kwiberg): Switch to absl::StrFormat, if binary size is ok. - char timestamp[50]; // Maximum string length of an int64_t is 20. - int len = - snprintf(timestamp, sizeof(timestamp), "[%03" PRId64 ":%03" PRId64 "]", - time / 1000, time % 1000); - RTC_DCHECK_LT(len, sizeof(timestamp)); - print_stream_ << timestamp; + log_line_.set_timestamp(webrtc::Timestamp::Millis(time)); } - if (thread_) { - PlatformThreadId id = CurrentThreadId(); - print_stream_ << "[" << id << "] "; + if (log_thread_) { + log_line_.set_thread_id(CurrentThreadId()); } if (file != nullptr) { + log_line_.set_filename(FilenameFromPath(file)); + log_line_.set_line(line); #if defined(WEBRTC_ANDROID) - tag_ = FilenameFromPath(file); - print_stream_ << "(line " << line << "): "; -#else - print_stream_ << "(" << FilenameFromPath(file) << ":" << line << "): "; + log_line_.set_tag(log_line_.filename()); #endif } @@ -172,51 +197,32 @@ LogMessage::LogMessage(const char* file, int line, LoggingSeverity sev, const char* tag) - : LogMessage(file, line, sev, ERRCTX_NONE, 0 /* err */) { - tag_ = tag; + : LogMessage(file, line, sev, ERRCTX_NONE, /*err=*/0) { + log_line_.set_tag(tag); print_stream_ << tag << ": "; } #endif -// DEPRECATED. Currently only used by downstream projects that use -// implementation details of logging.h. Work is ongoing to remove those -// dependencies. -LogMessage::LogMessage(const char* file, - int line, - LoggingSeverity sev, - const std::string& tag) - : LogMessage(file, line, sev) { - print_stream_ << tag << ": "; -} - LogMessage::~LogMessage() { FinishPrintStream(); - const std::string str = print_stream_.Release(); + log_line_.set_message(print_stream_.Release()); - if (severity_ >= g_dbg_sev) { -#if defined(WEBRTC_ANDROID) - OutputToDebug(str, severity_, tag_); -#else - OutputToDebug(str, severity_); -#endif + if (log_line_.severity() >= g_dbg_sev) { + OutputToDebug(log_line_); } - webrtc::MutexLock lock(&g_log_mutex_); + webrtc::MutexLock lock(&GetLoggingLock()); for (LogSink* entry = streams_; entry != nullptr; entry = entry->next_) { - if (severity_ >= entry->min_severity_) { -#if defined(WEBRTC_ANDROID) - entry->OnLogMessage(str, severity_, tag_); -#else - entry->OnLogMessage(str, severity_); -#endif + if (log_line_.severity() >= entry->min_severity_) { + entry->OnLogMessage(log_line_); } } } void LogMessage::AddTag(const char* tag) { #ifdef WEBRTC_ANDROID - tag_ = tag; + log_line_.set_tag(tag); #endif } @@ -242,16 +248,16 @@ uint32_t LogMessage::WallClockStartTime() { } void LogMessage::LogThreads(bool on) { - thread_ = on; + log_thread_ = on; } void LogMessage::LogTimestamps(bool on) { - timestamp_ = on; + log_timestamp_ = on; } void LogMessage::LogToDebug(LoggingSeverity min_sev) { g_dbg_sev = min_sev; - webrtc::MutexLock lock(&g_log_mutex_); + webrtc::MutexLock lock(&GetLoggingLock()); UpdateMinLogSeverity(); } @@ -260,7 +266,7 @@ void LogMessage::SetLogToStderr(bool log_to_stderr) { } int LogMessage::GetLogToStream(LogSink* stream) { - webrtc::MutexLock lock(&g_log_mutex_); + webrtc::MutexLock lock(&GetLoggingLock()); LoggingSeverity sev = LS_NONE; for (LogSink* entry = streams_; entry != nullptr; entry = entry->next_) { if (stream == nullptr || stream == entry) { @@ -271,7 +277,7 @@ int LogMessage::GetLogToStream(LogSink* stream) { } void LogMessage::AddLogToStream(LogSink* stream, LoggingSeverity min_sev) { - webrtc::MutexLock lock(&g_log_mutex_); + webrtc::MutexLock lock(&GetLoggingLock()); stream->min_severity_ = min_sev; stream->next_ = streams_; streams_ = stream; @@ -280,7 +286,7 @@ void LogMessage::AddLogToStream(LogSink* stream, LoggingSeverity min_sev) { } void LogMessage::RemoveLogToStream(LogSink* stream) { - webrtc::MutexLock lock(&g_log_mutex_); + webrtc::MutexLock lock(&GetLoggingLock()); for (LogSink** entry = &streams_; *entry != nullptr; entry = &(*entry)->next_) { if (*entry == stream) { @@ -292,7 +298,7 @@ void LogMessage::RemoveLogToStream(LogSink* stream) { UpdateMinLogSeverity(); } -void LogMessage::ConfigureLogging(const char* params) { +void LogMessage::ConfigureLogging(absl::string_view params) { LoggingSeverity current_level = LS_VERBOSE; LoggingSeverity debug_level = GetLogToDebug(); @@ -342,7 +348,7 @@ void LogMessage::ConfigureLogging(const char* params) { } void LogMessage::UpdateMinLogSeverity() - RTC_EXCLUSIVE_LOCKS_REQUIRED(g_log_mutex_) { + RTC_EXCLUSIVE_LOCKS_REQUIRED(GetLoggingLock()) { LoggingSeverity min_sev = g_dbg_sev; for (LogSink* entry = streams_; entry != nullptr; entry = entry->next_) { min_sev = std::min(min_sev, entry->min_severity_); @@ -350,46 +356,35 @@ void LogMessage::UpdateMinLogSeverity() g_min_sev = min_sev; } -#if defined(WEBRTC_ANDROID) -void LogMessage::OutputToDebug(const std::string& str, - LoggingSeverity severity, - const char* tag) { -#else -void LogMessage::OutputToDebug(const std::string& str, - LoggingSeverity severity) { -#endif +void LogMessage::OutputToDebug(const LogLineRef& log_line) { + std::string msg_str = log_line.DefaultLogLine(); bool log_to_stderr = log_to_stderr_; #if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && defined(NDEBUG) // On the Mac, all stderr output goes to the Console log and causes clutter. // So in opt builds, don't log to stderr unless the user specifically sets // a preference to do so. - CFStringRef key = CFStringCreateWithCString( - kCFAllocatorDefault, "logToStdErr", kCFStringEncodingUTF8); CFStringRef domain = CFBundleGetIdentifier(CFBundleGetMainBundle()); - if (key != nullptr && domain != nullptr) { + if (domain != nullptr) { Boolean exists_and_is_valid; - Boolean should_log = - CFPreferencesGetAppBooleanValue(key, domain, &exists_and_is_valid); + Boolean should_log = CFPreferencesGetAppBooleanValue( + CFSTR("logToStdErr"), domain, &exists_and_is_valid); // If the key doesn't exist or is invalid or is false, we will not log to // stderr. log_to_stderr = exists_and_is_valid && should_log; } - if (key != nullptr) { - CFRelease(key); - } #endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && defined(NDEBUG) #if defined(WEBRTC_WIN) // Always log to the debugger. // Perhaps stderr should be controlled by a preference, as on Mac? - OutputDebugStringA(str.c_str()); + OutputDebugStringA(msg_str.c_str()); if (log_to_stderr) { // This handles dynamically allocated consoles, too. if (HANDLE error_handle = ::GetStdHandle(STD_ERROR_HANDLE)) { log_to_stderr = false; DWORD written = 0; - ::WriteFile(error_handle, str.data(), static_cast(str.size()), - &written, 0); + ::WriteFile(error_handle, msg_str.c_str(), + static_cast(msg_str.size()), &written, 0); } } #endif // WEBRTC_WIN @@ -400,7 +395,7 @@ void LogMessage::OutputToDebug(const std::string& str, // Also write to stderr which maybe available to executable started // from the shell. int prio; - switch (severity) { + switch (log_line.severity()) { case LS_VERBOSE: prio = ANDROID_LOG_VERBOSE; break; @@ -417,27 +412,29 @@ void LogMessage::OutputToDebug(const std::string& str, prio = ANDROID_LOG_UNKNOWN; } - int size = str.size(); - int line = 0; + int size = msg_str.size(); + int current_line = 0; int idx = 0; const int max_lines = size / kMaxLogLineSize + 1; if (max_lines == 1) { - __android_log_print(prio, tag, "%.*s", size, str.c_str()); + __android_log_print(prio, log_line.tag().data(), "%.*s", size, + msg_str.c_str()); } else { while (size > 0) { const int len = std::min(size, kMaxLogLineSize); - // Use the size of the string in the format (str may have \0 in the + // Use the size of the string in the format (msg may have \0 in the // middle). - __android_log_print(prio, tag, "[%d/%d] %.*s", line + 1, max_lines, len, - str.c_str() + idx); + __android_log_print(prio, log_line.tag().data(), "[%d/%d] %.*s", + current_line + 1, max_lines, len, + msg_str.c_str() + idx); idx += len; size -= len; - ++line; + ++current_line; } } #endif // WEBRTC_ANDROID if (log_to_stderr) { - fprintf(stderr, "%s", str.c_str()); + fprintf(stderr, "%s", msg_str.c_str()); fflush(stderr); } } @@ -481,7 +478,7 @@ void Log(const LogArgType* fmt, ...) { } #endif default: { - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); va_end(args); return; } @@ -535,7 +532,7 @@ void Log(const LogArgType* fmt, ...) { reinterpret_cast(va_arg(args, const void*))); break; default: - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); va_end(args); return; } @@ -549,6 +546,16 @@ void Log(const LogArgType* fmt, ...) { #endif namespace rtc { +// Default implementation, override is recomended. +void LogSink::OnLogMessage(const LogLineRef& log_line) { +#if defined(WEBRTC_ANDROID) + OnLogMessage(log_line.DefaultLogLine(), log_line.severity(), + log_line.tag().data()); +#else + OnLogMessage(log_line.DefaultLogLine(), log_line.severity()); +#endif +} + // Inefficient default implementation, override is recommended. void LogSink::OnLogMessage(const std::string& msg, LoggingSeverity severity, @@ -560,4 +567,20 @@ void LogSink::OnLogMessage(const std::string& msg, LoggingSeverity /* severity */) { OnLogMessage(msg); } + +// Inefficient default implementation, override is recommended. +void LogSink::OnLogMessage(absl::string_view msg, + LoggingSeverity severity, + const char* tag) { + OnLogMessage(tag + (": " + std::string(msg)), severity); +} + +void LogSink::OnLogMessage(absl::string_view msg, + LoggingSeverity /* severity */) { + OnLogMessage(msg); +} + +void LogSink::OnLogMessage(absl::string_view msg) { + OnLogMessage(std::string(msg)); +} } // namespace rtc diff --git a/webrtc/rtc_base/logging.h b/webrtc/rtc_base/logging.h index d2607c2..b171cfe 100644 --- a/webrtc/rtc_base/logging.h +++ b/webrtc/rtc_base/logging.h @@ -21,9 +21,13 @@ // RTC_LOG(sev) logs the given stream at severity "sev", which must be a // compile-time constant of the LoggingSeverity type, without the namespace // prefix. +// RTC_LOG_IF(sev, condition) logs the given stream at severitye "sev" if +// "condition" is true. // RTC_LOG_V(sev) Like RTC_LOG(), but sev is a run-time variable of the // LoggingSeverity type (basically, it just doesn't prepend the namespace). // RTC_LOG_F(sev) Like RTC_LOG(), but includes the name of the current function. +// RTC_LOG_IF_F(sev, condition), Like RTC_LOG_IF(), but includes the name of +// the current function. // RTC_LOG_T(sev) Like RTC_LOG(), but includes the this pointer. // RTC_LOG_T_F(sev) Like RTC_LOG_F(), but includes the this pointer. // RTC_LOG_GLE(sev [, mod]) attempt to add a string description of the @@ -49,12 +53,15 @@ #include #include // no-presubmit-check TODO(webrtc:8982) #include +#include #include +#include "absl/base/attributes.h" #include "absl/meta/type_traits.h" #include "absl/strings/string_view.h" -#include "rtc_base/constructor_magic.h" -#include "rtc_base/deprecation.h" +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "rtc_base/platform_thread_types.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/system/inline.h" @@ -73,9 +80,7 @@ namespace rtc { ////////////////////////////////////////////////////////////////////// - -// Note that the non-standard LoggingSeverity aliases exist because they are -// still in broad use. The meanings of the levels are: +// The meanings of the levels are: // LS_VERBOSE: This level is for data which we do not want to appear in the // normal debug log, but should appear in diagnostic logs. // LS_INFO: Chatty level used in debugging for all sorts of things, the default @@ -89,9 +94,6 @@ enum LoggingSeverity { LS_WARNING, LS_ERROR, LS_NONE, - INFO = LS_INFO, - WARNING = LS_WARNING, - LERROR = LS_ERROR }; // LogErrorContext assists in interpreting the meaning of an error value. @@ -106,6 +108,50 @@ enum LogErrorContext { }; class LogMessage; + +// LogLineRef encapsulates all the information required to generate a log line. +// It is used both internally to LogMessage but also as a parameter to +// LogSink::OnLogMessage, allowing custom LogSinks to format the log in +// the most flexible way. +class LogLineRef { + public: + absl::string_view message() const { return message_; } + absl::string_view filename() const { return filename_; } + int line() const { return line_; } + absl::optional thread_id() const { return thread_id_; } + webrtc::Timestamp timestamp() const { return timestamp_; } + absl::string_view tag() const { return tag_; } + LoggingSeverity severity() const { return severity_; } + +#if RTC_LOG_ENABLED() + std::string DefaultLogLine() const; +#else + std::string DefaultLogLine() const { return ""; } +#endif + + private: + friend class LogMessage; + void set_message(std::string message) { message_ = std::move(message); } + void set_filename(absl::string_view filename) { filename_ = filename; } + void set_line(int line) { line_ = line; } + void set_thread_id(absl::optional thread_id) { + thread_id_ = thread_id; + } + void set_timestamp(webrtc::Timestamp timestamp) { timestamp_ = timestamp; } + void set_tag(absl::string_view tag) { tag_ = tag; } + void set_severity(LoggingSeverity severity) { severity_ = severity; } + + std::string message_; + absl::string_view filename_; + int line_ = 0; + absl::optional thread_id_; + webrtc::Timestamp timestamp_ = webrtc::Timestamp::MinusInfinity(); + // The default Android debug output tag. + absl::string_view tag_ = "libjingle"; + // The severity level of this message + LoggingSeverity severity_; +}; + // Virtual sink interface that can receive log messages. class LogSink { public: @@ -118,6 +164,14 @@ class LogSink { LoggingSeverity severity); virtual void OnLogMessage(const std::string& message) = 0; + virtual void OnLogMessage(absl::string_view msg, + LoggingSeverity severity, + const char* tag); + virtual void OnLogMessage(absl::string_view message, + LoggingSeverity severity); + virtual void OnLogMessage(absl::string_view message); + virtual void OnLogMessage(const LogLineRef& line); + private: friend class ::rtc::LogMessage; #if RTC_LOG_ENABLED() @@ -276,8 +330,15 @@ inline Val MakeVal( template struct has_to_log_string : std::false_type {}; template -struct has_to_log_string()))> - : std::true_type {}; +struct has_to_log_string())), + std::string>::value>> : std::true_type {}; + +template ::value>* = nullptr> +ToStringVal MakeVal(const T& x) { + return {ToLogString(x)}; +} // Handle arbitrary types other than the above by falling back to stringstream. // TODO(bugs.webrtc.org/9278): Get rid of this overload when callers don't need @@ -299,11 +360,6 @@ ToStringVal MakeVal(const T& x) { return {os.str()}; } -template ::value>* = nullptr> -ToStringVal MakeVal(const T& x) { - return {ToLogString(x)}; -} - #if RTC_LOG_ENABLED() void Log(const LogArgType* fmt, ...); #else @@ -431,16 +487,11 @@ class LogMessage { #if defined(WEBRTC_ANDROID) LogMessage(const char* file, int line, LoggingSeverity sev, const char* tag); #endif - // DEPRECATED - DO NOT USE - PLEASE USE THE MACROS INSTEAD OF THE CLASS. - // Android code should use the 'const char*' version since tags are static - // and we want to avoid allocating a std::string copy per log line. - RTC_DEPRECATED - LogMessage(const char* file, - int line, - LoggingSeverity sev, - const std::string& tag); ~LogMessage(); + LogMessage(const LogMessage&) = delete; + LogMessage& operator=(const LogMessage&) = delete; + void AddTag(const char* tag); rtc::StringBuilder& stream(); // Returns the time at which this function was called for the first time. @@ -449,7 +500,7 @@ class LogMessage { // which case the logging start time will be the time of the first LogMessage // instance is created. static int64_t LogStartTime(); - // Returns the wall clock equivalent of |LogStartTime|, in seconds from the + // Returns the wall clock equivalent of `LogStartTime`, in seconds from the // epoch. static uint32_t WallClockStartTime(); // LogThreads: Display the thread identifier of the current thread @@ -463,14 +514,14 @@ class LogMessage { // Sets whether logs will be directed to stderr in debug mode. static void SetLogToStderr(bool log_to_stderr); // Stream: Any non-blocking stream interface. - // Installs the |stream| to collect logs with severtiy |min_sev| or higher. - // |stream| must live until deinstalled by RemoveLogToStream. - // If |stream| is the first stream added to the system, we might miss some + // Installs the `stream` to collect logs with severtiy `min_sev` or higher. + // `stream` must live until deinstalled by RemoveLogToStream. + // If `stream` is the first stream added to the system, we might miss some // early concurrent log statement happening from another thread happening near // this instant. static void AddLogToStream(LogSink* stream, LoggingSeverity min_sev); // Removes the specified stream, without destroying it. When the method - // has completed, it's guaranteed that |stream| will receive no more logging + // has completed, it's guaranteed that `stream` will receive no more logging // calls. static void RemoveLogToStream(LogSink* stream); // Returns the severity for the specified stream, of if none is specified, @@ -481,10 +532,10 @@ class LogMessage { static int GetMinLogSeverity(); // Parses the provided parameter stream to configure the options above. // Useful for configuring logging from the command line. - static void ConfigureLogging(const char* params); - // Checks the current global debug severity and if the |streams_| collection - // is empty. If |severity| is smaller than the global severity and if the - // |streams_| collection is empty, the LogMessage will be considered a noop + static void ConfigureLogging(absl::string_view params); + // Checks the current global debug severity and if the `streams_` collection + // is empty. If `severity` is smaller than the global severity and if the + // `streams_` collection is empty, the LogMessage will be considered a noop // LogMessage. static bool IsNoop(LoggingSeverity severity); // Version of IsNoop that uses fewer instructions at the call site, since the @@ -505,14 +556,6 @@ class LogMessage { LogMessage(const char* file, int line, LoggingSeverity sev, const char* tag) { } #endif - // DEPRECATED - DO NOT USE - PLEASE USE THE MACROS INSTEAD OF THE CLASS. - // Android code should use the 'const char*' version since tags are static - // and we want to avoid allocating a std::string copy per log line. - RTC_DEPRECATED - LogMessage(const char* file, - int line, - LoggingSeverity sev, - const std::string& tag) {} ~LogMessage() = default; inline void AddTag(const char* tag) {} @@ -530,7 +573,7 @@ class LogMessage { inline static void RemoveLogToStream(LogSink* stream) {} inline static int GetLogToStream(LogSink* stream = nullptr) { return 0; } inline static int GetMinLogSeverity() { return 0; } - inline static void ConfigureLogging(const char* params) {} + inline static void ConfigureLogging(absl::string_view params) {} static constexpr bool IsNoop(LoggingSeverity severity) { return true; } template static constexpr bool IsNoop() { @@ -545,26 +588,14 @@ class LogMessage { // Updates min_sev_ appropriately when debug sinks change. static void UpdateMinLogSeverity(); -// These write out the actual log messages. -#if defined(WEBRTC_ANDROID) - static void OutputToDebug(const std::string& msg, - LoggingSeverity severity, - const char* tag); -#else - static void OutputToDebug(const std::string& msg, LoggingSeverity severity); -#endif // defined(WEBRTC_ANDROID) + // This writes out the actual log messages. + static void OutputToDebug(const LogLineRef& log_line_ref); // Called from the dtor (or from a test) to append optional extra error // information to the log stream and a newline character. void FinishPrintStream(); - // The severity level of this message - LoggingSeverity severity_; - -#if defined(WEBRTC_ANDROID) - // The default Android debug output tag. - const char* tag_ = "libjingle"; -#endif + LogLineRef log_line_; // String data generated in the constructor, that should be appended to // the message before output. @@ -573,14 +604,15 @@ class LogMessage { // The output streams and their associated severities static LogSink* streams_; - // Holds true with high probability if |streams_| is empty, false with high + // Holds true with high probability if `streams_` is empty, false with high // probability otherwise. Operated on with std::memory_order_relaxed because // it's ok to lose or log some additional statements near the instant streams // are added/removed. static std::atomic streams_empty_; - // Flags for formatting options - static bool thread_, timestamp_; + // Flags for formatting options and their potential values. + static bool log_thread_; + static bool log_timestamp_; // Determines if logs will be directed to stderr in debug mode. static bool log_to_stderr_; @@ -588,11 +620,15 @@ class LogMessage { // Next methods do nothing; no one will call these functions. inline static void UpdateMinLogSeverity() {} #if defined(WEBRTC_ANDROID) - inline static void OutputToDebug(const std::string& msg, + inline static void OutputToDebug(absl::string_view filename, + int line, + absl::string_view msg, LoggingSeverity severity, const char* tag) {} #else - inline static void OutputToDebug(const std::string& msg, + inline static void OutputToDebug(absl::string_view filename, + int line, + absl::string_view msg, LoggingSeverity severity) {} #endif // defined(WEBRTC_ANDROID) inline void FinishPrintStream() {} @@ -600,8 +636,6 @@ class LogMessage { // The stringbuilder that buffers the formatted message before output rtc::StringBuilder print_stream_; - - RTC_DISALLOW_COPY_AND_ASSIGN(LogMessage); }; ////////////////////////////////////////////////////////////////////// @@ -613,21 +647,29 @@ class LogMessage { ::rtc::webrtc_logging_impl::LogStreamer<>() \ << ::rtc::webrtc_logging_impl::LogMetadata(file, line, sev) -#define RTC_LOG(sev) \ - !rtc::LogMessage::IsNoop<::rtc::sev>() && \ +#define RTC_LOG(sev) \ + !::rtc::LogMessage::IsNoop<::rtc::sev>() && \ + RTC_LOG_FILE_LINE(::rtc::sev, __FILE__, __LINE__) + +#define RTC_LOG_IF(sev, condition) \ + !::rtc::LogMessage::IsNoop<::rtc::sev>() && (condition) && \ RTC_LOG_FILE_LINE(::rtc::sev, __FILE__, __LINE__) // The _V version is for when a variable is passed in. #define RTC_LOG_V(sev) \ - !rtc::LogMessage::IsNoop(sev) && RTC_LOG_FILE_LINE(sev, __FILE__, __LINE__) + !::rtc::LogMessage::IsNoop(sev) && RTC_LOG_FILE_LINE(sev, __FILE__, __LINE__) // The _F version prefixes the message with the current function name. #if (defined(__GNUC__) && !defined(NDEBUG)) || defined(WANT_PRETTY_LOG_F) #define RTC_LOG_F(sev) RTC_LOG(sev) << __PRETTY_FUNCTION__ << ": " +#define RTC_LOG_IF_F(sev, condition) \ + RTC_LOG_IF(sev, condition) << __PRETTY_FUNCTION__ << ": " #define RTC_LOG_T_F(sev) \ RTC_LOG(sev) << this << ": " << __PRETTY_FUNCTION__ << ": " #else #define RTC_LOG_F(sev) RTC_LOG(sev) << __FUNCTION__ << ": " +#define RTC_LOG_IF_F(sev, condition) \ + RTC_LOG_IF(sev, condition) << __FUNCTION__ << ": " #define RTC_LOG_T_F(sev) RTC_LOG(sev) << this << ": " << __FUNCTION__ << ": " #endif @@ -639,7 +681,7 @@ inline bool LogCheckLevel(LoggingSeverity sev) { } #define RTC_LOG_E(sev, ctx, err) \ - !rtc::LogMessage::IsNoop<::rtc::sev>() && \ + !::rtc::LogMessage::IsNoop<::rtc::sev>() && \ ::rtc::webrtc_logging_impl::LogCall() & \ ::rtc::webrtc_logging_impl::LogStreamer<>() \ << ::rtc::webrtc_logging_impl::LogMetadataErr { \ @@ -677,7 +719,7 @@ inline const char* AdaptString(const std::string& str) { } // namespace webrtc_logging_impl #define RTC_LOG_TAG(sev, tag) \ - !rtc::LogMessage::IsNoop(sev) && \ + !::rtc::LogMessage::IsNoop(sev) && \ ::rtc::webrtc_logging_impl::LogCall() & \ ::rtc::webrtc_logging_impl::LogStreamer<>() \ << ::rtc::webrtc_logging_impl::LogMetadataTag { \ @@ -695,16 +737,20 @@ inline const char* AdaptString(const std::string& str) { // they only generate code in debug builds. #if RTC_DLOG_IS_ON #define RTC_DLOG(sev) RTC_LOG(sev) +#define RTC_DLOG_IF(sev, condition) RTC_LOG_IF(sev, condition) #define RTC_DLOG_V(sev) RTC_LOG_V(sev) #define RTC_DLOG_F(sev) RTC_LOG_F(sev) +#define RTC_DLOG_IF_F(sev, condition) RTC_LOG_IF_F(sev, condition) #else #define RTC_DLOG_EAT_STREAM_PARAMS() \ while (false) \ ::rtc::webrtc_logging_impl::LogMessageVoidify() & \ (::rtc::webrtc_logging_impl::LogStreamer<>()) #define RTC_DLOG(sev) RTC_DLOG_EAT_STREAM_PARAMS() +#define RTC_DLOG_IF(sev, condition) RTC_DLOG_EAT_STREAM_PARAMS() #define RTC_DLOG_V(sev) RTC_DLOG_EAT_STREAM_PARAMS() #define RTC_DLOG_F(sev) RTC_DLOG_EAT_STREAM_PARAMS() +#define RTC_DLOG_IF_F(sev, condition) RTC_DLOG_EAT_STREAM_PARAMS() #endif } // namespace rtc diff --git a/webrtc/rtc_base/memory/BUILD.gn b/webrtc/rtc_base/memory/BUILD.gn index 838fbc6..ee3baa4 100644 --- a/webrtc/rtc_base/memory/BUILD.gn +++ b/webrtc/rtc_base/memory/BUILD.gn @@ -21,24 +21,22 @@ rtc_library("aligned_malloc") { } # Test only utility. -# TODO: Tag with `testonly = true` once all depending targets are correctly -# tagged. rtc_library("fifo_buffer") { + testonly = true visibility = [ ":unittests", "..:rtc_base_tests_utils", "..:rtc_base_unittests", - "../../p2p:rtc_p2p", # This needs to be fixed. ] sources = [ "fifo_buffer.cc", "fifo_buffer.h", ] deps = [ - "..:rtc_base", + "..:stream", + "..:threading", + "../../api/task_queue:pending_task_safety_flag", "../synchronization:mutex", - "../task_utils:pending_task_safety_flag", - "../task_utils:to_queued_task", ] } @@ -46,11 +44,18 @@ rtc_library("unittests") { testonly = true sources = [ "aligned_malloc_unittest.cc", + "always_valid_pointer_unittest.cc", "fifo_buffer_unittest.cc", ] deps = [ ":aligned_malloc", + ":always_valid_pointer", ":fifo_buffer", "../../test:test_support", ] } + +rtc_source_set("always_valid_pointer") { + sources = [ "always_valid_pointer.h" ] + deps = [ "..:checks" ] +} diff --git a/webrtc/rtc_base/memory/aligned_malloc.cc b/webrtc/rtc_base/memory/aligned_malloc.cc index b00fab2..7add079 100644 --- a/webrtc/rtc_base/memory/aligned_malloc.cc +++ b/webrtc/rtc_base/memory/aligned_malloc.cc @@ -26,7 +26,7 @@ namespace webrtc { uintptr_t GetRightAlign(uintptr_t start_pos, size_t alignment) { - // The pointer should be aligned with |alignment| bytes. The - 1 guarantees + // The pointer should be aligned with `alignment` bytes. The - 1 guarantees // that it is aligned towards the closest higher (right) address. return (start_pos + alignment - 1) & ~(alignment - 1); } diff --git a/webrtc/rtc_base/memory/aligned_malloc.h b/webrtc/rtc_base/memory/aligned_malloc.h index 42a6daa..1c7d303 100644 --- a/webrtc/rtc_base/memory/aligned_malloc.h +++ b/webrtc/rtc_base/memory/aligned_malloc.h @@ -21,13 +21,13 @@ namespace webrtc { -// Returns a pointer to the first boundry of |alignment| bytes following the -// address of |ptr|. +// Returns a pointer to the first boundry of `alignment` bytes following the +// address of `ptr`. // Note that there is no guarantee that the memory in question is available. -// |ptr| has no requirements other than it can't be NULL. +// `ptr` has no requirements other than it can't be NULL. void* GetRightAlign(const void* ptr, size_t alignment); -// Allocates memory of |size| bytes aligned on an |alignment| boundry. +// Allocates memory of `size` bytes aligned on an `alignment` boundry. // The return value is a pointer to the memory. Note that the memory must // be de-allocated using AlignedFree. void* AlignedMalloc(size_t size, size_t alignment); diff --git a/webrtc/rtc_base/meson.build b/webrtc/rtc_base/meson.build index 43b1054..71215d2 100644 --- a/webrtc/rtc_base/meson.build +++ b/webrtc/rtc_base/meson.build @@ -1,5 +1,6 @@ base_sources = [ 'checks.cc', + 'containers/flat_tree.cc', 'event.cc', 'event_tracer.cc', 'experiments/field_trial_parser.cc', @@ -8,15 +9,16 @@ base_sources = [ 'platform_thread.cc', 'platform_thread_types.cc', 'race_checker.cc', + 'random.cc', 'string_encode.cc', 'string_to_number.cc', 'string_utils.cc', 'strings/string_builder.cc', - 'synchronization/mutex.cc', - 'synchronization/rw_lock_wrapper.cc', + 'synchronization/sequence_checker_internal.cc', 'synchronization/yield.cc', 'synchronization/yield_policy.cc', 'system/file_wrapper.cc', + 'system_time.cc', 'time_utils.cc', 'zero_memory.cc', ] @@ -24,8 +26,6 @@ base_sources = [ base_headers = [ [ '', 'arraysize.h' ], [ '', 'checks.h' ], - [ '', 'constructor_magic.h' ], - [ '', 'deprecation.h' ], [ '', 'ref_count.h' ], [ '', 'type_traits.h' ], [ 'numerics', 'safe_compare.h' ], @@ -34,16 +34,6 @@ base_headers = [ [ 'system', 'rtc_export.h' ], ] -if have_posix - base_sources += [ - 'synchronization/rw_lock_posix.cc', - ] -elif have_win - base_sources += [ - 'synchronization/rw_lock_win.cc', - ] -endif - foreach h : base_headers install_headers( join_paths(h[0], h[1]), @@ -64,6 +54,10 @@ elif host_system == 'ios' 'UIKit', ] ) +elif host_system == 'android' + base_sources += [ + 'system/warn_current_thread_is_deadlocked.cc', + ] endif libbase = static_library('libbase', diff --git a/webrtc/rtc_base/numerics/divide_round.h b/webrtc/rtc_base/numerics/divide_round.h new file mode 100644 index 0000000..90c67fc --- /dev/null +++ b/webrtc/rtc_base/numerics/divide_round.h @@ -0,0 +1,60 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_NUMERICS_DIVIDE_ROUND_H_ +#define RTC_BASE_NUMERICS_DIVIDE_ROUND_H_ + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_compare.h" + +namespace webrtc { + +template +inline auto constexpr DivideRoundUp(Dividend dividend, Divisor divisor) { + static_assert(std::is_integral(), ""); + static_assert(std::is_integral(), ""); + RTC_DCHECK_GE(dividend, 0); + RTC_DCHECK_GT(divisor, 0); + + auto quotient = dividend / divisor; + auto remainder = dividend % divisor; + return quotient + (remainder > 0 ? 1 : 0); +} + +template +inline auto constexpr DivideRoundToNearest(Dividend dividend, Divisor divisor) { + static_assert(std::is_integral(), ""); + static_assert(std::is_integral(), ""); + RTC_DCHECK_GT(divisor, 0); + + if (dividend < Dividend{0}) { + auto half_of_divisor = divisor / 2; + auto quotient = dividend / divisor; + auto remainder = dividend % divisor; + if (rtc::SafeGt(-remainder, half_of_divisor)) { + --quotient; + } + return quotient; + } + + auto half_of_divisor = (divisor - 1) / 2; + auto quotient = dividend / divisor; + auto remainder = dividend % divisor; + if (rtc::SafeGt(remainder, half_of_divisor)) { + ++quotient; + } + return quotient; +} + +} // namespace webrtc + +#endif // RTC_BASE_NUMERICS_DIVIDE_ROUND_H_ diff --git a/webrtc/rtc_base/numerics/safe_conversions.h b/webrtc/rtc_base/numerics/safe_conversions.h index 5d58672..e00219c 100644 --- a/webrtc/rtc_base/numerics/safe_conversions.h +++ b/webrtc/rtc_base/numerics/safe_conversions.h @@ -63,12 +63,10 @@ inline constexpr Dst saturated_cast(Src value) { // Should fail only on attempting to assign NaN to a saturated integer. case internal::TYPE_INVALID: - FATAL(); - return std::numeric_limits::max(); + RTC_CHECK_NOTREACHED(); } - FATAL(); - return static_cast(value); + RTC_CHECK_NOTREACHED(); } } // namespace rtc diff --git a/webrtc/rtc_base/numerics/safe_minmax.h b/webrtc/rtc_base/numerics/safe_minmax.h index 6c41dfd..8356536 100644 --- a/webrtc/rtc_base/numerics/safe_minmax.h +++ b/webrtc/rtc_base/numerics/safe_minmax.h @@ -325,9 +325,9 @@ R2 SafeClamp(T x, L min, H max) { static_assert(IsIntlike::value || std::is_floating_point::value, "The third argument must be integral or floating-point"); RTC_DCHECK_LE(min, max); - return SafeLe(x, min) - ? static_cast(min) - : SafeGe(x, max) ? static_cast(max) : static_cast(x); + return SafeLe(x, min) ? static_cast(min) + : SafeGe(x, max) ? static_cast(max) + : static_cast(x); } } // namespace rtc diff --git a/webrtc/rtc_base/platform_thread.cc b/webrtc/rtc_base/platform_thread.cc index 8a5f2c9..6d369d7 100644 --- a/webrtc/rtc_base/platform_thread.cc +++ b/webrtc/rtc_base/platform_thread.cc @@ -10,131 +10,37 @@ #include "rtc_base/platform_thread.h" +#include +#include + #if !defined(WEBRTC_WIN) #include #endif -#include -#include - -#include #include "rtc_base/checks.h" namespace rtc { namespace { -#if !defined(WEBRTC_WIN) -struct ThreadAttributes { - ThreadAttributes() { pthread_attr_init(&attr); } - ~ThreadAttributes() { pthread_attr_destroy(&attr); } - pthread_attr_t* operator&() { return &attr; } - pthread_attr_t attr; -}; -#endif // defined(WEBRTC_WIN) -} // namespace - -PlatformThread::PlatformThread(ThreadRunFunction func, - void* obj, - absl::string_view thread_name, - ThreadPriority priority /*= kNormalPriority*/) - : run_function_(func), priority_(priority), obj_(obj), name_(thread_name) { - RTC_DCHECK(func); - RTC_DCHECK(!name_.empty()); - // TODO(tommi): Consider lowering the limit to 15 (limit on Linux). - RTC_DCHECK(name_.length() < 64); - spawned_thread_checker_.Detach(); -} - -PlatformThread::~PlatformThread() { - RTC_DCHECK(thread_checker_.IsCurrent()); -#if defined(WEBRTC_WIN) - RTC_DCHECK(!thread_); - RTC_DCHECK(!thread_id_); -#endif // defined(WEBRTC_WIN) -} #if defined(WEBRTC_WIN) -DWORD WINAPI PlatformThread::StartThread(void* param) { - // The GetLastError() function only returns valid results when it is called - // after a Win32 API function that returns a "failed" result. A crash dump - // contains the result from GetLastError() and to make sure it does not - // falsely report a Windows error we call SetLastError here. - ::SetLastError(ERROR_SUCCESS); - static_cast(param)->Run(); - return 0; +int Win32PriorityFromThreadPriority(ThreadPriority priority) { + switch (priority) { + case ThreadPriority::kLow: + return THREAD_PRIORITY_BELOW_NORMAL; + case ThreadPriority::kNormal: + return THREAD_PRIORITY_NORMAL; + case ThreadPriority::kHigh: + return THREAD_PRIORITY_ABOVE_NORMAL; + case ThreadPriority::kRealtime: + return THREAD_PRIORITY_TIME_CRITICAL; + } } -#else -void* PlatformThread::StartThread(void* param) { - static_cast(param)->Run(); - return 0; -} -#endif // defined(WEBRTC_WIN) +#endif -void PlatformThread::Start() { - RTC_DCHECK(thread_checker_.IsCurrent()); - RTC_DCHECK(!thread_) << "Thread already started?"; +bool SetPriority(ThreadPriority priority) { #if defined(WEBRTC_WIN) - // See bug 2902 for background on STACK_SIZE_PARAM_IS_A_RESERVATION. - // Set the reserved stack stack size to 1M, which is the default on Windows - // and Linux. - thread_ = ::CreateThread(nullptr, 1024 * 1024, &StartThread, this, - STACK_SIZE_PARAM_IS_A_RESERVATION, &thread_id_); - RTC_CHECK(thread_) << "CreateThread failed"; - RTC_DCHECK(thread_id_); -#else - ThreadAttributes attr; - // Set the stack stack size to 1M. - pthread_attr_setstacksize(&attr, 1024 * 1024); - RTC_CHECK_EQ(0, pthread_create(&thread_, &attr, &StartThread, this)); -#endif // defined(WEBRTC_WIN) -} - -bool PlatformThread::IsRunning() const { - RTC_DCHECK(thread_checker_.IsCurrent()); -#if defined(WEBRTC_WIN) - return thread_ != nullptr; -#else - return thread_ != 0; -#endif // defined(WEBRTC_WIN) -} - -PlatformThreadRef PlatformThread::GetThreadRef() const { -#if defined(WEBRTC_WIN) - return thread_id_; -#else - return thread_; -#endif // defined(WEBRTC_WIN) -} - -void PlatformThread::Stop() { - RTC_DCHECK(thread_checker_.IsCurrent()); - if (!IsRunning()) - return; - -#if defined(WEBRTC_WIN) - WaitForSingleObject(thread_, INFINITE); - CloseHandle(thread_); - thread_ = nullptr; - thread_id_ = 0; -#else - RTC_CHECK_EQ(0, pthread_join(thread_, nullptr)); - thread_ = 0; -#endif // defined(WEBRTC_WIN) - spawned_thread_checker_.Detach(); -} - -void PlatformThread::Run() { - // Attach the worker thread checker to this thread. - RTC_DCHECK(spawned_thread_checker_.IsCurrent()); - rtc::SetCurrentThreadName(name_.c_str()); - SetPriority(priority_); - run_function_(obj_); -} - -bool PlatformThread::SetPriority(ThreadPriority priority) { - RTC_DCHECK(spawned_thread_checker_.IsCurrent()); - -#if defined(WEBRTC_WIN) - return SetThreadPriority(thread_, priority) != FALSE; + return SetThreadPriority(GetCurrentThread(), + Win32PriorityFromThreadPriority(priority)) != FALSE; #elif defined(__native_client__) || defined(WEBRTC_FUCHSIA) // Setting thread priorities is not supported in NaCl or Fuchsia. return true; @@ -158,35 +64,148 @@ bool PlatformThread::SetPriority(ThreadPriority priority) { const int top_prio = max_prio - 1; const int low_prio = min_prio + 1; switch (priority) { - case kLowPriority: + case ThreadPriority::kLow: param.sched_priority = low_prio; break; - case kNormalPriority: + case ThreadPriority::kNormal: // The -1 ensures that the kHighPriority is always greater or equal to // kNormalPriority. param.sched_priority = (low_prio + top_prio - 1) / 2; break; - case kHighPriority: + case ThreadPriority::kHigh: param.sched_priority = std::max(top_prio - 2, low_prio); break; - case kHighestPriority: - param.sched_priority = std::max(top_prio - 1, low_prio); - break; - case kRealtimePriority: + case ThreadPriority::kRealtime: param.sched_priority = top_prio; break; } - return pthread_setschedparam(thread_, policy, ¶m) == 0; + return pthread_setschedparam(pthread_self(), policy, ¶m) == 0; #endif // defined(WEBRTC_WIN) } #if defined(WEBRTC_WIN) -bool PlatformThread::QueueAPC(PAPCFUNC function, ULONG_PTR data) { - RTC_DCHECK(thread_checker_.IsCurrent()); - RTC_DCHECK(IsRunning()); +DWORD WINAPI RunPlatformThread(void* param) { + // The GetLastError() function only returns valid results when it is called + // after a Win32 API function that returns a "failed" result. A crash dump + // contains the result from GetLastError() and to make sure it does not + // falsely report a Windows error we call SetLastError here. + ::SetLastError(ERROR_SUCCESS); + auto function = static_cast*>(param); + (*function)(); + delete function; + return 0; +} +#else +void* RunPlatformThread(void* param) { + auto function = static_cast*>(param); + (*function)(); + delete function; + return 0; +} +#endif // defined(WEBRTC_WIN) - return QueueUserAPC(function, thread_, data) != FALSE; +} // namespace + +PlatformThread::PlatformThread(Handle handle, bool joinable) + : handle_(handle), joinable_(joinable) {} + +PlatformThread::PlatformThread(PlatformThread&& rhs) + : handle_(rhs.handle_), joinable_(rhs.joinable_) { + rhs.handle_ = absl::nullopt; +} + +PlatformThread& PlatformThread::operator=(PlatformThread&& rhs) { + Finalize(); + handle_ = rhs.handle_; + joinable_ = rhs.joinable_; + rhs.handle_ = absl::nullopt; + return *this; +} + +PlatformThread::~PlatformThread() { + Finalize(); +} + +PlatformThread PlatformThread::SpawnJoinable( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes) { + return SpawnThread(std::move(thread_function), name, attributes, + /*joinable=*/true); +} + +PlatformThread PlatformThread::SpawnDetached( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes) { + return SpawnThread(std::move(thread_function), name, attributes, + /*joinable=*/false); +} + +absl::optional PlatformThread::GetHandle() const { + return handle_; +} + +#if defined(WEBRTC_WIN) +bool PlatformThread::QueueAPC(PAPCFUNC function, ULONG_PTR data) { + RTC_DCHECK(handle_.has_value()); + return handle_.has_value() ? QueueUserAPC(function, *handle_, data) != FALSE + : false; } #endif +void PlatformThread::Finalize() { + if (!handle_.has_value()) + return; +#if defined(WEBRTC_WIN) + if (joinable_) + WaitForSingleObject(*handle_, INFINITE); + CloseHandle(*handle_); +#else + if (joinable_) + RTC_CHECK_EQ(0, pthread_join(*handle_, nullptr)); +#endif + handle_ = absl::nullopt; +} + +PlatformThread PlatformThread::SpawnThread( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes, + bool joinable) { + RTC_DCHECK(thread_function); + RTC_DCHECK(!name.empty()); + // TODO(tommi): Consider lowering the limit to 15 (limit on Linux). + RTC_DCHECK(name.length() < 64); + auto start_thread_function_ptr = + new std::function([thread_function = std::move(thread_function), + name = std::string(name), attributes] { + rtc::SetCurrentThreadName(name.c_str()); + SetPriority(attributes.priority); + thread_function(); + }); +#if defined(WEBRTC_WIN) + // See bug 2902 for background on STACK_SIZE_PARAM_IS_A_RESERVATION. + // Set the reserved stack stack size to 1M, which is the default on Windows + // and Linux. + DWORD thread_id = 0; + PlatformThread::Handle handle = ::CreateThread( + nullptr, 1024 * 1024, &RunPlatformThread, start_thread_function_ptr, + STACK_SIZE_PARAM_IS_A_RESERVATION, &thread_id); + RTC_CHECK(handle) << "CreateThread failed"; +#else + pthread_attr_t attr; + pthread_attr_init(&attr); + // Set the stack stack size to 1M. + pthread_attr_setstacksize(&attr, 1024 * 1024); + pthread_attr_setdetachstate( + &attr, joinable ? PTHREAD_CREATE_JOINABLE : PTHREAD_CREATE_DETACHED); + PlatformThread::Handle handle; + RTC_CHECK_EQ(0, pthread_create(&handle, &attr, &RunPlatformThread, + start_thread_function_ptr)); + pthread_attr_destroy(&attr); +#endif // defined(WEBRTC_WIN) + return PlatformThread(handle, joinable); +} + } // namespace rtc diff --git a/webrtc/rtc_base/platform_thread.h b/webrtc/rtc_base/platform_thread.h index 4968de9..befd618 100644 --- a/webrtc/rtc_base/platform_thread.h +++ b/webrtc/rtc_base/platform_thread.h @@ -11,92 +11,108 @@ #ifndef RTC_BASE_PLATFORM_THREAD_H_ #define RTC_BASE_PLATFORM_THREAD_H_ -#ifndef WEBRTC_WIN +#include +#include +#if !defined(WEBRTC_WIN) #include #endif -#include #include "absl/strings/string_view.h" -#include "rtc_base/constructor_magic.h" +#include "absl/types/optional.h" #include "rtc_base/platform_thread_types.h" -#include "rtc_base/thread_checker.h" namespace rtc { -// Callback function that the spawned thread will enter once spawned. -typedef void (*ThreadRunFunction)(void*); - -enum ThreadPriority { -#ifdef WEBRTC_WIN - kLowPriority = THREAD_PRIORITY_BELOW_NORMAL, - kNormalPriority = THREAD_PRIORITY_NORMAL, - kHighPriority = THREAD_PRIORITY_ABOVE_NORMAL, - kHighestPriority = THREAD_PRIORITY_HIGHEST, - kRealtimePriority = THREAD_PRIORITY_TIME_CRITICAL -#else - kLowPriority = 1, - kNormalPriority = 2, - kHighPriority = 3, - kHighestPriority = 4, - kRealtimePriority = 5 -#endif +enum class ThreadPriority { + kLow = 1, + kNormal, + kHigh, + kRealtime, }; -// Represents a simple worker thread. The implementation must be assumed -// to be single threaded, meaning that all methods of the class, must be -// called from the same thread, including instantiation. -class PlatformThread { +struct ThreadAttributes { + ThreadPriority priority = ThreadPriority::kNormal; + ThreadAttributes& SetPriority(ThreadPriority priority_param) { + priority = priority_param; + return *this; + } +}; + +// Represents a simple worker thread. +class PlatformThread final { public: - PlatformThread(ThreadRunFunction func, - void* obj, - absl::string_view thread_name, - ThreadPriority priority = kNormalPriority); + // Handle is the base platform thread handle. +#if defined(WEBRTC_WIN) + using Handle = HANDLE; +#else + using Handle = pthread_t; +#endif // defined(WEBRTC_WIN) + // This ctor creates the PlatformThread with an unset handle (returning true + // in empty()) and is provided for convenience. + // TODO(bugs.webrtc.org/12727) Look into if default and move support can be + // removed. + PlatformThread() = default; + + // Moves `rhs` into this, storing an empty state in `rhs`. + // TODO(bugs.webrtc.org/12727) Look into if default and move support can be + // removed. + PlatformThread(PlatformThread&& rhs); + + // Copies won't work since we'd have problems with joinable threads. + PlatformThread(const PlatformThread&) = delete; + PlatformThread& operator=(const PlatformThread&) = delete; + + // Moves `rhs` into this, storing an empty state in `rhs`. + // TODO(bugs.webrtc.org/12727) Look into if default and move support can be + // removed. + PlatformThread& operator=(PlatformThread&& rhs); + + // For a PlatformThread that's been spawned joinable, the destructor suspends + // the calling thread until the created thread exits unless the thread has + // already exited. virtual ~PlatformThread(); - const std::string& name() const { return name_; } + // Finalizes any allocated resources. + // For a PlatformThread that's been spawned joinable, Finalize() suspends + // the calling thread until the created thread exits unless the thread has + // already exited. + // empty() returns true after completion. + void Finalize(); - // Spawns a thread and tries to set thread priority according to the priority - // from when CreateThread was called. - void Start(); + // Returns true if default constructed, moved from, or Finalize()ed. + bool empty() const { return !handle_.has_value(); } - bool IsRunning() const; + // Creates a started joinable thread which will be joined when the returned + // PlatformThread destructs or Finalize() is called. + static PlatformThread SpawnJoinable( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes = ThreadAttributes()); - // Returns an identifier for the worker thread that can be used to do - // thread checks. - PlatformThreadRef GetThreadRef() const; + // Creates a started detached thread. The caller has to use external + // synchronization as nothing is provided by the PlatformThread construct. + static PlatformThread SpawnDetached( + std::function thread_function, + absl::string_view name, + ThreadAttributes attributes = ThreadAttributes()); - // Stops (joins) the spawned thread. - void Stop(); + // Returns the base platform thread handle of this thread. + absl::optional GetHandle() const; - protected: #if defined(WEBRTC_WIN) - // Exposed to derived classes to allow for special cases specific to Windows. + // Queue a Windows APC function that runs when the thread is alertable. bool QueueAPC(PAPCFUNC apc_function, ULONG_PTR data); #endif private: - void Run(); - bool SetPriority(ThreadPriority priority); + PlatformThread(Handle handle, bool joinable); + static PlatformThread SpawnThread(std::function thread_function, + absl::string_view name, + ThreadAttributes attributes, + bool joinable); - ThreadRunFunction const run_function_ = nullptr; - const ThreadPriority priority_ = kNormalPriority; - void* const obj_; - // TODO(pbos): Make sure call sites use string literals and update to a const - // char* instead of a std::string. - const std::string name_; - rtc::ThreadChecker thread_checker_; - rtc::ThreadChecker spawned_thread_checker_; -#if defined(WEBRTC_WIN) - static DWORD WINAPI StartThread(void* param); - - HANDLE thread_ = nullptr; - DWORD thread_id_ = 0; -#else - static void* StartThread(void* param); - - pthread_t thread_ = 0; -#endif // defined(WEBRTC_WIN) - RTC_DISALLOW_COPY_AND_ASSIGN(PlatformThread); + absl::optional handle_; + bool joinable_ = false; }; } // namespace rtc diff --git a/webrtc/rtc_base/platform_thread_types.cc b/webrtc/rtc_base/platform_thread_types.cc index ea1dd61..d64ea68 100644 --- a/webrtc/rtc_base/platform_thread_types.cc +++ b/webrtc/rtc_base/platform_thread_types.cc @@ -25,6 +25,13 @@ typedef HRESULT(WINAPI* RTC_SetThreadDescription)(HANDLE hThread, PCWSTR lpThreadDescription); #endif +#if defined(WEBRTC_FUCHSIA) +#include +#include + +#include "rtc_base/checks.h" +#endif + namespace rtc { PlatformThreadId CurrentThreadId() { @@ -99,18 +106,20 @@ void SetCurrentThreadName(const char* name) { #pragma warning(push) #pragma warning(disable : 6320 6322) -#ifndef __MINGW32__ __try { ::RaiseException(0x406D1388, 0, sizeof(threadname_info) / sizeof(ULONG_PTR), reinterpret_cast(&threadname_info)); } __except (EXCEPTION_EXECUTE_HANDLER) { // NOLINT } -#endif #pragma warning(pop) #elif defined(WEBRTC_LINUX) || defined(WEBRTC_ANDROID) prctl(PR_SET_NAME, reinterpret_cast(name)); // NOLINT #elif defined(WEBRTC_MAC) || defined(WEBRTC_IOS) pthread_setname_np(name); +#elif defined(WEBRTC_FUCHSIA) + zx_status_t status = zx_object_set_property(zx_thread_self(), ZX_PROP_NAME, + name, strlen(name)); + RTC_DCHECK_EQ(status, ZX_OK); #endif } diff --git a/webrtc/rtc_base/race_checker.cc b/webrtc/rtc_base/race_checker.cc index bf9dfdc..f0d4e86 100644 --- a/webrtc/rtc_base/race_checker.cc +++ b/webrtc/rtc_base/race_checker.cc @@ -25,7 +25,9 @@ RaceChecker::RaceChecker() {} bool RaceChecker::Acquire() const { const PlatformThreadRef current_thread = CurrentThreadRef(); // Set new accessing thread if this is a new use. - if (access_count_++ == 0) + const int current_access_count = access_count_; + access_count_ = access_count_ + 1; + if (current_access_count == 0) accessing_thread_ = current_thread; // If this is being used concurrently this check will fail for the second // thread entering since it won't set the thread. Recursive use of checked @@ -35,7 +37,7 @@ bool RaceChecker::Acquire() const { } void RaceChecker::Release() const { - --access_count_; + access_count_ = access_count_ - 1; } namespace internal { diff --git a/webrtc/rtc_base/random.cc b/webrtc/rtc_base/random.cc new file mode 100644 index 0000000..5206b81 --- /dev/null +++ b/webrtc/rtc_base/random.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "rtc_base/random.h" + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +Random::Random(uint64_t seed) { + RTC_DCHECK(seed != 0x0ull); + state_ = seed; +} + +uint32_t Random::Rand(uint32_t t) { + // Casting the output to 32 bits will give an almost uniform number. + // Pr[x=0] = (2^32-1) / (2^64-1) + // Pr[x=k] = 2^32 / (2^64-1) for k!=0 + // Uniform would be Pr[x=k] = 2^32 / 2^64 for all 32-bit integers k. + uint32_t x = NextOutput(); + // If x / 2^32 is uniform on [0,1), then x / 2^32 * (t+1) is uniform on + // the interval [0,t+1), so the integer part is uniform on [0,t]. + uint64_t result = x * (static_cast(t) + 1); + result >>= 32; + return result; +} + +uint32_t Random::Rand(uint32_t low, uint32_t high) { + RTC_DCHECK(low <= high); + return Rand(high - low) + low; +} + +int32_t Random::Rand(int32_t low, int32_t high) { + RTC_DCHECK(low <= high); + const int64_t low_i64{low}; + return rtc::dchecked_cast( + Rand(rtc::dchecked_cast(high - low_i64)) + low_i64); +} + +template <> +float Random::Rand() { + double result = NextOutput() - 1; + result = result / static_cast(0xFFFFFFFFFFFFFFFFull); + return static_cast(result); +} + +template <> +double Random::Rand() { + double result = NextOutput() - 1; + result = result / static_cast(0xFFFFFFFFFFFFFFFFull); + return result; +} + +template <> +bool Random::Rand() { + return Rand(0, 1) == 1; +} + +double Random::Gaussian(double mean, double standard_deviation) { + // Creating a Normal distribution variable from two independent uniform + // variables based on the Box-Muller transform, which is defined on the + // interval (0, 1]. Note that we rely on NextOutput to generate integers + // in the range [1, 2^64-1]. Normally this behavior is a bit frustrating, + // but here it is exactly what we need. + const double kPi = 3.14159265358979323846; + double u1 = static_cast(NextOutput()) / + static_cast(0xFFFFFFFFFFFFFFFFull); + double u2 = static_cast(NextOutput()) / + static_cast(0xFFFFFFFFFFFFFFFFull); + return mean + standard_deviation * sqrt(-2 * log(u1)) * cos(2 * kPi * u2); +} + +double Random::Exponential(double lambda) { + double uniform = Rand(); + return -log(uniform) / lambda; +} + +} // namespace webrtc diff --git a/webrtc/rtc_base/random.h b/webrtc/rtc_base/random.h new file mode 100644 index 0000000..b3b9fd1 --- /dev/null +++ b/webrtc/rtc_base/random.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_RANDOM_H_ +#define RTC_BASE_RANDOM_H_ + +#include + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +class Random { + public: + // TODO(tommi): Change this so that the seed can be initialized internally, + // e.g. by offering two ways of constructing or offer a static method that + // returns a seed that's suitable for initialization. + // The problem now is that callers are calling clock_->TimeInMicroseconds() + // which calls TickTime::Now().Ticks(), which can return a very low value on + // Mac and can result in a seed of 0 after conversion to microseconds. + // Besides the quality of the random seed being poor, this also requires + // the client to take on extra dependencies to generate a seed. + // If we go for a static seed generator in Random, we can use something from + // webrtc/rtc_base and make sure that it works the same way across platforms. + // See also discussion here: https://codereview.webrtc.org/1623543002/ + explicit Random(uint64_t seed); + + Random() = delete; + Random(const Random&) = delete; + Random& operator=(const Random&) = delete; + + // Return pseudo-random integer of the specified type. + // We need to limit the size to 32 bits to keep the output close to uniform. + template + T Rand() { + static_assert(std::numeric_limits::is_integer && + std::numeric_limits::radix == 2 && + std::numeric_limits::digits <= 32, + "Rand is only supported for built-in integer types that are " + "32 bits or smaller."); + return static_cast(NextOutput()); + } + + // Uniformly distributed pseudo-random number in the interval [0, t]. + uint32_t Rand(uint32_t t); + + // Uniformly distributed pseudo-random number in the interval [low, high]. + uint32_t Rand(uint32_t low, uint32_t high); + + // Uniformly distributed pseudo-random number in the interval [low, high]. + int32_t Rand(int32_t low, int32_t high); + + // Normal Distribution. + double Gaussian(double mean, double standard_deviation); + + // Exponential Distribution. + double Exponential(double lambda); + + private: + // Outputs a nonzero 64-bit random number using Xorshift algorithm. + // https://en.wikipedia.org/wiki/Xorshift + uint64_t NextOutput() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + RTC_DCHECK(state_ != 0x0ULL); + return state_ * 2685821657736338717ull; + } + + uint64_t state_; +}; + +// Return pseudo-random number in the interval [0.0, 1.0). +template <> +float Random::Rand(); + +// Return pseudo-random number in the interval [0.0, 1.0). +template <> +double Random::Rand(); + +// Return pseudo-random boolean value. +template <> +bool Random::Rand(); + +} // namespace webrtc + +#endif // RTC_BASE_RANDOM_H_ diff --git a/webrtc/rtc_base/ref_counted_object.h b/webrtc/rtc_base/ref_counted_object.h index ce18379..418c3d8 100644 --- a/webrtc/rtc_base/ref_counted_object.h +++ b/webrtc/rtc_base/ref_counted_object.h @@ -10,10 +10,7 @@ #ifndef RTC_BASE_REF_COUNTED_OBJECT_H_ #define RTC_BASE_REF_COUNTED_OBJECT_H_ -#include -#include - -#include "rtc_base/constructor_magic.h" +#include "api/scoped_refptr.h" #include "rtc_base/ref_count.h" #include "rtc_base/ref_counter.h" @@ -24,6 +21,9 @@ class RefCountedObject : public T { public: RefCountedObject() {} + RefCountedObject(const RefCountedObject&) = delete; + RefCountedObject& operator=(const RefCountedObject&) = delete; + template explicit RefCountedObject(P0&& p0) : T(std::forward(p0)) {} @@ -33,9 +33,9 @@ class RefCountedObject : public T { std::forward(p1), std::forward(args)...) {} - virtual void AddRef() const { ref_count_.IncRef(); } + void AddRef() const override { ref_count_.IncRef(); } - virtual RefCountReleaseStatus Release() const { + RefCountReleaseStatus Release() const override { const auto status = ref_count_.DecRef(); if (status == RefCountReleaseStatus::kDroppedLastRef) { delete this; @@ -52,11 +52,36 @@ class RefCountedObject : public T { virtual bool HasOneRef() const { return ref_count_.HasOneRef(); } protected: - virtual ~RefCountedObject() {} + ~RefCountedObject() override {} mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; +}; - RTC_DISALLOW_COPY_AND_ASSIGN(RefCountedObject); +template +class FinalRefCountedObject final : public T { + public: + using T::T; + // Above using declaration propagates a default move constructor + // FinalRefCountedObject(FinalRefCountedObject&& other), but we also need + // move construction from T. + explicit FinalRefCountedObject(T&& other) : T(std::move(other)) {} + FinalRefCountedObject(const FinalRefCountedObject&) = delete; + FinalRefCountedObject& operator=(const FinalRefCountedObject&) = delete; + + void AddRef() const { ref_count_.IncRef(); } + RefCountReleaseStatus Release() const { + const auto status = ref_count_.DecRef(); + if (status == RefCountReleaseStatus::kDroppedLastRef) { + delete this; + } + return status; + } + bool HasOneRef() const { return ref_count_.HasOneRef(); } + + private: + ~FinalRefCountedObject() = default; + + mutable webrtc::webrtc_impl::RefCounter ref_count_{0}; }; } // namespace rtc diff --git a/webrtc/rtc_base/string_encode.cc b/webrtc/rtc_base/string_encode.cc index 1570b93..434d1e6 100644 --- a/webrtc/rtc_base/string_encode.cc +++ b/webrtc/rtc_base/string_encode.cc @@ -12,6 +12,8 @@ #include +#include "absl/strings/string_view.h" +#include "api/array_view.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" @@ -49,18 +51,18 @@ size_t hex_encode_output_length(size_t srclen, char delimiter) { } // hex_encode shows the hex representation of binary data in ascii, with -// |delimiter| between bytes, or none if |delimiter| == 0. +// `delimiter` between bytes, or none if `delimiter` == 0. void hex_encode_with_delimiter(char* buffer, - const char* csource, - size_t srclen, + absl::string_view source, char delimiter) { RTC_DCHECK(buffer); // Init and check bounds. const unsigned char* bsource = - reinterpret_cast(csource); + reinterpret_cast(source.data()); size_t srcpos = 0, bufpos = 0; + size_t srclen = source.length(); while (srcpos < srclen) { unsigned char ch = bsource[srcpos++]; buffer[bufpos] = hex_encode((ch >> 4) & 0xF); @@ -77,43 +79,30 @@ void hex_encode_with_delimiter(char* buffer, } // namespace -std::string hex_encode(const std::string& str) { - return hex_encode(str.c_str(), str.size()); +std::string hex_encode(absl::string_view str) { + return hex_encode_with_delimiter(str, 0); } -std::string hex_encode(const char* source, size_t srclen) { - return hex_encode_with_delimiter(source, srclen, 0); -} - -std::string hex_encode_with_delimiter(const char* source, - size_t srclen, +std::string hex_encode_with_delimiter(absl::string_view source, char delimiter) { - std::string s(hex_encode_output_length(srclen, delimiter), 0); - hex_encode_with_delimiter(&s[0], source, srclen, delimiter); + std::string s(hex_encode_output_length(source.length(), delimiter), 0); + hex_encode_with_delimiter(&s[0], source, delimiter); return s; } -size_t hex_decode(char* cbuffer, - size_t buflen, - const char* source, - size_t srclen) { - return hex_decode_with_delimiter(cbuffer, buflen, source, srclen, 0); -} - -size_t hex_decode_with_delimiter(char* cbuffer, - size_t buflen, - const char* source, - size_t srclen, +size_t hex_decode_with_delimiter(ArrayView cbuffer, + absl::string_view source, char delimiter) { - RTC_DCHECK(cbuffer); // TODO(kwiberg): estimate output size - if (buflen == 0) + if (cbuffer.empty()) return 0; // Init and bounds check. - unsigned char* bbuffer = reinterpret_cast(cbuffer); + unsigned char* bbuffer = reinterpret_cast(cbuffer.data()); size_t srcpos = 0, bufpos = 0; + size_t srclen = source.length(); + size_t needed = (delimiter) ? (srclen + 1) / 3 : srclen / 2; - if (buflen < needed) + if (cbuffer.size() < needed) return 0; while (srcpos < srclen) { @@ -141,18 +130,11 @@ size_t hex_decode_with_delimiter(char* cbuffer, return bufpos; } -size_t hex_decode(char* buffer, size_t buflen, const std::string& source) { - return hex_decode_with_delimiter(buffer, buflen, source, 0); -} -size_t hex_decode_with_delimiter(char* buffer, - size_t buflen, - const std::string& source, - char delimiter) { - return hex_decode_with_delimiter(buffer, buflen, source.c_str(), - source.length(), delimiter); +size_t hex_decode(ArrayView buffer, absl::string_view source) { + return hex_decode_with_delimiter(buffer, source, 0); } -size_t tokenize(const std::string& source, +size_t tokenize(absl::string_view source, char delimiter, std::vector* fields) { fields->clear(); @@ -160,146 +142,61 @@ size_t tokenize(const std::string& source, for (size_t i = 0; i < source.length(); ++i) { if (source[i] == delimiter) { if (i != last) { - fields->push_back(source.substr(last, i - last)); + fields->emplace_back(source.substr(last, i - last)); } last = i + 1; } } if (last != source.length()) { - fields->push_back(source.substr(last, source.length() - last)); + fields->emplace_back(source.substr(last, source.length() - last)); } return fields->size(); } -size_t tokenize_with_empty_tokens(const std::string& source, - char delimiter, - std::vector* fields) { - fields->clear(); - size_t last = 0; - for (size_t i = 0; i < source.length(); ++i) { - if (source[i] == delimiter) { - fields->push_back(source.substr(last, i - last)); - last = i + 1; - } - } - fields->push_back(source.substr(last, source.length() - last)); - return fields->size(); -} - -size_t tokenize_append(const std::string& source, - char delimiter, - std::vector* fields) { - if (!fields) - return 0; - - std::vector new_fields; - tokenize(source, delimiter, &new_fields); - fields->insert(fields->end(), new_fields.begin(), new_fields.end()); - return fields->size(); -} - -size_t tokenize(const std::string& source, - char delimiter, - char start_mark, - char end_mark, - std::vector* fields) { - if (!fields) - return 0; - fields->clear(); - - std::string remain_source = source; - while (!remain_source.empty()) { - size_t start_pos = remain_source.find(start_mark); - if (std::string::npos == start_pos) - break; - std::string pre_mark; - if (start_pos > 0) { - pre_mark = remain_source.substr(0, start_pos - 1); - } - - ++start_pos; - size_t end_pos = remain_source.find(end_mark, start_pos); - if (std::string::npos == end_pos) - break; - - // We have found the matching marks. First tokenize the pre-mask. Then add - // the marked part as a single field. Finally, loop back for the post-mark. - tokenize_append(pre_mark, delimiter, fields); - fields->push_back(remain_source.substr(start_pos, end_pos - start_pos)); - remain_source = remain_source.substr(end_pos + 1); - } - - return tokenize_append(remain_source, delimiter, fields); -} - -bool tokenize_first(const std::string& source, +bool tokenize_first(absl::string_view source, const char delimiter, std::string* token, std::string* rest) { // Find the first delimiter size_t left_pos = source.find(delimiter); - if (left_pos == std::string::npos) { + if (left_pos == absl::string_view::npos) { return false; } // Look for additional occurrances of delimiter. size_t right_pos = left_pos + 1; - while (source[right_pos] == delimiter) { + while (right_pos < source.size() && source[right_pos] == delimiter) { right_pos++; } - *token = source.substr(0, left_pos); - *rest = source.substr(right_pos); + *token = std::string(source.substr(0, left_pos)); + *rest = std::string(source.substr(right_pos)); return true; } -std::string join(const std::vector& source, char delimiter) { - if (source.size() == 0) { - return std::string(); - } - // Find length of the string to be returned to pre-allocate memory. - size_t source_string_length = 0; - for (size_t i = 0; i < source.size(); ++i) { - source_string_length += source[i].length(); - } - - // Build the joined string. - std::string joined_string; - joined_string.reserve(source_string_length + source.size() - 1); - for (size_t i = 0; i < source.size(); ++i) { - if (i != 0) { - joined_string += delimiter; - } - joined_string += source[i]; - } - return joined_string; -} - -size_t split(const std::string& source, - char delimiter, - std::vector* fields) { - RTC_DCHECK(fields); - fields->clear(); +std::vector split(absl::string_view source, char delimiter) { + std::vector fields; size_t last = 0; for (size_t i = 0; i < source.length(); ++i) { if (source[i] == delimiter) { - fields->push_back(source.substr(last, i - last)); + fields.push_back(source.substr(last, i - last)); last = i + 1; } } - fields->push_back(source.substr(last, source.length() - last)); - return fields->size(); + fields.push_back(source.substr(last)); + return fields; } std::string ToString(const bool b) { return b ? "true" : "false"; } -std::string ToString(const char* const s) { +std::string ToString(absl::string_view s) { return std::string(s); } -std::string ToString(const std::string s) { - return s; + +std::string ToString(const char* s) { + return std::string(s); } std::string ToString(const short s) { @@ -372,7 +269,7 @@ std::string ToString(const void* const p) { return std::string(&buf[0], len); } -bool FromString(const std::string& s, bool* b) { +bool FromString(absl::string_view s, bool* b) { if (s == "false") { *b = false; return true; diff --git a/webrtc/rtc_base/string_encode.h b/webrtc/rtc_base/string_encode.h index c1401b9..82a9dfd 100644 --- a/webrtc/rtc_base/string_encode.h +++ b/webrtc/rtc_base/string_encode.h @@ -17,7 +17,9 @@ #include #include +#include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "api/array_view.h" #include "rtc_base/checks.h" #include "rtc_base/string_to_number.h" @@ -27,79 +29,36 @@ namespace rtc { // String Encoding Utilities ////////////////////////////////////////////////////////////////////// -std::string hex_encode(const std::string& str); -std::string hex_encode(const char* source, size_t srclen); -std::string hex_encode_with_delimiter(const char* source, - size_t srclen, - char delimiter); +std::string hex_encode(absl::string_view str); +std::string hex_encode_with_delimiter(absl::string_view source, char delimiter); // hex_decode converts ascii hex to binary. -size_t hex_decode(char* buffer, - size_t buflen, - const char* source, - size_t srclen); +size_t hex_decode(ArrayView buffer, absl::string_view source); // hex_decode, assuming that there is a delimiter between every byte // pair. -// |delimiter| == 0 means no delimiter +// `delimiter` == 0 means no delimiter // If the buffer is too short or the data is invalid, we return 0. -size_t hex_decode_with_delimiter(char* buffer, - size_t buflen, - const char* source, - size_t srclen, +size_t hex_decode_with_delimiter(ArrayView buffer, + absl::string_view source, char delimiter); -// Helper functions for hex_decode. -size_t hex_decode(char* buffer, size_t buflen, const std::string& source); -size_t hex_decode_with_delimiter(char* buffer, - size_t buflen, - const std::string& source, - char delimiter); - -// Joins the source vector of strings into a single string, with each -// field in source being separated by delimiter. No trailing delimiter is added. -std::string join(const std::vector& source, char delimiter); - // Splits the source string into multiple fields separated by delimiter, -// with duplicates of delimiter creating empty fields. -size_t split(const std::string& source, - char delimiter, - std::vector* fields); +// with duplicates of delimiter creating empty fields. Empty input produces a +// single, empty, field. +std::vector split(absl::string_view source, char delimiter); // Splits the source string into multiple fields separated by delimiter, // with duplicates of delimiter ignored. Trailing delimiter ignored. -size_t tokenize(const std::string& source, +size_t tokenize(absl::string_view source, char delimiter, std::vector* fields); -// Tokenize, including the empty tokens. -size_t tokenize_with_empty_tokens(const std::string& source, - char delimiter, - std::vector* fields); - -// Tokenize and append the tokens to fields. Return the new size of fields. -size_t tokenize_append(const std::string& source, - char delimiter, - std::vector* fields); - -// Splits the source string into multiple fields separated by delimiter, with -// duplicates of delimiter ignored. Trailing delimiter ignored. A substring in -// between the start_mark and the end_mark is treated as a single field. Return -// the size of fields. For example, if source is "filename -// \"/Library/Application Support/media content.txt\"", delimiter is ' ', and -// the start_mark and end_mark are '"', this method returns two fields: -// "filename" and "/Library/Application Support/media content.txt". -size_t tokenize(const std::string& source, - char delimiter, - char start_mark, - char end_mark, - std::vector* fields); - // Extract the first token from source as separated by delimiter, with // duplicates of delimiter ignored. Return false if the delimiter could not be // found, otherwise return true. -bool tokenize_first(const std::string& source, - const char delimiter, +bool tokenize_first(absl::string_view source, + char delimiter, std::string* token, std::string* rest); @@ -107,8 +66,10 @@ bool tokenize_first(const std::string& source, // TODO(jonasolsson): Remove these when absl::StrCat becomes available. std::string ToString(bool b); +std::string ToString(absl::string_view s); +// The const char* overload is needed for correct overload resolution because of +// the const void* version of ToString() below. std::string ToString(const char* s); -std::string ToString(std::string t); std::string ToString(short s); std::string ToString(unsigned short s); @@ -128,7 +89,7 @@ template ::value && !std::is_same::value, int>::type = 0> -static bool FromString(const std::string& s, T* t) { +static bool FromString(absl::string_view s, T* t) { RTC_DCHECK(t); absl::optional result = StringToNumber(s); @@ -138,10 +99,10 @@ static bool FromString(const std::string& s, T* t) { return result.has_value(); } -bool FromString(const std::string& s, bool* b); +bool FromString(absl::string_view s, bool* b); template -static inline T FromString(const std::string& str) { +static inline T FromString(absl::string_view str) { T val; FromString(str, &val); return val; diff --git a/webrtc/rtc_base/string_to_number.cc b/webrtc/rtc_base/string_to_number.cc index 351610f..1209ece 100644 --- a/webrtc/rtc_base/string_to_number.cc +++ b/webrtc/rtc_base/string_to_number.cc @@ -20,30 +20,41 @@ namespace rtc { namespace string_to_number_internal { -absl::optional ParseSigned(const char* str, int base) { - RTC_DCHECK(str); - if (isdigit(str[0]) || str[0] == '-') { +absl::optional ParseSigned(absl::string_view str, int base) { + if (str.empty()) + return absl::nullopt; + + if (isdigit(static_cast(str[0])) || str[0] == '-') { + std::string str_str(str); char* end = nullptr; errno = 0; - const signed_type value = std::strtoll(str, &end, base); - if (end && *end == '\0' && errno == 0) { + const signed_type value = std::strtoll(str_str.c_str(), &end, base); + // Check for errors and also make sure that there were no embedded nuls in + // the input string. + if (end == str_str.c_str() + str_str.size() && errno == 0) { return value; } } return absl::nullopt; } -absl::optional ParseUnsigned(const char* str, int base) { - RTC_DCHECK(str); - if (isdigit(str[0]) || str[0] == '-') { +absl::optional ParseUnsigned(absl::string_view str, int base) { + if (str.empty()) + return absl::nullopt; + + if (isdigit(static_cast(str[0])) || str[0] == '-') { + std::string str_str(str); // Explicitly discard negative values. std::strtoull parsing causes unsigned // wraparound. We cannot just reject values that start with -, though, since // -0 is perfectly fine, as is -0000000000000000000000000000000. const bool is_negative = str[0] == '-'; char* end = nullptr; errno = 0; - const unsigned_type value = std::strtoull(str, &end, base); - if (end && *end == '\0' && errno == 0 && (value == 0 || !is_negative)) { + const unsigned_type value = std::strtoull(str_str.c_str(), &end, base); + // Check for errors and also make sure that there were no embedded nuls in + // the input string. + if (end == str_str.c_str() + str_str.size() && errno == 0 && + (value == 0 || !is_negative)) { return value; } } @@ -69,22 +80,25 @@ inline long double StrToT(const char* str, char** str_end) { } template -absl::optional ParseFloatingPoint(const char* str) { - RTC_DCHECK(str); - if (*str == '\0') +absl::optional ParseFloatingPoint(absl::string_view str) { + if (str.empty()) return absl::nullopt; + + if (str[0] == '\0') + return absl::nullopt; + std::string str_str(str); char* end = nullptr; errno = 0; - const T value = StrToT(str, &end); - if (end && *end == '\0' && errno == 0) { + const T value = StrToT(str_str.c_str(), &end); + if (end == str_str.c_str() + str_str.size() && errno == 0) { return value; } return absl::nullopt; } -template absl::optional ParseFloatingPoint(const char* str); -template absl::optional ParseFloatingPoint(const char* str); -template absl::optional ParseFloatingPoint(const char* str); +template absl::optional ParseFloatingPoint(absl::string_view str); +template absl::optional ParseFloatingPoint(absl::string_view str); +template absl::optional ParseFloatingPoint(absl::string_view str); } // namespace string_to_number_internal } // namespace rtc diff --git a/webrtc/rtc_base/string_to_number.h b/webrtc/rtc_base/string_to_number.h index 4cb5215..1d704ee 100644 --- a/webrtc/rtc_base/string_to_number.h +++ b/webrtc/rtc_base/string_to_number.h @@ -15,6 +15,7 @@ #include #include +#include "absl/strings/string_view.h" #include "absl/types/optional.h" namespace rtc { @@ -25,10 +26,9 @@ namespace rtc { // functions (std::stoi, etc.) indicate errors by throwing exceptions, which // are disabled in WebRTC. // -// Integers are parsed using one of the following functions: -// absl::optional StringToNumber(const char* str, int base = 10); -// absl::optional StringToNumber(const std::string& str, -// int base = 10); +// Integers are parsed using: +// absl::optional StringToNumber(absl::string_view str, +// int base = 10); // // These functions parse a value from the beginning of a string into one of the // fundamental integer types, or returns an empty Optional if parsing @@ -38,26 +38,23 @@ namespace rtc { // By setting base to 0, one of octal, decimal or hexadecimal will be // detected from the string's prefix (0, nothing or 0x, respectively). // If non-zero, base can be set to a value between 2 and 36 inclusively. -// -// If desired, this interface could be extended with support for floating-point -// types. namespace string_to_number_internal { // These must be (unsigned) long long, to match the signature of strto(u)ll. using unsigned_type = unsigned long long; // NOLINT(runtime/int) using signed_type = long long; // NOLINT(runtime/int) -absl::optional ParseSigned(const char* str, int base); -absl::optional ParseUnsigned(const char* str, int base); +absl::optional ParseSigned(absl::string_view str, int base); +absl::optional ParseUnsigned(absl::string_view str, int base); template -absl::optional ParseFloatingPoint(const char* str); +absl::optional ParseFloatingPoint(absl::string_view str); } // namespace string_to_number_internal template typename std::enable_if::value && std::is_signed::value, absl::optional>::type -StringToNumber(const char* str, int base = 10) { +StringToNumber(absl::string_view str, int base = 10) { using string_to_number_internal::signed_type; static_assert( std::numeric_limits::max() <= @@ -78,7 +75,7 @@ template typename std::enable_if::value && std::is_unsigned::value, absl::optional>::type -StringToNumber(const char* str, int base = 10) { +StringToNumber(absl::string_view str, int base = 10) { using string_to_number_internal::unsigned_type; static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), @@ -95,7 +92,7 @@ StringToNumber(const char* str, int base = 10) { template typename std::enable_if::value, absl::optional>::type -StringToNumber(const char* str, int base = 10) { +StringToNumber(absl::string_view str, int base = 10) { static_assert( std::numeric_limits::max() <= std::numeric_limits::max(), "StringToNumber only supports floating-point numbers as large " @@ -103,14 +100,6 @@ StringToNumber(const char* str, int base = 10) { return string_to_number_internal::ParseFloatingPoint(str); } -// The std::string overloads only exists if there is a matching const char* -// version. -template -auto StringToNumber(const std::string& str, int base = 10) - -> decltype(StringToNumber(str.c_str(), base)) { - return StringToNumber(str.c_str(), base); -} - } // namespace rtc #endif // RTC_BASE_STRING_TO_NUMBER_H_ diff --git a/webrtc/rtc_base/string_utils.cc b/webrtc/rtc_base/string_utils.cc index 1720c62..b93e615 100644 --- a/webrtc/rtc_base/string_utils.cc +++ b/webrtc/rtc_base/string_utils.cc @@ -10,39 +10,23 @@ #include "rtc_base/string_utils.h" +#include "absl/strings/string_view.h" + namespace rtc { -size_t strcpyn(char* buffer, - size_t buflen, - const char* source, - size_t srclen /* = SIZE_UNKNOWN */) { +size_t strcpyn(char* buffer, size_t buflen, absl::string_view source) { if (buflen <= 0) return 0; - if (srclen == SIZE_UNKNOWN) { - srclen = strlen(source); - } + size_t srclen = source.length(); if (srclen >= buflen) { srclen = buflen - 1; } - memcpy(buffer, source, srclen); + memcpy(buffer, source.data(), srclen); buffer[srclen] = 0; return srclen; } -static const char kWhitespace[] = " \n\r\t"; - -std::string string_trim(const std::string& s) { - std::string::size_type first = s.find_first_not_of(kWhitespace); - std::string::size_type last = s.find_last_not_of(kWhitespace); - - if (first == std::string::npos || last == std::string::npos) { - return std::string(""); - } - - return s.substr(first, last - first + 1); -} - std::string ToHex(const int i) { char buffer[50]; snprintf(buffer, sizeof(buffer), "%x", i); diff --git a/webrtc/rtc_base/string_utils.h b/webrtc/rtc_base/string_utils.h index 23c55cb..9534d59 100644 --- a/webrtc/rtc_base/string_utils.h +++ b/webrtc/rtc_base/string_utils.h @@ -11,11 +11,11 @@ #ifndef RTC_BASE_STRING_UTILS_H_ #define RTC_BASE_STRING_UTILS_H_ -#include -#include #include #include +#include "absl/strings/string_view.h" + #if defined(WEBRTC_WIN) #include #include @@ -30,15 +30,26 @@ #include +#include "absl/strings/string_view.h" + namespace rtc { const size_t SIZE_UNKNOWN = static_cast(-1); +// An absl::string_view comparator functor for use with container types such as +// std::map that support heterogenous lookup. +// +// Example usage: +// std::map my_map; +struct AbslStringViewCmp { + using is_transparent = void; + bool operator()(absl::string_view a, absl::string_view b) const { + return a < b; + } +}; + // Safe version of strncpy that always nul-terminate. -size_t strcpyn(char* buffer, - size_t buflen, - const char* source, - size_t srclen = SIZE_UNKNOWN); +size_t strcpyn(char* buffer, size_t buflen, absl::string_view source); /////////////////////////////////////////////////////////////////////////////// // UTF helpers (Windows only) @@ -57,7 +68,7 @@ inline std::wstring ToUtf16(const char* utf8, size_t len) { return ws; } -inline std::wstring ToUtf16(const std::string& str) { +inline std::wstring ToUtf16(absl::string_view str) { return ToUtf16(str.data(), str.length()); } @@ -82,11 +93,45 @@ inline std::string ToUtf8(const std::wstring& wstr) { #endif // WEBRTC_WIN -// Remove leading and trailing whitespaces. -std::string string_trim(const std::string& s); - // TODO(jonasolsson): replace with absl::Hex when that becomes available. -std::string ToHex(const int i); +std::string ToHex(int i); + +// CompileTimeString comprises of a string-like object which can be used as a +// regular const char* in compile time and supports concatenation. Useful for +// concatenating constexpr strings in for example macro declarations. +namespace rtc_base_string_utils_internal { +template +struct CompileTimeString { + char string[NPlus1] = {0}; + constexpr CompileTimeString() = default; + template + explicit constexpr CompileTimeString(const char (&chars)[MPlus1]) { + char* chars_pointer = string; + for (auto c : chars) + *chars_pointer++ = c; + } + template + constexpr auto Concat(CompileTimeString b) { + CompileTimeString result; + char* chars_pointer = result.string; + for (auto c : string) + *chars_pointer++ = c; + chars_pointer = result.string + NPlus1 - 1; + for (auto c : b.string) + *chars_pointer++ = c; + result.string[NPlus1 + MPlus1 - 2] = 0; + return result; + } + constexpr operator const char*() { return string; } +}; +} // namespace rtc_base_string_utils_internal + +// Makes a constexpr CompileTimeString without having to specify X +// explicitly. +template +constexpr auto MakeCompileTimeString(const char (&a)[N]) { + return rtc_base_string_utils_internal::CompileTimeString(a); +} } // namespace rtc diff --git a/webrtc/rtc_base/strings/string_builder.cc b/webrtc/rtc_base/strings/string_builder.cc index caa931b..a419b0b 100644 --- a/webrtc/rtc_base/strings/string_builder.cc +++ b/webrtc/rtc_base/strings/string_builder.cc @@ -15,6 +15,7 @@ #include #include +#include "absl/strings/string_view.h" #include "rtc_base/checks.h" #include "rtc_base/numerics/safe_minmax.h" @@ -26,16 +27,20 @@ SimpleStringBuilder::SimpleStringBuilder(rtc::ArrayView buffer) RTC_DCHECK(IsConsistent()); } -SimpleStringBuilder& SimpleStringBuilder::operator<<(const char* str) { - return Append(str, strlen(str)); -} - SimpleStringBuilder& SimpleStringBuilder::operator<<(char ch) { - return Append(&ch, 1); + return operator<<(absl::string_view(&ch, 1)); } -SimpleStringBuilder& SimpleStringBuilder::operator<<(const std::string& str) { - return Append(str.c_str(), str.length()); +SimpleStringBuilder& SimpleStringBuilder::operator<<(absl::string_view str) { + RTC_DCHECK_LT(size_ + str.length(), buffer_.size()) + << "Buffer size was insufficient"; + const size_t chars_added = + rtc::SafeMin(str.length(), buffer_.size() - size_ - 1); + memcpy(&buffer_[size_], str.data(), chars_added); + size_ += chars_added; + buffer_[size_] = '\0'; + RTC_DCHECK(IsConsistent()); + return *this; } // Numeric conversion routines. @@ -98,7 +103,7 @@ SimpleStringBuilder& SimpleStringBuilder::AppendFormat(const char* fmt, ...) { } else { // This should never happen, but we're paranoid, so re-write the // terminator in case vsnprintf() overwrote it. - RTC_NOTREACHED(); + RTC_DCHECK_NOTREACHED(); buffer_[size_] = '\0'; } va_end(args); @@ -106,18 +111,6 @@ SimpleStringBuilder& SimpleStringBuilder::AppendFormat(const char* fmt, ...) { return *this; } -SimpleStringBuilder& SimpleStringBuilder::Append(const char* str, - size_t length) { - RTC_DCHECK_LT(size_ + length, buffer_.size()) - << "Buffer size was insufficient"; - const size_t chars_added = rtc::SafeMin(length, buffer_.size() - size_ - 1); - memcpy(&buffer_[size_], str, chars_added); - size_ += chars_added; - buffer_[size_] = '\0'; - RTC_DCHECK(IsConsistent()); - return *this; -} - StringBuilder& StringBuilder::AppendFormat(const char* fmt, ...) { va_list args, copy; va_start(args, fmt); diff --git a/webrtc/rtc_base/strings/string_builder.h b/webrtc/rtc_base/strings/string_builder.h index e528cf2..0098637 100644 --- a/webrtc/rtc_base/strings/string_builder.h +++ b/webrtc/rtc_base/strings/string_builder.h @@ -25,16 +25,15 @@ namespace rtc { // when you might otherwise be tempted to use a stringstream (discouraged for // anything except logging). It uses a fixed-size buffer provided by the caller // and concatenates strings and numbers into it, allowing the results to be -// read via |str()|. +// read via `str()`. class SimpleStringBuilder { public: explicit SimpleStringBuilder(rtc::ArrayView buffer); SimpleStringBuilder(const SimpleStringBuilder&) = delete; SimpleStringBuilder& operator=(const SimpleStringBuilder&) = delete; - SimpleStringBuilder& operator<<(const char* str); SimpleStringBuilder& operator<<(char ch); - SimpleStringBuilder& operator<<(const std::string& str); + SimpleStringBuilder& operator<<(absl::string_view str); SimpleStringBuilder& operator<<(int i); SimpleStringBuilder& operator<<(unsigned i); SimpleStringBuilder& operator<<(long i); // NOLINT @@ -45,12 +44,12 @@ class SimpleStringBuilder { SimpleStringBuilder& operator<<(double f); SimpleStringBuilder& operator<<(long double f); - // Returns a pointer to the built string. The name |str()| is borrowed for + // Returns a pointer to the built string. The name `str()` is borrowed for // compatibility reasons as we replace usage of stringstream throughout the // code base. const char* str() const { return buffer_.data(); } - // Returns the length of the string. The name |size()| is picked for STL + // Returns the length of the string. The name `size()` is picked for STL // compatibility reasons. size_t size() const { return size_; } @@ -61,10 +60,6 @@ class SimpleStringBuilder { SimpleStringBuilder& AppendFormat(const char* fmt, ...); - // An alternate way from operator<<() to append a string. This variant is - // slightly more efficient when the length of the string to append, is known. - SimpleStringBuilder& Append(const char* str, size_t length); - private: bool IsConsistent() const { return size_ <= buffer_.size() - 1 && buffer_[size_] == '\0'; diff --git a/webrtc/rtc_base/swap_queue.h b/webrtc/rtc_base/swap_queue.h index 9eac49a..3c8149c 100644 --- a/webrtc/rtc_base/swap_queue.h +++ b/webrtc/rtc_base/swap_queue.h @@ -17,8 +17,8 @@ #include #include +#include "absl/base/attributes.h" #include "rtc_base/checks.h" -#include "rtc_base/system/unused.h" namespace webrtc { @@ -127,7 +127,7 @@ class SwapQueue { // When specified, the T given in *input must pass the ItemVerifier() test. // The contents of *input after the call are then also guaranteed to pass the // ItemVerifier() test. - bool Insert(T* input) RTC_WARN_UNUSED_RESULT { + ABSL_MUST_USE_RESULT bool Insert(T* input) { RTC_DCHECK(input); RTC_DCHECK(queue_item_verifier_(*input)); @@ -168,7 +168,7 @@ class SwapQueue { // empty). When specified, The T given in *output must pass the ItemVerifier() // test and the contents of *output after the call are then also guaranteed to // pass the ItemVerifier() test. - bool Remove(T* output) RTC_WARN_UNUSED_RESULT { + ABSL_MUST_USE_RESULT bool Remove(T* output) { RTC_DCHECK(output); RTC_DCHECK(queue_item_verifier_(*output)); diff --git a/webrtc/rtc_base/synchronization/BUILD.gn b/webrtc/rtc_base/synchronization/BUILD.gn index a79a048..5cab524 100644 --- a/webrtc/rtc_base/synchronization/BUILD.gn +++ b/webrtc/rtc_base/synchronization/BUILD.gn @@ -20,9 +20,8 @@ rtc_library("yield") { deps = [] } -rtc_library("mutex") { +rtc_source_set("mutex") { sources = [ - "mutex.cc", "mutex.h", "mutex_critical_section.h", "mutex_pthread.h", @@ -36,7 +35,7 @@ rtc_library("mutex") { "..:checks", "..:macromagic", "..:platform_thread_types", - "../system:unused", + "../system:no_unique_address", ] absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] if (rtc_use_absl_mutex) { @@ -44,33 +43,15 @@ rtc_library("mutex") { } } -rtc_library("rw_lock_wrapper") { - public = [ "rw_lock_wrapper.h" ] - sources = [ "rw_lock_wrapper.cc" ] - deps = [ "..:macromagic" ] - if (is_win) { - sources += [ - "rw_lock_win.cc", - "rw_lock_win.h", - ] - deps += [ "..:logging" ] - } else { - sources += [ - "rw_lock_posix.cc", - "rw_lock_posix.h", - ] - } -} - -rtc_library("sequence_checker") { +rtc_library("sequence_checker_internal") { + visibility = [ "../../api:sequence_checker" ] sources = [ - "sequence_checker.cc", - "sequence_checker.h", + "sequence_checker_internal.cc", + "sequence_checker_internal.h", ] deps = [ ":mutex", "..:checks", - "..:criticalsection", "..:macromagic", "..:platform_thread_types", "..:stringutils", @@ -91,7 +72,7 @@ rtc_library("yield_policy") { ] } -if (rtc_include_tests) { +if (rtc_include_tests && rtc_enable_google_benchmarks) { rtc_library("synchronization_unittests") { testonly = true sources = [ @@ -104,8 +85,9 @@ if (rtc_include_tests) { ":yield_policy", "..:checks", "..:macromagic", - "..:rtc_base", + "..:platform_thread", "..:rtc_event", + "..:threading", "../../test:test_support", "//third_party/google_benchmark", ] @@ -120,19 +102,4 @@ if (rtc_include_tests) { "//third_party/google_benchmark", ] } - - rtc_library("sequence_checker_unittests") { - testonly = true - - sources = [ "sequence_checker_unittest.cc" ] - deps = [ - ":sequence_checker", - "..:checks", - "..:rtc_base_approved", - "..:task_queue_for_test", - "../../api:function_view", - "../../test:test_main", - "../../test:test_support", - ] - } } diff --git a/webrtc/rtc_base/synchronization/mutex.cc b/webrtc/rtc_base/synchronization/mutex.cc deleted file mode 100644 index 6c2d6ff..0000000 --- a/webrtc/rtc_base/synchronization/mutex.cc +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 The WebRTC Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "rtc_base/synchronization/mutex.h" - -#include "rtc_base/checks.h" -#include "rtc_base/synchronization/yield.h" - -namespace webrtc { - -#if !defined(WEBRTC_ABSL_MUTEX) -void GlobalMutex::Lock() { - while (mutex_locked_.exchange(1)) { - YieldCurrentThread(); - } -} - -void GlobalMutex::Unlock() { - int old = mutex_locked_.exchange(0); - RTC_DCHECK_EQ(old, 1) << "Unlock called without calling Lock first"; -} - -GlobalMutexLock::GlobalMutexLock(GlobalMutex* mutex) : mutex_(mutex) { - mutex_->Lock(); -} - -GlobalMutexLock::~GlobalMutexLock() { - mutex_->Unlock(); -} -#endif // #if !defined(WEBRTC_ABSL_MUTEX) - -} // namespace webrtc diff --git a/webrtc/rtc_base/synchronization/mutex.h b/webrtc/rtc_base/synchronization/mutex.h index 620fe74..104f4fd 100644 --- a/webrtc/rtc_base/synchronization/mutex.h +++ b/webrtc/rtc_base/synchronization/mutex.h @@ -13,9 +13,9 @@ #include +#include "absl/base/attributes.h" #include "absl/base/const_init.h" #include "rtc_base/checks.h" -#include "rtc_base/system/unused.h" #include "rtc_base/thread_annotations.h" #if defined(WEBRTC_ABSL_MUTEX) @@ -38,15 +38,15 @@ class RTC_LOCKABLE Mutex final { Mutex(const Mutex&) = delete; Mutex& operator=(const Mutex&) = delete; - void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION() { - impl_.Lock(); - } - RTC_WARN_UNUSED_RESULT bool TryLock() RTC_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION() { impl_.Lock(); } + ABSL_MUST_USE_RESULT bool TryLock() RTC_EXCLUSIVE_TRYLOCK_FUNCTION(true) { return impl_.TryLock(); } - void Unlock() RTC_UNLOCK_FUNCTION() { - impl_.Unlock(); - } + // Return immediately if this thread holds the mutex, or RTC_DCHECK_IS_ON==0. + // Otherwise, may report an error (typically by crashing with a diagnostic), + // or may return immediately. + void AssertHeld() const RTC_ASSERT_EXCLUSIVE_LOCK() { impl_.AssertHeld(); } + void Unlock() RTC_UNLOCK_FUNCTION() { impl_.Unlock(); } private: MutexImpl impl_; @@ -68,41 +68,6 @@ class RTC_SCOPED_LOCKABLE MutexLock final { Mutex* mutex_; }; -// A mutex used to protect global variables. Do NOT use for other purposes. -#if defined(WEBRTC_ABSL_MUTEX) -using GlobalMutex = absl::Mutex; -using GlobalMutexLock = absl::MutexLock; -#else -class RTC_LOCKABLE GlobalMutex final { - public: - GlobalMutex(const GlobalMutex&) = delete; - GlobalMutex& operator=(const GlobalMutex&) = delete; - - constexpr explicit GlobalMutex(absl::ConstInitType /*unused*/) - : mutex_locked_(0) {} - - void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION(); - void Unlock() RTC_UNLOCK_FUNCTION(); - - private: - std::atomic mutex_locked_; // 0 means lock not taken, 1 means taken. -}; - -// GlobalMutexLock, for serializing execution through a scope. -class RTC_SCOPED_LOCKABLE GlobalMutexLock final { - public: - GlobalMutexLock(const GlobalMutexLock&) = delete; - GlobalMutexLock& operator=(const GlobalMutexLock&) = delete; - - explicit GlobalMutexLock(GlobalMutex* mutex) - RTC_EXCLUSIVE_LOCK_FUNCTION(mutex_); - ~GlobalMutexLock() RTC_UNLOCK_FUNCTION(); - - private: - GlobalMutex* mutex_; -}; -#endif // if defined(WEBRTC_ABSL_MUTEX) - } // namespace webrtc #endif // RTC_BASE_SYNCHRONIZATION_MUTEX_H_ diff --git a/webrtc/rtc_base/synchronization/mutex_critical_section.h b/webrtc/rtc_base/synchronization/mutex_critical_section.h index d206794..b6a3c4a 100644 --- a/webrtc/rtc_base/synchronization/mutex_critical_section.h +++ b/webrtc/rtc_base/synchronization/mutex_critical_section.h @@ -23,6 +23,7 @@ #include // must come after windows headers. // clang-format on +#include "absl/base/attributes.h" #include "rtc_base/thread_annotations.h" namespace webrtc { @@ -37,9 +38,10 @@ class RTC_LOCKABLE MutexImpl final { void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION() { EnterCriticalSection(&critical_section_); } - RTC_WARN_UNUSED_RESULT bool TryLock() RTC_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + ABSL_MUST_USE_RESULT bool TryLock() RTC_EXCLUSIVE_TRYLOCK_FUNCTION(true) { return TryEnterCriticalSection(&critical_section_) != FALSE; } + void AssertHeld() const RTC_ASSERT_EXCLUSIVE_LOCK() {} void Unlock() RTC_UNLOCK_FUNCTION() { LeaveCriticalSection(&critical_section_); } diff --git a/webrtc/rtc_base/synchronization/mutex_pthread.h b/webrtc/rtc_base/synchronization/mutex_pthread.h index c9496e7..c749a20 100644 --- a/webrtc/rtc_base/synchronization/mutex_pthread.h +++ b/webrtc/rtc_base/synchronization/mutex_pthread.h @@ -18,6 +18,8 @@ #include #endif +#include "absl/base/attributes.h" +#include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread_annotations.h" namespace webrtc { @@ -38,14 +40,60 @@ class RTC_LOCKABLE MutexImpl final { MutexImpl& operator=(const MutexImpl&) = delete; ~MutexImpl() { pthread_mutex_destroy(&mutex_); } - void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION() { pthread_mutex_lock(&mutex_); } - RTC_WARN_UNUSED_RESULT bool TryLock() RTC_EXCLUSIVE_TRYLOCK_FUNCTION(true) { - return pthread_mutex_trylock(&mutex_) == 0; + void Lock() RTC_EXCLUSIVE_LOCK_FUNCTION() { + pthread_mutex_lock(&mutex_); + owner_.SetOwner(); + } + ABSL_MUST_USE_RESULT bool TryLock() RTC_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + if (pthread_mutex_trylock(&mutex_) != 0) { + return false; + } + owner_.SetOwner(); + return true; + } + void AssertHeld() const RTC_ASSERT_EXCLUSIVE_LOCK() { owner_.AssertOwned(); } + void Unlock() RTC_UNLOCK_FUNCTION() { + owner_.ClearOwner(); + pthread_mutex_unlock(&mutex_); } - void Unlock() RTC_UNLOCK_FUNCTION() { pthread_mutex_unlock(&mutex_); } private: + class OwnerRecord { + public: +#if !RTC_DCHECK_IS_ON + void SetOwner() {} + void ClearOwner() {} + void AssertOwned() const {} +#else + void SetOwner() { + latest_owner_ = pthread_self(); + is_owned_ = true; + } + void ClearOwner() { is_owned_ = false; } + void AssertOwned() const { + RTC_CHECK(is_owned_); + RTC_CHECK(pthread_equal(latest_owner_, pthread_self())); + } + + private: + // Use two separate primitive types, rather than absl::optional, since the + // data race described below might invalidate absl::optional invariants. + bool is_owned_ = false; + pthread_t latest_owner_ = pthread_self(); +#endif + }; + pthread_mutex_t mutex_; + // This record is modified only with the mutex held, and hence, calls to + // AssertHeld where mutex is held are race-free and will always succeed. + // + // The failure case is more subtle: If AssertHeld is called from some thread + // not holding the mutex, and RTC_DCHECK_IS_ON==1, we have a data race. It is + // highly likely that the calling thread will see `is_owned_` false or + // `latest_owner_` different from itself, and crash. But it may fail to crash, + // and invoke some other undefined behavior (still, this race can happen only + // when RTC_DCHECK_IS_ON==1). + RTC_NO_UNIQUE_ADDRESS OwnerRecord owner_; }; } // namespace webrtc diff --git a/webrtc/rtc_base/synchronization/rw_lock_posix.cc b/webrtc/rtc_base/synchronization/rw_lock_posix.cc deleted file mode 100644 index 15ef3d7..0000000 --- a/webrtc/rtc_base/synchronization/rw_lock_posix.cc +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "rtc_base/synchronization/rw_lock_posix.h" - -#include - -namespace webrtc { - -RWLockPosix::RWLockPosix() : lock_() {} - -RWLockPosix::~RWLockPosix() { - pthread_rwlock_destroy(&lock_); -} - -RWLockPosix* RWLockPosix::Create() { - RWLockPosix* ret_val = new RWLockPosix(); - if (!ret_val->Init()) { - delete ret_val; - return NULL; - } - return ret_val; -} - -bool RWLockPosix::Init() { - return pthread_rwlock_init(&lock_, 0) == 0; -} - -void RWLockPosix::AcquireLockExclusive() { - pthread_rwlock_wrlock(&lock_); -} - -void RWLockPosix::ReleaseLockExclusive() { - pthread_rwlock_unlock(&lock_); -} - -void RWLockPosix::AcquireLockShared() { - pthread_rwlock_rdlock(&lock_); -} - -void RWLockPosix::ReleaseLockShared() { - pthread_rwlock_unlock(&lock_); -} - -} // namespace webrtc diff --git a/webrtc/rtc_base/synchronization/rw_lock_posix.h b/webrtc/rtc_base/synchronization/rw_lock_posix.h deleted file mode 100644 index a103fe7..0000000 --- a/webrtc/rtc_base/synchronization/rw_lock_posix.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef RTC_BASE_SYNCHRONIZATION_RW_LOCK_POSIX_H_ -#define RTC_BASE_SYNCHRONIZATION_RW_LOCK_POSIX_H_ - -#include - -#include "rtc_base/synchronization/rw_lock_wrapper.h" - -namespace webrtc { - -class RWLockPosix : public RWLockWrapper { - public: - static RWLockPosix* Create(); - ~RWLockPosix() override; - - void AcquireLockExclusive() override; - void ReleaseLockExclusive() override; - - void AcquireLockShared() override; - void ReleaseLockShared() override; - - private: - RWLockPosix(); - bool Init(); - - pthread_rwlock_t lock_; -}; - -} // namespace webrtc - -#endif // RTC_BASE_SYNCHRONIZATION_RW_LOCK_POSIX_H_ diff --git a/webrtc/rtc_base/synchronization/rw_lock_win.cc b/webrtc/rtc_base/synchronization/rw_lock_win.cc deleted file mode 100644 index 3274c78..0000000 --- a/webrtc/rtc_base/synchronization/rw_lock_win.cc +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "rtc_base/synchronization/rw_lock_win.h" - -#include "rtc_base/logging.h" - -namespace webrtc { - -RWLockWin::RWLockWin() { - InitializeSRWLock(&lock_); -} - -RWLockWin* RWLockWin::Create() { - return new RWLockWin(); -} - -void RWLockWin::AcquireLockExclusive() { - AcquireSRWLockExclusive(&lock_); -} - -void RWLockWin::ReleaseLockExclusive() { - ReleaseSRWLockExclusive(&lock_); -} - -void RWLockWin::AcquireLockShared() { - AcquireSRWLockShared(&lock_); -} - -void RWLockWin::ReleaseLockShared() { - ReleaseSRWLockShared(&lock_); -} - -} // namespace webrtc diff --git a/webrtc/rtc_base/synchronization/rw_lock_win.h b/webrtc/rtc_base/synchronization/rw_lock_win.h deleted file mode 100644 index c79e61e..0000000 --- a/webrtc/rtc_base/synchronization/rw_lock_win.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef RTC_BASE_SYNCHRONIZATION_RW_LOCK_WIN_H_ -#define RTC_BASE_SYNCHRONIZATION_RW_LOCK_WIN_H_ - -#include - -#include "rtc_base/synchronization/rw_lock_wrapper.h" - -namespace webrtc { - -class RWLockWin : public RWLockWrapper { - public: - static RWLockWin* Create(); - - void AcquireLockExclusive() override; - void ReleaseLockExclusive() override; - - void AcquireLockShared() override; - void ReleaseLockShared() override; - - private: - RWLockWin(); - - SRWLOCK lock_; -}; - -} // namespace webrtc - -#endif // RTC_BASE_SYNCHRONIZATION_RW_LOCK_WIN_H_ diff --git a/webrtc/rtc_base/synchronization/rw_lock_wrapper.h b/webrtc/rtc_base/synchronization/rw_lock_wrapper.h deleted file mode 100644 index 39f52fc..0000000 --- a/webrtc/rtc_base/synchronization/rw_lock_wrapper.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef RTC_BASE_SYNCHRONIZATION_RW_LOCK_WRAPPER_H_ -#define RTC_BASE_SYNCHRONIZATION_RW_LOCK_WRAPPER_H_ - -#include "rtc_base/thread_annotations.h" - -// Note, Windows pre-Vista version of RW locks are not supported natively. For -// these OSs regular critical sections have been used to approximate RW lock -// functionality and will therefore have worse performance. - -namespace webrtc { - -class RTC_LOCKABLE RWLockWrapper { - public: - static RWLockWrapper* CreateRWLock(); - virtual ~RWLockWrapper() {} - - virtual void AcquireLockExclusive() RTC_EXCLUSIVE_LOCK_FUNCTION() = 0; - virtual void ReleaseLockExclusive() RTC_UNLOCK_FUNCTION() = 0; - - virtual void AcquireLockShared() RTC_SHARED_LOCK_FUNCTION() = 0; - virtual void ReleaseLockShared() RTC_UNLOCK_FUNCTION() = 0; -}; - -// RAII extensions of the RW lock. Prevents Acquire/Release missmatches and -// provides more compact locking syntax. -class RTC_SCOPED_LOCKABLE ReadLockScoped { - public: - explicit ReadLockScoped(RWLockWrapper& rw_lock) - RTC_SHARED_LOCK_FUNCTION(rw_lock) - : rw_lock_(rw_lock) { - rw_lock_.AcquireLockShared(); - } - - ~ReadLockScoped() RTC_UNLOCK_FUNCTION() { rw_lock_.ReleaseLockShared(); } - - private: - RWLockWrapper& rw_lock_; -}; - -class RTC_SCOPED_LOCKABLE WriteLockScoped { - public: - explicit WriteLockScoped(RWLockWrapper& rw_lock) - RTC_EXCLUSIVE_LOCK_FUNCTION(rw_lock) - : rw_lock_(rw_lock) { - rw_lock_.AcquireLockExclusive(); - } - - ~WriteLockScoped() RTC_UNLOCK_FUNCTION() { rw_lock_.ReleaseLockExclusive(); } - - private: - RWLockWrapper& rw_lock_; -}; - -} // namespace webrtc - -#endif // RTC_BASE_SYNCHRONIZATION_RW_LOCK_WRAPPER_H_ diff --git a/webrtc/rtc_base/synchronization/sequence_checker.h b/webrtc/rtc_base/synchronization/sequence_checker.h deleted file mode 100644 index ecf8490..0000000 --- a/webrtc/rtc_base/synchronization/sequence_checker.h +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#ifndef RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_H_ -#define RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_H_ - -#include - -#include "api/task_queue/task_queue_base.h" -#include "rtc_base/platform_thread_types.h" -#include "rtc_base/synchronization/mutex.h" -#include "rtc_base/system/rtc_export.h" -#include "rtc_base/thread_annotations.h" - -namespace webrtc { -// Real implementation of SequenceChecker, for use in debug mode, or -// for temporary use in release mode (e.g. to RTC_CHECK on a threading issue -// seen only in the wild). -// -// Note: You should almost always use the SequenceChecker class to get the -// right version for your build configuration. -class RTC_EXPORT SequenceCheckerImpl { - public: - SequenceCheckerImpl(); - ~SequenceCheckerImpl(); - - bool IsCurrent() const; - // Changes the task queue or thread that is checked for in IsCurrent. This can - // be useful when an object may be created on one task queue / thread and then - // used exclusively on another thread. - void Detach(); - - // Returns a string that is formatted to match with the error string printed - // by RTC_CHECK() when a condition is not met. - // This is used in conjunction with the RTC_DCHECK_RUN_ON() macro. - std::string ExpectationToString() const; - - private: - mutable Mutex lock_; - // These are mutable so that IsCurrent can set them. - mutable bool attached_ RTC_GUARDED_BY(lock_); - mutable rtc::PlatformThreadRef valid_thread_ RTC_GUARDED_BY(lock_); - mutable const TaskQueueBase* valid_queue_ RTC_GUARDED_BY(lock_); - mutable const void* valid_system_queue_ RTC_GUARDED_BY(lock_); -}; - -// Do nothing implementation, for use in release mode. -// -// Note: You should almost always use the SequenceChecker class to get the -// right version for your build configuration. -class SequenceCheckerDoNothing { - public: - bool IsCurrent() const { return true; } - void Detach() {} -}; - -// SequenceChecker is a helper class used to help verify that some methods -// of a class are called on the same task queue or thread. A -// SequenceChecker is bound to a a task queue if the object is -// created on a task queue, or a thread otherwise. -// -// -// Example: -// class MyClass { -// public: -// void Foo() { -// RTC_DCHECK_RUN_ON(sequence_checker_); -// ... (do stuff) ... -// } -// -// private: -// SequenceChecker sequence_checker_; -// } -// -// In Release mode, IsCurrent will always return true. -#if RTC_DCHECK_IS_ON -class RTC_LOCKABLE SequenceChecker : public SequenceCheckerImpl {}; -#else -class RTC_LOCKABLE SequenceChecker : public SequenceCheckerDoNothing {}; -#endif // RTC_ENABLE_THREAD_CHECKER - -namespace webrtc_seq_check_impl { -// Helper class used by RTC_DCHECK_RUN_ON (see example usage below). -class RTC_SCOPED_LOCKABLE SequenceCheckerScope { - public: - template - explicit SequenceCheckerScope(const ThreadLikeObject* thread_like_object) - RTC_EXCLUSIVE_LOCK_FUNCTION(thread_like_object) {} - SequenceCheckerScope(const SequenceCheckerScope&) = delete; - SequenceCheckerScope& operator=(const SequenceCheckerScope&) = delete; - ~SequenceCheckerScope() RTC_UNLOCK_FUNCTION() {} - - template - static bool IsCurrent(const ThreadLikeObject* thread_like_object) { - return thread_like_object->IsCurrent(); - } -}; -} // namespace webrtc_seq_check_impl -} // namespace webrtc - -// RTC_RUN_ON/RTC_GUARDED_BY/RTC_DCHECK_RUN_ON macros allows to annotate -// variables are accessed from same thread/task queue. -// Using tools designed to check mutexes, it checks at compile time everywhere -// variable is access, there is a run-time dcheck thread/task queue is correct. -// -// class ThreadExample { -// public: -// void NeedVar1() { -// RTC_DCHECK_RUN_ON(network_thread_); -// transport_->Send(); -// } -// -// private: -// rtc::Thread* network_thread_; -// int transport_ RTC_GUARDED_BY(network_thread_); -// }; -// -// class SequenceCheckerExample { -// public: -// int CalledFromPacer() RTC_RUN_ON(pacer_sequence_checker_) { -// return var2_; -// } -// -// void CallMeFromPacer() { -// RTC_DCHECK_RUN_ON(&pacer_sequence_checker_) -// << "Should be called from pacer"; -// CalledFromPacer(); -// } -// -// private: -// int pacer_var_ RTC_GUARDED_BY(pacer_sequence_checker_); -// SequenceChecker pacer_sequence_checker_; -// }; -// -// class TaskQueueExample { -// public: -// class Encoder { -// public: -// rtc::TaskQueue* Queue() { return encoder_queue_; } -// void Encode() { -// RTC_DCHECK_RUN_ON(encoder_queue_); -// DoSomething(var_); -// } -// -// private: -// rtc::TaskQueue* const encoder_queue_; -// Frame var_ RTC_GUARDED_BY(encoder_queue_); -// }; -// -// void Encode() { -// // Will fail at runtime when DCHECK is enabled: -// // encoder_->Encode(); -// // Will work: -// rtc::scoped_refptr encoder = encoder_; -// encoder_->Queue()->PostTask([encoder] { encoder->Encode(); }); -// } -// -// private: -// rtc::scoped_refptr encoder_; -// } - -// Document if a function expected to be called from same thread/task queue. -#define RTC_RUN_ON(x) \ - RTC_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(x)) - -namespace webrtc { -std::string ExpectationToString(const webrtc::SequenceChecker* checker); - -// Catch-all implementation for types other than explicitly supported above. -template -std::string ExpectationToString(const ThreadLikeObject*) { - return std::string(); -} - -} // namespace webrtc - -#define RTC_DCHECK_RUN_ON(x) \ - webrtc::webrtc_seq_check_impl::SequenceCheckerScope seq_check_scope(x); \ - RTC_DCHECK((x)->IsCurrent()) << webrtc::ExpectationToString(x) - -#endif // RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_H_ diff --git a/webrtc/rtc_base/synchronization/sequence_checker.cc b/webrtc/rtc_base/synchronization/sequence_checker_internal.cc similarity index 59% rename from webrtc/rtc_base/synchronization/sequence_checker.cc rename to webrtc/rtc_base/synchronization/sequence_checker_internal.cc index 1de26cf..4b9583d 100644 --- a/webrtc/rtc_base/synchronization/sequence_checker.cc +++ b/webrtc/rtc_base/synchronization/sequence_checker_internal.cc @@ -7,61 +7,39 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/synchronization/sequence_checker_internal.h" -#if defined(WEBRTC_MAC) -#include -#endif +#include +#include "rtc_base/checks.h" #include "rtc_base/strings/string_builder.h" namespace webrtc { -namespace { -// On Mac, returns the label of the current dispatch queue; elsewhere, return -// null. -const void* GetSystemQueueRef() { -#if defined(WEBRTC_MAC) - return dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL); -#else - return nullptr; -#endif -} +namespace webrtc_sequence_checker_internal { -} // namespace - -std::string ExpectationToString(const webrtc::SequenceChecker* checker) { -#if RTC_DCHECK_IS_ON - return checker->ExpectationToString(); -#endif - return std::string(); -} - -SequenceCheckerImpl::SequenceCheckerImpl() - : attached_(true), +SequenceCheckerImpl::SequenceCheckerImpl(bool attach_to_current_thread) + : attached_(attach_to_current_thread), valid_thread_(rtc::CurrentThreadRef()), - valid_queue_(TaskQueueBase::Current()), - valid_system_queue_(GetSystemQueueRef()) {} + valid_queue_(TaskQueueBase::Current()) {} -SequenceCheckerImpl::~SequenceCheckerImpl() = default; +SequenceCheckerImpl::SequenceCheckerImpl(TaskQueueBase* attached_queue) + : attached_(attached_queue != nullptr), + valid_thread_(rtc::PlatformThreadRef()), + valid_queue_(attached_queue) {} bool SequenceCheckerImpl::IsCurrent() const { const TaskQueueBase* const current_queue = TaskQueueBase::Current(); const rtc::PlatformThreadRef current_thread = rtc::CurrentThreadRef(); - const void* const current_system_queue = GetSystemQueueRef(); MutexLock scoped_lock(&lock_); if (!attached_) { // Previously detached. attached_ = true; valid_thread_ = current_thread; valid_queue_ = current_queue; - valid_system_queue_ = current_system_queue; return true; } - if (valid_queue_ || current_queue) { + if (valid_queue_) { return valid_queue_ == current_queue; } - if (valid_system_queue_ && valid_system_queue_ == current_system_queue) { - return true; - } return rtc::IsThreadRefEqual(valid_thread_, current_thread); } @@ -76,7 +54,6 @@ void SequenceCheckerImpl::Detach() { std::string SequenceCheckerImpl::ExpectationToString() const { const TaskQueueBase* const current_queue = TaskQueueBase::Current(); const rtc::PlatformThreadRef current_thread = rtc::CurrentThreadRef(); - const void* const current_system_queue = GetSystemQueueRef(); MutexLock scoped_lock(&lock_); if (!attached_) return "Checker currently not attached."; @@ -90,17 +67,13 @@ std::string SequenceCheckerImpl::ExpectationToString() const { rtc::StringBuilder message; message.AppendFormat( - "# Expected: TQ: %p SysQ: %p Thread: %p\n" - "# Actual: TQ: %p SysQ: %p Thread: %p\n", - valid_queue_, valid_system_queue_, - reinterpret_cast(valid_thread_), current_queue, - current_system_queue, reinterpret_cast(current_thread)); + "# Expected: TQ: %p Thread: %p\n" + "# Actual: TQ: %p Thread: %p\n", + valid_queue_, reinterpret_cast(valid_thread_), current_queue, + reinterpret_cast(current_thread)); if ((valid_queue_ || current_queue) && valid_queue_ != current_queue) { message << "TaskQueue doesn't match\n"; - } else if (valid_system_queue_ && - valid_system_queue_ != current_system_queue) { - message << "System queue doesn't match\n"; } else if (!rtc::IsThreadRefEqual(valid_thread_, current_thread)) { message << "Threads don't match\n"; } @@ -109,4 +82,5 @@ std::string SequenceCheckerImpl::ExpectationToString() const { } #endif // RTC_DCHECK_IS_ON +} // namespace webrtc_sequence_checker_internal } // namespace webrtc diff --git a/webrtc/rtc_base/synchronization/sequence_checker_internal.h b/webrtc/rtc_base/synchronization/sequence_checker_internal.h new file mode 100644 index 0000000..a23ac08 --- /dev/null +++ b/webrtc/rtc_base/synchronization/sequence_checker_internal.h @@ -0,0 +1,90 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_INTERNAL_H_ +#define RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_INTERNAL_H_ + +#include +#include + +#include "api/task_queue/task_queue_base.h" +#include "rtc_base/platform_thread_types.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/rtc_export.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_sequence_checker_internal { + +// Real implementation of SequenceChecker, for use in debug mode, or +// for temporary use in release mode (e.g. to RTC_CHECK on a threading issue +// seen only in the wild). +// +// Note: You should almost always use the SequenceChecker class to get the +// right version for your build configuration. +class RTC_EXPORT SequenceCheckerImpl { + public: + explicit SequenceCheckerImpl(bool attach_to_current_thread); + explicit SequenceCheckerImpl(TaskQueueBase* attached_queue); + ~SequenceCheckerImpl() = default; + + bool IsCurrent() const; + // Changes the task queue or thread that is checked for in IsCurrent. This can + // be useful when an object may be created on one task queue / thread and then + // used exclusively on another thread. + void Detach(); + + // Returns a string that is formatted to match with the error string printed + // by RTC_CHECK() when a condition is not met. + // This is used in conjunction with the RTC_DCHECK_RUN_ON() macro. + std::string ExpectationToString() const; + + private: + mutable Mutex lock_; + // These are mutable so that IsCurrent can set them. + mutable bool attached_ RTC_GUARDED_BY(lock_); + mutable rtc::PlatformThreadRef valid_thread_ RTC_GUARDED_BY(lock_); + mutable const TaskQueueBase* valid_queue_ RTC_GUARDED_BY(lock_); +}; + +// Do nothing implementation, for use in release mode. +// +// Note: You should almost always use the SequenceChecker class to get the +// right version for your build configuration. +class SequenceCheckerDoNothing { + public: + explicit SequenceCheckerDoNothing(bool attach_to_current_thread) {} + explicit SequenceCheckerDoNothing(TaskQueueBase* attached_queue) {} + bool IsCurrent() const { return true; } + void Detach() {} +}; + +template +std::enable_if_t, + std::string> +ExpectationToString(const ThreadLikeObject* checker) { +#if RTC_DCHECK_IS_ON + return checker->ExpectationToString(); +#else + return std::string(); +#endif +} + +// Catch-all implementation for types other than explicitly supported above. +template +std::enable_if_t, + std::string> +ExpectationToString(const ThreadLikeObject*) { + return std::string(); +} + +} // namespace webrtc_sequence_checker_internal +} // namespace webrtc + +#endif // RTC_BASE_SYNCHRONIZATION_SEQUENCE_CHECKER_INTERNAL_H_ diff --git a/webrtc/rtc_base/system/BUILD.gn b/webrtc/rtc_base/system/BUILD.gn new file mode 100644 index 0000000..77f5139 --- /dev/null +++ b/webrtc/rtc_base/system/BUILD.gn @@ -0,0 +1,111 @@ +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") +if (is_android) { + import("//build/config/android/config.gni") + import("//build/config/android/rules.gni") +} + +rtc_source_set("arch") { + sources = [ "arch.h" ] +} + +rtc_source_set("asm_defines") { + sources = [ "asm_defines.h" ] +} + +rtc_library("file_wrapper") { + sources = [ + "file_wrapper.cc", + "file_wrapper.h", + ] + deps = [ + "..:checks", + "..:criticalsection", + "..:safe_conversions", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +if (rtc_include_tests) { + rtc_library("file_wrapper_unittests") { + testonly = true + sources = [ "file_wrapper_unittest.cc" ] + deps = [ + ":file_wrapper", + "//rtc_base:checks", + "//test:fileutils", + "//test:test_support", + ] + } +} + +rtc_source_set("ignore_warnings") { + sources = [ "ignore_warnings.h" ] +} + +rtc_source_set("inline") { + sources = [ "inline.h" ] +} + +rtc_source_set("unused") { + sources = [ "unused.h" ] +} + +rtc_source_set("assume") { + sources = [ "assume.h" ] +} + +rtc_source_set("rtc_export") { + sources = [ + "rtc_export.h", + "rtc_export_template.h", + ] +} + +rtc_source_set("no_unique_address") { + sources = [ "no_unique_address.h" ] +} + +rtc_source_set("no_cfi_icall") { + sources = [ "no_cfi_icall.h" ] + + deps = [ "..:sanitizer" ] +} + +if (is_mac || is_ios) { + rtc_library("cocoa_threading") { + sources = [ + "cocoa_threading.h", + "cocoa_threading.mm", + ] + deps = [ "..:checks" ] + frameworks = [ "Foundation.framework" ] + } + + rtc_library("gcd_helpers") { + sources = [ + "gcd_helpers.h", + "gcd_helpers.m", + ] + include_dirs = [ "../.." ] + } +} + +rtc_source_set("warn_current_thread_is_deadlocked") { + sources = [ "warn_current_thread_is_deadlocked.h" ] + deps = [] + if (is_android && !build_with_chromium) { + sources += [ "warn_current_thread_is_deadlocked.cc" ] + deps += [ + "..:logging", + "../../sdk/android:native_api_stacktrace", + ] + } +} diff --git a/webrtc/rtc_base/system/arch.h b/webrtc/rtc_base/system/arch.h index be2367b..9d945ef 100644 --- a/webrtc/rtc_base/system/arch.h +++ b/webrtc/rtc_base/system/arch.h @@ -73,6 +73,16 @@ #elif defined(__riscv) && __riscv_xlen == 32 #define WEBRTC_ARCH_32_BITS #define WEBRTC_ARCH_LITTLE_ENDIAN +#elif defined(__loongarch32) +#define WEBRTC_ARCH_LOONG_FAMILY +#define WEBRTC_ARCH_LOONG32 +#define WEBRTC_ARCH_32_BITS +#define WEBRTC_ARCH_LITTLE_ENDIAN +#elif defined(__loongarch64) +#define WEBRTC_ARCH_LOONG_FAMILY +#define WEBRTC_ARCH_LOONG64 +#define WEBRTC_ARCH_64_BITS +#define WEBRTC_ARCH_LITTLE_ENDIAN #elif defined(__pnacl__) #define WEBRTC_ARCH_32_BITS #define WEBRTC_ARCH_LITTLE_ENDIAN diff --git a/webrtc/rtc_base/system/file_wrapper.cc b/webrtc/rtc_base/system/file_wrapper.cc index 3b5f744..af34d0e 100644 --- a/webrtc/rtc_base/system/file_wrapper.cc +++ b/webrtc/rtc_base/system/file_wrapper.cc @@ -9,28 +9,37 @@ */ #include "rtc_base/system/file_wrapper.h" -#include "rtc_base/numerics/safe_conversions.h" + +#include #include +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" #ifdef _WIN32 -#include +#include #else -#include #endif #include namespace webrtc { namespace { -FILE* FileOpen(const char* file_name_utf8, bool read_only, int* error) { +FILE* FileOpen(absl::string_view file_name_utf8, bool read_only, int* error) { + RTC_CHECK_EQ(file_name_utf8.find_first_of('\0'), absl::string_view::npos) + << "Invalid filename, containing NUL character"; + std::string file_name(file_name_utf8); #if defined(_WIN32) - int len = MultiByteToWideChar(CP_UTF8, 0, file_name_utf8, -1, nullptr, 0); + int len = MultiByteToWideChar(CP_UTF8, 0, file_name.c_str(), -1, nullptr, 0); std::wstring wstr(len, 0); - MultiByteToWideChar(CP_UTF8, 0, file_name_utf8, -1, &wstr[0], len); + MultiByteToWideChar(CP_UTF8, 0, file_name.c_str(), -1, &wstr[0], len); FILE* file = _wfopen(wstr.c_str(), read_only ? L"rb" : L"wb"); #else - FILE* file = fopen(file_name_utf8, read_only ? "rb" : "wb"); + FILE* file = fopen(file_name.c_str(), read_only ? "rb" : "wb"); #endif if (!file && error) { *error = errno; @@ -38,36 +47,19 @@ FILE* FileOpen(const char* file_name_utf8, bool read_only, int* error) { return file; } -const char* GetCstrCheckNoEmbeddedNul(const std::string& s) { - const char* p = s.c_str(); - RTC_CHECK_EQ(strlen(p), s.size()) - << "Invalid filename, containing NUL character"; - return p; -} } // namespace // static -FileWrapper FileWrapper::OpenReadOnly(const char* file_name_utf8) { +FileWrapper FileWrapper::OpenReadOnly(absl::string_view file_name_utf8) { return FileWrapper(FileOpen(file_name_utf8, true, nullptr)); } // static -FileWrapper FileWrapper::OpenReadOnly(const std::string& file_name_utf8) { - return OpenReadOnly(GetCstrCheckNoEmbeddedNul(file_name_utf8)); -} - -// static -FileWrapper FileWrapper::OpenWriteOnly(const char* file_name_utf8, +FileWrapper FileWrapper::OpenWriteOnly(absl::string_view file_name_utf8, int* error /*=nullptr*/) { return FileWrapper(FileOpen(file_name_utf8, false, error)); } -// static -FileWrapper FileWrapper::OpenWriteOnly(const std::string& file_name_utf8, - int* error /*=nullptr*/) { - return OpenWriteOnly(GetCstrCheckNoEmbeddedNul(file_name_utf8), error); -} - FileWrapper::FileWrapper(FileWrapper&& other) { operator=(std::move(other)); } @@ -89,6 +81,22 @@ bool FileWrapper::SeekTo(int64_t position) { return fseek(file_, rtc::checked_cast(position), SEEK_SET) == 0; } +long FileWrapper::FileSize() { + if (file_ == nullptr) + return -1; + long original_position = ftell(file_); + if (original_position < 0) + return -1; + int seek_error = fseek(file_, 0, SEEK_END); + if (seek_error) + return -1; + long file_size = ftell(file_); + seek_error = fseek(file_, original_position, SEEK_SET); + if (seek_error) + return -1; + return file_size; +} + bool FileWrapper::Flush() { RTC_DCHECK(file_); return fflush(file_) == 0; diff --git a/webrtc/rtc_base/system/file_wrapper.h b/webrtc/rtc_base/system/file_wrapper.h index bfd8fcd..7e73577 100644 --- a/webrtc/rtc_base/system/file_wrapper.h +++ b/webrtc/rtc_base/system/file_wrapper.h @@ -12,11 +12,14 @@ #define RTC_BASE_SYSTEM_FILE_WRAPPER_H_ #include +#include #include #include #include +#include "absl/strings/string_view.h" + // Implementation that can read (exclusive) or write from/to a file. namespace webrtc { @@ -33,20 +36,16 @@ class FileWrapper final { public: // Opens a file, in read or write mode. Use the is_open() method on the // returned object to check if the open operation was successful. On failure, - // and if |error| is non-null, the system errno value is stored at |*error|. + // and if `error` is non-null, the system errno value is stored at |*error|. // The file is closed by the destructor. - static FileWrapper OpenReadOnly(const char* file_name_utf8); - static FileWrapper OpenReadOnly(const std::string& file_name_utf8); - static FileWrapper OpenWriteOnly(const char* file_name_utf8, - int* error = nullptr); - - static FileWrapper OpenWriteOnly(const std::string& file_name_utf8, + static FileWrapper OpenReadOnly(absl::string_view file_name_utf8); + static FileWrapper OpenWriteOnly(absl::string_view file_name_utf8, int* error = nullptr); FileWrapper() = default; - // Takes over ownership of |file|, closing it on destruction. Calling with - // null |file| is allowed, and results in a FileWrapper with is_open() false. + // Takes over ownership of `file`, closing it on destruction. Calling with + // null `file` is allowed, and results in a FileWrapper with is_open() false. explicit FileWrapper(FILE* file) : file_(file) {} ~FileWrapper() { Close(); } @@ -88,6 +87,11 @@ class FileWrapper final { // Seek to given position. bool SeekTo(int64_t position); + // Returns the file size or -1 if a size could not be determined. + // (A file size might not exists for non-seekable files or file-like + // objects, for example /dev/tty on unix.) + long FileSize(); + // Returns number of bytes read. Short count indicates EOF or error. size_t Read(void* buf, size_t length); diff --git a/webrtc/rtc_base/system/no_unique_address.h b/webrtc/rtc_base/system/no_unique_address.h new file mode 100644 index 0000000..a40db34 --- /dev/null +++ b/webrtc/rtc_base/system/no_unique_address.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_SYSTEM_NO_UNIQUE_ADDRESS_H_ +#define RTC_BASE_SYSTEM_NO_UNIQUE_ADDRESS_H_ + +// RTC_NO_UNIQUE_ADDRESS is a portable annotation to tell the compiler that +// a data member need not have an address distinct from all other non-static +// data members of its class. +// It allows empty types to actually occupy zero bytes as class members, +// instead of occupying at least one byte just so that they get their own +// address. There is almost never any reason not to use it on class members +// that could possibly be empty. +// The macro expands to [[no_unique_address]] if the compiler supports the +// attribute, it expands to nothing otherwise. +// Clang should supports this attribute since C++11, while other compilers +// should add support for it starting from C++20. Among clang compilers, +// clang-cl doesn't support it yet and support is unclear also when the target +// platform is iOS. +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif +#if __has_cpp_attribute(no_unique_address) +// NOLINTNEXTLINE(whitespace/braces) +#define RTC_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else +#define RTC_NO_UNIQUE_ADDRESS +#endif + +#endif // RTC_BASE_SYSTEM_NO_UNIQUE_ADDRESS_H_ diff --git a/webrtc/rtc_base/system/unused.h b/webrtc/rtc_base/system/unused.h index a0add4e..03d0c2f 100644 --- a/webrtc/rtc_base/system/unused.h +++ b/webrtc/rtc_base/system/unused.h @@ -11,29 +11,18 @@ #ifndef RTC_BASE_SYSTEM_UNUSED_H_ #define RTC_BASE_SYSTEM_UNUSED_H_ -// Annotate a function indicating the caller must examine the return value. -// Use like: -// int foo() RTC_WARN_UNUSED_RESULT; -// To explicitly ignore a result, cast to void. -// TODO(kwiberg): Remove when we can use [[nodiscard]] from C++17. -#if defined(__clang__) -#define RTC_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) -#elif defined(__GNUC__) -// gcc has a __warn_unused_result__ attribute, but you can't quiet it by -// casting to void, so we don't use it. -#define RTC_WARN_UNUSED_RESULT -#else -#define RTC_WARN_UNUSED_RESULT -#endif - // Prevent the compiler from warning about an unused variable. For example: // int result = DoSomething(); -// assert(result == 17); +// RTC_DCHECK(result == 17); // RTC_UNUSED(result); // Note: In most cases it is better to remove the unused variable rather than // suppressing the compiler warning. #ifndef RTC_UNUSED +#ifdef __cplusplus #define RTC_UNUSED(x) static_cast(x) +#else +#define RTC_UNUSED(x) (void)(x) +#endif #endif // RTC_UNUSED #endif // RTC_BASE_SYSTEM_UNUSED_H_ diff --git a/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.cc b/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.cc new file mode 100644 index 0000000..614208a --- /dev/null +++ b/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.cc @@ -0,0 +1,28 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/system/warn_current_thread_is_deadlocked.h" + +#include "rtc_base/logging.h" +#if 0 +# commented out to prevent pulling in extra android files +#include "sdk/android/native_api/stacktrace/stacktrace.h" +#endif + +namespace webrtc { + +void WarnThatTheCurrentThreadIsProbablyDeadlocked() { + RTC_LOG(LS_WARNING) << "Probable deadlock:"; +#if 0 + RTC_LOG(LS_WARNING) << StackTraceToString(GetStackTrace()); +#endif +} + +} // namespace webrtc diff --git a/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.h b/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.h index ae4313e..4a0ba9d 100644 --- a/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.h +++ b/webrtc/rtc_base/system/warn_current_thread_is_deadlocked.h @@ -14,11 +14,7 @@ namespace webrtc { #if defined(WEBRTC_ANDROID) && !defined(WEBRTC_CHROMIUM_BUILD) -/* webrtc-audio-processing: - * We don't support the Android SDK integration for this, so stub out void WarnThatTheCurrentThreadIsProbablyDeadlocked(); - */ -inline void WarnThatTheCurrentThreadIsProbablyDeadlocked() {} #else inline void WarnThatTheCurrentThreadIsProbablyDeadlocked() {} #endif diff --git a/webrtc/rtc_base/system_time.cc b/webrtc/rtc_base/system_time.cc new file mode 100644 index 0000000..058e6c2 --- /dev/null +++ b/webrtc/rtc_base/system_time.cc @@ -0,0 +1,102 @@ +/* + * Copyright 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// If WEBRTC_EXCLUDE_SYSTEM_TIME is set, an implementation of +// rtc::SystemTimeNanos() must be provided externally. +#ifndef WEBRTC_EXCLUDE_SYSTEM_TIME + +#include + +#include + +#if defined(WEBRTC_POSIX) +#include +#if defined(WEBRTC_MAC) +#include +#endif +#endif + +#if defined(WEBRTC_WIN) +// clang-format off +// clang formatting would put last, +// which leads to compilation failure. +#include +#include +#include +// clang-format on +#endif + +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/system_time.h" +#include "rtc_base/time_utils.h" + +namespace rtc { + +int64_t SystemTimeNanos() { + int64_t ticks; +#if defined(WEBRTC_MAC) + static mach_timebase_info_data_t timebase; + if (timebase.denom == 0) { + // Get the timebase if this is the first time we run. + // Recommended by Apple's QA1398. + if (mach_timebase_info(&timebase) != KERN_SUCCESS) { + RTC_DCHECK_NOTREACHED(); + } + } + // Use timebase to convert absolute time tick units into nanoseconds. + const auto mul = [](uint64_t a, uint32_t b) -> int64_t { + RTC_DCHECK_NE(b, 0); + RTC_DCHECK_LE(a, std::numeric_limits::max() / b) + << "The multiplication " << a << " * " << b << " overflows"; + return rtc::dchecked_cast(a * b); + }; + ticks = mul(mach_absolute_time(), timebase.numer) / timebase.denom; +#elif defined(WEBRTC_POSIX) + struct timespec ts; + // TODO(deadbeef): Do we need to handle the case when CLOCK_MONOTONIC is not + // supported? + clock_gettime(CLOCK_MONOTONIC, &ts); + ticks = kNumNanosecsPerSec * static_cast(ts.tv_sec) + + static_cast(ts.tv_nsec); +#elif defined(WINUWP) + ticks = WinUwpSystemTimeNanos(); +#elif defined(WEBRTC_WIN) + // TODO(webrtc:14601): Fix the volatile increment instead of suppressing the + // warning. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-volatile" + static volatile LONG last_timegettime = 0; + static volatile int64_t num_wrap_timegettime = 0; + volatile LONG* last_timegettime_ptr = &last_timegettime; + DWORD now = timeGetTime(); + // Atomically update the last gotten time + DWORD old = InterlockedExchange(last_timegettime_ptr, now); + if (now < old) { + // If now is earlier than old, there may have been a race between threads. + // 0x0fffffff ~3.1 days, the code will not take that long to execute + // so it must have been a wrap around. + if (old > 0xf0000000 && now < 0x0fffffff) { + num_wrap_timegettime++; + } + } + ticks = now + (num_wrap_timegettime << 32); + // TODO(deadbeef): Calculate with nanosecond precision. Otherwise, we're + // just wasting a multiply and divide when doing Time() on Windows. + ticks = ticks * kNumNanosecsPerMillisec; +#pragma clang diagnostic pop +#else +#error Unsupported platform. +#endif + return ticks; +} + +} // namespace rtc +#endif // WEBRTC_EXCLUDE_SYSTEM_TIME diff --git a/webrtc/rtc_base/system_time.h b/webrtc/rtc_base/system_time.h new file mode 100644 index 0000000..c0ebc2a --- /dev/null +++ b/webrtc/rtc_base/system_time.h @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_SYSTEM_TIME_H_ +#define RTC_BASE_SYSTEM_TIME_H_ + +#include + +namespace rtc { + +// Returns the actual system time, even if a clock is set for testing. +// Useful for timeouts while using a test clock, or for logging. +int64_t SystemTimeNanos(); + +} // namespace rtc + +#endif // RTC_BASE_SYSTEM_TIME_H_ diff --git a/webrtc/rtc_base/thread_annotations.h b/webrtc/rtc_base/thread_annotations.h index 8569fab..689f668 100644 --- a/webrtc/rtc_base/thread_annotations.h +++ b/webrtc/rtc_base/thread_annotations.h @@ -88,6 +88,9 @@ #define RTC_UNLOCK_FUNCTION(...) \ RTC_THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) +#define RTC_ASSERT_EXCLUSIVE_LOCK(...) \ + RTC_THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) + // An escape hatch for thread safety analysis to ignore the annotated function. #define RTC_NO_THREAD_SAFETY_ANALYSIS \ RTC_THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) diff --git a/webrtc/rtc_base/thread_checker.h b/webrtc/rtc_base/thread_checker.h deleted file mode 100644 index 876a08e..0000000 --- a/webrtc/rtc_base/thread_checker.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -// Borrowed from Chromium's src/base/threading/thread_checker.h. - -#ifndef RTC_BASE_THREAD_CHECKER_H_ -#define RTC_BASE_THREAD_CHECKER_H_ - -#include "rtc_base/deprecation.h" -#include "rtc_base/synchronization/sequence_checker.h" - -namespace rtc { -// TODO(srte): Replace usages of this with SequenceChecker. -class ThreadChecker : public webrtc::SequenceChecker { - public: - RTC_DEPRECATED bool CalledOnValidThread() const { return IsCurrent(); } - RTC_DEPRECATED void DetachFromThread() { Detach(); } -}; -} // namespace rtc -#endif // RTC_BASE_THREAD_CHECKER_H_ diff --git a/webrtc/rtc_base/time_utils.cc b/webrtc/rtc_base/time_utils.cc index 11c9d5a..9f112e4 100644 --- a/webrtc/rtc_base/time_utils.cc +++ b/webrtc/rtc_base/time_utils.cc @@ -12,27 +12,29 @@ #if defined(WEBRTC_POSIX) #include -#if defined(WEBRTC_MAC) -#include -#endif -#endif - -#if defined(WEBRTC_WIN) -// clang-format off -// clang formatting would put last, -// which leads to compilation failure. -#include -#include -#include -// clang-format on #endif #include "rtc_base/checks.h" #include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/system_time.h" #include "rtc_base/time_utils.h" +#if defined(WEBRTC_WIN) +#include "rtc_base/win32.h" +#endif +#if defined(WEBRTC_WIN) +#include +#endif namespace rtc { +#if defined(WEBRTC_WIN) || defined(WINUWP) +// FileTime (January 1st 1601) to Unix time (January 1st 1970) +// offset in units of 100ns. +static constexpr uint64_t kFileTimeToUnixTimeEpochOffset = + 116444736000000000ULL; +static constexpr uint64_t kFileTimeToMicroSeconds = 10LL; +#endif + ClockInterface* g_clock = nullptr; ClockInterface* SetClockForTesting(ClockInterface* clock) { @@ -123,8 +125,6 @@ class TimeHelper final { } private: - static constexpr uint64_t kFileTimeToUnixTimeEpochOffset = - 116444736000000000ULL; static constexpr uint64_t kNTPTimeToUnixTimeEpochOffset = 2208988800000L; // The number of nanoseconds since unix system epoch @@ -141,61 +141,12 @@ void SyncWithNtp(int64_t time_from_ntp_server_ms) { TimeHelper::SyncWithNtp(time_from_ntp_server_ms); } -#endif // defined(WINUWP) - -int64_t SystemTimeNanos() { - int64_t ticks; -#if defined(WEBRTC_MAC) - static mach_timebase_info_data_t timebase; - if (timebase.denom == 0) { - // Get the timebase if this is the first time we run. - // Recommended by Apple's QA1398. - if (mach_timebase_info(&timebase) != KERN_SUCCESS) { - RTC_NOTREACHED(); - } - } - // Use timebase to convert absolute time tick units into nanoseconds. - const auto mul = [](uint64_t a, uint32_t b) -> int64_t { - RTC_DCHECK_NE(b, 0); - RTC_DCHECK_LE(a, std::numeric_limits::max() / b) - << "The multiplication " << a << " * " << b << " overflows"; - return rtc::dchecked_cast(a * b); - }; - ticks = mul(mach_absolute_time(), timebase.numer) / timebase.denom; -#elif defined(WEBRTC_POSIX) - struct timespec ts; - // TODO(deadbeef): Do we need to handle the case when CLOCK_MONOTONIC is not - // supported? - clock_gettime(CLOCK_MONOTONIC, &ts); - ticks = kNumNanosecsPerSec * static_cast(ts.tv_sec) + - static_cast(ts.tv_nsec); -#elif defined(WINUWP) - ticks = TimeHelper::TicksNs(); -#elif defined(WEBRTC_WIN) - static volatile LONG last_timegettime = 0; - static volatile int64_t num_wrap_timegettime = 0; - volatile LONG* last_timegettime_ptr = &last_timegettime; - DWORD now = timeGetTime(); - // Atomically update the last gotten time - DWORD old = InterlockedExchange(last_timegettime_ptr, now); - if (now < old) { - // If now is earlier than old, there may have been a race between threads. - // 0x0fffffff ~3.1 days, the code will not take that long to execute - // so it must have been a wrap around. - if (old > 0xf0000000 && now < 0x0fffffff) { - num_wrap_timegettime++; - } - } - ticks = now + (num_wrap_timegettime << 32); - // TODO(deadbeef): Calculate with nanosecond precision. Otherwise, we're - // just wasting a multiply and divide when doing Time() on Windows. - ticks = ticks * kNumNanosecsPerMillisec; -#else -#error Unsupported platform. -#endif - return ticks; +int64_t WinUwpSystemTimeNanos() { + return TimeHelper::TicksNs(); } +#endif // defined(WINUWP) + int64_t SystemTimeMillis() { return static_cast(SystemTimeNanos() / kNumNanosecsPerMillisec); } @@ -232,28 +183,6 @@ int64_t TimeDiff(int64_t later, int64_t earlier) { return later - earlier; } -TimestampWrapAroundHandler::TimestampWrapAroundHandler() - : last_ts_(0), num_wrap_(-1) {} - -int64_t TimestampWrapAroundHandler::Unwrap(uint32_t ts) { - if (num_wrap_ == -1) { - last_ts_ = ts; - num_wrap_ = 0; - return ts; - } - - if (ts < last_ts_) { - if (last_ts_ >= 0xf0000000 && ts < 0x0fffffff) - ++num_wrap_; - } else if ((ts - last_ts_) > 0xf0000000) { - // Backwards wrap. Unwrap with last wrap count and don't update last_ts_. - return ts + (num_wrap_ - 1) * (int64_t{1} << 32); - } - - last_ts_ = ts; - return ts + (num_wrap_ << 32); -} - int64_t TmToSeconds(const tm& tm) { static short int mdays[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; static short int cumul_mdays[12] = {0, 31, 59, 90, 120, 151, @@ -289,11 +218,11 @@ int64_t TmToSeconds(const tm& tm) { // We will have added one day too much above if expiration is during a leap // year, and expiration is in January or February. - if (expiry_in_leap_year && month <= 2 - 1) // |month| is zero based. + if (expiry_in_leap_year && month <= 2 - 1) // `month` is zero based. day -= 1; - // Combine all variables into seconds from 1970-01-01 00:00 (except |month| - // which was accumulated into |day| above). + // Combine all variables into seconds from 1970-01-01 00:00 (except `month` + // which was accumulated into `day` above). return (((static_cast(year - 1970) * 365 + day) * 24 + hour) * 60 + min) * 60 + @@ -310,13 +239,15 @@ int64_t TimeUTCMicros() { // Convert from second (1.0) and microsecond (1e-6). return (static_cast(time.tv_sec) * rtc::kNumMicrosecsPerSec + time.tv_usec); - #elif defined(WEBRTC_WIN) - struct _timeb time; - _ftime(&time); - // Convert from second (1.0) and milliseconds (1e-3). - return (static_cast(time.time) * rtc::kNumMicrosecsPerSec + - static_cast(time.millitm) * rtc::kNumMicrosecsPerMillisec); + FILETIME ft; + // This will give us system file in UTC format in multiples of 100ns. + GetSystemTimeAsFileTime(&ft); + LARGE_INTEGER li; + li.HighPart = ft.dwHighDateTime; + li.LowPart = ft.dwLowDateTime; + return (li.QuadPart - kFileTimeToUnixTimeEpochOffset) / + kFileTimeToMicroSeconds; #endif } diff --git a/webrtc/rtc_base/time_utils.h b/webrtc/rtc_base/time_utils.h index 147ab8d..271c1d6 100644 --- a/webrtc/rtc_base/time_utils.h +++ b/webrtc/rtc_base/time_utils.h @@ -16,6 +16,7 @@ #include "rtc_base/checks.h" #include "rtc_base/system/rtc_export.h" +#include "rtc_base/system_time.h" namespace rtc { @@ -30,6 +31,12 @@ static const int64_t kNumNanosecsPerMillisec = static const int64_t kNumNanosecsPerMicrosec = kNumNanosecsPerSec / kNumMicrosecsPerSec; +// Elapsed milliseconds between NTP base, 1900 January 1 00:00 GMT +// (see https://tools.ietf.org/html/rfc868), and January 1 00:00 GMT 1970 +// epoch. This is useful when converting between the NTP time base and the +// time base used in RTCP reports. +constexpr int64_t kNtpJan1970Millisecs = 2'208'988'800 * kNumMillisecsPerSec; + // TODO(honghaiz): Define a type for the time value specifically. class ClockInterface { @@ -61,11 +68,16 @@ RTC_EXPORT ClockInterface* GetClockForTesting(); // Synchronizes the current clock based upon an NTP server's epoch in // milliseconds. void SyncWithNtp(int64_t time_from_ntp_server_ms); + +// Returns the current time in nanoseconds. The clock is synchonized with the +// system wall clock time upon instatiation. It may also be synchronized using +// the SyncWithNtp() function above. Please note that the clock will most likely +// drift away from the system wall clock time as time goes by. +int64_t WinUwpSystemTimeNanos(); #endif // defined(WINUWP) // Returns the actual system time, even if a clock is set for testing. // Useful for timeouts while using a test clock, or for logging. -int64_t SystemTimeNanos(); int64_t SystemTimeMillis(); // Returns the current time in milliseconds in 32 bits. @@ -102,17 +114,6 @@ inline int64_t TimeUntil(int64_t later) { return later - TimeMillis(); } -class TimestampWrapAroundHandler { - public: - TimestampWrapAroundHandler(); - - int64_t Unwrap(uint32_t ts); - - private: - uint32_t last_ts_; - int64_t num_wrap_; -}; - // Convert from tm, which is relative to 1900-01-01 00:00 to number of // seconds from 1970-01-01 00:00 ("epoch"). Don't return time_t since that // is still 32 bits on many systems. diff --git a/webrtc/rtc_base/trace_event.h b/webrtc/rtc_base/trace_event.h index a0b788f..6689bc0 100644 --- a/webrtc/rtc_base/trace_event.h +++ b/webrtc/rtc_base/trace_event.h @@ -21,13 +21,13 @@ #endif // Type values for identifying types in the TraceValue union. -#define TRACE_VALUE_TYPE_BOOL (static_cast(1)) -#define TRACE_VALUE_TYPE_UINT (static_cast(2)) -#define TRACE_VALUE_TYPE_INT (static_cast(3)) -#define TRACE_VALUE_TYPE_DOUBLE (static_cast(4)) -#define TRACE_VALUE_TYPE_POINTER (static_cast(5)) -#define TRACE_VALUE_TYPE_STRING (static_cast(6)) -#define TRACE_VALUE_TYPE_COPY_STRING (static_cast(7)) +#define TRACE_VALUE_TYPE_BOOL (static_cast(1)) +#define TRACE_VALUE_TYPE_UINT (static_cast(2)) +#define TRACE_VALUE_TYPE_INT (static_cast(3)) +#define TRACE_VALUE_TYPE_DOUBLE (static_cast(4)) +#define TRACE_VALUE_TYPE_POINTER (static_cast(5)) +#define TRACE_VALUE_TYPE_STRING (static_cast(6)) +#define TRACE_VALUE_TYPE_COPY_STRING (static_cast(7)) #if RTC_TRACE_EVENTS_ENABLED @@ -136,10 +136,10 @@ // TRACE_EVENT_INSTANT0("SUBSYSTEM", str); // BAD! // free(str); // Trace system now has dangling pointer // -// To avoid this issue with the |name| and |arg_name| parameters, use the +// To avoid this issue with the `name` and `arg_name` parameters, use the // TRACE_EVENT_COPY_XXX overloads of the macros at additional runtime overhead. // Notes: The category must always be in a long-lived char* (i.e. static const). -// The |arg_values|, when used, are always deep copied with the _COPY +// The `arg_values`, when used, are always deep copied with the _COPY // macros. // // When are string argument values copied: @@ -161,7 +161,7 @@ // By default, const char* argument values are assumed to have long-lived scope // and will not be copied. Use this macro to force a const char* to be copied. #define TRACE_STR_COPY(str) \ - webrtc::trace_event_internal::TraceStringWithCopy(str) + webrtc::trace_event_internal::TraceStringWithCopy(str) // This will mark the trace event as disabled by default. The user will need // to explicitly enable the event. @@ -170,7 +170,7 @@ // By default, uint64 ID argument values are not mangled with the Process ID in // TRACE_EVENT_ASYNC macros. Use this macro to force Process ID mangling. #define TRACE_ID_MANGLE(id) \ - webrtc::trace_event_internal::TraceID::ForceMangle(id) + webrtc::trace_event_internal::TraceID::ForceMangle(id) // Records a pair of begin and end events called "name" for the current // scope, with 0, 1 or 2 associated arguments. If the category is not @@ -178,175 +178,174 @@ // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. #define TRACE_EVENT0(category, name) \ - INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name) + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name) #define TRACE_EVENT1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val) + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val) #define TRACE_EVENT2(category, name, arg1_name, arg1_val, arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) // Records a single event called "name" immediately, with 0, 1 or 2 // associated arguments. If the category is not enabled, then this // does nothing. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -#define TRACE_EVENT_INSTANT0(category, name) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, \ - category, name, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_INSTANT1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, \ - category, name, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_INSTANT2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, \ - category, name, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ - arg2_name, arg2_val) -#define TRACE_EVENT_COPY_INSTANT0(category, name) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, \ - category, name, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_INSTANT0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_INSTANT1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +#define TRACE_EVENT_INSTANT2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) +#define TRACE_EVENT_COPY_INSTANT0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_COPY) #define TRACE_EVENT_COPY_INSTANT1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, \ - category, name, TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) #define TRACE_EVENT_COPY_INSTANT2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, \ - category, name, TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ - arg2_name, arg2_val) + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ + arg2_name, arg2_val) // Records a single BEGIN event called "name" immediately, with 0, 1 or 2 // associated arguments. If the category is not enabled, then this // does nothing. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -#define TRACE_EVENT_BEGIN0(category, name) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, \ - category, name, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, \ - category, name, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, \ - category, name, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ - arg2_name, arg2_val) -#define TRACE_EVENT_COPY_BEGIN0(category, name) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, \ - category, name, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_BEGIN0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +#define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) +#define TRACE_EVENT_COPY_BEGIN0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_COPY) #define TRACE_EVENT_COPY_BEGIN1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, \ - category, name, TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) -#define TRACE_EVENT_COPY_BEGIN2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, \ - category, name, TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ - arg2_name, arg2_val) + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) +#define TRACE_EVENT_COPY_BEGIN2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ + arg2_name, arg2_val) // Records a single END event for "name" immediately. If the category // is not enabled, then this does nothing. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -#define TRACE_EVENT_END0(category, name) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, \ - category, name, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, \ - category, name, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, \ - category, name, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ - arg2_name, arg2_val) -#define TRACE_EVENT_COPY_END0(category, name) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, \ - category, name, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_END0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +#define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) +#define TRACE_EVENT_COPY_END0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_COPY) #define TRACE_EVENT_COPY_END1(category, name, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, \ - category, name, TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) -#define TRACE_EVENT_COPY_END2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, \ - category, name, TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ - arg2_name, arg2_val) + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) +#define TRACE_EVENT_COPY_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ + arg2_name, arg2_val) // Records the value of a counter called "name" immediately. Value // must be representable as a 32 bit integer. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -#define TRACE_COUNTER1(category, name, value) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, \ - category, name, TRACE_EVENT_FLAG_NONE, \ - "value", static_cast(value)) -#define TRACE_COPY_COUNTER1(category, name, value) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, \ - category, name, TRACE_EVENT_FLAG_COPY, \ - "value", static_cast(value)) +#define TRACE_COUNTER1(category, name, value) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_NONE, "value", \ + static_cast(value)) +#define TRACE_COPY_COUNTER1(category, name, value) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_COPY, "value", \ + static_cast(value)) // Records the values of a multi-parted counter called "name" immediately. // The UI will treat value1 and value2 as parts of a whole, displaying their // values as a stacked-bar chart. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -#define TRACE_COUNTER2(category, name, value1_name, value1_val, \ - value2_name, value2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, \ - category, name, TRACE_EVENT_FLAG_NONE, \ - value1_name, static_cast(value1_val), \ - value2_name, static_cast(value2_val)) -#define TRACE_COPY_COUNTER2(category, name, value1_name, value1_val, \ - value2_name, value2_val) \ - INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, \ - category, name, TRACE_EVENT_FLAG_COPY, \ - value1_name, static_cast(value1_val), \ - value2_name, static_cast(value2_val)) +#define TRACE_COUNTER2(category, name, value1_name, value1_val, value2_name, \ + value2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_NONE, value1_name, \ + static_cast(value1_val), value2_name, \ + static_cast(value2_val)) +#define TRACE_COPY_COUNTER2(category, name, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_COPY, value1_name, \ + static_cast(value1_val), value2_name, \ + static_cast(value2_val)) // Records the value of a counter called "name" immediately. Value // must be representable as a 32 bit integer. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -// - |id| is used to disambiguate counters with the same name. It must either +// - `id` is used to disambiguate counters with the same name. It must either // be a pointer or an integer value up to 64 bits. If it's a pointer, the bits // will be xored with a hash of the process ID so that the same pointer on // two different processes will not collide. -#define TRACE_COUNTER_ID1(category, name, id, value) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, \ - category, name, id, TRACE_EVENT_FLAG_NONE, \ - "value", static_cast(value)) -#define TRACE_COPY_COUNTER_ID1(category, name, id, value) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - "value", static_cast(value)) +#define TRACE_COUNTER_ID1(category, name, id, value) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, name, \ + id, TRACE_EVENT_FLAG_NONE, "value", \ + static_cast(value)) +#define TRACE_COPY_COUNTER_ID1(category, name, id, value) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, name, \ + id, TRACE_EVENT_FLAG_COPY, "value", \ + static_cast(value)) // Records the values of a multi-parted counter called "name" immediately. // The UI will treat value1 and value2 as parts of a whole, displaying their // values as a stacked-bar chart. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -// - |id| is used to disambiguate counters with the same name. It must either +// - `id` is used to disambiguate counters with the same name. It must either // be a pointer or an integer value up to 64 bits. If it's a pointer, the bits // will be xored with a hash of the process ID so that the same pointer on // two different processes will not collide. -#define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ - value2_name, value2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, \ - category, name, id, TRACE_EVENT_FLAG_NONE, \ - value1_name, static_cast(value1_val), \ - value2_name, static_cast(value2_val)) -#define TRACE_COPY_COUNTER_ID2(category, name, id, value1_name, value1_val, \ - value2_name, value2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - value1_name, static_cast(value1_val), \ - value2_name, static_cast(value2_val)) - +#define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, name, \ + id, TRACE_EVENT_FLAG_NONE, value1_name, \ + static_cast(value1_val), value2_name, \ + static_cast(value2_val)) +#define TRACE_COPY_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, name, \ + id, TRACE_EVENT_FLAG_COPY, value1_name, \ + static_cast(value1_val), value2_name, \ + static_cast(value2_val)) // Records a single ASYNC_BEGIN event called "name" immediately, with 0, 1 or 2 // associated arguments. If the category is not enabled, then this // does nothing. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -// - |id| is used to match the ASYNC_BEGIN event with the ASYNC_END event. ASYNC +// - `id` is used to match the ASYNC_BEGIN event with the ASYNC_END event. ASYNC // events are considered to match if their category, name and id values all -// match. |id| must either be a pointer or an integer value up to 64 bits. If +// match. `id` must either be a pointer or an integer value up to 64 bits. If // it's a pointer, the bits will be xored with a hash of the process ID so // that the same pointer on two different processes will not collide. // An asynchronous operation can consist of multiple phases. The first phase is @@ -354,89 +353,91 @@ // ASYNC_STEP macros. When the operation completes, call ASYNC_END. // An ASYNC trace typically occur on a single thread (if not, they will only be // drawn on the thread defined in the ASYNC_BEGIN event), but all events in that -// operation must use the same |name| and |id|. Each event can have its own +// operation must use the same `name` and `id`. Each event can have its own // args. -#define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_NONE, \ - arg1_name, arg1_val, arg2_name, arg2_val) -#define TRACE_EVENT_COPY_ASYNC_BEGIN0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val) +#define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val, arg2_name, arg2_val) +#define TRACE_EVENT_COPY_ASYNC_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY) #define TRACE_EVENT_COPY_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val) + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val) #define TRACE_EVENT_COPY_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val, arg2_name, arg2_val) + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val, arg2_name, arg2_val) -// Records a single ASYNC_STEP event for |step| immediately. If the category -// is not enabled, then this does nothing. The |name| and |id| must match the -// ASYNC_BEGIN event above. The |step| param identifies this step within the +// Records a single ASYNC_STEP event for `step` immediately. If the category +// is not enabled, then this does nothing. The `name` and `id` must match the +// ASYNC_BEGIN event above. The `step` param identifies this step within the // async event. This should be called at the beginning of the next phase of an // asynchronous operation. -#define TRACE_EVENT_ASYNC_STEP0(category, name, id, step) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, \ - category, name, id, TRACE_EVENT_FLAG_NONE, "step", step) -#define TRACE_EVENT_ASYNC_STEP1(category, name, id, step, \ - arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, \ - category, name, id, TRACE_EVENT_FLAG_NONE, "step", step, \ - arg1_name, arg1_val) -#define TRACE_EVENT_COPY_ASYNC_STEP0(category, name, id, step) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, \ - category, name, id, TRACE_EVENT_FLAG_COPY, "step", step) -#define TRACE_EVENT_COPY_ASYNC_STEP1(category, name, id, step, \ - arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, \ - category, name, id, TRACE_EVENT_FLAG_COPY, "step", step, \ - arg1_name, arg1_val) +#define TRACE_EVENT_ASYNC_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step) +#define TRACE_EVENT_ASYNC_STEP1(category, name, id, step, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step, arg1_name, arg1_val) +#define TRACE_EVENT_COPY_ASYNC_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step) +#define TRACE_EVENT_COPY_ASYNC_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step, arg1_name, arg1_val) // Records a single ASYNC_END event for "name" immediately. If the category // is not enabled, then this does nothing. -#define TRACE_EVENT_ASYNC_END0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, \ - category, name, id, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, \ - category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, \ - category, name, id, TRACE_EVENT_FLAG_NONE, \ - arg1_name, arg1_val, arg2_name, arg2_val) -#define TRACE_EVENT_COPY_ASYNC_END0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, \ - category, name, id, TRACE_EVENT_FLAG_COPY) -#define TRACE_EVENT_COPY_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val) -#define TRACE_EVENT_COPY_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val, arg2_name, arg2_val) - +#define TRACE_EVENT_ASYNC_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val) +#define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val, arg2_name, arg2_val) +#define TRACE_EVENT_COPY_ASYNC_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_COPY_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val) +#define TRACE_EVENT_COPY_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val, arg2_name, arg2_val) // Records a single FLOW_BEGIN event called "name" immediately, with 0, 1 or 2 // associated arguments. If the category is not enabled, then this // does nothing. // - category and name strings must have application lifetime (statics or // literals). They may not include " chars. -// - |id| is used to match the FLOW_BEGIN event with the FLOW_END event. FLOW +// - `id` is used to match the FLOW_BEGIN event with the FLOW_END event. FLOW // events are considered to match if their category, name and id values all -// match. |id| must either be a pointer or an integer value up to 64 bits. If +// match. `id` must either be a pointer or an integer value up to 64 bits. If // it's a pointer, the bits will be xored with a hash of the process ID so // that the same pointer on two different processes will not collide. // FLOW events are different from ASYNC events in how they are drawn by the @@ -447,79 +448,81 @@ // by the FLOW_BEGIN calls. Additional phases can be defined using the FLOW_STEP // macros. When the operation completes, call FLOW_END. An async operation can // span threads and processes, but all events in that operation must use the -// same |name| and |id|. Each event can have its own args. -#define TRACE_EVENT_FLOW_BEGIN0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_NONE, \ - arg1_name, arg1_val, arg2_name, arg2_val) -#define TRACE_EVENT_COPY_FLOW_BEGIN0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_COPY) -#define TRACE_EVENT_COPY_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val) -#define TRACE_EVENT_COPY_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val, arg2_name, arg2_val) +// same `name` and `id`. Each event can have its own args. +#define TRACE_EVENT_FLOW_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val) +#define TRACE_EVENT_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val, arg2_name, arg2_val) +#define TRACE_EVENT_COPY_FLOW_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_COPY_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val) +#define TRACE_EVENT_COPY_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val, arg2_name, arg2_val) -// Records a single FLOW_STEP event for |step| immediately. If the category -// is not enabled, then this does nothing. The |name| and |id| must match the -// FLOW_BEGIN event above. The |step| param identifies this step within the +// Records a single FLOW_STEP event for `step` immediately. If the category +// is not enabled, then this does nothing. The `name` and `id` must match the +// FLOW_BEGIN event above. The `step` param identifies this step within the // async event. This should be called at the beginning of the next phase of an // asynchronous operation. -#define TRACE_EVENT_FLOW_STEP0(category, name, id, step) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, \ - category, name, id, TRACE_EVENT_FLAG_NONE, "step", step) -#define TRACE_EVENT_FLOW_STEP1(category, name, id, step, \ - arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, \ - category, name, id, TRACE_EVENT_FLAG_NONE, "step", step, \ - arg1_name, arg1_val) -#define TRACE_EVENT_COPY_FLOW_STEP0(category, name, id, step) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, \ - category, name, id, TRACE_EVENT_FLAG_COPY, "step", step) -#define TRACE_EVENT_COPY_FLOW_STEP1(category, name, id, step, \ - arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, \ - category, name, id, TRACE_EVENT_FLAG_COPY, "step", step, \ - arg1_name, arg1_val) +#define TRACE_EVENT_FLOW_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step) +#define TRACE_EVENT_FLOW_STEP1(category, name, id, step, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step, arg1_name, arg1_val) +#define TRACE_EVENT_COPY_FLOW_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step) +#define TRACE_EVENT_COPY_FLOW_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step, arg1_name, arg1_val) // Records a single FLOW_END event for "name" immediately. If the category // is not enabled, then this does nothing. -#define TRACE_EVENT_FLOW_END0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, \ - category, name, id, TRACE_EVENT_FLAG_NONE) -#define TRACE_EVENT_FLOW_END1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, \ - category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) -#define TRACE_EVENT_FLOW_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, \ - category, name, id, TRACE_EVENT_FLAG_NONE, \ - arg1_name, arg1_val, arg2_name, arg2_val) -#define TRACE_EVENT_COPY_FLOW_END0(category, name, id) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, \ - category, name, id, TRACE_EVENT_FLAG_COPY) -#define TRACE_EVENT_COPY_FLOW_END1(category, name, id, arg1_name, arg1_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val) -#define TRACE_EVENT_COPY_FLOW_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) \ - INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, \ - category, name, id, TRACE_EVENT_FLAG_COPY, \ - arg1_name, arg1_val, arg2_name, arg2_val) - +#define TRACE_EVENT_FLOW_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, name, \ + id, TRACE_EVENT_FLAG_NONE) +#define TRACE_EVENT_FLOW_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, name, \ + id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val) +#define TRACE_EVENT_FLOW_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, name, \ + id, TRACE_EVENT_FLAG_NONE, arg1_name, \ + arg1_val, arg2_name, arg2_val) +#define TRACE_EVENT_COPY_FLOW_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, name, \ + id, TRACE_EVENT_FLAG_COPY) +#define TRACE_EVENT_COPY_FLOW_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, name, \ + id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val) +#define TRACE_EVENT_COPY_FLOW_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, name, \ + id, TRACE_EVENT_FLAG_COPY, arg1_name, \ + arg1_val, arg2_name, arg2_val) //////////////////////////////////////////////////////////////////////////////// // Implementation specific tracing API definitions. @@ -535,7 +538,7 @@ // const unsigned char* // TRACE_EVENT_API_GET_CATEGORY_ENABLED(const char* category_name) #define TRACE_EVENT_API_GET_CATEGORY_ENABLED \ - webrtc::EventTracer::GetCategoryEnabled + webrtc::EventTracer::GetCategoryEnabled // Add a trace event to the platform tracing system. // void TRACE_EVENT_API_ADD_TRACE_EVENT( @@ -555,12 +558,10 @@ // Implementation detail: trace event macros create temporary variables // to keep instrumentation overhead low. These macros give each temporary // variable a unique name based on the line number to prevent name collissions. -#define INTERNAL_TRACE_EVENT_UID3(a,b) \ - trace_event_unique_##a##b -#define INTERNAL_TRACE_EVENT_UID2(a,b) \ - INTERNAL_TRACE_EVENT_UID3(a,b) +#define INTERNAL_TRACE_EVENT_UID3(a, b) trace_event_unique_##a##b +#define INTERNAL_TRACE_EVENT_UID2(a, b) INTERNAL_TRACE_EVENT_UID3(a, b) #define INTERNAL_TRACE_EVENT_UID(name_prefix) \ - INTERNAL_TRACE_EVENT_UID2(name_prefix, __LINE__) + INTERNAL_TRACE_EVENT_UID2(name_prefix, __LINE__) #if WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS #define INTERNAL_TRACE_EVENT_INFO_TYPE const unsigned char* @@ -569,55 +570,53 @@ #endif // WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS // Implementation detail: internal macro to create static category. -#define INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category) \ - INTERNAL_TRACE_EVENT_INFO_TYPE INTERNAL_TRACE_EVENT_UID(catstatic) = \ - TRACE_EVENT_API_GET_CATEGORY_ENABLED(category); +#define INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category) \ + INTERNAL_TRACE_EVENT_INFO_TYPE INTERNAL_TRACE_EVENT_UID(catstatic) = \ + TRACE_EVENT_API_GET_CATEGORY_ENABLED(category); // Implementation detail: internal macro to create static category and add // event if the category is enabled. -#define INTERNAL_TRACE_EVENT_ADD(phase, category, name, flags, ...) \ - do { \ - INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ - if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ - webrtc::trace_event_internal::AddTraceEvent( \ - phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ - webrtc::trace_event_internal::kNoEventId, flags, ##__VA_ARGS__); \ - } \ - } while (0) +#define INTERNAL_TRACE_EVENT_ADD(phase, category, name, flags, ...) \ + do { \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + webrtc::trace_event_internal::AddTraceEvent( \ + phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + webrtc::trace_event_internal::kNoEventId, flags, ##__VA_ARGS__); \ + } \ + } while (0) // Implementation detail: internal macro to create static category and add begin // event if the category is enabled. Also adds the end event when the scope // ends. -#define INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, ...) \ - INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ - webrtc::trace_event_internal::TraceEndOnScopeClose \ - INTERNAL_TRACE_EVENT_UID(profileScope); \ - if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ - webrtc::trace_event_internal::AddTraceEvent( \ - TRACE_EVENT_PHASE_BEGIN, \ - INTERNAL_TRACE_EVENT_UID(catstatic), \ - name, webrtc::trace_event_internal::kNoEventId, \ - TRACE_EVENT_FLAG_NONE, ##__VA_ARGS__); \ - INTERNAL_TRACE_EVENT_UID(profileScope).Initialize( \ - INTERNAL_TRACE_EVENT_UID(catstatic), name); \ - } +#define INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, ...) \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + webrtc::trace_event_internal::TraceEndOnScopeClose INTERNAL_TRACE_EVENT_UID( \ + profileScope); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + webrtc::trace_event_internal::AddTraceEvent( \ + TRACE_EVENT_PHASE_BEGIN, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + webrtc::trace_event_internal::kNoEventId, TRACE_EVENT_FLAG_NONE, \ + ##__VA_ARGS__); \ + INTERNAL_TRACE_EVENT_UID(profileScope) \ + .Initialize(INTERNAL_TRACE_EVENT_UID(catstatic), name); \ + } // Implementation detail: internal macro to create static category and add // event if the category is enabled. #define INTERNAL_TRACE_EVENT_ADD_WITH_ID(phase, category, name, id, flags, \ - ...) \ - do { \ - INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ - if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ - unsigned char trace_event_flags = flags | TRACE_EVENT_FLAG_HAS_ID; \ - webrtc::trace_event_internal::TraceID trace_event_trace_id( \ - id, &trace_event_flags); \ - webrtc::trace_event_internal::AddTraceEvent( \ - phase, INTERNAL_TRACE_EVENT_UID(catstatic), \ - name, trace_event_trace_id.data(), trace_event_flags, \ - ##__VA_ARGS__); \ - } \ - } while (0) + ...) \ + do { \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + unsigned char trace_event_flags = flags | TRACE_EVENT_FLAG_HAS_ID; \ + webrtc::trace_event_internal::TraceID trace_event_trace_id( \ + id, &trace_event_flags); \ + webrtc::trace_event_internal::AddTraceEvent( \ + phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + trace_event_trace_id.data(), trace_event_flags, ##__VA_ARGS__); \ + } \ + } while (0) // Notes regarding the following definitions: // New values can be added and propagated to third party libraries, but existing @@ -625,23 +624,23 @@ // definitions. // Phase indicates the nature of an event entry. E.g. part of a begin/end pair. -#define TRACE_EVENT_PHASE_BEGIN ('B') -#define TRACE_EVENT_PHASE_END ('E') -#define TRACE_EVENT_PHASE_INSTANT ('I') +#define TRACE_EVENT_PHASE_BEGIN ('B') +#define TRACE_EVENT_PHASE_END ('E') +#define TRACE_EVENT_PHASE_INSTANT ('I') #define TRACE_EVENT_PHASE_ASYNC_BEGIN ('S') -#define TRACE_EVENT_PHASE_ASYNC_STEP ('T') -#define TRACE_EVENT_PHASE_ASYNC_END ('F') +#define TRACE_EVENT_PHASE_ASYNC_STEP ('T') +#define TRACE_EVENT_PHASE_ASYNC_END ('F') #define TRACE_EVENT_PHASE_FLOW_BEGIN ('s') -#define TRACE_EVENT_PHASE_FLOW_STEP ('t') -#define TRACE_EVENT_PHASE_FLOW_END ('f') +#define TRACE_EVENT_PHASE_FLOW_STEP ('t') +#define TRACE_EVENT_PHASE_FLOW_END ('f') #define TRACE_EVENT_PHASE_METADATA ('M') -#define TRACE_EVENT_PHASE_COUNTER ('C') +#define TRACE_EVENT_PHASE_COUNTER ('C') // Flags for changing the behavior of TRACE_EVENT_API_ADD_TRACE_EVENT. -#define TRACE_EVENT_FLAG_NONE (static_cast(0)) -#define TRACE_EVENT_FLAG_COPY (static_cast(1 << 0)) -#define TRACE_EVENT_FLAG_HAS_ID (static_cast(1 << 1)) -#define TRACE_EVENT_FLAG_MANGLE_ID (static_cast(1 << 2)) +#define TRACE_EVENT_FLAG_NONE (static_cast(0)) +#define TRACE_EVENT_FLAG_COPY (static_cast(1 << 0)) +#define TRACE_EVENT_FLAG_HAS_ID (static_cast(1 << 1)) +#define TRACE_EVENT_FLAG_MANGLE_ID (static_cast(1 << 2)) namespace webrtc { namespace trace_event_internal { @@ -657,57 +656,71 @@ const unsigned long long kNoEventId = 0; class TraceID { public: class ForceMangle { - public: - explicit ForceMangle(unsigned long long id) : data_(id) {} - explicit ForceMangle(unsigned long id) : data_(id) {} - explicit ForceMangle(unsigned int id) : data_(id) {} - explicit ForceMangle(unsigned short id) : data_(id) {} - explicit ForceMangle(unsigned char id) : data_(id) {} - explicit ForceMangle(long long id) - : data_(static_cast(id)) {} - explicit ForceMangle(long id) - : data_(static_cast(id)) {} - explicit ForceMangle(int id) - : data_(static_cast(id)) {} - explicit ForceMangle(short id) - : data_(static_cast(id)) {} - explicit ForceMangle(signed char id) - : data_(static_cast(id)) {} + public: + explicit ForceMangle(unsigned long long id) : data_(id) {} + explicit ForceMangle(unsigned long id) : data_(id) {} + explicit ForceMangle(unsigned int id) : data_(id) {} + explicit ForceMangle(unsigned short id) : data_(id) {} + explicit ForceMangle(unsigned char id) : data_(id) {} + explicit ForceMangle(long long id) + : data_(static_cast(id)) {} + explicit ForceMangle(long id) + : data_(static_cast(id)) {} + explicit ForceMangle(int id) : data_(static_cast(id)) {} + explicit ForceMangle(short id) + : data_(static_cast(id)) {} + explicit ForceMangle(signed char id) + : data_(static_cast(id)) {} - unsigned long long data() const { return data_; } + unsigned long long data() const { return data_; } - private: - unsigned long long data_; + private: + unsigned long long data_; }; explicit TraceID(const void* id, unsigned char* flags) - : data_(static_cast( - reinterpret_cast(id))) { + : data_( + static_cast(reinterpret_cast(id))) { *flags |= TRACE_EVENT_FLAG_MANGLE_ID; } explicit TraceID(ForceMangle id, unsigned char* flags) : data_(id.data()) { *flags |= TRACE_EVENT_FLAG_MANGLE_ID; } - explicit TraceID(unsigned long long id, unsigned char* flags) - : data_(id) { (void)flags; } - explicit TraceID(unsigned long id, unsigned char* flags) - : data_(id) { (void)flags; } - explicit TraceID(unsigned int id, unsigned char* flags) - : data_(id) { (void)flags; } - explicit TraceID(unsigned short id, unsigned char* flags) - : data_(id) { (void)flags; } - explicit TraceID(unsigned char id, unsigned char* flags) - : data_(id) { (void)flags; } + explicit TraceID(unsigned long long id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned long id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned int id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned short id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned char id, unsigned char* flags) : data_(id) { + (void)flags; + } explicit TraceID(long long id, unsigned char* flags) - : data_(static_cast(id)) { (void)flags; } + : data_(static_cast(id)) { + (void)flags; + } explicit TraceID(long id, unsigned char* flags) - : data_(static_cast(id)) { (void)flags; } + : data_(static_cast(id)) { + (void)flags; + } explicit TraceID(int id, unsigned char* flags) - : data_(static_cast(id)) { (void)flags; } + : data_(static_cast(id)) { + (void)flags; + } explicit TraceID(short id, unsigned char* flags) - : data_(static_cast(id)) { (void)flags; } + : data_(static_cast(id)) { + (void)flags; + } explicit TraceID(signed char id, unsigned char* flags) - : data_(static_cast(id)) { (void)flags; } + : data_(static_cast(id)) { + (void)flags; + } unsigned long long data() const { return data_; } @@ -729,7 +742,8 @@ union TraceValueUnion { class TraceStringWithCopy { public: explicit TraceStringWithCopy(const char* str) : str_(str) {} - operator const char* () const { return str_; } + operator const char*() const { return str_; } + private: const char* str_; }; @@ -737,26 +751,22 @@ class TraceStringWithCopy { // Define SetTraceValue for each allowed type. It stores the type and // value in the return arguments. This allows this API to avoid declaring any // structures so that it is portable to third_party libraries. -#define INTERNAL_DECLARE_SET_TRACE_VALUE(actual_type, \ - union_member, \ - value_type_id) \ - static inline void SetTraceValue(actual_type arg, \ - unsigned char* type, \ - unsigned long long* value) { \ - TraceValueUnion type_value; \ - type_value.union_member = arg; \ - *type = value_type_id; \ - *value = type_value.as_uint; \ - } +#define INTERNAL_DECLARE_SET_TRACE_VALUE(actual_type, union_member, \ + value_type_id) \ + static inline void SetTraceValue(actual_type arg, unsigned char* type, \ + unsigned long long* value) { \ + TraceValueUnion type_value; \ + type_value.union_member = arg; \ + *type = value_type_id; \ + *value = type_value.as_uint; \ + } // Simpler form for int types that can be safely casted. -#define INTERNAL_DECLARE_SET_TRACE_VALUE_INT(actual_type, \ - value_type_id) \ - static inline void SetTraceValue(actual_type arg, \ - unsigned char* type, \ - unsigned long long* value) { \ - *type = value_type_id; \ - *value = static_cast(arg); \ - } +#define INTERNAL_DECLARE_SET_TRACE_VALUE_INT(actual_type, value_type_id) \ + static inline void SetTraceValue(actual_type arg, unsigned char* type, \ + unsigned long long* value) { \ + *type = value_type_id; \ + *value = static_cast(arg); \ + } INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned long long, TRACE_VALUE_TYPE_UINT) INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned long, TRACE_VALUE_TYPE_UINT) @@ -770,11 +780,14 @@ INTERNAL_DECLARE_SET_TRACE_VALUE_INT(short, TRACE_VALUE_TYPE_INT) INTERNAL_DECLARE_SET_TRACE_VALUE_INT(signed char, TRACE_VALUE_TYPE_INT) INTERNAL_DECLARE_SET_TRACE_VALUE(bool, as_bool, TRACE_VALUE_TYPE_BOOL) INTERNAL_DECLARE_SET_TRACE_VALUE(double, as_double, TRACE_VALUE_TYPE_DOUBLE) -INTERNAL_DECLARE_SET_TRACE_VALUE(const void*, as_pointer, +INTERNAL_DECLARE_SET_TRACE_VALUE(const void*, + as_pointer, TRACE_VALUE_TYPE_POINTER) -INTERNAL_DECLARE_SET_TRACE_VALUE(const char*, as_string, +INTERNAL_DECLARE_SET_TRACE_VALUE(const char*, + as_string, TRACE_VALUE_TYPE_STRING) -INTERNAL_DECLARE_SET_TRACE_VALUE(const TraceStringWithCopy&, as_string, +INTERNAL_DECLARE_SET_TRACE_VALUE(const TraceStringWithCopy&, + as_string, TRACE_VALUE_TYPE_COPY_STRING) #undef INTERNAL_DECLARE_SET_TRACE_VALUE @@ -797,53 +810,49 @@ static inline void SetTraceValue(const std::string& arg, // these procedures. static inline void AddTraceEvent(char phase, - const unsigned char* category_enabled, - const char* name, - unsigned long long id, - unsigned char flags) { + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + unsigned char flags) { TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, kZeroNumArgs, nullptr, nullptr, nullptr, flags); } -template +template static inline void AddTraceEvent(char phase, - const unsigned char* category_enabled, - const char* name, - unsigned long long id, - unsigned char flags, - const char* arg1_name, - const ARG1_TYPE& arg1_val) { + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + unsigned char flags, + const char* arg1_name, + const ARG1_TYPE& arg1_val) { const int num_args = 1; unsigned char arg_types[1]; unsigned long long arg_values[1]; SetTraceValue(arg1_val, &arg_types[0], &arg_values[0]); - TRACE_EVENT_API_ADD_TRACE_EVENT( - phase, category_enabled, name, id, - num_args, &arg1_name, arg_types, arg_values, - flags); + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, num_args, + &arg1_name, arg_types, arg_values, flags); } -template +template static inline void AddTraceEvent(char phase, - const unsigned char* category_enabled, - const char* name, - unsigned long long id, - unsigned char flags, - const char* arg1_name, - const ARG1_TYPE& arg1_val, - const char* arg2_name, - const ARG2_TYPE& arg2_val) { + const unsigned char* category_enabled, + const char* name, + unsigned long long id, + unsigned char flags, + const char* arg1_name, + const ARG1_TYPE& arg1_val, + const char* arg2_name, + const ARG2_TYPE& arg2_val) { const int num_args = 2; - const char* arg_names[2] = { arg1_name, arg2_name }; + const char* arg_names[2] = {arg1_name, arg2_name}; unsigned char arg_types[2]; unsigned long long arg_values[2]; SetTraceValue(arg1_val, &arg_types[0], &arg_values[0]); SetTraceValue(arg2_val, &arg_types[1], &arg_values[1]); - TRACE_EVENT_API_ADD_TRACE_EVENT( - phase, category_enabled, name, id, - num_args, arg_names, arg_types, arg_values, - flags); + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, num_args, + arg_names, arg_types, arg_values, flags); } // Used by TRACE_EVENTx macro. Do not use directly. @@ -856,8 +865,7 @@ class TraceEndOnScopeClose { AddEventIfEnabled(); } - void Initialize(const unsigned char* category_enabled, - const char* name) { + void Initialize(const unsigned char* category_enabled, const char* name) { data_.category_enabled = category_enabled; data_.name = name; p_data_ = &data_; @@ -896,7 +904,9 @@ class TraceEndOnScopeClose { // This section defines no-op alternatives to the tracing macros when // RTC_DISABLE_TRACE_EVENTS is defined. -#define RTC_NOOP() do {} while (0) +#define RTC_NOOP() \ + do { \ + } while (0) #define TRACE_STR_COPY(str) RTC_NOOP() @@ -907,111 +917,131 @@ class TraceEndOnScopeClose { #define TRACE_EVENT0(category, name) RTC_NOOP() #define TRACE_EVENT1(category, name, arg1_name, arg1_val) RTC_NOOP() #define TRACE_EVENT2(category, name, arg1_name, arg1_val, arg2_name, arg2_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_INSTANT0(category, name) RTC_NOOP() #define TRACE_EVENT_INSTANT1(category, name, arg1_name, arg1_val) RTC_NOOP() -#define TRACE_EVENT_INSTANT2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() +#define TRACE_EVENT_INSTANT2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_INSTANT0(category, name) RTC_NOOP() #define TRACE_EVENT_COPY_INSTANT1(category, name, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_COPY_INSTANT2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_BEGIN0(category, name) RTC_NOOP() #define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) RTC_NOOP() -#define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() +#define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_BEGIN0(category, name) RTC_NOOP() #define TRACE_EVENT_COPY_BEGIN1(category, name, arg1_name, arg1_val) RTC_NOOP() #define TRACE_EVENT_COPY_BEGIN2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_END0(category, name) RTC_NOOP() #define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) RTC_NOOP() -#define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() +#define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_END0(category, name) RTC_NOOP() #define TRACE_EVENT_COPY_END1(category, name, arg1_name, arg1_val) RTC_NOOP() -#define TRACE_EVENT_COPY_END2(category, name, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() +#define TRACE_EVENT_COPY_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() #define TRACE_COUNTER1(category, name, value) RTC_NOOP() #define TRACE_COPY_COUNTER1(category, name, value) RTC_NOOP() -#define TRACE_COUNTER2(category, name, value1_name, value1_val, \ - value2_name, value2_val) RTC_NOOP() +#define TRACE_COUNTER2(category, name, value1_name, value1_val, value2_name, \ + value2_val) \ + RTC_NOOP() #define TRACE_COPY_COUNTER2(category, name, value1_name, value1_val, \ - value2_name, value2_val) RTC_NOOP() + value2_name, value2_val) \ + RTC_NOOP() #define TRACE_COUNTER_ID1(category, name, id, value) RTC_NOOP() #define TRACE_COPY_COUNTER_ID1(category, name, id, value) RTC_NOOP() #define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ - value2_name, value2_val) RTC_NOOP() + value2_name, value2_val) \ + RTC_NOOP() #define TRACE_COPY_COUNTER_ID2(category, name, id, value1_name, value1_val, \ - value2_name, value2_val) RTC_NOOP() + value2_name, value2_val) \ + RTC_NOOP() #define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) RTC_NOOP() #define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_BEGIN0(category, name, id) RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_ASYNC_STEP0(category, name, id, step) RTC_NOOP() -#define TRACE_EVENT_ASYNC_STEP1(category, name, id, step, \ - arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT_ASYNC_STEP1(category, name, id, step, arg1_name, arg1_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_STEP0(category, name, id, step) RTC_NOOP() -#define TRACE_EVENT_COPY_ASYNC_STEP1(category, name, id, step, \ - arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT_COPY_ASYNC_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() #define TRACE_EVENT_ASYNC_END0(category, name, id) RTC_NOOP() #define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_END0(category, name, id) RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_COPY_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_FLOW_BEGIN0(category, name, id) RTC_NOOP() #define TRACE_EVENT_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_BEGIN0(category, name, id) RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_FLOW_STEP0(category, name, id, step) RTC_NOOP() -#define TRACE_EVENT_FLOW_STEP1(category, name, id, step, \ - arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT_FLOW_STEP1(category, name, id, step, arg1_name, arg1_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_STEP0(category, name, id, step) RTC_NOOP() -#define TRACE_EVENT_COPY_FLOW_STEP1(category, name, id, step, \ - arg1_name, arg1_val) RTC_NOOP() +#define TRACE_EVENT_COPY_FLOW_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() #define TRACE_EVENT_FLOW_END0(category, name, id) RTC_NOOP() #define TRACE_EVENT_FLOW_END1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_FLOW_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_END0(category, name, id) RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_END1(category, name, id, arg1_name, arg1_val) \ - RTC_NOOP() + RTC_NOOP() #define TRACE_EVENT_COPY_FLOW_END2(category, name, id, arg1_name, arg1_val, \ - arg2_name, arg2_val) RTC_NOOP() + arg2_name, arg2_val) \ + RTC_NOOP() #define TRACE_EVENT_API_GET_CATEGORY_ENABLED "" @@ -1019,4 +1049,4 @@ class TraceEndOnScopeClose { #endif // RTC_TRACE_EVENTS_ENABLED -#endif // RTC_BASE_TRACE_EVENT_H_ \ No newline at end of file +#endif // RTC_BASE_TRACE_EVENT_H_ diff --git a/webrtc/rtc_base/units/BUILD.gn b/webrtc/rtc_base/units/BUILD.gn index e2ab873..bbb87a0 100644 --- a/webrtc/rtc_base/units/BUILD.gn +++ b/webrtc/rtc_base/units/BUILD.gn @@ -10,13 +10,14 @@ import("../../webrtc.gni") rtc_source_set("unit_base") { visibility = [ - "../../api/units:*", ":*", + "../../api/units:*", ] sources = [ "unit_base.h" ] deps = [ "../../rtc_base:checks", + "../../rtc_base:divide_round", "../../rtc_base:safe_conversions", ] } diff --git a/webrtc/rtc_base/units/unit_base.h b/webrtc/rtc_base/units/unit_base.h index 7196bae..a6bdbf5 100644 --- a/webrtc/rtc_base/units/unit_base.h +++ b/webrtc/rtc_base/units/unit_base.h @@ -18,6 +18,7 @@ #include #include "rtc_base/checks.h" +#include "rtc_base/numerics/divide_round.h" #include "rtc_base/numerics/safe_conversions.h" namespace webrtc { @@ -50,22 +51,22 @@ class UnitBase { return value_ == MinusInfinityVal(); } - constexpr bool operator==(const Unit_T& other) const { + constexpr bool operator==(const UnitBase& other) const { return value_ == other.value_; } - constexpr bool operator!=(const Unit_T& other) const { + constexpr bool operator!=(const UnitBase& other) const { return value_ != other.value_; } - constexpr bool operator<=(const Unit_T& other) const { + constexpr bool operator<=(const UnitBase& other) const { return value_ <= other.value_; } - constexpr bool operator>=(const Unit_T& other) const { + constexpr bool operator>=(const UnitBase& other) const { return value_ >= other.value_; } - constexpr bool operator>(const Unit_T& other) const { + constexpr bool operator>(const UnitBase& other) const { return value_ > other.value_; } - constexpr bool operator<(const Unit_T& other) const { + constexpr bool operator<(const UnitBase& other) const { return value_ < other.value_; } constexpr Unit_T RoundTo(const Unit_T& resolution) const { @@ -140,10 +141,9 @@ class UnitBase { template constexpr typename std::enable_if::value, T>::type ToValue() const { - return IsPlusInfinity() - ? std::numeric_limits::infinity() - : IsMinusInfinity() ? -std::numeric_limits::infinity() - : value_; + return IsPlusInfinity() ? std::numeric_limits::infinity() + : IsMinusInfinity() ? -std::numeric_limits::infinity() + : value_; } template constexpr T ToValueOr(T fallback_value) const { @@ -154,12 +154,7 @@ class UnitBase { constexpr typename std::enable_if::value, T>::type ToFraction() const { RTC_DCHECK(IsFinite()); - if (Unit_T::one_sided) { - return rtc::dchecked_cast( - DivRoundPositiveToNearest(value_, Denominator)); - } else { - return rtc::dchecked_cast(DivRoundToNearest(value_, Denominator)); - } + return rtc::dchecked_cast(DivideRoundToNearest(value_, Denominator)); } template constexpr typename std::enable_if::value, T>::type @@ -169,9 +164,7 @@ class UnitBase { template constexpr int64_t ToFractionOr(int64_t fallback_value) const { - return IsFinite() ? Unit_T::one_sided - ? DivRoundPositiveToNearest(value_, Denominator) - : DivRoundToNearest(value_, Denominator) + return IsFinite() ? DivideRoundToNearest(value_, Denominator) : fallback_value; } @@ -205,14 +198,6 @@ class UnitBase { constexpr const Unit_T& AsSubClassRef() const { return static_cast(*this); } - // Assumes that n >= 0 and d > 0. - static constexpr int64_t DivRoundPositiveToNearest(int64_t n, int64_t d) { - return (n + d / 2) / d; - } - // Assumes that d > 0. - static constexpr int64_t DivRoundToNearest(int64_t n, int64_t d) { - return (n + (n >= 0 ? d / 2 : -d / 2)) / d; - } int64_t value_; }; @@ -266,14 +251,18 @@ class RelativeUnit : public UnitBase { return UnitBase::template ToValue() / other.template ToValue(); } - template - constexpr typename std::enable_if::value, Unit_T>::type - operator/(const T& scalar) const { - return UnitBase::FromValue( - std::round(UnitBase::template ToValue() / scalar)); + template >* = nullptr> + constexpr Unit_T operator/(T scalar) const { + return UnitBase::FromValue(std::llround(this->ToValue() / scalar)); + } + template >* = nullptr> + constexpr Unit_T operator/(T scalar) const { + return UnitBase::FromValue(this->ToValue() / scalar); } constexpr Unit_T operator*(double scalar) const { - return UnitBase::FromValue(std::round(this->ToValue() * scalar)); + return UnitBase::FromValue(std::llround(this->ToValue() * scalar)); } constexpr Unit_T operator*(int64_t scalar) const { return UnitBase::FromValue(this->ToValue() * scalar); @@ -281,6 +270,9 @@ class RelativeUnit : public UnitBase { constexpr Unit_T operator*(int32_t scalar) const { return UnitBase::FromValue(this->ToValue() * scalar); } + constexpr Unit_T operator*(size_t scalar) const { + return UnitBase::FromValue(this->ToValue() * scalar); + } protected: using UnitBase::UnitBase; @@ -298,6 +290,19 @@ template inline constexpr Unit_T operator*(int32_t scalar, RelativeUnit other) { return other * scalar; } +template +inline constexpr Unit_T operator*(size_t scalar, RelativeUnit other) { + return other * scalar; +} + +template +inline constexpr Unit_T operator-(RelativeUnit other) { + if (other.IsPlusInfinity()) + return UnitBase::MinusInfinity(); + if (other.IsMinusInfinity()) + return UnitBase::PlusInfinity(); + return -1 * other; +} } // namespace rtc_units_impl diff --git a/webrtc/rtc_base/win32.h b/webrtc/rtc_base/win32.h index c90296e..6e8d287 100644 --- a/webrtc/rtc_base/win32.h +++ b/webrtc/rtc_base/win32.h @@ -20,11 +20,10 @@ #define NOMINMAX #endif -// clang-format off -// clang formating would change include order. -#include // must come first +#include + +// Must be after winsock2.h. #include -// clang-format on typedef int socklen_t; @@ -39,66 +38,11 @@ typedef struct _TOKEN_MANDATORY_LABEL { #undef SetPort -#include - namespace rtc { const char* win32_inet_ntop(int af, const void* src, char* dst, socklen_t size); int win32_inet_pton(int af, const char* src, void* dst); -enum WindowsMajorVersions { - kWindows2000 = 5, - kWindowsVista = 6, - kWindows10 = 10, -}; - -#if !defined(WINUWP) -bool GetOsVersion(int* major, int* minor, int* build); - -inline bool IsWindowsVistaOrLater() { - int major; - return (GetOsVersion(&major, nullptr, nullptr) && major >= kWindowsVista); -} - -inline bool IsWindowsXpOrLater() { - int major, minor; - return (GetOsVersion(&major, &minor, nullptr) && - (major >= kWindowsVista || (major == kWindows2000 && minor >= 1))); -} - -inline bool IsWindows8OrLater() { - int major, minor; - return (GetOsVersion(&major, &minor, nullptr) && - (major > kWindowsVista || (major == kWindowsVista && minor >= 2))); -} - -inline bool IsWindows10OrLater() { - int major; - return (GetOsVersion(&major, nullptr, nullptr) && (major >= kWindows10)); -} - -#else - -// When targetting WinUWP the OS must be Windows 10 (or greater) as lesser -// Windows OS targets are not supported. -inline bool IsWindowsVistaOrLater() { - return true; -} - -inline bool IsWindowsXpOrLater() { - return true; -} - -inline bool IsWindows8OrLater() { - return true; -} - -inline bool IsWindows10OrLater() { - return true; -} - -#endif // !defined(WINUWP) - } // namespace rtc #endif // RTC_BASE_WIN32_H_ diff --git a/webrtc/system_wrappers/BUILD.gn b/webrtc/system_wrappers/BUILD.gn index b446648..2576d4e 100644 --- a/webrtc/system_wrappers/BUILD.gn +++ b/webrtc/system_wrappers/BUILD.gn @@ -36,8 +36,10 @@ rtc_library("system_wrappers") { "../api/units:timestamp", "../modules:module_api_public", "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:safe_conversions", + "../rtc_base:timeutils", "../rtc_base/synchronization:mutex", - "../rtc_base/synchronization:rw_lock_wrapper", "../rtc_base/system:arch", "../rtc_base/system:rtc_export", ] @@ -52,7 +54,7 @@ rtc_library("system_wrappers") { ] } else { sources += [ "source/cpu_features_android.cc" ] - deps += [ "//third_party/android_sdk:cpu_features" ] + deps += [ "//third_party/cpu_features:ndk_compat" ] } libs += [ "log" ] @@ -74,10 +76,7 @@ rtc_library("system_wrappers") { deps += [ "../rtc_base:win32" ] } - deps += [ - "../rtc_base:rtc_base_approved", - "../rtc_base:rtc_numerics", - ] + deps += [ "../rtc_base:rtc_numerics" ] } rtc_library("field_trial") { @@ -88,11 +87,16 @@ rtc_library("field_trial") { defines = [ "WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT" ] } deps = [ + "../experiments:registered_field_trials", "../rtc_base:checks", "../rtc_base:logging", "../rtc_base:stringutils", + "../rtc_base/containers:flat_set", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("metrics") { @@ -104,16 +108,32 @@ rtc_library("metrics") { } deps = [ "../rtc_base:checks", - "../rtc_base:rtc_base_approved", + "../rtc_base:macromagic", + "../rtc_base:stringutils", "../rtc_base/synchronization:mutex", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } -if (rtc_include_tests) { +rtc_library("denormal_disabler") { + visibility = [ "*" ] + public = [ "include/denormal_disabler.h" ] + sources = [ "source/denormal_disabler.cc" ] + deps = [ + "../rtc_base:checks", + "../rtc_base/system:arch", + ] + if (is_clang) { + cflags_cc = [ "-Wno-unused-private-field" ] + } +} + +if (rtc_include_tests && !build_with_chromium) { rtc_test("system_wrappers_unittests") { testonly = true sources = [ "source/clock_unittest.cc", + "source/denormal_disabler_unittest.cc", "source/field_trial_unittest.cc", "source/metrics_default_unittest.cc", "source/metrics_unittest.cc", @@ -122,18 +142,21 @@ if (rtc_include_tests) { ] deps = [ + ":denormal_disabler", ":field_trial", ":metrics", ":system_wrappers", "../rtc_base:checks", - "../rtc_base:rtc_base_approved", + "../rtc_base:random", + "../rtc_base:stringutils", "../test:rtc_expect_death", "../test:test_main", "../test:test_support", "//testing/gtest", - "//third_party/abseil-cpp/absl/strings", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + if (is_android) { deps += [ "//testing/android/native_test:native_test_support" ] diff --git a/webrtc/system_wrappers/include/cpu_features_wrapper.h b/webrtc/system_wrappers/include/cpu_features_wrapper.h index 612b4a5..254e2d8 100644 --- a/webrtc/system_wrappers/include/cpu_features_wrapper.h +++ b/webrtc/system_wrappers/include/cpu_features_wrapper.h @@ -16,7 +16,7 @@ namespace webrtc { // List of features in x86. -typedef enum { kSSE2, kSSE3, kAVX2 } CPUFeature; +typedef enum { kSSE2, kSSE3, kAVX2, kFMA3 } CPUFeature; // List of features in ARM. enum { diff --git a/webrtc/system_wrappers/include/denormal_disabler.h b/webrtc/system_wrappers/include/denormal_disabler.h new file mode 100644 index 0000000..bd3d401 --- /dev/null +++ b/webrtc/system_wrappers/include/denormal_disabler.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ +#define SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ + +#include "rtc_base/system/arch.h" + +namespace webrtc { + +// Activates the hardware (HW) way to flush denormals (see [1]) to zero as they +// can very seriously impact performance. At destruction time restores the +// denormals handling state read by the ctor; hence, supports nested calls. +// Equals a no-op if the architecture is not x86 or ARM or if the compiler is +// not CLANG. +// [1] https://en.wikipedia.org/wiki/Denormal_number +// +// Example usage: +// +// void Foo() { +// DenormalDisabler d; +// ... +// } +class DenormalDisabler { + public: + // Ctor. If architecture and compiler are supported, stores the HW settings + // for denormals, disables denormals and sets `disabling_activated_` to true. + // Otherwise, only sets `disabling_activated_` to false. + DenormalDisabler(); + // Ctor. Same as above, but also requires `enabled` to be true to disable + // denormals. + explicit DenormalDisabler(bool enabled); + DenormalDisabler(const DenormalDisabler&) = delete; + DenormalDisabler& operator=(const DenormalDisabler&) = delete; + // Dtor. If `disabling_activated_` is true, restores the denormals HW settings + // read by the ctor before denormals were disabled. Otherwise it's a no-op. + ~DenormalDisabler(); + + // Returns true if architecture and compiler are supported. + static bool IsSupported(); + + private: + const int status_word_; + const bool disabling_activated_; +}; + +} // namespace webrtc + +#endif // SYSTEM_WRAPPERS_INCLUDE_DENORMAL_DISABLER_H_ diff --git a/webrtc/system_wrappers/include/field_trial.h b/webrtc/system_wrappers/include/field_trial.h index 52db33b..8d0ad25 100644 --- a/webrtc/system_wrappers/include/field_trial.h +++ b/webrtc/system_wrappers/include/field_trial.h @@ -13,6 +13,9 @@ #include +#include "absl/strings/string_view.h" +#include "rtc_base/containers/flat_set.h" + // Field trials allow webrtc clients (such as Chrome) to turn on feature code // in binaries out in the field and gather information with that. // @@ -24,7 +27,7 @@ // WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT (if GN is used this can be achieved // by setting the GN arg rtc_exclude_field_trial_default to true). // 2. Provide an implementation of: -// std::string webrtc::field_trial::FindFullName(const std::string& trial). +// std::string webrtc::field_trial::FindFullName(absl::string_view trial). // // They are designed to wire up directly to chrome field trials and to speed up // developers by reducing the need to wire APIs to control whether a feature is @@ -61,18 +64,18 @@ namespace field_trial { // if the trial does not exists. // // Note: To keep things tidy append all the trial names with WebRTC. -std::string FindFullName(const std::string& name); +std::string FindFullName(absl::string_view name); // Convenience method, returns true iff FindFullName(name) return a string that // starts with "Enabled". // TODO(tommi): Make sure all implementations support this. -inline bool IsEnabled(const char* name) { +inline bool IsEnabled(absl::string_view name) { return FindFullName(name).find("Enabled") == 0; } // Convenience method, returns true iff FindFullName(name) return a string that // starts with "Disabled". -inline bool IsDisabled(const char* name) { +inline bool IsDisabled(absl::string_view name) { return FindFullName(name).find("Disabled") == 0; } @@ -84,17 +87,28 @@ void InitFieldTrialsFromString(const char* trials_string); const char* GetFieldTrialString(); -#ifndef WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT // Validates the given field trial string. -bool FieldTrialsStringIsValid(const char* trials_string); +bool FieldTrialsStringIsValid(absl::string_view trials_string); // Merges two field trial strings. // // If a key (trial) exists twice with conflicting values (groups), the value // in 'second' takes precedence. // Shall only be called with valid FieldTrial strings. -std::string MergeFieldTrialsStrings(const char* first, const char* second); -#endif // WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT +std::string MergeFieldTrialsStrings(absl::string_view first, + absl::string_view second); + +// This helper allows to temporary "register" a field trial within the current +// scope. This is only useful for tests that use the global field trial string, +// otherwise you can use `webrtc::FieldTrialsRegistry`. +// +// If you want to isolate changes to the global field trial string itself within +// the current scope you should use `webrtc::test::ScopedFieldTrials`. +class FieldTrialsAllowedInScopeForTesting { + public: + explicit FieldTrialsAllowedInScopeForTesting(flat_set keys); + ~FieldTrialsAllowedInScopeForTesting(); +}; } // namespace field_trial } // namespace webrtc diff --git a/webrtc/system_wrappers/include/metrics.h b/webrtc/system_wrappers/include/metrics.h index 8e0f084..6e2da1b 100644 --- a/webrtc/system_wrappers/include/metrics.h +++ b/webrtc/system_wrappers/include/metrics.h @@ -13,12 +13,14 @@ #include +#include #include #include #include -#include "rtc_base/atomic_ops.h" +#include "absl/strings/string_view.h" #include "rtc_base/checks.h" +#include "rtc_base/string_utils.h" #if defined(RTC_DISABLE_METRICS) #define RTC_METRICS_ENABLED 0 @@ -30,8 +32,8 @@ namespace webrtc { namespace metrics_impl { template void NoOp(const Ts&...) {} -} -} +} // namespace metrics_impl +} // namespace webrtc #if RTC_METRICS_ENABLED #define EXPECT_METRIC_EQ(val1, val2) EXPECT_EQ(val1, val2) @@ -44,12 +46,16 @@ void NoOp(const Ts&...) {} #define EXPECT_METRIC_THAT(value, matcher) EXPECT_THAT(value, matcher) #else #define EXPECT_METRIC_EQ(val1, val2) webrtc::metrics_impl::NoOp(val1, val2) -#define EXPECT_METRIC_EQ_WAIT(val1, val2, timeout) webrtc::metrics_impl::NoOp(val1, val2, timeout) +#define EXPECT_METRIC_EQ_WAIT(val1, val2, timeout) \ + webrtc::metrics_impl::NoOp(val1, val2, timeout) #define EXPECT_METRIC_GT(val1, val2) webrtc::metrics_impl::NoOp(val1, val2) #define EXPECT_METRIC_LE(val1, val2) webrtc::metrics_impl::NoOp(val1, val2) -#define EXPECT_METRIC_TRUE(condition) webrtc::metrics_impl::NoOp(condition || true) -#define EXPECT_METRIC_FALSE(condition) webrtc::metrics_impl::NoOp(condition && false) -#define EXPECT_METRIC_THAT(value, matcher) webrtc::metrics_impl::NoOp(value, testing::_) +#define EXPECT_METRIC_TRUE(condition) \ + webrtc::metrics_impl::NoOp(condition || true) +#define EXPECT_METRIC_FALSE(condition) \ + webrtc::metrics_impl::NoOp(condition && false) +#define EXPECT_METRIC_THAT(value, matcher) \ + webrtc::metrics_impl::NoOp(value, testing::_) #endif #if RTC_METRICS_ENABLED @@ -76,12 +82,12 @@ void NoOp(const Ts&...) {} // by setting the GN arg rtc_exclude_metrics_default to true). // 2. Provide implementations of: // Histogram* webrtc::metrics::HistogramFactoryGetCounts( -// const std::string& name, int sample, int min, int max, +// absl::string_view name, int sample, int min, int max, // int bucket_count); // Histogram* webrtc::metrics::HistogramFactoryGetEnumeration( -// const std::string& name, int sample, int boundary); +// absl::string_view name, int sample, int boundary); // void webrtc::metrics::HistogramAdd( -// Histogram* histogram_pointer, const std::string& name, int sample); +// Histogram* histogram_pointer, absl::string_view name, int sample); // // Example usage: // @@ -163,7 +169,7 @@ void NoOp(const Ts&...) {} RTC_HISTOGRAM_ENUMERATION_SPARSE(name, sample, 2) // Histogram for enumerators (evenly spaced buckets). -// |boundary| should be above the max enumerator sample. +// `boundary` should be above the max enumerator sample. // // TODO(qingsi): Refactor the default implementation given by RtcHistogram, // which is already sparse, and remove the boundary argument from the macro. @@ -181,33 +187,29 @@ void NoOp(const Ts&...) {} RTC_HISTOGRAM_ENUMERATION(name, sample, 2) // Histogram for enumerators (evenly spaced buckets). -// |boundary| should be above the max enumerator sample. +// `boundary` should be above the max enumerator sample. #define RTC_HISTOGRAM_ENUMERATION(name, sample, boundary) \ RTC_HISTOGRAM_COMMON_BLOCK_SLOW( \ name, sample, \ webrtc::metrics::HistogramFactoryGetEnumeration(name, boundary)) // The name of the histogram should not vary. -// TODO(asapersson): Consider changing string to const char*. -#define RTC_HISTOGRAM_COMMON_BLOCK(constant_name, sample, \ - factory_get_invocation) \ - do { \ - static webrtc::metrics::Histogram* atomic_histogram_pointer = nullptr; \ - webrtc::metrics::Histogram* histogram_pointer = \ - rtc::AtomicOps::AcquireLoadPtr(&atomic_histogram_pointer); \ - if (!histogram_pointer) { \ - histogram_pointer = factory_get_invocation; \ - webrtc::metrics::Histogram* prev_pointer = \ - rtc::AtomicOps::CompareAndSwapPtr( \ - &atomic_histogram_pointer, \ - static_cast(nullptr), \ - histogram_pointer); \ - RTC_DCHECK(prev_pointer == nullptr || \ - prev_pointer == histogram_pointer); \ - } \ - if (histogram_pointer) { \ - webrtc::metrics::HistogramAdd(histogram_pointer, sample); \ - } \ +#define RTC_HISTOGRAM_COMMON_BLOCK(constant_name, sample, \ + factory_get_invocation) \ + do { \ + static std::atomic atomic_histogram_pointer( \ + nullptr); \ + webrtc::metrics::Histogram* histogram_pointer = \ + atomic_histogram_pointer.load(std::memory_order_acquire); \ + if (!histogram_pointer) { \ + histogram_pointer = factory_get_invocation; \ + webrtc::metrics::Histogram* null_histogram = nullptr; \ + atomic_histogram_pointer.compare_exchange_strong(null_histogram, \ + histogram_pointer); \ + } \ + if (histogram_pointer) { \ + webrtc::metrics::HistogramAdd(histogram_pointer, sample); \ + } \ } while (0) // The histogram is constructed/found for each call. @@ -223,7 +225,7 @@ void NoOp(const Ts&...) {} // Helper macros. // Macros for calling a histogram with varying name (e.g. when using a metric // in different modes such as real-time vs screenshare). Fast, because pointer -// is cached. |index| should be different for different names. Allowed |index| +// is cached. `index` should be different for different names. Allowed `index` // values are 0, 1, and 2. #define RTC_HISTOGRAMS_COUNTS_100(index, name, sample) \ RTC_HISTOGRAMS_COMMON(index, name, sample, \ @@ -270,7 +272,7 @@ void NoOp(const Ts&...) {} macro_invocation; \ break; \ default: \ - RTC_NOTREACHED(); \ + RTC_DCHECK_NOTREACHED(); \ } \ } while (0) @@ -280,17 +282,23 @@ void NoOp(const Ts&...) {} // This section defines no-op alternatives to the metrics macros when // RTC_METRICS_ENABLED is defined. -#define RTC_HISTOGRAM_COUNTS_100(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_100(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_200(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_200(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_500(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_500(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_1000(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_1000(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_10000(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_10000(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_100000(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_100000(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) #define RTC_HISTOGRAM_COUNTS(name, sample, min, max, bucket_count) \ webrtc::metrics_impl::NoOp(name, sample, min, max, bucket_count) @@ -298,31 +306,41 @@ void NoOp(const Ts&...) {} #define RTC_HISTOGRAM_COUNTS_LINEAR(name, sample, min, max, bucket_count) \ webrtc::metrics_impl::NoOp(name, sample, min, max, bucket_count) -#define RTC_HISTOGRAM_COUNTS_SPARSE_100(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_SPARSE_100(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_SPARSE_200(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_SPARSE_200(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_SPARSE_500(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_SPARSE_500(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_SPARSE_1000(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_SPARSE_1000(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_SPARSE_10000(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_SPARSE_10000(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_COUNTS_SPARSE_100000(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_COUNTS_SPARSE_100000(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) #define RTC_HISTOGRAM_COUNTS_SPARSE(name, sample, min, max, bucket_count) \ webrtc::metrics_impl::NoOp(name, sample, min, max, bucket_count) -#define RTC_HISTOGRAM_PERCENTAGE_SPARSE(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_PERCENTAGE_SPARSE(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_BOOLEAN_SPARSE(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_BOOLEAN_SPARSE(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) #define RTC_HISTOGRAM_ENUMERATION_SPARSE(name, sample, boundary) \ webrtc::metrics_impl::NoOp(name, sample, boundary) -#define RTC_HISTOGRAM_PERCENTAGE(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_PERCENTAGE(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) -#define RTC_HISTOGRAM_BOOLEAN(name, sample) webrtc::metrics_impl::NoOp(name, sample) +#define RTC_HISTOGRAM_BOOLEAN(name, sample) \ + webrtc::metrics_impl::NoOp(name, sample) #define RTC_HISTOGRAM_ENUMERATION(name, sample, boundary) \ webrtc::metrics_impl::NoOp(name, sample, boundary) @@ -334,11 +352,14 @@ void NoOp(const Ts&...) {} #define RTC_HISTOGRAM_COMMON_BLOCK_SLOW(name, sample, factory_get_invocation) \ webrtc::metrics_impl::NoOp(name, sample, factory_get_invocation) -#define RTC_HISTOGRAMS_COUNTS_100(index, name, sample) webrtc::metrics_impl::NoOp(index, name, sample) +#define RTC_HISTOGRAMS_COUNTS_100(index, name, sample) \ + webrtc::metrics_impl::NoOp(index, name, sample) -#define RTC_HISTOGRAMS_COUNTS_200(index, name, sample) webrtc::metrics_impl::NoOp(index, name, sample) +#define RTC_HISTOGRAMS_COUNTS_200(index, name, sample) \ + webrtc::metrics_impl::NoOp(index, name, sample) -#define RTC_HISTOGRAMS_COUNTS_500(index, name, sample) webrtc::metrics_impl::NoOp(index, name, sample) +#define RTC_HISTOGRAMS_COUNTS_500(index, name, sample) \ + webrtc::metrics_impl::NoOp(index, name, sample) #define RTC_HISTOGRAMS_COUNTS_1000(index, name, sample) \ webrtc::metrics_impl::NoOp(index, name, sample) @@ -352,7 +373,8 @@ void NoOp(const Ts&...) {} #define RTC_HISTOGRAMS_ENUMERATION(index, name, sample, boundary) \ webrtc::metrics_impl::NoOp(index, name, sample, boundary) -#define RTC_HISTOGRAMS_PERCENTAGE(index, name, sample) webrtc::metrics_impl::NoOp(index, name, sample) +#define RTC_HISTOGRAMS_PERCENTAGE(index, name, sample) \ + webrtc::metrics_impl::NoOp(index, name, sample) #define RTC_HISTOGRAMS_COMMON(index, name, sample, macro_invocation) \ webrtc::metrics_impl::NoOp(index, name, sample, macro_invocation) @@ -363,7 +385,7 @@ namespace webrtc { namespace metrics { // Time that should have elapsed for stats that are gathered once per call. -enum { kMinRunTimeInSeconds = 10 }; +constexpr int kMinRunTimeInSeconds = 10; class Histogram; @@ -371,32 +393,31 @@ class Histogram; // histogram). // Get histogram for counters. -Histogram* HistogramFactoryGetCounts(const std::string& name, +Histogram* HistogramFactoryGetCounts(absl::string_view name, int min, int max, int bucket_count); // Get histogram for counters with linear bucket spacing. -Histogram* HistogramFactoryGetCountsLinear(const std::string& name, +Histogram* HistogramFactoryGetCountsLinear(absl::string_view name, int min, int max, int bucket_count); // Get histogram for enumerators. -// |boundary| should be above the max enumerator sample. -Histogram* HistogramFactoryGetEnumeration(const std::string& name, - int boundary); +// `boundary` should be above the max enumerator sample. +Histogram* HistogramFactoryGetEnumeration(absl::string_view name, int boundary); // Get sparse histogram for enumerators. -// |boundary| should be above the max enumerator sample. -Histogram* SparseHistogramFactoryGetEnumeration(const std::string& name, +// `boundary` should be above the max enumerator sample. +Histogram* SparseHistogramFactoryGetEnumeration(absl::string_view name, int boundary); -// Function for adding a |sample| to a histogram. +// Function for adding a `sample` to a histogram. void HistogramAdd(Histogram* histogram_pointer, int sample); struct SampleInfo { - SampleInfo(const std::string& name, int min, int max, size_t bucket_count); + SampleInfo(absl::string_view name, int min, int max, size_t bucket_count); ~SampleInfo(); const std::string name; @@ -412,25 +433,26 @@ void Enable(); // Gets histograms and clears all samples. void GetAndReset( - std::map>* histograms); + std::map, rtc::AbslStringViewCmp>* + histograms); // Functions below are mainly for testing. // Clears all samples. void Reset(); -// Returns the number of times the |sample| has been added to the histogram. -int NumEvents(const std::string& name, int sample); +// Returns the number of times the `sample` has been added to the histogram. +int NumEvents(absl::string_view name, int sample); // Returns the total number of added samples to the histogram. -int NumSamples(const std::string& name); +int NumSamples(absl::string_view name); // Returns the minimum sample value (or -1 if the histogram has no samples). -int MinSample(const std::string& name); +int MinSample(absl::string_view name); // Returns a map with keys the samples with at least one event and values the // number of events for that sample. -std::map Samples(const std::string& name); +std::map Samples(absl::string_view name); } // namespace metrics } // namespace webrtc diff --git a/webrtc/system_wrappers/meson.build b/webrtc/system_wrappers/meson.build index b969413..4a4e43e 100644 --- a/webrtc/system_wrappers/meson.build +++ b/webrtc/system_wrappers/meson.build @@ -1,5 +1,6 @@ system_wrappers_sources = [ 'source/cpu_features.cc', + 'source/denormal_disabler.cc', 'source/field_trial.cc', 'source/metrics.cc', 'source/sleep.cc', diff --git a/webrtc/system_wrappers/source/cpu_features.cc b/webrtc/system_wrappers/source/cpu_features.cc index 0f81212..8cd701e 100644 --- a/webrtc/system_wrappers/source/cpu_features.cc +++ b/webrtc/system_wrappers/source/cpu_features.cc @@ -30,7 +30,7 @@ int GetCPUInfoNoASM(CPUFeature feature) { #if defined(WEBRTC_ENABLE_AVX2) // xgetbv returns the value of an Intel Extended Control Register (XCR). -// Currently only XCR0 is defined by Intel so |xcr| should always be zero. +// Currently only XCR0 is defined by Intel so `xcr` should always be zero. static uint64_t xgetbv(uint32_t xcr) { #if defined(_MSC_VER) return _xgetbv(xcr); @@ -93,15 +93,19 @@ int GetCPUInfo(CPUFeature feature) { // a) AVX are supported by the CPU, // b) XSAVE is supported by the CPU, // c) XSAVE is enabled by the kernel. - // See http://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled - // AVX2 support needs (avx_support && (cpu_info7[1] & 0x00000020) != 0;). - return (cpu_info[2] & 0x10000000) != 0 && + // Compiling with MSVC and /arch:AVX2 surprisingly generates BMI2 + // instructions (see crbug.com/1315519). + return (cpu_info[2] & 0x10000000) != 0 /* AVX */ && (cpu_info[2] & 0x04000000) != 0 /* XSAVE */ && (cpu_info[2] & 0x08000000) != 0 /* OSXSAVE */ && (xgetbv(0) & 0x00000006) == 6 /* XSAVE enabled by kernel */ && - (cpu_info7[1] & 0x00000020) != 0; + (cpu_info7[1] & 0x00000020) != 0 /* AVX2 */ && + (cpu_info7[1] & 0x00000100) != 0 /* BMI2 */; } #endif // WEBRTC_ENABLE_AVX2 + if (feature == kFMA3) { + return 0 != (cpu_info[2] & 0x00001000); + } return 0; } #else diff --git a/webrtc/system_wrappers/source/denormal_disabler.cc b/webrtc/system_wrappers/source/denormal_disabler.cc new file mode 100644 index 0000000..bb9c056 --- /dev/null +++ b/webrtc/system_wrappers/source/denormal_disabler.cc @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "system_wrappers/include/denormal_disabler.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +#if defined(WEBRTC_ARCH_X86_FAMILY) && defined(__clang__) +#define WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED +#endif + +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) || \ + defined(WEBRTC_ARCH_ARM_FAMILY) +#define WEBRTC_DENORMAL_DISABLER_SUPPORTED +#endif + +constexpr int kUnspecifiedStatusWord = -1; + +#if defined(WEBRTC_DENORMAL_DISABLER_SUPPORTED) + +// Control register bit mask to disable denormals on the hardware. +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) +// On x86 two bits are used: flush-to-zero (FTZ) and denormals-are-zero (DAZ). +constexpr int kDenormalBitMask = 0x8040; +#elif defined(WEBRTC_ARCH_ARM_FAMILY) +// On ARM one bit is used: flush-to-zero (FTZ). +constexpr int kDenormalBitMask = 1 << 24; +#endif + +// Reads the relevant CPU control register and returns its value for supported +// architectures and compilers. Otherwise returns `kUnspecifiedStatusWord`. +int ReadStatusWord() { + int result = kUnspecifiedStatusWord; +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) + asm volatile("stmxcsr %0" : "=m"(result)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_32_BITS) + asm volatile("vmrs %[result], FPSCR" : [result] "=r"(result)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_64_BITS) + asm volatile("mrs %x[result], FPCR" : [result] "=r"(result)); +#endif + return result; +} + +// Writes `status_word` in the relevant CPU control register if the architecture +// and the compiler are supported. +void SetStatusWord(int status_word) { +#if defined(WEBRTC_DENORMAL_DISABLER_X86_SUPPORTED) + asm volatile("ldmxcsr %0" : : "m"(status_word)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_32_BITS) + asm volatile("vmsr FPSCR, %[src]" : : [src] "r"(status_word)); +#elif defined(WEBRTC_ARCH_ARM_FAMILY) && defined(WEBRTC_ARCH_64_BITS) + asm volatile("msr FPCR, %x[src]" : : [src] "r"(status_word)); +#endif +} + +// Returns true if the status word indicates that denormals are enabled. +constexpr bool DenormalsEnabled(int status_word) { + return (status_word & kDenormalBitMask) != kDenormalBitMask; +} + +#endif // defined(WEBRTC_DENORMAL_DISABLER_SUPPORTED) + +} // namespace + +#if defined(WEBRTC_DENORMAL_DISABLER_SUPPORTED) +DenormalDisabler::DenormalDisabler() : DenormalDisabler(/*enabled=*/true) {} + +DenormalDisabler::DenormalDisabler(bool enabled) + : status_word_(enabled ? ReadStatusWord() : kUnspecifiedStatusWord), + disabling_activated_(enabled && DenormalsEnabled(status_word_)) { + if (disabling_activated_) { + RTC_DCHECK_NE(status_word_, kUnspecifiedStatusWord); + SetStatusWord(status_word_ | kDenormalBitMask); + RTC_DCHECK(!DenormalsEnabled(ReadStatusWord())); + } +} + +bool DenormalDisabler::IsSupported() { + return true; +} + +DenormalDisabler::~DenormalDisabler() { + if (disabling_activated_) { + RTC_DCHECK_NE(status_word_, kUnspecifiedStatusWord); + SetStatusWord(status_word_); + } +} +#else +DenormalDisabler::DenormalDisabler() : DenormalDisabler(/*enabled=*/false) {} + +DenormalDisabler::DenormalDisabler(bool enabled) + : status_word_(kUnspecifiedStatusWord), disabling_activated_(false) {} + +bool DenormalDisabler::IsSupported() { + return false; +} + +DenormalDisabler::~DenormalDisabler() = default; +#endif + +} // namespace webrtc diff --git a/webrtc/system_wrappers/source/field_trial.cc b/webrtc/system_wrappers/source/field_trial.cc index f1dccc9..a1d21fa 100644 --- a/webrtc/system_wrappers/source/field_trial.cc +++ b/webrtc/system_wrappers/source/field_trial.cc @@ -13,9 +13,13 @@ #include #include +#include +#include "absl/algorithm/container.h" #include "absl/strings/string_view.h" +#include "experiments/registered_field_trials.h" #include "rtc_base/checks.h" +#include "rtc_base/containers/flat_set.h" #include "rtc_base/logging.h" #include "rtc_base/string_encode.h" @@ -26,9 +30,15 @@ namespace field_trial { static const char* trials_init_string = NULL; -#ifndef WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT namespace { + constexpr char kPersistentStringSeparator = '/'; + +flat_set& TestKeys() { + static auto* test_keys = new flat_set(); + return *test_keys; +} + // Validates the given field trial string. // E.g.: // "WebRTC-experimentFoo/Enabled/WebRTC-experimentBar/Enabled100kbps/" @@ -68,9 +78,10 @@ bool FieldTrialsStringIsValidInternal(const absl::string_view trials) { return true; } + } // namespace -bool FieldTrialsStringIsValid(const char* trials_string) { +bool FieldTrialsStringIsValid(absl::string_view trials_string) { return FieldTrialsStringIsValidInternal(trials_string); } @@ -78,18 +89,19 @@ void InsertOrReplaceFieldTrialStringsInMap( std::map* fieldtrial_map, const absl::string_view trials_string) { if (FieldTrialsStringIsValidInternal(trials_string)) { - std::vector tokens; - rtc::split(std::string(trials_string), '/', &tokens); + std::vector tokens = rtc::split(trials_string, '/'); // Skip last token which is empty due to trailing '/'. for (size_t idx = 0; idx < tokens.size() - 1; idx += 2) { - (*fieldtrial_map)[tokens[idx]] = tokens[idx + 1]; + (*fieldtrial_map)[std::string(tokens[idx])] = + std::string(tokens[idx + 1]); } } else { - RTC_DCHECK(false) << "Invalid field trials string:" << trials_string; + RTC_DCHECK_NOTREACHED() << "Invalid field trials string:" << trials_string; } } -std::string MergeFieldTrialsStrings(const char* first, const char* second) { +std::string MergeFieldTrialsStrings(absl::string_view first, + absl::string_view second) { std::map fieldtrial_map; InsertOrReplaceFieldTrialStringsInMap(&fieldtrial_map, first); InsertOrReplaceFieldTrialStringsInMap(&fieldtrial_map, second); @@ -102,11 +114,23 @@ std::string MergeFieldTrialsStrings(const char* first, const char* second) { return merged; } -std::string FindFullName(const std::string& name) { +#ifndef WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT +std::string FindFullName(absl::string_view name) { +#if WEBRTC_STRICT_FIELD_TRIALS == 1 + RTC_DCHECK(absl::c_linear_search(kRegisteredFieldTrials, name) || + TestKeys().contains(name)) + << name << " is not registered, see g3doc/field-trials.md."; +#elif WEBRTC_STRICT_FIELD_TRIALS == 2 + RTC_LOG_IF(LS_WARNING, + !(absl::c_linear_search(kRegisteredFieldTrials, name) || + TestKeys().contains(name))) + << name << " is not registered, see g3doc/field-trials.md."; +#endif + if (trials_init_string == NULL) return std::string(); - std::string trials_string(trials_init_string); + absl::string_view trials_string(trials_init_string); if (trials_string.empty()) return std::string(); @@ -122,14 +146,14 @@ std::string FindFullName(const std::string& name) { if (field_value_end == trials_string.npos || field_value_end == field_name_end + 1) break; - std::string field_name(trials_string, next_item, - field_name_end - next_item); - std::string field_value(trials_string, field_name_end + 1, - field_value_end - field_name_end - 1); + absl::string_view field_name = + trials_string.substr(next_item, field_name_end - next_item); + absl::string_view field_value = trials_string.substr( + field_name_end + 1, field_value_end - field_name_end - 1); next_item = field_value_end + 1; if (name == field_name) - return field_value; + return std::string(field_value); } return std::string(); } @@ -138,12 +162,10 @@ std::string FindFullName(const std::string& name) { // Optionally initialize field trial from a string. void InitFieldTrialsFromString(const char* trials_string) { RTC_LOG(LS_INFO) << "Setting field trial string:" << trials_string; -#ifndef WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT if (trials_string) { RTC_DCHECK(FieldTrialsStringIsValidInternal(trials_string)) << "Invalid field trials string:" << trials_string; }; -#endif // WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT trials_init_string = trials_string; } @@ -151,5 +173,14 @@ const char* GetFieldTrialString() { return trials_init_string; } +FieldTrialsAllowedInScopeForTesting::FieldTrialsAllowedInScopeForTesting( + flat_set keys) { + TestKeys() = std::move(keys); +} + +FieldTrialsAllowedInScopeForTesting::~FieldTrialsAllowedInScopeForTesting() { + TestKeys().clear(); +} + } // namespace field_trial } // namespace webrtc diff --git a/webrtc/system_wrappers/source/metrics.cc b/webrtc/system_wrappers/source/metrics.cc index d428336..39ca590 100644 --- a/webrtc/system_wrappers/source/metrics.cc +++ b/webrtc/system_wrappers/source/metrics.cc @@ -11,7 +11,8 @@ #include -#include "rtc_base/constructor_magic.h" +#include "absl/strings/string_view.h" +#include "rtc_base/string_utils.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" @@ -30,11 +31,14 @@ const int kMaxSampleMapSize = 300; class RtcHistogram { public: - RtcHistogram(const std::string& name, int min, int max, int bucket_count) + RtcHistogram(absl::string_view name, int min, int max, int bucket_count) : min_(min), max_(max), info_(name, min, max, bucket_count) { RTC_DCHECK_GT(bucket_count, 0); } + RtcHistogram(const RtcHistogram&) = delete; + RtcHistogram& operator=(const RtcHistogram&) = delete; + void Add(int sample) { sample = std::min(sample, max_); sample = std::max(sample, min_ - 1); // Underflow bucket. @@ -99,8 +103,6 @@ class RtcHistogram { const int min_; const int max_; SampleInfo info_ RTC_GUARDED_BY(mutex_); - - RTC_DISALLOW_COPY_AND_ASSIGN(RtcHistogram); }; class RtcHistogramMap { @@ -108,7 +110,10 @@ class RtcHistogramMap { RtcHistogramMap() {} ~RtcHistogramMap() {} - Histogram* GetCountsHistogram(const std::string& name, + RtcHistogramMap(const RtcHistogramMap&) = delete; + RtcHistogramMap& operator=(const RtcHistogramMap&) = delete; + + Histogram* GetCountsHistogram(absl::string_view name, int min, int max, int bucket_count) { @@ -118,23 +123,24 @@ class RtcHistogramMap { return reinterpret_cast(it->second.get()); RtcHistogram* hist = new RtcHistogram(name, min, max, bucket_count); - map_[name].reset(hist); + map_.emplace(name, hist); return reinterpret_cast(hist); } - Histogram* GetEnumerationHistogram(const std::string& name, int boundary) { + Histogram* GetEnumerationHistogram(absl::string_view name, int boundary) { MutexLock lock(&mutex_); const auto& it = map_.find(name); if (it != map_.end()) return reinterpret_cast(it->second.get()); RtcHistogram* hist = new RtcHistogram(name, 1, boundary, boundary + 1); - map_[name].reset(hist); + map_.emplace(name, hist); return reinterpret_cast(hist); } - void GetAndReset( - std::map>* histograms) { + void GetAndReset(std::map, + rtc::AbslStringViewCmp>* histograms) { MutexLock lock(&mutex_); for (const auto& kv : map_) { std::unique_ptr info = kv.second->GetAndReset(); @@ -150,25 +156,25 @@ class RtcHistogramMap { kv.second->Reset(); } - int NumEvents(const std::string& name, int sample) const { + int NumEvents(absl::string_view name, int sample) const { MutexLock lock(&mutex_); const auto& it = map_.find(name); return (it == map_.end()) ? 0 : it->second->NumEvents(sample); } - int NumSamples(const std::string& name) const { + int NumSamples(absl::string_view name) const { MutexLock lock(&mutex_); const auto& it = map_.find(name); return (it == map_.end()) ? 0 : it->second->NumSamples(); } - int MinSample(const std::string& name) const { + int MinSample(absl::string_view name) const { MutexLock lock(&mutex_); const auto& it = map_.find(name); return (it == map_.end()) ? -1 : it->second->MinSample(); } - std::map Samples(const std::string& name) const { + std::map Samples(absl::string_view name) const { MutexLock lock(&mutex_); const auto& it = map_.find(name); return (it == map_.end()) ? std::map() : it->second->Samples(); @@ -176,25 +182,21 @@ class RtcHistogramMap { private: mutable Mutex mutex_; - std::map> map_ - RTC_GUARDED_BY(mutex_); - - RTC_DISALLOW_COPY_AND_ASSIGN(RtcHistogramMap); + std::map, rtc::AbslStringViewCmp> + map_ RTC_GUARDED_BY(mutex_); }; // RtcHistogramMap is allocated upon call to Enable(). // The histogram getter functions, which return pointer values to the histograms // in the map, are cached in WebRTC. Therefore, this memory is not freed by the // application (the memory will be reclaimed by the OS). -static RtcHistogramMap* volatile g_rtc_histogram_map = nullptr; +static std::atomic g_rtc_histogram_map(nullptr); void CreateMap() { - RtcHistogramMap* map = rtc::AtomicOps::AcquireLoadPtr(&g_rtc_histogram_map); + RtcHistogramMap* map = g_rtc_histogram_map.load(std::memory_order_acquire); if (map == nullptr) { RtcHistogramMap* new_map = new RtcHistogramMap(); - RtcHistogramMap* old_map = rtc::AtomicOps::CompareAndSwapPtr( - &g_rtc_histogram_map, static_cast(nullptr), new_map); - if (old_map != nullptr) + if (!g_rtc_histogram_map.compare_exchange_strong(map, new_map)) delete new_map; } } @@ -202,15 +204,15 @@ void CreateMap() { // Set the first time we start using histograms. Used to make sure Enable() is // not called thereafter. #if RTC_DCHECK_IS_ON -static volatile int g_rtc_histogram_called = 0; +static std::atomic g_rtc_histogram_called(0); #endif // Gets the map (or nullptr). RtcHistogramMap* GetMap() { #if RTC_DCHECK_IS_ON - rtc::AtomicOps::ReleaseStore(&g_rtc_histogram_called, 1); + g_rtc_histogram_called.store(1, std::memory_order_release); #endif - return g_rtc_histogram_map; + return g_rtc_histogram_map.load(); } } // namespace @@ -222,7 +224,7 @@ RtcHistogramMap* GetMap() { // Creates (or finds) histogram. // The returned histogram pointer is cached (and used for adding samples in // subsequent calls). -Histogram* HistogramFactoryGetCounts(const std::string& name, +Histogram* HistogramFactoryGetCounts(absl::string_view name, int min, int max, int bucket_count) { @@ -235,7 +237,7 @@ Histogram* HistogramFactoryGetCounts(const std::string& name, // Creates (or finds) histogram. // The returned histogram pointer is cached (and used for adding samples in // subsequent calls). -Histogram* HistogramFactoryGetCountsLinear(const std::string& name, +Histogram* HistogramFactoryGetCountsLinear(absl::string_view name, int min, int max, int bucket_count) { @@ -250,7 +252,7 @@ Histogram* HistogramFactoryGetCountsLinear(const std::string& name, // Creates (or finds) histogram. // The returned histogram pointer is cached (and used for adding samples in // subsequent calls). -Histogram* HistogramFactoryGetEnumeration(const std::string& name, +Histogram* HistogramFactoryGetEnumeration(absl::string_view name, int boundary) { RtcHistogramMap* map = GetMap(); if (!map) @@ -260,12 +262,12 @@ Histogram* HistogramFactoryGetEnumeration(const std::string& name, } // Our default implementation reuses the non-sparse histogram. -Histogram* SparseHistogramFactoryGetEnumeration(const std::string& name, +Histogram* SparseHistogramFactoryGetEnumeration(absl::string_view name, int boundary) { return HistogramFactoryGetEnumeration(name, boundary); } -// Fast path. Adds |sample| to cached |histogram_pointer|. +// Fast path. Adds `sample` to cached `histogram_pointer`. void HistogramAdd(Histogram* histogram_pointer, int sample) { RtcHistogram* ptr = reinterpret_cast(histogram_pointer); ptr->Add(sample); @@ -273,7 +275,7 @@ void HistogramAdd(Histogram* histogram_pointer, int sample) { #endif // WEBRTC_EXCLUDE_METRICS_DEFAULT -SampleInfo::SampleInfo(const std::string& name, +SampleInfo::SampleInfo(absl::string_view name, int min, int max, size_t bucket_count) @@ -283,15 +285,16 @@ SampleInfo::~SampleInfo() {} // Implementation of global functions in metrics.h. void Enable() { - RTC_DCHECK(g_rtc_histogram_map == nullptr); + RTC_DCHECK(g_rtc_histogram_map.load() == nullptr); #if RTC_DCHECK_IS_ON - RTC_DCHECK_EQ(0, rtc::AtomicOps::AcquireLoad(&g_rtc_histogram_called)); + RTC_DCHECK_EQ(0, g_rtc_histogram_called.load(std::memory_order_acquire)); #endif CreateMap(); } void GetAndReset( - std::map>* histograms) { + std::map, rtc::AbslStringViewCmp>* + histograms) { histograms->clear(); RtcHistogramMap* map = GetMap(); if (map) @@ -304,22 +307,22 @@ void Reset() { map->Reset(); } -int NumEvents(const std::string& name, int sample) { +int NumEvents(absl::string_view name, int sample) { RtcHistogramMap* map = GetMap(); return map ? map->NumEvents(name, sample) : 0; } -int NumSamples(const std::string& name) { +int NumSamples(absl::string_view name) { RtcHistogramMap* map = GetMap(); return map ? map->NumSamples(name) : 0; } -int MinSample(const std::string& name) { +int MinSample(absl::string_view name) { RtcHistogramMap* map = GetMap(); return map ? map->MinSample(name) : -1; } -std::map Samples(const std::string& name) { +std::map Samples(absl::string_view name) { RtcHistogramMap* map = GetMap(); return map ? map->Samples(name) : std::map(); } diff --git a/webrtc/third_party/pffft/BUILD.gn b/webrtc/third_party/pffft/BUILD.gn index 7d6c537..0203735 100644 --- a/webrtc/third_party/pffft/BUILD.gn +++ b/webrtc/third_party/pffft/BUILD.gn @@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,6 +7,8 @@ import("//testing/libfuzzer/fuzzer_test.gni") import("//testing/test.gni") config("common_config") { + cflags = [ "-Wno-shadow" ] + if (is_win) { defines = [ # Required to use math constants from math.h. @@ -17,7 +19,8 @@ config("common_config") { # PFFFT doesn't support SIMD on some cpus, so build a scalar version. if ((current_cpu == "arm" && !arm_use_neon) || current_cpu == "mipsel" || current_cpu == "mips64el" || current_cpu == "ppc64" || - current_cpu == "s390x") { + current_cpu == "riscv64" || current_cpu == "s390x" || + current_cpu == "loong64") { defines = [ "PFFFT_SIMD_DISABLE" ] } } diff --git a/webrtc/third_party/pffft/LICENSE b/webrtc/third_party/pffft/LICENSE index b37aae8..ea8623f 100644 --- a/webrtc/third_party/pffft/LICENSE +++ b/webrtc/third_party/pffft/LICENSE @@ -1 +1,45 @@ -Q29weXJpZ2h0IChjKSAyMDEzICBKdWxpZW4gUG9tbWllciAoIHBvbW1pZXJAbW9kYXJ0dC5jb20gKQoKQmFzZWQgb24gb3JpZ2luYWwgZm9ydHJhbiA3NyBjb2RlIGZyb20gRkZUUEFDS3Y0IGZyb20gTkVUTElCLAphdXRob3JlZCBieSBEciBQYXVsIFN3YXJ6dHJhdWJlciBvZiBOQ0FSLCBpbiAxOTg1LgoKQXMgY29uZmlybWVkIGJ5IHRoZSBOQ0FSIGZmdHBhY2sgc29mdHdhcmUgY3VyYXRvcnMsIHRoZSBmb2xsb3dpbmcKRkZUUEFDS3Y1IGxpY2Vuc2UgYXBwbGllcyB0byBGRlRQQUNLdjQgc291cmNlcy4gTXkgY2hhbmdlcyBhcmUKcmVsZWFzZWQgdW5kZXIgdGhlIHNhbWUgdGVybXMuCgpGRlRQQUNLIGxpY2Vuc2U6CgpodHRwOi8vd3d3LmNpc2wudWNhci5lZHUvY3NzL3NvZnR3YXJlL2ZmdHBhY2s1L2Z0cGsuaHRtbAoKQ29weXJpZ2h0IChjKSAyMDA0IHRoZSBVbml2ZXJzaXR5IENvcnBvcmF0aW9uIGZvciBBdG1vc3BoZXJpYwpSZXNlYXJjaCAoIlVDQVIiKS4gQWxsIHJpZ2h0cyByZXNlcnZlZC4gRGV2ZWxvcGVkIGJ5IE5DQVIncwpDb21wdXRhdGlvbmFsIGFuZCBJbmZvcm1hdGlvbiBTeXN0ZW1zIExhYm9yYXRvcnksIFVDQVIsCnd3dy5jaXNsLnVjYXIuZWR1LgoKUmVkaXN0cmlidXRpb24gYW5kIHVzZSBvZiB0aGUgU29mdHdhcmUgaW4gc291cmNlIGFuZCBiaW5hcnkgZm9ybXMsCndpdGggb3Igd2l0aG91dCBtb2RpZmljYXRpb24sIGlzIHBlcm1pdHRlZCBwcm92aWRlZCB0aGF0IHRoZQpmb2xsb3dpbmcgY29uZGl0aW9ucyBhcmUgbWV0OgoKLSBOZWl0aGVyIHRoZSBuYW1lcyBvZiBOQ0FSJ3MgQ29tcHV0YXRpb25hbCBhbmQgSW5mb3JtYXRpb24gU3lzdGVtcwpMYWJvcmF0b3J5LCB0aGUgVW5pdmVyc2l0eSBDb3Jwb3JhdGlvbiBmb3IgQXRtb3NwaGVyaWMgUmVzZWFyY2gsCm5vciB0aGUgbmFtZXMgb2YgaXRzIHNwb25zb3JzIG9yIGNvbnRyaWJ1dG9ycyBtYXkgYmUgdXNlZCB0bwplbmRvcnNlIG9yIHByb21vdGUgcHJvZHVjdHMgZGVyaXZlZCBmcm9tIHRoaXMgU29mdHdhcmUgd2l0aG91dApzcGVjaWZpYyBwcmlvciB3cml0dGVuIHBlcm1pc3Npb24uCgotIFJlZGlzdHJpYnV0aW9ucyBvZiBzb3VyY2UgY29kZSBtdXN0IHJldGFpbiB0aGUgYWJvdmUgY29weXJpZ2h0Cm5vdGljZXMsIHRoaXMgbGlzdCBvZiBjb25kaXRpb25zLCBhbmQgdGhlIGRpc2NsYWltZXIgYmVsb3cuCgotIFJlZGlzdHJpYnV0aW9ucyBpbiBiaW5hcnkgZm9ybSBtdXN0IHJlcHJvZHVjZSB0aGUgYWJvdmUgY29weXJpZ2h0Cm5vdGljZSwgdGhpcyBsaXN0IG9mIGNvbmRpdGlvbnMsIGFuZCB0aGUgZGlzY2xhaW1lciBiZWxvdyBpbiB0aGUKZG9jdW1lbnRhdGlvbiBhbmQvb3Igb3RoZXIgbWF0ZXJpYWxzIHByb3ZpZGVkIHdpdGggdGhlCmRpc3RyaWJ1dGlvbi4KClRISVMgU09GVFdBUkUgSVMgUFJPVklERUQgIkFTIElTIiwgV0lUSE9VVCBXQVJSQU5UWSBPRiBBTlkgS0lORCwKRVhQUkVTUyBPUiBJTVBMSUVELCBJTkNMVURJTkcsIEJVVCBOT1QgTElNSVRFRCBUTyBUSEUgV0FSUkFOVElFUyBPRgpNRVJDSEFOVEFCSUxJVFksIEZJVE5FU1MgRk9SIEEgUEFSVElDVUxBUiBQVVJQT1NFIEFORApOT05JTkZSSU5HRU1FTlQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRSBDT05UUklCVVRPUlMgT1IgQ09QWVJJR0hUCkhPTERFUlMgQkUgTElBQkxFIEZPUiBBTlkgQ0xBSU0sIElORElSRUNULCBJTkNJREVOVEFMLCBTUEVDSUFMLApFWEVNUExBUlksIE9SIENPTlNFUVVFTlRJQUwgREFNQUdFUyBPUiBPVEhFUiBMSUFCSUxJVFksIFdIRVRIRVIgSU4gQU4KQUNUSU9OIE9GIENPTlRSQUNULCBUT1JUIE9SIE9USEVSV0lTRSwgQVJJU0lORyBGUk9NLCBPVVQgT0YgT1IgSU4KQ09OTkVDVElPTiBXSVRIIFRIRSBTT0ZUV0FSRSBPUiBUSEUgVVNFIE9SIE9USEVSIERFQUxJTkdTIFdJVEggVEhFClNPRlRXQVJFLgo= \ No newline at end of file +Copyright (c) 2013 Julien Pommier ( pommier@modartt.com ) + +Based on original fortran 77 code from FFTPACKv4 from NETLIB, +authored by Dr Paul Swarztrauber of NCAR, in 1985. + +As confirmed by the NCAR fftpack software curators, the following +FFTPACKv5 license applies to FFTPACKv4 sources. My changes are +released under the same terms. + +FFTPACK license: + +http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html + +Copyright (c) 2004 the University Corporation for Atmospheric +Research ("UCAR"). All rights reserved. Developed by NCAR's +Computational and Information Systems Laboratory, UCAR, +www.cisl.ucar.edu. + +Redistribution and use of the Software in source and binary forms, +with or without modification, is permitted provided that the +following conditions are met: + +- Neither the names of NCAR's Computational and Information Systems +Laboratory, the University Corporation for Atmospheric Research, +nor the names of its sponsors or contributors may be used to +endorse or promote products derived from this Software without +specific prior written permission. + +- Redistributions of source code must retain the above copyright +notices, this list of conditions, and the disclaimer below. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions, and the disclaimer below in the +documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. diff --git a/webrtc/third_party/pffft/README.md b/webrtc/third_party/pffft/README.md new file mode 100644 index 0000000..e373a57 --- /dev/null +++ b/webrtc/third_party/pffft/README.md @@ -0,0 +1,69 @@ +# Notes on PFFFT +We strongly recommend to **read this file** before using PFFFT and to **always wrap** the original C library within a C++ wrapper. + +[Example of PFFFT wrapper](https://cs.chromium.org/chromium/src/third_party/webrtc/modules/audio_processing/utility/pffft_wrapper.h). + +## Scratch buffer +The caller can optionally provide a scratch buffer. When not provided, VLA is used to provide a thread-safe option. +However, it is recommended to write a C++ wrapper which allocates its own scratch buffer. +Note that the scratch buffer has the same memory alignment requirements of the input and output vectors. + +## Output layout +PFFFT computes the forward transform with two possible output layouts: +1. ordered +2. unordered + +### Ordered layout +Calling `pffft_transform_ordered` produces an array of **interleaved real and imaginary parts**. +The last Fourier coefficient is purely real and stored in the imaginary part of the DC component (which is also purely real). + +### Unordered layout +Calling `pffft_transform` produces an array with a more complex structure, but in a more efficient way than `pffft_transform_ordered`. +Below, the output produced by Matlab and that produced by PFFFT are compared. +The comparison is made for a 32 point transform of a 16 sample buffer. +A 32 point transform has been chosen as this is the minimum supported by PFFFT. + +Important notes: +- In Matlab the DC (Matlab index 1 [R1, I1]]) and Nyquist (Matlab index 17 [R17, I17]) values are not repeated as complex conjugates. +- In PFFFT the Nyquist real and imaginary parts ([R17, I17]) are omitted entirely. +- In PFFFT the final 8 values (4 real and 4 imaginary) are not in the same order as all of the others. +- In PFFFT all imaginary parts are stored as negatives (like second half in Matlab). + +``` ++-------+-----------+-------+-------+ +| Index | Matlab | Index | PFFFT | ++-------+-----------+-------+-------+ +| 1 | R1 + I1 | 0 | R1 | +| 2 | R2+ I2 | 1 | R2 | +| 3 | R3 + I3 | 2 | R3 | +| 4 | R4 + I4 | 3 | R4 | +| 5 | R5 + I5 | 4 | -I1 | +| 6 | R6 + I6 | 5 | -I2 | +| 7 | R7 + I7 | 6 | -I3 | +| 8 | R8 + I8 | 7 | -I4 | +| 9 | R9 + I9 | 8 | R5 | +| 10 | R10 + I10 | 9 | R6 | +| 11 | R11 + I11 | 10 | R7 | +| 12 | R12 + I12 | 11 | R8 | +| 13 | R13 + I13 | 12 | -I5 | +| 14 | R14 + I14 | 13 | -I6 | +| 15 | R15 + I15 | 14 | -I7 | +| 16 | R16 + I16 | 15 | -I8 | +| 17 | R17 + I17 | 16 | R9 | +| 18 | R16 - I16 | 17 | R10 | +| 19 | R15 - I15 | 18 | R11 | +| 20 | R14 - I14 | 19 | R12 | +| 21 | R13 - I13 | 20 | -I9 | +| 22 | R12 - I12 | 21 | -I10 | +| 23 | R11 - I11 | 22 | -I11 | +| 24 | R10 - I10 | 23 | -I12 | +| 25 | R9 - I9 | 24 | R13 | +| 26 | R8 - I8 | 25 | R16 | +| 27 | R7 - I7 | 26 | R15 | +| 28 | R6 - I6 | 27 | R14 | +| 29 | R5 - I5 | 28 | -I13 | +| 30 | R4 - I4 | 29 | -I16 | +| 31 | R3 - I3 | 30 | -I15 | +| 32 | R2 - I2 | 31 | -I14 | ++-------+-----------+-------+-------+ +``` diff --git a/webrtc/third_party/pffft/README.txt b/webrtc/third_party/pffft/README.txt new file mode 100644 index 0000000..878f4a6 --- /dev/null +++ b/webrtc/third_party/pffft/README.txt @@ -0,0 +1,379 @@ +PFFFT: a pretty fast FFT. + +TL;DR +-- + +PFFFT does 1D Fast Fourier Transforms, of single precision real and +complex vectors. It tries do it fast, it tries to be correct, and it +tries to be small. Computations do take advantage of SSE1 instructions +on x86 cpus, Altivec on powerpc cpus, and NEON on ARM cpus. The +license is BSD-like. + + +Why does it exist: +-- + +I was in search of a good performing FFT library , preferably very +small and with a very liberal license. + +When one says "fft library", FFTW ("Fastest Fourier Transform in the +West") is probably the first name that comes to mind -- I guess that +99% of open-source projects that need a FFT do use FFTW, and are happy +with it. However, it is quite a large library , which does everything +fft related (2d transforms, 3d transforms, other transformations such +as discrete cosine , or fast hartley). And it is licensed under the +GNU GPL , which means that it cannot be used in non open-source +products. + +An alternative to FFTW that is really small, is the venerable FFTPACK +v4, which is available on NETLIB. A more recent version (v5) exists, +but it is larger as it deals with multi-dimensional transforms. This +is a library that is written in FORTRAN 77, a language that is now +considered as a bit antiquated by many. FFTPACKv4 was written in 1985, +by Dr Paul Swarztrauber of NCAR, more than 25 years ago ! And despite +its age, benchmarks show it that it still a very good performing FFT +library, see for example the 1d single precision benchmarks here: +http://www.fftw.org/speed/opteron-2.2GHz-32bit/ . It is however not +competitive with the fastest ones, such as FFTW, Intel MKL, AMD ACML, +Apple vDSP. The reason for that is that those libraries do take +advantage of the SSE SIMD instructions available on Intel CPUs, +available since the days of the Pentium III. These instructions deal +with small vectors of 4 floats at a time, instead of a single float +for a traditionnal FPU, so when using these instructions one may expect +a 4-fold performance improvement. + +The idea was to take this fortran fftpack v4 code, translate to C, +modify it to deal with those SSE instructions, and check that the +final performance is not completely ridiculous when compared to other +SIMD FFT libraries. Translation to C was performed with f2c ( +http://www.netlib.org/f2c/ ). The resulting file was a bit edited in +order to remove the thousands of gotos that were introduced by +f2c. You will find the fftpack.h and fftpack.c sources in the +repository, this a complete translation of +http://www.netlib.org/fftpack/ , with the discrete cosine transform +and the test program. There is no license information in the netlib +repository, but it was confirmed to me by the fftpack v5 curators that +the same terms do apply to fftpack v4: +http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html . This is a +"BSD-like" license, it is compatible with proprietary projects. + +Adapting fftpack to deal with the SIMD 4-element vectors instead of +scalar single precision numbers was more complex than I originally +thought, especially with the real transforms, and I ended up writing +more code than I planned.. + + +The code: +-- + +Only two files, in good old C, pffft.c and pffft.h . The API is very +very simple, just make sure that you read the comments in pffft.h. + + +Comparison with other FFTs: +-- + +The idea was not to break speed records, but to get a decently fast +fft that is at least 50% as fast as the fastest FFT -- especially on +slowest computers . I'm more focused on getting the best performance +on slow cpus (Atom, Intel Core 1, old Athlons, ARM Cortex-A9...), than +on getting top performance on today fastest cpus. + +It can be used in a real-time context as the fft functions do not +perform any memory allocation -- that is why they accept a 'work' +array in their arguments. + +It is also a bit focused on performing 1D convolutions, that is why it +provides "unordered" FFTs , and a fourier domain convolution +operation. + + +Benchmark results (cpu tested: core i7 2600, core 2 quad, core 1 duo, atom N270, cortex-A9) +-- + +The benchmark shows the performance of various fft implementations measured in +MFlops, with the number of floating point operations being defined as 5Nlog2(N) +for a length N complex fft, and 2.5*Nlog2(N) for a real fft. +See http://www.fftw.org/speed/method.html for an explanation of these formulas. + +MacOS Lion, gcc 4.2, 64-bit, fftw 3.3 on a 3.4 GHz core i7 2600 + +Built with: + + gcc-4.2 -o test_pffft -arch x86_64 -O3 -Wall -W pffft.c test_pffft.c fftpack.c -L/usr/local/lib -I/usr/local/include/ -DHAVE_VECLIB -framework veclib -DHAVE_FFTW -lfftw3f + +| input len |real FFTPack| real vDSP | real FFTW | real PFFFT | |cplx FFTPack| cplx vDSP | cplx FFTW | cplx PFFFT | +|-----------+------------+------------+------------+------------| |------------+------------+------------+------------| +| 64 | 2816 | 8596 | 7329 | 8187 | | 2887 | 14898 | 14668 | 11108 | +| 96 | 3298 | n/a | 8378 | 7727 | | 3953 | n/a | 15680 | 10878 | +| 128 | 3507 | 11575 | 9266 | 10108 | | 4233 | 17598 | 16427 | 12000 | +| 160 | 3391 | n/a | 9838 | 10711 | | 4220 | n/a | 16653 | 11187 | +| 192 | 3919 | n/a | 9868 | 10956 | | 4297 | n/a | 15770 | 12540 | +| 256 | 4283 | 13179 | 10694 | 13128 | | 4545 | 19550 | 16350 | 13822 | +| 384 | 3136 | n/a | 10810 | 12061 | | 3600 | n/a | 16103 | 13240 | +| 480 | 3477 | n/a | 10632 | 12074 | | 3536 | n/a | 11630 | 12522 | +| 512 | 3783 | 15141 | 11267 | 13838 | | 3649 | 20002 | 16560 | 13580 | +| 640 | 3639 | n/a | 11164 | 13946 | | 3695 | n/a | 15416 | 13890 | +| 768 | 3800 | n/a | 11245 | 13495 | | 3590 | n/a | 15802 | 14552 | +| 800 | 3440 | n/a | 10499 | 13301 | | 3659 | n/a | 12056 | 13268 | +| 1024 | 3924 | 15605 | 11450 | 15339 | | 3769 | 20963 | 13941 | 15467 | +| 2048 | 4518 | 16195 | 11551 | 15532 | | 4258 | 20413 | 13723 | 15042 | +| 2400 | 4294 | n/a | 10685 | 13078 | | 4093 | n/a | 12777 | 13119 | +| 4096 | 4750 | 16596 | 11672 | 15817 | | 4157 | 19662 | 14316 | 14336 | +| 8192 | 3820 | 16227 | 11084 | 12555 | | 3691 | 18132 | 12102 | 13813 | +| 9216 | 3864 | n/a | 10254 | 12870 | | 3586 | n/a | 12119 | 13994 | +| 16384 | 3822 | 15123 | 10454 | 12822 | | 3613 | 16874 | 12370 | 13881 | +| 32768 | 4175 | 14512 | 10662 | 11095 | | 3881 | 14702 | 11619 | 11524 | +| 262144 | 3317 | 11429 | 6269 | 9517 | | 2810 | 11729 | 7757 | 10179 | +| 1048576 | 2913 | 10551 | 4730 | 5867 | | 2661 | 7881 | 3520 | 5350 | +|-----------+------------+------------+------------+------------| |------------+------------+------------+------------| + + +Debian 6, gcc 4.4.5, 64-bit, fftw 3.3.1 on a 3.4 GHz core i7 2600 + +Built with: +gcc -o test_pffft -DHAVE_FFTW -msse2 -O3 -Wall -W pffft.c test_pffft.c fftpack.c -L$HOME/local/lib -I$HOME/local/include/ -lfftw3f -lm + +| N (input length) | real FFTPack | real FFTW | real PFFFT | | cplx FFTPack | cplx FFTW | cplx PFFFT | +|------------------+--------------+--------------+--------------| |--------------+--------------+--------------| +| 64 | 3840 | 7680 | 8777 | | 4389 | 20480 | 11171 | +| 96 | 4214 | 9633 | 8429 | | 4816 | 22477 | 11238 | +| 128 | 3584 | 10240 | 10240 | | 5120 | 23893 | 11947 | +| 192 | 4854 | 11095 | 12945 | | 4854 | 22191 | 14121 | +| 256 | 4096 | 11703 | 16384 | | 5120 | 23406 | 13653 | +| 384 | 4395 | 14651 | 12558 | | 4884 | 19535 | 14651 | +| 512 | 5760 | 13166 | 15360 | | 4608 | 23040 | 15360 | +| 768 | 4907 | 14020 | 16357 | | 4461 | 19628 | 14020 | +| 1024 | 5120 | 14629 | 14629 | | 5120 | 20480 | 15754 | +| 2048 | 5632 | 14080 | 18773 | | 4693 | 12516 | 16091 | +| 4096 | 5120 | 13653 | 17554 | | 4726 | 7680 | 14456 | +| 8192 | 4160 | 7396 | 13312 | | 4437 | 14791 | 13312 | +| 9216 | 4210 | 6124 | 13473 | | 4491 | 7282 | 14970 | +| 16384 | 3976 | 11010 | 14313 | | 4210 | 11450 | 13631 | +| 32768 | 4260 | 10224 | 10954 | | 4260 | 6816 | 11797 | +| 262144 | 3736 | 6896 | 9961 | | 2359 | 8965 | 9437 | +| 1048576 | 2796 | 4534 | 6453 | | 1864 | 3078 | 5592 | +|------------------+--------------+--------------+--------------| |--------------+--------------+--------------| + + + +MacOS Snow Leopard, gcc 4.0, 32-bit, fftw 3.3 on a 1.83 GHz core 1 duo + +Built with: + + gcc -o test_pffft -DHAVE_FFTW -DHAVE_VECLIB -O3 -Wall -W pffft.c test_pffft.c fftpack.c -L/usr/local/lib -I/usr/local/include/ -lfftw3f -framework veclib + +| input len |real FFTPack| real vDSP | real FFTW | real PFFFT | |cplx FFTPack| cplx vDSP | cplx FFTW | cplx PFFFT | +|-----------+------------+------------+------------+------------| |------------+------------+------------+------------| +| 64 | 745 | 2145 | 1706 | 2028 | | 961 | 3356 | 3313 | 2300 | +| 96 | 877 | n/a | 1976 | 1978 | | 1059 | n/a | 3333 | 2233 | +| 128 | 951 | 2808 | 2213 | 2279 | | 1202 | 3803 | 3739 | 2494 | +| 192 | 1002 | n/a | 2456 | 2429 | | 1186 | n/a | 3701 | 2508 | +| 256 | 1065 | 3205 | 2641 | 2793 | | 1302 | 4013 | 3912 | 2663 | +| 384 | 845 | n/a | 2759 | 2499 | | 948 | n/a | 3729 | 2504 | +| 512 | 900 | 3476 | 2956 | 2759 | | 974 | 4057 | 3954 | 2645 | +| 768 | 910 | n/a | 2912 | 2737 | | 975 | n/a | 3837 | 2614 | +| 1024 | 936 | 3583 | 3107 | 3009 | | 1006 | 4124 | 3821 | 2697 | +| 2048 | 1057 | 3585 | 3091 | 2837 | | 1089 | 3889 | 3701 | 2513 | +| 4096 | 1083 | 3524 | 3092 | 2733 | | 1039 | 3617 | 3462 | 2364 | +| 8192 | 874 | 3252 | 2967 | 2363 | | 911 | 3106 | 2789 | 2302 | +| 9216 | 898 | n/a | 2420 | 2290 | | 865 | n/a | 2676 | 2204 | +| 16384 | 903 | 2892 | 2506 | 2421 | | 899 | 3026 | 2797 | 2289 | +| 32768 | 965 | 2837 | 2550 | 2358 | | 920 | 2922 | 2763 | 2240 | +| 262144 | 738 | 2422 | 1589 | 1708 | | 610 | 2038 | 1436 | 1091 | +| 1048576 | 528 | 1207 | 845 | 880 | | 606 | 1020 | 669 | 1036 | +|-----------+------------+------------+------------+------------| |------------+------------+------------+------------| + + + +Ubuntu 11.04, gcc 4.5, 32-bit, fftw 3.2 on a 2.66 core 2 quad + +Built with: +gcc -o test_pffft -DHAVE_FFTW -msse -mfpmath=sse -O3 -Wall -W pffft.c test_pffft.c fftpack.c -L/usr/local/lib -I/usr/local/include/ -lfftw3f -lm + +| input len |real FFTPack| real FFTW | real PFFFT | |cplx FFTPack| cplx FFTW | cplx PFFFT | +|-----------+------------+------------+------------| |------------+------------+------------| +| 64 | 1920 | 3614 | 5120 | | 2194 | 7680 | 6467 | +| 96 | 1873 | 3549 | 5187 | | 2107 | 8429 | 5863 | +| 128 | 2240 | 3773 | 5514 | | 2560 | 7964 | 6827 | +| 192 | 1765 | 4569 | 7767 | | 2284 | 9137 | 7061 | +| 256 | 2048 | 5461 | 7447 | | 2731 | 9638 | 7802 | +| 384 | 1998 | 5861 | 6762 | | 2313 | 9253 | 7644 | +| 512 | 2095 | 6144 | 7680 | | 2194 | 10240 | 7089 | +| 768 | 2230 | 5773 | 7549 | | 2045 | 10331 | 7010 | +| 1024 | 2133 | 6400 | 8533 | | 2133 | 10779 | 7877 | +| 2048 | 2011 | 7040 | 8665 | | 1942 | 10240 | 7768 | +| 4096 | 2194 | 6827 | 8777 | | 1755 | 9452 | 6827 | +| 8192 | 1849 | 6656 | 6656 | | 1752 | 7831 | 6827 | +| 9216 | 1871 | 5858 | 6416 | | 1643 | 6909 | 6266 | +| 16384 | 1883 | 6223 | 6506 | | 1664 | 7340 | 6982 | +| 32768 | 1826 | 6390 | 6667 | | 1631 | 7481 | 6971 | +| 262144 | 1546 | 4075 | 5977 | | 1299 | 3415 | 3551 | +| 1048576 | 1104 | 2071 | 1730 | | 1104 | 1149 | 1834 | +|-----------+------------+------------+------------| |------------+------------+------------| + + + +Ubuntu 11.04, gcc 4.5, 32-bit, fftw 3.3 on a 1.6 GHz Atom N270 + +Built with: +gcc -o test_pffft -DHAVE_FFTW -msse -mfpmath=sse -O3 -Wall -W pffft.c test_pffft.c fftpack.c -L/usr/local/lib -I/usr/local/include/ -lfftw3f -lm + +| N (input length) | real FFTPack | real FFTW | real PFFFT | | cplx FFTPack | cplx FFTW | cplx PFFFT | +|------------------+--------------+--------------+--------------| |--------------+--------------+--------------| +| 64 | 452 | 1041 | 1336 | | 549 | 2318 | 1781 | +| 96 | 444 | 1297 | 1297 | | 503 | 2408 | 1686 | +| 128 | 527 | 1525 | 1707 | | 543 | 2655 | 1886 | +| 192 | 498 | 1653 | 1849 | | 539 | 2678 | 1942 | +| 256 | 585 | 1862 | 2156 | | 594 | 2777 | 2244 | +| 384 | 499 | 1870 | 1998 | | 511 | 2586 | 1890 | +| 512 | 562 | 2095 | 2194 | | 542 | 2973 | 2194 | +| 768 | 545 | 2045 | 2133 | | 545 | 2365 | 2133 | +| 1024 | 595 | 2133 | 2438 | | 569 | 2695 | 2179 | +| 2048 | 587 | 2125 | 2347 | | 521 | 2230 | 1707 | +| 4096 | 495 | 1890 | 1834 | | 492 | 1876 | 1672 | +| 8192 | 469 | 1548 | 1729 | | 438 | 1740 | 1664 | +| 9216 | 468 | 1663 | 1663 | | 446 | 1585 | 1531 | +| 16384 | 453 | 1608 | 1767 | | 398 | 1476 | 1664 | +| 32768 | 456 | 1420 | 1503 | | 387 | 1388 | 1345 | +| 262144 | 309 | 385 | 726 | | 262 | 415 | 840 | +| 1048576 | 280 | 351 | 739 | | 261 | 313 | 797 | +|------------------+--------------+--------------+--------------| |--------------+--------------+--------------| + + + +Windows 7, visual c++ 2010 on a 1.6 GHz Atom N270 + +Built with: +cl /Ox -D_USE_MATH_DEFINES /arch:SSE test_pffft.c pffft.c fftpack.c + +(visual c++ is definitively not very good with SSE intrinsics...) + +| N (input length) | real FFTPack | real PFFFT | | cplx FFTPack | cplx PFFFT | +|------------------+--------------+--------------| |--------------+--------------| +| 64 | 173 | 1009 | | 174 | 1159 | +| 96 | 169 | 1029 | | 188 | 1201 | +| 128 | 195 | 1242 | | 191 | 1275 | +| 192 | 178 | 1312 | | 184 | 1276 | +| 256 | 196 | 1591 | | 186 | 1281 | +| 384 | 172 | 1409 | | 181 | 1281 | +| 512 | 187 | 1640 | | 181 | 1313 | +| 768 | 171 | 1614 | | 176 | 1258 | +| 1024 | 186 | 1812 | | 178 | 1223 | +| 2048 | 190 | 1707 | | 186 | 1099 | +| 4096 | 182 | 1446 | | 177 | 975 | +| 8192 | 175 | 1345 | | 169 | 1034 | +| 9216 | 165 | 1271 | | 168 | 1023 | +| 16384 | 166 | 1396 | | 165 | 949 | +| 32768 | 172 | 1311 | | 161 | 881 | +| 262144 | 136 | 632 | | 134 | 629 | +| 1048576 | 134 | 698 | | 127 | 623 | +|------------------+--------------+--------------| |--------------+--------------| + + + +Ubuntu 12.04, gcc-4.7.3, 32-bit, with fftw 3.3.3 (built with --enable-neon), on a 1.2GHz ARM Cortex A9 (Tegra 3) + +Built with: +gcc-4.7 -O3 -DHAVE_FFTW -march=armv7-a -mtune=cortex-a9 -mfloat-abi=hard -mfpu=neon -ffast-math test_pffft.c pffft.c -o test_pffft_arm fftpack.c -lm -I/usr/local/include/ -L/usr/local/lib/ -lfftw3f + +| input len |real FFTPack| real FFTW | real PFFFT | |cplx FFTPack| cplx FFTW | cplx PFFFT | +|-----------+------------+------------+------------| |------------+------------+------------| +| 64 | 549 | 452 | 731 | | 512 | 602 | 640 | +| 96 | 421 | 272 | 702 | | 496 | 571 | 602 | +| 128 | 498 | 512 | 815 | | 597 | 618 | 652 | +| 160 | 521 | 536 | 815 | | 586 | 669 | 625 | +| 192 | 539 | 571 | 883 | | 485 | 597 | 626 | +| 256 | 640 | 539 | 975 | | 569 | 611 | 671 | +| 384 | 499 | 610 | 879 | | 499 | 602 | 637 | +| 480 | 518 | 507 | 877 | | 496 | 661 | 616 | +| 512 | 524 | 591 | 1002 | | 549 | 678 | 668 | +| 640 | 542 | 612 | 955 | | 568 | 663 | 645 | +| 768 | 557 | 613 | 981 | | 491 | 663 | 598 | +| 800 | 514 | 353 | 882 | | 514 | 360 | 574 | +| 1024 | 640 | 640 | 1067 | | 492 | 683 | 602 | +| 2048 | 587 | 640 | 908 | | 486 | 640 | 552 | +| 2400 | 479 | 368 | 777 | | 422 | 376 | 518 | +| 4096 | 511 | 614 | 853 | | 426 | 640 | 534 | +| 8192 | 415 | 584 | 708 | | 386 | 622 | 516 | +| 9216 | 419 | 571 | 687 | | 364 | 586 | 506 | +| 16384 | 426 | 577 | 716 | | 398 | 606 | 530 | +| 32768 | 417 | 572 | 673 | | 399 | 572 | 468 | +| 262144 | 219 | 380 | 293 | | 255 | 431 | 343 | +| 1048576 | 202 | 274 | 237 | | 265 | 282 | 355 | +|-----------+------------+------------+------------| |------------+------------+------------| + +Same platform as above, but this time pffft and fftpack are built with clang 3.2: + +clang -O3 -DHAVE_FFTW -march=armv7-a -mtune=cortex-a9 -mfloat-abi=hard -mfpu=neon -ffast-math test_pffft.c pffft.c -o test_pffft_arm fftpack.c -lm -I/usr/local/include/ -L/usr/local/lib/ -lfftw3f + +| input len |real FFTPack| real FFTW | real PFFFT | |cplx FFTPack| cplx FFTW | cplx PFFFT | +|-----------+------------+------------+------------| |------------+------------+------------| +| 64 | 427 | 452 | 853 | | 427 | 602 | 1024 | +| 96 | 351 | 276 | 843 | | 337 | 571 | 963 | +| 128 | 373 | 512 | 996 | | 390 | 618 | 1054 | +| 160 | 426 | 536 | 987 | | 375 | 669 | 914 | +| 192 | 404 | 571 | 1079 | | 388 | 588 | 1079 | +| 256 | 465 | 539 | 1205 | | 445 | 602 | 1170 | +| 384 | 366 | 610 | 1099 | | 343 | 594 | 1099 | +| 480 | 356 | 507 | 1140 | | 335 | 651 | 931 | +| 512 | 411 | 591 | 1213 | | 384 | 649 | 1124 | +| 640 | 398 | 612 | 1193 | | 373 | 654 | 901 | +| 768 | 409 | 613 | 1227 | | 383 | 663 | 1044 | +| 800 | 411 | 348 | 1073 | | 353 | 358 | 809 | +| 1024 | 427 | 640 | 1280 | | 413 | 692 | 1004 | +| 2048 | 414 | 626 | 1126 | | 371 | 640 | 853 | +| 2400 | 399 | 373 | 898 | | 319 | 368 | 653 | +| 4096 | 404 | 602 | 1059 | | 357 | 633 | 778 | +| 8192 | 332 | 584 | 792 | | 308 | 616 | 716 | +| 9216 | 322 | 561 | 783 | | 299 | 586 | 687 | +| 16384 | 344 | 568 | 778 | | 314 | 617 | 745 | +| 32768 | 342 | 564 | 737 | | 314 | 552 | 629 | +| 262144 | 201 | 383 | 313 | | 227 | 435 | 413 | +| 1048576 | 187 | 262 | 251 | | 228 | 281 | 409 | +|-----------+------------+------------+------------| |------------+------------+------------| + +So it looks like, on ARM, gcc 4.7 is the best at scalar floating point +(the fftpack performance numbers are better with gcc), while clang is +the best with neon intrinsics (see how pffft perf has improved with +clang 3.2). + + +NVIDIA Jetson TK1 board, gcc-4.8.2. The cpu is a 2.3GHz cortex A15 (Tegra K1). + +Built with: +gcc -O3 -march=armv7-a -mtune=native -mfloat-abi=hard -mfpu=neon -ffast-math test_pffft.c pffft.c -o test_pffft_arm fftpack.c -lm + +| input len |real FFTPack| real PFFFT | |cplx FFTPack| cplx PFFFT | +|-----------+------------+------------| |------------+------------| +| 64 | 1735 | 3308 | | 1994 | 3744 | +| 96 | 1596 | 3448 | | 1987 | 3572 | +| 128 | 1807 | 4076 | | 2255 | 3960 | +| 160 | 1769 | 4083 | | 2071 | 3845 | +| 192 | 1990 | 4233 | | 2017 | 3939 | +| 256 | 2191 | 4882 | | 2254 | 4346 | +| 384 | 1878 | 4492 | | 2073 | 4012 | +| 480 | 1748 | 4398 | | 1923 | 3951 | +| 512 | 2030 | 5064 | | 2267 | 4195 | +| 640 | 1918 | 4756 | | 2094 | 4184 | +| 768 | 2099 | 4907 | | 2048 | 4297 | +| 800 | 1822 | 4555 | | 1880 | 4063 | +| 1024 | 2232 | 5355 | | 2187 | 4420 | +| 2048 | 2176 | 4983 | | 2027 | 3602 | +| 2400 | 1741 | 4256 | | 1710 | 3344 | +| 4096 | 1816 | 3914 | | 1851 | 3349 | +| 8192 | 1716 | 3481 | | 1700 | 3255 | +| 9216 | 1735 | 3589 | | 1653 | 3094 | +| 16384 | 1567 | 3483 | | 1637 | 3244 | +| 32768 | 1624 | 3240 | | 1655 | 3156 | +| 262144 | 1012 | 1898 | | 983 | 1503 | +| 1048576 | 876 | 1154 | | 868 | 1341 | +|-----------+------------+------------| |------------+------------| + +The performance on the tegra K1 is pretty impressive. I'm not +including the FFTW numbers as they as slightly below the scalar +fftpack numbers, so something must be wrong (however it seems to be +correctly configured and is using neon simd instructions). + +When using clang 3.4 the pffft version is even a bit faster, reaching +5.7 GFlops for real ffts of size 1024. diff --git a/webrtc/third_party/pffft/patches/01-rmv_printf.diff b/webrtc/third_party/pffft/patches/01-rmv_printf.diff new file mode 100644 index 0000000..3a91494 --- /dev/null +++ b/webrtc/third_party/pffft/patches/01-rmv_printf.diff @@ -0,0 +1,60 @@ +diff --git a/third_party/pffft/src/pffft.c b/third_party/pffft/src/pffft.c +index 7934db448a09..2e0c2f651438 100644 +--- a/third_party/pffft/src/pffft.c ++++ b/third_party/pffft/src/pffft.c +@@ -59,7 +59,7 @@ + + #include "pffft.h" + #include +-#include ++// #include + #include + #include + +@@ -222,31 +222,35 @@ void validate_pffft_simd() { + memcpy(a3.f, f+12, 4*sizeof(float)); + + t = a0; u = a1; t.v = VZERO(); +- printf("VZERO=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 0, 0, 0, 0); ++ // printf("VZERO=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); ++ assertv4(t, 0, 0, 0, 0); + t.v = VADD(a1.v, a2.v); +- printf("VADD(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 12, 14, 16, 18); ++ // printf("VADD(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); ++ assertv4(t, 12, 14, 16, 18); + t.v = VMUL(a1.v, a2.v); +- printf("VMUL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 32, 45, 60, 77); ++ // printf("VMUL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); ++ assertv4(t, 32, 45, 60, 77); + t.v = VMADD(a1.v, a2.v,a0.v); +- printf("VMADD(4:7,8:11,0:3)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 32, 46, 62, 80); ++ // printf("VMADD(4:7,8:11,0:3)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); ++ assertv4(t, 32, 46, 62, 80); + + INTERLEAVE2(a1.v,a2.v,t.v,u.v); +- printf("INTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]); ++ // printf("INTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]); + assertv4(t, 4, 8, 5, 9); assertv4(u, 6, 10, 7, 11); + UNINTERLEAVE2(a1.v,a2.v,t.v,u.v); +- printf("UNINTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]); ++ // printf("UNINTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]); + assertv4(t, 4, 6, 8, 10); assertv4(u, 5, 7, 9, 11); + + t.v=LD_PS1(f[15]); +- printf("LD_PS1(15)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); ++ // printf("LD_PS1(15)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 15, 15, 15, 15); + t.v = VSWAPHL(a1.v, a2.v); +- printf("VSWAPHL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); ++ // printf("VSWAPHL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 8, 9, 6, 7); + VTRANSPOSE4(a0.v, a1.v, a2.v, a3.v); +- printf("VTRANSPOSE4(0:3,4:7,8:11,12:15)=[%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", +- a0.f[0], a0.f[1], a0.f[2], a0.f[3], a1.f[0], a1.f[1], a1.f[2], a1.f[3], +- a2.f[0], a2.f[1], a2.f[2], a2.f[3], a3.f[0], a3.f[1], a3.f[2], a3.f[3]); ++ // printf("VTRANSPOSE4(0:3,4:7,8:11,12:15)=[%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", ++ // a0.f[0], a0.f[1], a0.f[2], a0.f[3], a1.f[0], a1.f[1], a1.f[2], a1.f[3], ++ // a2.f[0], a2.f[1], a2.f[2], a2.f[3], a3.f[0], a3.f[1], a3.f[2], a3.f[3]); + assertv4(a0, 0, 4, 8, 12); assertv4(a1, 1, 5, 9, 13); assertv4(a2, 2, 6, 10, 14); assertv4(a3, 3, 7, 11, 15); + } + #endif //!PFFFT_SIMD_DISABLE diff --git a/webrtc/third_party/pffft/patches/02-decl_validate_simd.diff b/webrtc/third_party/pffft/patches/02-decl_validate_simd.diff new file mode 100644 index 0000000..4531511 --- /dev/null +++ b/webrtc/third_party/pffft/patches/02-decl_validate_simd.diff @@ -0,0 +1,16 @@ +diff --git a/third_party/pffft/src/pffft.h b/third_party/pffft/src/pffft.h +index 2bfa7b3ebcfb..bb6f78d4b795 100644 +--- a/third_party/pffft/src/pffft.h ++++ b/third_party/pffft/src/pffft.h +@@ -83,6 +83,11 @@ + extern "C" { + #endif + ++#ifndef PFFFT_SIMD_DISABLE ++ // Detects compiler bugs with respect to simd instruction. ++ void validate_pffft_simd(); ++#endif ++ + /* opaque struct holding internal stuff (precomputed twiddle factors) + this struct can be shared by many threads as it contains only + read-only data. diff --git a/webrtc/third_party/pffft/patches/03-malloca.diff b/webrtc/third_party/pffft/patches/03-malloca.diff new file mode 100644 index 0000000..ba7c4cf --- /dev/null +++ b/webrtc/third_party/pffft/patches/03-malloca.diff @@ -0,0 +1,82 @@ +diff --git a/third_party/pffft/src/pffft.c b/third_party/pffft/src/pffft.c +index 776f564aa28c..643836626c0f 100644 +--- a/third_party/pffft/src/pffft.c ++++ b/third_party/pffft/src/pffft.c +@@ -59,7 +59,6 @@ + + #include "pffft.h" + #include +-// #include + #include + #include + +@@ -75,11 +74,14 @@ + # define NEVER_INLINE(return_type) return_type __attribute__ ((noinline)) + # define RESTRICT __restrict + # define VLA_ARRAY_ON_STACK(type__, varname__, size__) type__ varname__[size__]; ++# define VLA_ARRAY_ON_STACK_FREE(varname__) + #elif defined(COMPILER_MSVC) ++#include + # define ALWAYS_INLINE(return_type) __forceinline return_type + # define NEVER_INLINE(return_type) __declspec(noinline) return_type + # define RESTRICT __restrict +-# define VLA_ARRAY_ON_STACK(type__, varname__, size__) type__ *varname__ = (type__*)_alloca(size__ * sizeof(type__)) ++# define VLA_ARRAY_ON_STACK(type__, varname__, size__) type__ *varname__ = (type__*)_malloca(size__ * sizeof(type__)) ++# define VLA_ARRAY_ON_STACK_FREE(varname__) _freea(varname__) + #endif + + +@@ -219,35 +221,24 @@ void validate_pffft_simd() { + memcpy(a3.f, f+12, 4*sizeof(float)); + + t = a0; u = a1; t.v = VZERO(); +- // printf("VZERO=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 0, 0, 0, 0); + t.v = VADD(a1.v, a2.v); +- // printf("VADD(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 12, 14, 16, 18); + t.v = VMUL(a1.v, a2.v); +- // printf("VMUL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 32, 45, 60, 77); + t.v = VMADD(a1.v, a2.v,a0.v); +- // printf("VMADD(4:7,8:11,0:3)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 32, 46, 62, 80); + + INTERLEAVE2(a1.v,a2.v,t.v,u.v); +- // printf("INTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]); + assertv4(t, 4, 8, 5, 9); assertv4(u, 6, 10, 7, 11); + UNINTERLEAVE2(a1.v,a2.v,t.v,u.v); +- // printf("UNINTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]); + assertv4(t, 4, 6, 8, 10); assertv4(u, 5, 7, 9, 11); + + t.v=LD_PS1(f[15]); +- // printf("LD_PS1(15)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 15, 15, 15, 15); + t.v = VSWAPHL(a1.v, a2.v); +- // printf("VSWAPHL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); + assertv4(t, 8, 9, 6, 7); + VTRANSPOSE4(a0.v, a1.v, a2.v, a3.v); +- // printf("VTRANSPOSE4(0:3,4:7,8:11,12:15)=[%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", +- // a0.f[0], a0.f[1], a0.f[2], a0.f[3], a1.f[0], a1.f[1], a1.f[2], a1.f[3], +- // a2.f[0], a2.f[1], a2.f[2], a2.f[3], a3.f[0], a3.f[1], a3.f[2], a3.f[3]); + assertv4(a0, 0, 4, 8, 12); assertv4(a1, 1, 5, 9, 13); assertv4(a2, 2, 6, 10, 14); assertv4(a3, 3, 7, 11, 15); + } + #endif //!PFFFT_SIMD_DISABLE +@@ -1674,6 +1665,8 @@ void pffft_transform_internal(PFFFT_Setup *setup, const float *finput, float *fo + ib = !ib; + } + assert(buff[ib] == voutput); ++ ++ VLA_ARRAY_ON_STACK_FREE(scratch_on_stack); + } + + void pffft_zconvolve_accumulate(PFFFT_Setup *s, const float *a, const float *b, float *ab, float scaling) { +@@ -1851,6 +1844,8 @@ void pffft_transform_internal_nosimd(PFFFT_Setup *setup, const float *input, flo + ib = !ib; + } + assert(buff[ib] == output); ++ ++ VLA_ARRAY_ON_STACK_FREE(scratch_on_stack); + } + + #define pffft_zconvolve_accumulate_nosimd pffft_zconvolve_accumulate diff --git a/webrtc/third_party/pffft/patches/04-fix_ptr_cast.diff b/webrtc/third_party/pffft/patches/04-fix_ptr_cast.diff new file mode 100644 index 0000000..ae8ec0c --- /dev/null +++ b/webrtc/third_party/pffft/patches/04-fix_ptr_cast.diff @@ -0,0 +1,48 @@ +diff --git a/third_party/pffft/src/pffft.c b/third_party/pffft/src/pffft.c +index 643836626c0f..3033e61b813e 100644 +--- a/third_party/pffft/src/pffft.c ++++ b/third_party/pffft/src/pffft.c +@@ -58,6 +58,7 @@ + */ + + #include "pffft.h" ++#include + #include + #include + #include +@@ -125,7 +126,7 @@ inline v4sf ld_ps1(const float *p) { v4sf v=vec_lde(0,p); return vec_splat(vec_p + x3 = vec_mergel(y1, y3); \ + } + # define VSWAPHL(a,b) vec_perm(a,b, (vector unsigned char)(16,17,18,19,20,21,22,23,8,9,10,11,12,13,14,15)) +-# define VALIGNED(ptr) ((((long)(ptr)) & 0xF) == 0) ++# define VALIGNED(ptr) ((((uintptr_t)(ptr)) & 0xF) == 0) + + /* + SSE1 support macros +@@ -145,7 +146,7 @@ typedef __m128 v4sf; + # define UNINTERLEAVE2(in1, in2, out1, out2) { v4sf tmp__ = _mm_shuffle_ps(in1, in2, _MM_SHUFFLE(2,0,2,0)); out2 = _mm_shuffle_ps(in1, in2, _MM_SHUFFLE(3,1,3,1)); out1 = tmp__; } + # define VTRANSPOSE4(x0,x1,x2,x3) _MM_TRANSPOSE4_PS(x0,x1,x2,x3) + # define VSWAPHL(a,b) _mm_shuffle_ps(b, a, _MM_SHUFFLE(3,2,1,0)) +-# define VALIGNED(ptr) ((((long)(ptr)) & 0xF) == 0) ++# define VALIGNED(ptr) ((((uintptr_t)(ptr)) & 0xF) == 0) + + /* + ARM NEON support macros +@@ -172,7 +173,7 @@ typedef float32x4_t v4sf; + // marginally faster version + //# define VTRANSPOSE4(x0,x1,x2,x3) { asm("vtrn.32 %q0, %q1;\n vtrn.32 %q2,%q3\n vswp %f0,%e2\n vswp %f1,%e3" : "+w"(x0), "+w"(x1), "+w"(x2), "+w"(x3)::); } + # define VSWAPHL(a,b) vcombine_f32(vget_low_f32(b), vget_high_f32(a)) +-# define VALIGNED(ptr) ((((long)(ptr)) & 0x3) == 0) ++# define VALIGNED(ptr) ((((uintptr_t)(ptr)) & 0x3) == 0) + #else + # if !defined(PFFFT_SIMD_DISABLE) + # warning "building with simd disabled !\n"; +@@ -190,7 +191,7 @@ typedef float v4sf; + # define VMADD(a,b,c) ((a)*(b)+(c)) + # define VSUB(a,b) ((a)-(b)) + # define LD_PS1(p) (p) +-# define VALIGNED(ptr) ((((long)(ptr)) & 0x3) == 0) ++# define VALIGNED(ptr) ((((uintptr_t)(ptr)) & 0x3) == 0) + #endif + + // shortcuts for complex multiplcations diff --git a/webrtc/third_party/pffft/patches/05-fix-arch-detection.diff b/webrtc/third_party/pffft/patches/05-fix-arch-detection.diff new file mode 100644 index 0000000..d6903ff --- /dev/null +++ b/webrtc/third_party/pffft/patches/05-fix-arch-detection.diff @@ -0,0 +1,22 @@ +diff --git a/third_party/pffft/src/pffft.c b/third_party/pffft/src/pffft.c +index 3033e61b813e..bdac4d784999 100644 +--- a/third_party/pffft/src/pffft.c ++++ b/third_party/pffft/src/pffft.c +@@ -131,7 +131,7 @@ inline v4sf ld_ps1(const float *p) { v4sf v=vec_lde(0,p); return vec_splat(vec_p + /* + SSE1 support macros + */ +-#elif !defined(PFFFT_SIMD_DISABLE) && (defined(__x86_64__) || defined(_M_X64) || defined(i386) || defined(_M_IX86)) ++#elif !defined(PFFFT_SIMD_DISABLE) && (defined(__x86_64__) || defined(_M_X64) || defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #include + typedef __m128 v4sf; +@@ -151,7 +151,7 @@ typedef __m128 v4sf; + /* + ARM NEON support macros + */ +-#elif !defined(PFFFT_SIMD_DISABLE) && defined(__arm__) ++#elif !defined(PFFFT_SIMD_DISABLE) && (defined(__arm__) || defined(__ARMEL__) || defined(__aarch64__) || defined(_M_ARM64)) + # include + typedef float32x4_t v4sf; + # define SIMD_SZ 4 diff --git a/webrtc/third_party/rnnoise/BUILD.gn b/webrtc/third_party/rnnoise/BUILD.gn index 7bfa784..ffa502e 100644 --- a/webrtc/third_party/rnnoise/BUILD.gn +++ b/webrtc/third_party/rnnoise/BUILD.gn @@ -1 +1,17 @@ -IyBDb3B5cmlnaHQgMjAxOCBUaGUgQ2hyb21pdW0gQXV0aG9ycy4gQWxsIHJpZ2h0cyByZXNlcnZlZC4KIyBVc2Ugb2YgdGhpcyBzb3VyY2UgY29kZSBpcyBnb3Zlcm5lZCBieSBhIEJTRC1zdHlsZSBsaWNlbnNlIHRoYXQgY2FuIGJlCiMgZm91bmQgaW4gdGhlIExJQ0VOU0UgZmlsZS4KCmltcG9ydCgiLy90ZXN0aW5nL3Rlc3QuZ25pIikKCmdyb3VwKCJybm5vaXNlIikgewogIGRlcHMgPSBbICI6cm5uX3ZhZCIgXQp9Cgpzb3VyY2Vfc2V0KCJybm5fdmFkIikgewogIHNvdXJjZXMgPSBbCiAgICAic3JjL3Jubl9hY3RpdmF0aW9ucy5oIiwKICAgICJzcmMvcm5uX3ZhZF93ZWlnaHRzLmNjIiwKICAgICJzcmMvcm5uX3ZhZF93ZWlnaHRzLmgiLAogIF0KfQo= \ No newline at end of file +# Copyright 2018 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//testing/test.gni") + +group("rnnoise") { + deps = [ ":rnn_vad" ] +} + +source_set("rnn_vad") { + sources = [ + "src/rnn_activations.h", + "src/rnn_vad_weights.cc", + "src/rnn_vad_weights.h", + ] +}