audio_processor.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_WEBRTC_AUDIO_PROCESSOR_H_
  5. #define MEDIA_WEBRTC_AUDIO_PROCESSOR_H_
  6. #include <memory>
  7. #include "base/callback.h"
  8. #include "base/component_export.h"
  9. #include "base/files/file.h"
  10. #include "base/memory/scoped_refptr.h"
  11. #include "base/sequence_checker.h"
  12. #include "base/thread_annotations.h"
  13. #include "base/time/time.h"
  14. #include "media/base/audio_parameters.h"
  15. #include "media/base/audio_processing.h"
  16. #include "media/base/audio_push_fifo.h"
  17. #include "media/webrtc/audio_delay_stats_reporter.h"
  18. #include "third_party/abseil-cpp/absl/types/optional.h"
  19. #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
  20. #include "third_party/webrtc/modules/audio_processing/include/audio_processing_statistics.h"
  21. #include "third_party/webrtc/rtc_base/task_queue.h"
  22. namespace media {
  23. class AudioBus;
  24. class AudioProcessorCaptureBus;
  25. class AudioProcessorCaptureFifo;
  26. // This class applies audio processing effects such as echo cancellation and
  27. // noise suppression to input capture audio (such as a microphone signal).
  28. // Effects are applied based on configuration from AudioProcessingSettings, and
  29. // mainly rely on an instance of the webrtc::AudioProcessing module (APM) owned
  30. // by the AudioProcessor.
  31. //
  32. // The AudioProcessor can handle up to three threads (in practice, sequences):
  33. // - An owning sequence, which performs construction, destruction, diagnostic
  34. // recordings, and similar signals.
  35. // - A capture thread, which calls ProcessCapturedAudio().
  36. // - A playout thread, which calls OnPlayoutData().
  37. //
  38. // All member functions must be called on the owning sequence unless
  39. // specifically documented otherwise.
  40. //
  41. // Thread-safe exceptions to this scheme are explicitly documented as such.
  42. class COMPONENT_EXPORT(MEDIA_WEBRTC) AudioProcessor {
  43. public:
  44. // Callback for consuming processed capture audio.
  45. // |audio_bus| contains the most recent processed capture audio.
  46. // |new_volume| specifies a new microphone volume from the AGC. The new
  47. // microphone volume range is [0.0, 1.0], and is only set if the microphone
  48. // volume should be adjusted.
  49. // Called on the capture thread.
  50. using DeliverProcessedAudioCallback =
  51. base::RepeatingCallback<void(const media::AudioBus& audio_bus,
  52. base::TimeTicks audio_capture_time,
  53. absl::optional<double> new_volume)>;
  54. using LogCallback = base::RepeatingCallback<void(base::StringPiece)>;
  55. // |deliver_processed_audio_callback| is used to deliver frames of processed
  56. // capture audio, from ProcessCapturedAudio(), and has to be valid for as long
  57. // as ProcessCapturedAudio() may be called.
  58. // |log_callback| is used for logging messages on the owning sequence.
  59. // |input_format| specifies the format of the incoming capture data.
  60. // |output_format| specifies the output format. If
  61. // |settings|.NeedWebrtcAudioProcessing() is true, then the output must be in
  62. // 10 ms chunks: the formats must specify |sample rate|/100 samples per buffer
  63. // (rounded down). Sample rates which are not divisible by 100 are supported
  64. // on a best-effort basis, audio quality and stability may suffer.
  65. static std::unique_ptr<AudioProcessor> Create(
  66. DeliverProcessedAudioCallback deliver_processed_audio_callback,
  67. LogCallback log_callback,
  68. const AudioProcessingSettings& settings,
  69. const media::AudioParameters& input_format,
  70. const media::AudioParameters& output_format);
  71. // See Create() for details.
  72. AudioProcessor(
  73. DeliverProcessedAudioCallback deliver_processed_audio_callback,
  74. LogCallback log_callback,
  75. const media::AudioParameters& input_format,
  76. const media::AudioParameters& output_format,
  77. rtc::scoped_refptr<webrtc::AudioProcessing> webrtc_audio_processing,
  78. bool stereo_mirroring);
  79. ~AudioProcessor();
  80. AudioProcessor(const AudioProcessor&) = delete;
  81. AudioProcessor& operator=(const AudioProcessor&) = delete;
  82. // Processes and delivers capture audio in chunks of <= 10 ms to
  83. // |deliver_processed_audio_callback_|: Each call to ProcessCapturedAudio()
  84. // method triggers zero or more calls to |deliver_processed_audio_callback_|,
  85. // depending on internal FIFO size and content. |num_preferred_channels| is
  86. // the highest number of channels that any sink is interested in. This can be
  87. // different from the number of channels in the output format. A value of -1
  88. // means an unknown number. If |settings|.multi_channel_capture_processing is
  89. // true, the number of channels of the output of the Audio Processing Module
  90. // (APM) will be equal to the highest observed value of num_preferred_channels
  91. // as long as it does not exceed the number of channels of the output format.
  92. // |volume| specifies the current microphone volume, in the range [0.0, 1.0].
  93. // Must be called on the capture thread.
  94. void ProcessCapturedAudio(const media::AudioBus& audio_source,
  95. base::TimeTicks audio_capture_time,
  96. int num_preferred_channels,
  97. double volume,
  98. bool key_pressed);
  99. // Analyzes playout audio for e.g. echo cancellation.
  100. // Must be called on the playout thread.
  101. void OnPlayoutData(const media::AudioBus& audio_bus,
  102. int sample_rate,
  103. base::TimeDelta audio_delay);
  104. // Accessor to check if WebRTC audio processing is enabled or not.
  105. bool has_webrtc_audio_processing() const {
  106. DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_);
  107. return !!webrtc_audio_processing_;
  108. }
  109. // Instructs the Audio Processing Module (APM) to reduce its complexity when
  110. // |muted| is true. This mode is triggered when all audio tracks are disabled.
  111. // The default APM complexity mode is restored by |muted| set to false.
  112. void SetOutputWillBeMuted(bool muted);
  113. // Starts a new diagnostic audio recording (aecdump). If an aecdump recording
  114. // is already ongoing, it is stopped before starting the new one.
  115. void OnStartDump(base::File dump_file);
  116. // Stops any ongoing aecdump.
  117. void OnStopDump();
  118. // Returns any available statistics from the WebRTC audio processing module.
  119. // May be called on any thread.
  120. webrtc::AudioProcessingStats GetStats();
  121. absl::optional<webrtc::AudioProcessing::Config>
  122. GetAudioProcessingModuleConfigForTesting() const {
  123. if (webrtc_audio_processing_) {
  124. return webrtc_audio_processing_->GetConfig();
  125. }
  126. return absl::nullopt;
  127. }
  128. // The format of audio input to and output from the processor; constant
  129. // throughout AudioProcessor lifetime.
  130. const media::AudioParameters& input_format() const { return input_format_; }
  131. const media::AudioParameters& output_format() const { return output_format_; }
  132. // Returns an input format compatible with the specified audio processing
  133. // settings and device parameters. Returns nullopt if no compatible format can
  134. // be produced.
  135. static absl::optional<AudioParameters> ComputeInputFormat(
  136. const AudioParameters& device_format,
  137. const AudioProcessingSettings& settings);
  138. // Returns an output format that minimizes delay and resampling for given
  139. // input format and audio processing settings.
  140. static AudioParameters GetDefaultOutputFormat(
  141. const AudioParameters& input_format,
  142. const AudioProcessingSettings& settings);
  143. private:
  144. friend class AudioProcessorTest;
  145. // Called by ProcessCapturedAudio().
  146. // Returns the new microphone volume in the range of |0.0, 1.0], or unset if
  147. // the volume should not be updated.
  148. // |num_preferred_channels| is the highest number of channels that any sink is
  149. // interested in. This can be different from the number of channels in the
  150. // output format. A value of -1 means an unknown number. If
  151. // |settings|.multi_channel_capture_processing is true, the number of
  152. // channels of the output of the Audio Processing Module (APM) will be equal
  153. // to the highest observed value of num_preferred_channels as long as it does
  154. // not exceed the number of channels of the output format.
  155. // Called on the capture thread.
  156. absl::optional<double> ProcessData(const float* const* process_ptrs,
  157. int process_frames,
  158. base::TimeDelta capture_delay,
  159. double volume,
  160. bool key_pressed,
  161. int num_preferred_channels,
  162. float* const* output_ptrs);
  163. // Used as callback from |playout_fifo_| in OnPlayoutData().
  164. // Called on the playout thread.
  165. void AnalyzePlayoutData(const AudioBus& audio_bus, int frame_delay);
  166. void SendLogMessage(const std::string& message)
  167. VALID_CONTEXT_REQUIRED(owning_sequence_);
  168. SEQUENCE_CHECKER(owning_sequence_);
  169. // The WebRTC audio processing module (APM). Performs the bulk of the audio
  170. // processing and resampling algorithms.
  171. const rtc::scoped_refptr<webrtc::AudioProcessing> webrtc_audio_processing_;
  172. // If true, then the audio processor should swap the left and right channel of
  173. // captured stereo audio.
  174. const bool stereo_mirroring_;
  175. // Members accessed only by the owning sequence:
  176. // Used by SendLogMessage.
  177. const LogCallback log_callback_ GUARDED_BY_CONTEXT(owning_sequence_);
  178. // Low-priority task queue for doing AEC dump recordings. It has to
  179. // created/destroyed on the same sequence and it must outlive
  180. // any aecdump recording in |webrtc_audio_processing_|.
  181. std::unique_ptr<rtc::TaskQueue> worker_queue_
  182. GUARDED_BY_CONTEXT(owning_sequence_);
  183. // Cached value for the playout delay latency. Updated on the playout thread
  184. // and read on the capture thread.
  185. std::atomic<base::TimeDelta> playout_delay_{base::TimeDelta()};
  186. // Members configured on the owning sequence in the constructor and
  187. // used on the capture thread:
  188. // FIFO to provide capture audio in chunks of up to 10 ms.
  189. std::unique_ptr<AudioProcessorCaptureFifo> capture_fifo_;
  190. // Receives APM processing output.
  191. std::unique_ptr<AudioProcessorCaptureBus> output_bus_;
  192. // Input and output formats for capture processing.
  193. const media::AudioParameters input_format_;
  194. const media::AudioParameters output_format_;
  195. // Members accessed only on the capture thread:
  196. // Consumer of processed capture audio in ProcessCapturedAudio().
  197. const DeliverProcessedAudioCallback deliver_processed_audio_callback_;
  198. // Observed maximum number of preferred output channels. Used for not
  199. // performing audio processing on more channels than the sinks are interested
  200. // in. The value is a maximum over time and can increase but never decrease.
  201. // If |settings|.multi_channel_capture_processing is true, Audio Processing
  202. // Module (APM) will output max_num_preferred_output_channels_ channels as
  203. // long as it does not exceed the number of channels of the output format.
  204. int max_num_preferred_output_channels_ = 1;
  205. // For reporting audio delay stats.
  206. media::AudioDelayStatsReporter audio_delay_stats_reporter_;
  207. // Members accessed only on the playout thread:
  208. // FIFO to provide playout audio in 10 ms chunks.
  209. AudioPushFifo playout_fifo_;
  210. // Cached value of the playout delay before adjusting for delay introduced by
  211. // |playout_fifo_|.
  212. base::TimeDelta unbuffered_playout_delay_ = base::TimeDelta();
  213. // The sample rate of incoming playout audio.
  214. absl::optional<int> playout_sample_rate_hz_ = absl::nullopt;
  215. // Counters to avoid excessively logging errors on a real-time thread.
  216. size_t apm_playout_error_code_log_count_ = 0;
  217. size_t large_delay_log_count_ = 0;
  218. };
  219. } // namespace media
  220. #endif // MEDIA_WEBRTC_AUDIO_PROCESSOR_H_