input_controller.h 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef SERVICES_AUDIO_INPUT_CONTROLLER_H_
  5. #define SERVICES_AUDIO_INPUT_CONTROLLER_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <memory>
  9. #include <string>
  10. #include "base/memory/raw_ptr.h"
  11. #include "base/memory/weak_ptr.h"
  12. #include "base/strings/string_piece.h"
  13. #include "base/threading/thread_checker.h"
  14. #include "base/time/time.h"
  15. #include "base/timer/timer.h"
  16. #include "build/build_config.h"
  17. #include "media/base/audio_parameters.h"
  18. #include "media/base/audio_processing.h"
  19. #include "media/media_buildflags.h"
  20. #include "media/mojo/mojom/audio_processing.mojom.h"
  21. #include "mojo/public/cpp/bindings/pending_receiver.h"
  22. #include "mojo/public/cpp/bindings/pending_remote.h"
  23. #include "mojo/public/cpp/bindings/receiver.h"
  24. #include "mojo/public/cpp/bindings/remote.h"
  25. #include "services/audio/stream_monitor.h"
  26. #include "third_party/abseil-cpp/absl/types/optional.h"
  27. namespace media {
  28. class AudioBus;
  29. class AudioInputStream;
  30. class AudioManager;
  31. class Snoopable;
  32. class UserInputMonitor;
  33. } // namespace media
  34. namespace audio {
  35. class AudioProcessorHandler;
  36. class AecdumpRecordingManager;
  37. class AudioCallback;
  38. class OutputTapper;
  39. class DeviceOutputListener;
  40. class InputStreamActivityMonitor;
  41. class ProcessingAudioFifo;
  42. // Only do power monitoring for non-mobile platforms to save resources.
  43. #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
  44. #define AUDIO_POWER_MONITORING
  45. #endif
  46. // All public methods of InputController must be called from the audio thread.
  47. //
  48. // Audio data flow through InputController:
  49. //
  50. // * Without any audio processing:
  51. // InputController::|audio_callback_|::OnData()
  52. // -> InputController::OnData()
  53. // --> InputController::|sync_writer_|::Write()
  54. //
  55. // * With audio processing but no dedicated processing thread:
  56. // InputController::|audio_callback_|::OnData()
  57. // -> InputController::OnData()
  58. // --> InputController::|audio_processor_handler_|::ProcessCapturedAudio()
  59. // ---> InputController::DeliverProcessedAudio()
  60. // ----> InputController::|sync_writer_|::Write()
  61. //
  62. // * With audio processing and a dedicated processing thread:
  63. // Audio capture device thread:
  64. // InputController::|audio_callback_|::OnData()
  65. // -> InputController::OnData()
  66. // --> InputController::|processing_fifo_|::PushData()
  67. // Audio processing thread:
  68. // ---> InputController::|audio_processor_handler_|::ProcessCapturedAudio()
  69. // ----> InputController::DeliverProcessedAudio()
  70. // -----> InputController::|sync_writer_|::Write()
  71. //
  72. // - InputController::|audio_processor_handler_| changes format from the
  73. // AudioInputStream format to |params| provided to
  74. // InputController::Create().
  75. //
  76. class InputController final : public StreamMonitor {
  77. public:
  78. // Error codes to make native logging more clear. These error codes are added
  79. // to generic error strings to provide a higher degree of details.
  80. // Changing these values can lead to problems when matching native debug
  81. // logs with the actual cause of error.
  82. enum ErrorCode {
  83. // An unspecified error occured.
  84. UNKNOWN_ERROR = 0,
  85. // Failed to create an audio input stream.
  86. STREAM_CREATE_ERROR, // = 1
  87. // Failed to open an audio input stream.
  88. STREAM_OPEN_ERROR, // = 2
  89. // Native input stream reports an error. Exact reason differs between
  90. // platforms.
  91. STREAM_ERROR, // = 3
  92. // Open failed due to lack of system permissions.
  93. STREAM_OPEN_SYSTEM_PERMISSIONS_ERROR, // = 4
  94. // Open failed due to device in use by another app.
  95. STREAM_OPEN_DEVICE_IN_USE_ERROR, // = 5
  96. };
  97. #if defined(AUDIO_POWER_MONITORING)
  98. // Used to log a silence report (see OnData).
  99. // Elements in this enum should not be deleted or rearranged; the only
  100. // permitted operation is to add new elements before SILENCE_STATE_MAX and
  101. // update SILENCE_STATE_MAX.
  102. // Possible silence state transitions:
  103. // SILENCE_STATE_AUDIO_AND_SILENCE
  104. // ^ ^
  105. // SILENCE_STATE_ONLY_AUDIO SILENCE_STATE_ONLY_SILENCE
  106. // ^ ^
  107. // SILENCE_STATE_NO_MEASUREMENT
  108. enum SilenceState {
  109. SILENCE_STATE_NO_MEASUREMENT = 0,
  110. SILENCE_STATE_ONLY_AUDIO = 1,
  111. SILENCE_STATE_ONLY_SILENCE = 2,
  112. SILENCE_STATE_AUDIO_AND_SILENCE = 3,
  113. SILENCE_STATE_MAX = SILENCE_STATE_AUDIO_AND_SILENCE
  114. };
  115. #endif
  116. // An event handler that receives events from the InputController. The
  117. // following methods are all called on the audio thread.
  118. class EventHandler {
  119. public:
  120. // The initial "muted" state of the underlying stream is sent along with the
  121. // OnCreated callback, to avoid the stream being treated as unmuted until an
  122. // OnMuted callback has had time to be processed.
  123. virtual void OnCreated(bool initially_muted) = 0;
  124. virtual void OnError(ErrorCode error_code) = 0;
  125. virtual void OnLog(base::StringPiece) = 0;
  126. // Called whenever the muted state of the underlying stream changes.
  127. virtual void OnMuted(bool is_muted) = 0;
  128. protected:
  129. virtual ~EventHandler() {}
  130. };
  131. // A synchronous writer interface used by InputController for
  132. // synchronous writing.
  133. class SyncWriter {
  134. public:
  135. virtual ~SyncWriter() {}
  136. // Write certain amount of data from |data|.
  137. virtual void Write(const media::AudioBus* data,
  138. double volume,
  139. bool key_pressed,
  140. base::TimeTicks capture_time) = 0;
  141. // Close this synchronous writer.
  142. virtual void Close() = 0;
  143. };
  144. // enum used for determining what UMA stats to report.
  145. enum StreamType {
  146. VIRTUAL = 0,
  147. HIGH_LATENCY = 1,
  148. LOW_LATENCY = 2,
  149. FAKE = 3,
  150. };
  151. InputController(const InputController&) = delete;
  152. InputController& operator=(const InputController&) = delete;
  153. ~InputController() final;
  154. media::AudioInputStream* stream_for_testing() { return stream_; }
  155. // |user_input_monitor| is used for typing detection and can be NULL.
  156. static std::unique_ptr<InputController> Create(
  157. media::AudioManager* audio_manager,
  158. EventHandler* event_handler,
  159. SyncWriter* sync_writer,
  160. media::UserInputMonitor* user_input_monitor,
  161. InputStreamActivityMonitor* activity_monitor,
  162. DeviceOutputListener* device_output_listener,
  163. AecdumpRecordingManager* aecdump_recording_manager,
  164. media::mojom::AudioProcessingConfigPtr processing_config,
  165. const media::AudioParameters& params,
  166. const std::string& device_id,
  167. bool agc_is_enabled);
  168. // Starts recording using the created audio input stream.
  169. void Record();
  170. // Closes the audio input stream, freeing the associated resources. Must be
  171. // called before destruction.
  172. void Close();
  173. // Sets the capture volume of the input stream. The value 0.0 corresponds
  174. // to muted and 1.0 to maximum volume.
  175. void SetVolume(double volume);
  176. // Sets the output device which will be used to cancel audio from, if this
  177. // input device supports echo cancellation.
  178. void SetOutputDeviceForAec(const std::string& output_device_id);
  179. // StreamMonitor implementation
  180. void OnStreamActive(Snoopable* snoopable) override;
  181. void OnStreamInactive(Snoopable* snoopable) override;
  182. private:
  183. friend class InputControllerTestHelper;
  184. // Used to log the result of capture startup.
  185. // This was previously logged as a boolean with only the no callback and OK
  186. // options. The enum order is kept to ensure backwards compatibility.
  187. // Elements in this enum should not be deleted or rearranged; the only
  188. // permitted operation is to add new elements before
  189. // CAPTURE_STARTUP_RESULT_MAX and update CAPTURE_STARTUP_RESULT_MAX.
  190. //
  191. // The NO_DATA_CALLBACK enum has been replaced with NEVER_GOT_DATA,
  192. // and there are also other histograms such as
  193. // Media.Audio.InputStartupSuccessMac to cover issues similar
  194. // to the ones the NO_DATA_CALLBACK was intended for.
  195. enum CaptureStartupResult {
  196. CAPTURE_STARTUP_OK = 0,
  197. CAPTURE_STARTUP_CREATE_STREAM_FAILED = 1,
  198. CAPTURE_STARTUP_OPEN_STREAM_FAILED = 2,
  199. CAPTURE_STARTUP_NEVER_GOT_DATA = 3,
  200. CAPTURE_STARTUP_STOPPED_EARLY = 4,
  201. CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_STOPPED_EARLY,
  202. };
  203. InputController(EventHandler* event_handler,
  204. SyncWriter* sync_writer,
  205. media::UserInputMonitor* user_input_monitor,
  206. InputStreamActivityMonitor* activity_monitor,
  207. DeviceOutputListener* device_output_listener,
  208. AecdumpRecordingManager* aecdump_recording_manager,
  209. media::mojom::AudioProcessingConfigPtr processing_config,
  210. const media::AudioParameters& output_params,
  211. const media::AudioParameters& device_params,
  212. StreamType type);
  213. void DoCreate(media::AudioManager* audio_manager,
  214. const media::AudioParameters& params,
  215. const std::string& device_id,
  216. bool enable_agc);
  217. void DoReportError();
  218. void DoLogAudioLevels(float level_dbfs, int microphone_volume_percent);
  219. #if defined(AUDIO_POWER_MONITORING)
  220. // Updates the silence state, see enum SilenceState above for state
  221. // transitions.
  222. void UpdateSilenceState(bool silence);
  223. // Logs the silence state as UMA stat.
  224. void LogSilenceState(SilenceState value);
  225. #endif
  226. // Logs the result of creating an InputController.
  227. void LogCaptureStartupResult(CaptureStartupResult result);
  228. // Logs whether an error was encountered suring the stream.
  229. void LogCallbackError();
  230. // Called by the stream with log messages.
  231. void LogMessage(const std::string& message);
  232. // Called on the hw callback thread. Checks for keyboard input if
  233. // |user_input_monitor_| is set otherwise returns false.
  234. bool CheckForKeyboardInput();
  235. // Does power monitoring on supported platforms.
  236. // Called on the hw callback thread.
  237. // Returns true iff average power and mic volume was returned and should
  238. // be posted to DoLogAudioLevels on the audio thread.
  239. // Returns false if either power measurements are disabled or aren't needed
  240. // right now (they're done periodically).
  241. bool CheckAudioPower(const media::AudioBus* source,
  242. double volume,
  243. float* average_power_dbfs,
  244. int* mic_volume_percent);
  245. void CheckMutedState();
  246. // Called once at first audio callback.
  247. void ReportIsAlive();
  248. // Receives new input data on the hw callback thread.
  249. void OnData(const media::AudioBus* source,
  250. base::TimeTicks capture_time,
  251. double volume);
  252. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  253. // Called from the constructor. Helper to isolate logic setting up audio
  254. // processing components.
  255. void MaybeSetUpAudioProcessing(
  256. media::mojom::AudioProcessingConfigPtr processing_config,
  257. const media::AudioParameters& processing_output_params,
  258. const media::AudioParameters& device_params,
  259. DeviceOutputListener* device_output_listener,
  260. AecdumpRecordingManager* aecdump_recording_manager);
  261. // Used as a callback for |audio_processor_handler_|.
  262. void DeliverProcessedAudio(const media::AudioBus& audio_bus,
  263. base::TimeTicks audio_capture_time,
  264. absl::optional<double> new_volume);
  265. #endif
  266. static StreamType ParamsToStreamType(const media::AudioParameters& params);
  267. // The task runner for the audio manager. All control methods should be called
  268. // via tasks run by this TaskRunner.
  269. const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  270. // Contains the InputController::EventHandler which receives state
  271. // notifications from this class.
  272. const raw_ptr<EventHandler> event_handler_;
  273. // Pointer to the audio input stream object.
  274. // Only used on the audio thread.
  275. raw_ptr<media::AudioInputStream> stream_ = nullptr;
  276. // SyncWriter is used only in low-latency mode for synchronous writing.
  277. const raw_ptr<SyncWriter> sync_writer_;
  278. StreamType type_;
  279. double max_volume_ = 0.0;
  280. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  281. // Handles audio processing effects applied to the microphone capture audio.
  282. std::unique_ptr<AudioProcessorHandler> audio_processor_handler_;
  283. // Offloads processing captured data to its own real time thread.
  284. // Note: Ordering is important, as |processing_fifo_| must be destroyed before
  285. // |audio_processing_handler_|.
  286. std::unique_ptr<ProcessingAudioFifo> processing_fifo_;
  287. // Manages the |audio_processor_handler_| subscription to output audio.
  288. std::unique_ptr<OutputTapper> output_tapper_;
  289. #endif
  290. const raw_ptr<media::UserInputMonitor> user_input_monitor_;
  291. // Notified when the stream starts/stops recording.
  292. const raw_ptr<InputStreamActivityMonitor> activity_monitor_;
  293. #if defined(AUDIO_POWER_MONITORING)
  294. // Whether the silence state and microphone levels should be checked and sent
  295. // as UMA stats.
  296. bool power_measurement_is_enabled_ = false;
  297. // Updated each time a power measurement is performed.
  298. base::TimeTicks last_audio_level_log_time_;
  299. // The silence report sent as UMA stat at the end of a session.
  300. SilenceState silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
  301. #endif
  302. size_t prev_key_down_count_ = 0;
  303. // Time when the stream started recording.
  304. base::TimeTicks stream_create_time_;
  305. bool is_muted_ = false;
  306. base::RepeatingTimer check_muted_state_timer_;
  307. // Holds a pointer to the callback object that receives audio data from
  308. // the lower audio layer. Valid only while 'recording' (between calls to
  309. // stream_->Start() and stream_->Stop()).
  310. // The value of this pointer is only set and read on the audio thread while
  311. // the callbacks themselves occur on the hw callback thread. More details
  312. // in the AudioCallback class in the cc file.
  313. std::unique_ptr<AudioCallback> audio_callback_;
  314. // A weak pointer factory that we use when posting tasks to the audio thread
  315. // that we want to be automatically discarded after Close() has been called
  316. // and that we do not want to keep the InputController instance alive
  317. // beyond what is desired by the user of the instance. An example of where
  318. // this is important is when we fire error notifications from the hw callback
  319. // thread, post them to the audio thread. In that case, we do not want the
  320. // error notification to keep the InputController alive for as long as
  321. // the error notification is pending and then make a callback from an
  322. // InputController that has already been closed.
  323. // All outstanding weak pointers are invalidated at the end of Close().
  324. base::WeakPtr<InputController> weak_this_;
  325. base::WeakPtrFactory<InputController> weak_ptr_factory_{this};
  326. };
  327. } // namespace audio
  328. #endif // SERVICES_AUDIO_INPUT_CONTROLLER_H_