input_controller.cc 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "services/audio/input_controller.h"
  5. #include <inttypes.h>
  6. #include <algorithm>
  7. #include <limits>
  8. #include <memory>
  9. #include <utility>
  10. #include "base/bind.h"
  11. #include "base/cxx17_backports.h"
  12. #include "base/logging.h"
  13. #include "base/memory/ptr_util.h"
  14. #include "base/memory/raw_ptr.h"
  15. #include "base/metrics/histogram_macros.h"
  16. #include "base/strings/string_number_conversions.h"
  17. #include "base/strings/stringprintf.h"
  18. #include "base/task/bind_post_task.h"
  19. #include "base/task/single_thread_task_runner.h"
  20. #include "base/threading/thread_restrictions.h"
  21. #include "base/threading/thread_task_runner_handle.h"
  22. #include "base/time/time.h"
  23. #include "base/trace_event/trace_event.h"
  24. #include "media/audio/audio_io.h"
  25. #include "media/audio/audio_manager.h"
  26. #include "media/base/audio_bus.h"
  27. #include "media/base/audio_processing.h"
  28. #include "media/base/media_switches.h"
  29. #include "media/base/user_input_monitor.h"
  30. #include "services/audio/audio_manager_power_user.h"
  31. #include "services/audio/concurrent_stream_metric_reporter.h"
  32. #include "services/audio/device_output_listener.h"
  33. #include "services/audio/output_tapper.h"
  34. #include "services/audio/processing_audio_fifo.h"
  35. #include "services/audio/reference_output.h"
  36. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  37. #include "services/audio/audio_processor_handler.h"
  38. #endif
  39. namespace audio {
  40. namespace {
  41. using OpenOutcome = media::AudioInputStream::OpenOutcome;
  42. const int kMaxInputChannels = 3;
  43. constexpr base::TimeDelta kCheckMutedStateInterval = base::Seconds(1);
  44. #if defined(AUDIO_POWER_MONITORING)
  45. // Time in seconds between two successive measurements of audio power levels.
  46. constexpr base::TimeDelta kPowerMonitorLogInterval = base::Seconds(15);
  47. // A warning will be logged when the microphone audio volume is below this
  48. // threshold.
  49. const int kLowLevelMicrophoneLevelPercent = 10;
  50. // Logs if the user has enabled the microphone mute or not. This is normally
  51. // done by marking a checkbox in an audio-settings UI which is unique for each
  52. // platform. Elements in this enum should not be added, deleted or rearranged.
  53. enum MicrophoneMuteResult {
  54. MICROPHONE_IS_MUTED = 0,
  55. MICROPHONE_IS_NOT_MUTED = 1,
  56. MICROPHONE_MUTE_MAX = MICROPHONE_IS_NOT_MUTED
  57. };
  58. void LogMicrophoneMuteResult(MicrophoneMuteResult result) {
  59. UMA_HISTOGRAM_ENUMERATION("Media.MicrophoneMuted", result,
  60. MICROPHONE_MUTE_MAX + 1);
  61. }
  62. const char* SilenceStateToString(InputController::SilenceState state) {
  63. switch (state) {
  64. case InputController::SILENCE_STATE_NO_MEASUREMENT:
  65. return "SILENCE_STATE_NO_MEASUREMENT";
  66. case InputController::SILENCE_STATE_ONLY_AUDIO:
  67. return "SILENCE_STATE_ONLY_AUDIO";
  68. case InputController::SILENCE_STATE_ONLY_SILENCE:
  69. return "SILENCE_STATE_ONLY_SILENCE";
  70. case InputController::SILENCE_STATE_AUDIO_AND_SILENCE:
  71. return "SILENCE_STATE_AUDIO_AND_SILENCE";
  72. default:
  73. NOTREACHED();
  74. }
  75. return "INVALID";
  76. }
  77. // Helper method which calculates the average power of an audio bus. Unit is in
  78. // dBFS, where 0 dBFS corresponds to all channels and samples equal to 1.0.
  79. float AveragePower(const media::AudioBus& buffer) {
  80. const int frames = buffer.frames();
  81. const int channels = buffer.channels();
  82. if (frames <= 0 || channels <= 0)
  83. return 0.0f;
  84. // Scan all channels and accumulate the sum of squares for all samples.
  85. float sum_power = 0.0f;
  86. for (int ch = 0; ch < channels; ++ch) {
  87. const float* channel_data = buffer.channel(ch);
  88. for (int i = 0; i < frames; i++) {
  89. const float sample = channel_data[i];
  90. sum_power += sample * sample;
  91. }
  92. }
  93. // Update accumulated average results, with clamping for sanity.
  94. const float average_power =
  95. base::clamp(sum_power / (frames * channels), 0.0f, 1.0f);
  96. // Convert average power level to dBFS units, and pin it down to zero if it
  97. // is insignificantly small.
  98. const float kInsignificantPower = 1.0e-10f; // -100 dBFS
  99. const float power_dbfs = average_power < kInsignificantPower
  100. ? -std::numeric_limits<float>::infinity()
  101. : 10.0f * log10f(average_power);
  102. return power_dbfs;
  103. }
  104. #endif // AUDIO_POWER_MONITORING
  105. } // namespace
  106. // This class implements the AudioInputCallback interface in place of the
  107. // InputController (AIC), so that
  108. // - The AIC itself does not publicly inherit AudioInputCallback.
  109. // - The lifetime of the AudioCallback (shorter than the AIC) matches the
  110. // interval during which hardware callbacks come.
  111. // - The callback class can gather information on what happened during capture
  112. // and store it in a state that can be fetched after stopping capture
  113. // (received_callback(), error_during_callback()).
  114. class AudioCallback : public media::AudioInputStream::AudioInputCallback {
  115. public:
  116. using OnDataCallback = base::RepeatingCallback<
  117. void(const media::AudioBus*, base::TimeTicks, double volume)>;
  118. using OnFirstDataCallback = base::OnceCallback<void()>;
  119. using OnErrorCallback = base::RepeatingCallback<void()>;
  120. // All callbacks are called on the hw callback thread.
  121. AudioCallback(OnDataCallback on_data_callback,
  122. OnFirstDataCallback on_first_data_callback,
  123. OnErrorCallback on_error_callback)
  124. : on_data_callback_(std::move(on_data_callback)),
  125. on_first_data_callback_(std::move(on_first_data_callback)),
  126. on_error_callback_(std::move(on_error_callback)) {
  127. DCHECK(on_data_callback_);
  128. DCHECK(on_first_data_callback_);
  129. DCHECK(on_error_callback_);
  130. }
  131. ~AudioCallback() override = default;
  132. // These should not be called when the stream is live.
  133. bool received_callback() const { return !on_first_data_callback_; }
  134. bool error_during_callback() const { return error_during_callback_; }
  135. private:
  136. void OnData(const media::AudioBus* source,
  137. base::TimeTicks capture_time,
  138. double volume) override {
  139. TRACE_EVENT1("audio", "InputController::OnData", "capture time (ms)",
  140. (capture_time - base::TimeTicks()).InMillisecondsF());
  141. if (on_first_data_callback_) {
  142. // Mark the stream as alive at first audio callback. Currently only used
  143. // for logging purposes.
  144. std::move(on_first_data_callback_).Run();
  145. }
  146. on_data_callback_.Run(source, capture_time, volume);
  147. }
  148. void OnError() override {
  149. error_during_callback_ = true;
  150. on_error_callback_.Run();
  151. }
  152. const OnDataCallback on_data_callback_;
  153. OnFirstDataCallback on_first_data_callback_;
  154. const OnErrorCallback on_error_callback_;
  155. bool error_during_callback_ = false;
  156. };
  157. InputController::InputController(
  158. EventHandler* event_handler,
  159. SyncWriter* sync_writer,
  160. media::UserInputMonitor* user_input_monitor,
  161. InputStreamActivityMonitor* activity_monitor,
  162. DeviceOutputListener* device_output_listener,
  163. AecdumpRecordingManager* aecdump_recording_manager,
  164. media::mojom::AudioProcessingConfigPtr processing_config,
  165. const media::AudioParameters& output_params,
  166. const media::AudioParameters& device_params,
  167. StreamType type)
  168. : task_runner_(base::ThreadTaskRunnerHandle::Get()),
  169. event_handler_(event_handler),
  170. stream_(nullptr),
  171. sync_writer_(sync_writer),
  172. type_(type),
  173. user_input_monitor_(user_input_monitor),
  174. activity_monitor_(activity_monitor) {
  175. DCHECK(task_runner_->BelongsToCurrentThread());
  176. DCHECK(event_handler_);
  177. DCHECK(sync_writer_);
  178. DCHECK(activity_monitor_);
  179. weak_this_ = weak_ptr_factory_.GetWeakPtr();
  180. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  181. MaybeSetUpAudioProcessing(std::move(processing_config), output_params,
  182. device_params, device_output_listener,
  183. aecdump_recording_manager);
  184. #endif
  185. if (!user_input_monitor_) {
  186. event_handler_->OnLog(
  187. "AIC::InputController() => (WARNING: keypress monitoring is disabled)");
  188. }
  189. }
  190. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  191. void InputController::MaybeSetUpAudioProcessing(
  192. media::mojom::AudioProcessingConfigPtr processing_config,
  193. const media::AudioParameters& processing_output_params,
  194. const media::AudioParameters& device_params,
  195. DeviceOutputListener* device_output_listener,
  196. AecdumpRecordingManager* aecdump_recording_manager) {
  197. if (!device_output_listener)
  198. return;
  199. if (!(processing_config &&
  200. processing_config->settings.NeedAudioModification())) {
  201. return;
  202. }
  203. absl::optional<media::AudioParameters> processing_input_params =
  204. media::AudioProcessor::ComputeInputFormat(device_params,
  205. processing_config->settings);
  206. if (!processing_input_params) {
  207. event_handler_->OnLog(base::StringPrintf(
  208. "AIC::MaybeSetupAudioProcessing() => (Unsupported device_params=%s, "
  209. "cannot do audio processing)",
  210. device_params.AsHumanReadableString().c_str()));
  211. return;
  212. }
  213. // In case fake audio input is requested.
  214. processing_input_params->set_format(processing_output_params.format());
  215. // Unretained() is safe, since |this| and |event_handler_| outlive
  216. // |audio_processor_handler_|.
  217. audio_processor_handler_ = std::make_unique<AudioProcessorHandler>(
  218. processing_config->settings, *processing_input_params,
  219. processing_output_params,
  220. base::BindRepeating(&EventHandler::OnLog,
  221. base::Unretained(event_handler_)),
  222. base::BindRepeating(&InputController::DeliverProcessedAudio,
  223. base::Unretained(this)),
  224. std::move(processing_config->controls_receiver),
  225. aecdump_recording_manager);
  226. // If the required processing is lightweight, there is no need to offload work
  227. // to a new thread.
  228. if (!processing_config->settings.NeedPlayoutReference())
  229. return;
  230. int fifo_size = media::kChromeWideEchoCancellationProcessingFifoSize.Get();
  231. // Only use the FIFO/new thread if its size is explicitly set.
  232. if (fifo_size) {
  233. // base::Unretained() is safe since both |audio_processor_handler_| and
  234. // |event_handler_| outlive |processing_fifo_|.
  235. processing_fifo_ = std::make_unique<ProcessingAudioFifo>(
  236. *processing_input_params, fifo_size,
  237. base::BindRepeating(&AudioProcessorHandler::ProcessCapturedAudio,
  238. base::Unretained(audio_processor_handler_.get())),
  239. base::BindRepeating(&EventHandler::OnLog,
  240. base::Unretained(event_handler_.get())));
  241. }
  242. // Unretained() is safe, since |event_handler_| outlives |output_tapper_|.
  243. output_tapper_ = std::make_unique<OutputTapper>(
  244. device_output_listener, audio_processor_handler_.get(),
  245. base::BindRepeating(&EventHandler::OnLog,
  246. base::Unretained(event_handler_)));
  247. }
  248. #endif
  249. InputController::~InputController() {
  250. DCHECK(task_runner_->BelongsToCurrentThread());
  251. DCHECK(!audio_callback_);
  252. DCHECK(!stream_);
  253. DCHECK(!check_muted_state_timer_.IsRunning());
  254. }
  255. // static
  256. std::unique_ptr<InputController> InputController::Create(
  257. media::AudioManager* audio_manager,
  258. EventHandler* event_handler,
  259. SyncWriter* sync_writer,
  260. media::UserInputMonitor* user_input_monitor,
  261. InputStreamActivityMonitor* activity_monitor,
  262. DeviceOutputListener* device_output_listener,
  263. AecdumpRecordingManager* aecdump_recording_manager,
  264. media::mojom::AudioProcessingConfigPtr processing_config,
  265. const media::AudioParameters& params,
  266. const std::string& device_id,
  267. bool enable_agc) {
  268. DCHECK(audio_manager);
  269. DCHECK(audio_manager->GetTaskRunner()->BelongsToCurrentThread());
  270. DCHECK(activity_monitor);
  271. DCHECK(sync_writer);
  272. DCHECK(event_handler);
  273. DCHECK(params.IsValid());
  274. if (params.channels() > kMaxInputChannels)
  275. return nullptr;
  276. const media::AudioParameters device_params =
  277. AudioManagerPowerUser(audio_manager).GetInputStreamParameters(device_id);
  278. // Create the InputController object and ensure that it runs on
  279. // the audio-manager thread.
  280. // Using `new` to access a non-public constructor.
  281. std::unique_ptr<InputController> controller =
  282. base::WrapUnique(new InputController(
  283. event_handler, sync_writer, user_input_monitor, activity_monitor,
  284. device_output_listener, aecdump_recording_manager,
  285. std::move(processing_config), params, device_params,
  286. ParamsToStreamType(params)));
  287. controller->DoCreate(audio_manager, params, device_id, enable_agc);
  288. return controller;
  289. }
  290. void InputController::Record() {
  291. DCHECK(task_runner_->BelongsToCurrentThread());
  292. SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
  293. if (!stream_ || audio_callback_)
  294. return;
  295. event_handler_->OnLog("AIC::Record()");
  296. if (user_input_monitor_) {
  297. user_input_monitor_->EnableKeyPressMonitoring();
  298. prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
  299. }
  300. stream_create_time_ = base::TimeTicks::Now();
  301. // Unretained() is safe, since |this| outlives |audio_callback_|.
  302. // |on_first_data_callback| and |on_error_callback| calls are posted on the
  303. // audio thread, since all AudioCallback callbacks run on the hw callback
  304. // thread.
  305. audio_callback_ = std::make_unique<AudioCallback>(
  306. /*on_data_callback=*/base::BindRepeating(&InputController::OnData,
  307. base::Unretained(this)),
  308. /*on_first_data_callback=*/
  309. base::BindPostTask(
  310. task_runner_,
  311. base::BindOnce(&InputController::ReportIsAlive, weak_this_)),
  312. /*on_error_callback=*/
  313. base::BindPostTask(
  314. task_runner_,
  315. base::BindRepeating(&InputController::DoReportError, weak_this_)));
  316. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  317. if (processing_fifo_)
  318. processing_fifo_->Start();
  319. if (output_tapper_)
  320. output_tapper_->Start();
  321. #endif
  322. stream_->Start(audio_callback_.get());
  323. activity_monitor_->OnInputStreamActive();
  324. return;
  325. }
  326. void InputController::Close() {
  327. DCHECK(task_runner_->BelongsToCurrentThread());
  328. SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
  329. if (!stream_)
  330. return;
  331. check_muted_state_timer_.AbandonAndStop();
  332. std::string log_string;
  333. static const char kLogStringPrefix[] = "AIC::Close => ";
  334. // Allow calling unconditionally and bail if we don't have a stream to close.
  335. if (audio_callback_) {
  336. // Calls to OnData() should stop beyond this point.
  337. stream_->Stop();
  338. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  339. if (output_tapper_)
  340. output_tapper_->Stop();
  341. if (processing_fifo_) {
  342. // Stop the FIFO after |stream_| is stopped, to guarantee there are no
  343. // more calls to OnData().
  344. // Note: destroying the FIFO will synchronously wait for the processing
  345. // thread to stop.
  346. processing_fifo_.reset();
  347. }
  348. #endif
  349. activity_monitor_->OnInputStreamInactive();
  350. // Sometimes a stream (and accompanying audio track) is created and
  351. // immediately closed or discarded. In this case they are registered as
  352. // 'stopped early' rather than 'never got data'.
  353. const base::TimeDelta duration =
  354. base::TimeTicks::Now() - stream_create_time_;
  355. CaptureStartupResult capture_startup_result =
  356. audio_callback_->received_callback()
  357. ? CAPTURE_STARTUP_OK
  358. : (duration.InMilliseconds() < 500
  359. ? CAPTURE_STARTUP_STOPPED_EARLY
  360. : CAPTURE_STARTUP_NEVER_GOT_DATA);
  361. LogCaptureStartupResult(capture_startup_result);
  362. LogCallbackError();
  363. log_string = base::StringPrintf("%s(stream duration=%" PRId64 " seconds%s",
  364. kLogStringPrefix, duration.InSeconds(),
  365. audio_callback_->received_callback()
  366. ? ")"
  367. : " - no callbacks received)");
  368. if (type_ == LOW_LATENCY) {
  369. if (audio_callback_->received_callback()) {
  370. UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDuration", duration);
  371. } else {
  372. UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDurationWithoutCallback",
  373. duration);
  374. }
  375. }
  376. if (user_input_monitor_)
  377. user_input_monitor_->DisableKeyPressMonitoring();
  378. audio_callback_.reset();
  379. } else {
  380. log_string = base::StringPrintf("%s(WARNING: recording never started)",
  381. kLogStringPrefix);
  382. }
  383. event_handler_->OnLog(log_string);
  384. stream_->Close();
  385. stream_ = nullptr;
  386. sync_writer_->Close();
  387. #if defined(AUDIO_POWER_MONITORING)
  388. // Send UMA stats if enabled.
  389. if (power_measurement_is_enabled_) {
  390. LogSilenceState(silence_state_);
  391. log_string = base::StringPrintf("%s(silence_state=%s)", kLogStringPrefix,
  392. SilenceStateToString(silence_state_));
  393. event_handler_->OnLog(log_string);
  394. }
  395. #endif
  396. max_volume_ = 0.0;
  397. weak_ptr_factory_.InvalidateWeakPtrs();
  398. }
  399. void InputController::SetVolume(double volume) {
  400. DCHECK(task_runner_->BelongsToCurrentThread());
  401. DCHECK_GE(volume, 0);
  402. DCHECK_LE(volume, 1.0);
  403. if (!stream_)
  404. return;
  405. event_handler_->OnLog(
  406. base::StringPrintf("AIC::SetVolume({volume=%.2f})", volume));
  407. // Only ask for the maximum volume at first call and use cached value
  408. // for remaining function calls.
  409. if (!max_volume_) {
  410. max_volume_ = stream_->GetMaxVolume();
  411. }
  412. if (max_volume_ == 0.0) {
  413. DLOG(WARNING) << "Failed to access input volume control";
  414. return;
  415. }
  416. // Set the stream volume and scale to a range matched to the platform.
  417. stream_->SetVolume(max_volume_ * volume);
  418. }
  419. void InputController::SetOutputDeviceForAec(
  420. const std::string& output_device_id) {
  421. DCHECK(task_runner_->BelongsToCurrentThread());
  422. if (stream_)
  423. stream_->SetOutputDeviceForAec(output_device_id);
  424. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  425. if (output_tapper_)
  426. output_tapper_->SetOutputDeviceForAec(output_device_id);
  427. #endif
  428. }
  429. void InputController::OnStreamActive(Snoopable* output_stream) {
  430. DCHECK(task_runner_->BelongsToCurrentThread());
  431. }
  432. void InputController::OnStreamInactive(Snoopable* output_stream) {
  433. DCHECK(task_runner_->BelongsToCurrentThread());
  434. }
  435. InputController::ErrorCode MapOpenOutcomeToErrorCode(OpenOutcome outcome) {
  436. switch (outcome) {
  437. case OpenOutcome::kFailedSystemPermissions:
  438. return InputController::STREAM_OPEN_SYSTEM_PERMISSIONS_ERROR;
  439. case OpenOutcome::kFailedInUse:
  440. return InputController::STREAM_OPEN_DEVICE_IN_USE_ERROR;
  441. default:
  442. return InputController::STREAM_OPEN_ERROR;
  443. }
  444. }
  445. void InputController::DoCreate(media::AudioManager* audio_manager,
  446. const media::AudioParameters& params,
  447. const std::string& device_id,
  448. bool enable_agc) {
  449. DCHECK(task_runner_->BelongsToCurrentThread());
  450. DCHECK(!stream_);
  451. SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
  452. event_handler_->OnLog("AIC::DoCreate({device_id=" + device_id + "})");
  453. #if defined(AUDIO_POWER_MONITORING)
  454. // We only do power measurements for UMA stats for low latency streams, and
  455. // only if agc is requested, to avoid adding logs and UMA for non-WebRTC
  456. // clients.
  457. power_measurement_is_enabled_ = (type_ == LOW_LATENCY && enable_agc);
  458. last_audio_level_log_time_ = base::TimeTicks::Now();
  459. #endif
  460. const media::AudioParameters audio_input_stream_params =
  461. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  462. audio_processor_handler_ ? audio_processor_handler_->input_format() :
  463. #endif
  464. params;
  465. // Unretained is safe since |this| owns |stream|.
  466. auto* stream = audio_manager->MakeAudioInputStream(
  467. audio_input_stream_params, device_id,
  468. base::BindRepeating(&InputController::LogMessage,
  469. base::Unretained(this)));
  470. if (!stream) {
  471. LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
  472. event_handler_->OnError(STREAM_CREATE_ERROR);
  473. return;
  474. }
  475. auto open_outcome = stream->Open();
  476. if (open_outcome != OpenOutcome::kSuccess) {
  477. stream->Close();
  478. LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
  479. event_handler_->OnError(MapOpenOutcomeToErrorCode(open_outcome));
  480. return;
  481. }
  482. #if defined(AUDIO_POWER_MONITORING)
  483. bool agc_is_supported = stream->SetAutomaticGainControl(enable_agc);
  484. // Disable power measurements on platforms that does not support AGC at a
  485. // lower level. AGC can fail on platforms where we don't support the
  486. // functionality to modify the input volume slider. One such example is
  487. // Windows XP.
  488. power_measurement_is_enabled_ &= agc_is_supported;
  489. event_handler_->OnLog(
  490. base::StringPrintf("AIC::DoCreate => (power_measurement_is_enabled=%d)",
  491. power_measurement_is_enabled_));
  492. #else
  493. stream->SetAutomaticGainControl(enable_agc);
  494. #endif
  495. // Finally, keep the stream pointer around, update the state and notify.
  496. stream_ = stream;
  497. // Send initial muted state along with OnCreated, to avoid races.
  498. is_muted_ = stream_->IsMuted();
  499. event_handler_->OnCreated(is_muted_);
  500. check_muted_state_timer_.Start(FROM_HERE, kCheckMutedStateInterval, this,
  501. &InputController::CheckMutedState);
  502. DCHECK(check_muted_state_timer_.IsRunning());
  503. }
  504. void InputController::DoReportError() {
  505. DCHECK(task_runner_->BelongsToCurrentThread());
  506. event_handler_->OnError(STREAM_ERROR);
  507. }
  508. void InputController::DoLogAudioLevels(float level_dbfs,
  509. int microphone_volume_percent) {
  510. #if defined(AUDIO_POWER_MONITORING)
  511. DCHECK(task_runner_->BelongsToCurrentThread());
  512. if (!stream_)
  513. return;
  514. // Detect if the user has enabled hardware mute by pressing the mute
  515. // button in audio settings for the selected microphone.
  516. const bool microphone_is_muted = stream_->IsMuted();
  517. if (microphone_is_muted) {
  518. LogMicrophoneMuteResult(MICROPHONE_IS_MUTED);
  519. event_handler_->OnLog("AIC::OnData => (microphone is muted)");
  520. } else {
  521. LogMicrophoneMuteResult(MICROPHONE_IS_NOT_MUTED);
  522. }
  523. std::string log_string = base::StringPrintf(
  524. "AIC::OnData => (average audio level=%.2f dBFS", level_dbfs);
  525. static const float kSilenceThresholdDBFS = -72.24719896f;
  526. if (level_dbfs < kSilenceThresholdDBFS)
  527. log_string += " <=> low audio input level";
  528. event_handler_->OnLog(log_string + ")");
  529. if (!microphone_is_muted) {
  530. UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
  531. }
  532. log_string = base::StringPrintf("AIC::OnData => (microphone volume=%d%%",
  533. microphone_volume_percent);
  534. if (microphone_volume_percent < kLowLevelMicrophoneLevelPercent)
  535. log_string += " <=> low microphone level";
  536. event_handler_->OnLog(log_string + ")");
  537. #endif
  538. }
  539. #if defined(AUDIO_POWER_MONITORING)
  540. void InputController::UpdateSilenceState(bool silence) {
  541. if (silence) {
  542. if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
  543. silence_state_ = SILENCE_STATE_ONLY_SILENCE;
  544. } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
  545. silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
  546. } else {
  547. DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
  548. silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
  549. }
  550. } else {
  551. if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
  552. silence_state_ = SILENCE_STATE_ONLY_AUDIO;
  553. } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
  554. silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
  555. } else {
  556. DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
  557. silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
  558. }
  559. }
  560. }
  561. void InputController::LogSilenceState(SilenceState value) {
  562. UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
  563. value, SILENCE_STATE_MAX + 1);
  564. }
  565. #endif
  566. void InputController::LogCaptureStartupResult(CaptureStartupResult result) {
  567. switch (type_) {
  568. case LOW_LATENCY:
  569. UMA_HISTOGRAM_ENUMERATION("Media.LowLatencyAudioCaptureStartupSuccess",
  570. result, CAPTURE_STARTUP_RESULT_MAX + 1);
  571. break;
  572. case HIGH_LATENCY:
  573. UMA_HISTOGRAM_ENUMERATION("Media.HighLatencyAudioCaptureStartupSuccess",
  574. result, CAPTURE_STARTUP_RESULT_MAX + 1);
  575. break;
  576. case VIRTUAL:
  577. UMA_HISTOGRAM_ENUMERATION("Media.VirtualAudioCaptureStartupSuccess",
  578. result, CAPTURE_STARTUP_RESULT_MAX + 1);
  579. break;
  580. default:
  581. break;
  582. }
  583. }
  584. void InputController::LogCallbackError() {
  585. bool error_during_callback = audio_callback_->error_during_callback();
  586. switch (type_) {
  587. case LOW_LATENCY:
  588. UMA_HISTOGRAM_BOOLEAN("Media.Audio.Capture.LowLatencyCallbackError",
  589. error_during_callback);
  590. break;
  591. case HIGH_LATENCY:
  592. UMA_HISTOGRAM_BOOLEAN("Media.Audio.Capture.HighLatencyCallbackError",
  593. error_during_callback);
  594. break;
  595. case VIRTUAL:
  596. UMA_HISTOGRAM_BOOLEAN("Media.Audio.Capture.VirtualCallbackError",
  597. error_during_callback);
  598. break;
  599. default:
  600. break;
  601. }
  602. }
  603. void InputController::LogMessage(const std::string& message) {
  604. DCHECK(task_runner_->BelongsToCurrentThread());
  605. event_handler_->OnLog(message);
  606. }
  607. bool InputController::CheckForKeyboardInput() {
  608. if (!user_input_monitor_)
  609. return false;
  610. const size_t current_count = user_input_monitor_->GetKeyPressCount();
  611. const bool key_pressed = current_count != prev_key_down_count_;
  612. prev_key_down_count_ = current_count;
  613. DVLOG_IF(6, key_pressed) << "Detected keypress.";
  614. return key_pressed;
  615. }
  616. bool InputController::CheckAudioPower(const media::AudioBus* source,
  617. double volume,
  618. float* average_power_dbfs,
  619. int* mic_volume_percent) {
  620. #if defined(AUDIO_POWER_MONITORING)
  621. // Only do power-level measurements if DoCreate() has been called. It will
  622. // ensure that logging will mainly be done for WebRTC and WebSpeech
  623. // clients.
  624. if (!power_measurement_is_enabled_)
  625. return false;
  626. // Perform periodic audio (power) level measurements.
  627. const auto now = base::TimeTicks::Now();
  628. if (now - last_audio_level_log_time_ <= kPowerMonitorLogInterval) {
  629. return false;
  630. }
  631. *average_power_dbfs = AveragePower(*source);
  632. *mic_volume_percent = static_cast<int>(100.0 * volume);
  633. last_audio_level_log_time_ = now;
  634. return true;
  635. #else
  636. return false;
  637. #endif
  638. }
  639. void InputController::CheckMutedState() {
  640. DCHECK(task_runner_->BelongsToCurrentThread());
  641. DCHECK(stream_);
  642. const bool new_state = stream_->IsMuted();
  643. if (new_state != is_muted_) {
  644. is_muted_ = new_state;
  645. event_handler_->OnMuted(is_muted_);
  646. std::string log_string =
  647. base::StringPrintf("AIC::OnMuted({is_muted=%d})", is_muted_);
  648. event_handler_->OnLog(log_string);
  649. }
  650. }
  651. void InputController::ReportIsAlive() {
  652. DCHECK(task_runner_->BelongsToCurrentThread());
  653. DCHECK(stream_);
  654. // Don't store any state, just log the event for now.
  655. event_handler_->OnLog("AIC::OnData => (stream is alive)");
  656. }
  657. void InputController::OnData(const media::AudioBus* source,
  658. base::TimeTicks capture_time,
  659. double volume) {
  660. const bool key_pressed = CheckForKeyboardInput();
  661. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  662. if (processing_fifo_) {
  663. DCHECK(audio_processor_handler_);
  664. processing_fifo_->PushData(source, capture_time, volume, key_pressed);
  665. } else if (audio_processor_handler_) {
  666. audio_processor_handler_->ProcessCapturedAudio(*source, capture_time,
  667. volume, key_pressed);
  668. } else
  669. #endif
  670. {
  671. sync_writer_->Write(source, volume, key_pressed, capture_time);
  672. }
  673. float average_power_dbfs;
  674. int mic_volume_percent;
  675. if (CheckAudioPower(source, volume, &average_power_dbfs,
  676. &mic_volume_percent)) {
  677. // Use event handler on the audio thread to relay a message to the ARIH
  678. // in content which does the actual logging on the IO thread.
  679. task_runner_->PostTask(
  680. FROM_HERE,
  681. base::BindOnce(&InputController::DoLogAudioLevels, weak_this_,
  682. average_power_dbfs, mic_volume_percent));
  683. }
  684. }
  685. #if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  686. void InputController::DeliverProcessedAudio(const media::AudioBus& audio_bus,
  687. base::TimeTicks audio_capture_time,
  688. absl::optional<double> new_volume) {
  689. // When processing is performed in the audio service, the consumer is not
  690. // expected to use the input volume and keypress information.
  691. sync_writer_->Write(&audio_bus, /*volume=*/1.0,
  692. /*key_pressed=*/false, audio_capture_time);
  693. if (new_volume) {
  694. task_runner_->PostTask(
  695. FROM_HERE,
  696. base::BindOnce(&InputController::SetVolume, weak_this_, *new_volume));
  697. }
  698. }
  699. #endif
  700. // static
  701. InputController::StreamType InputController::ParamsToStreamType(
  702. const media::AudioParameters& params) {
  703. switch (params.format()) {
  704. case media::AudioParameters::Format::AUDIO_PCM_LINEAR:
  705. return InputController::StreamType::HIGH_LATENCY;
  706. case media::AudioParameters::Format::AUDIO_PCM_LOW_LATENCY:
  707. return InputController::StreamType::LOW_LATENCY;
  708. default:
  709. // Currently, the remaining supported type is fake. Reconsider if other
  710. // formats become supported.
  711. return InputController::StreamType::FAKE;
  712. }
  713. }
  714. } // namespace audio