loopback_stream.cc 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "services/audio/loopback_stream.h"
  5. #include <algorithm>
  6. #include <string>
  7. #include "base/bind.h"
  8. #include "base/containers/contains.h"
  9. #include "base/sync_socket.h"
  10. #include "base/threading/sequenced_task_runner_handle.h"
  11. #include "base/time/default_tick_clock.h"
  12. #include "base/trace_event/trace_event.h"
  13. #include "media/base/audio_bus.h"
  14. #include "media/base/vector_math.h"
  15. #include "mojo/public/cpp/system/buffer.h"
  16. #include "mojo/public/cpp/system/platform_handle.h"
  17. #include "third_party/abseil-cpp/absl/utility/utility.h"
  18. namespace audio {
  19. namespace {
  20. // Start with a conservative, but reasonable capture delay that should work for
  21. // most platforms (i.e., not needing an increase during a loopback session).
  22. constexpr base::TimeDelta kInitialCaptureDelay = base::Milliseconds(20);
  23. } // namespace
  24. // static
  25. constexpr double LoopbackStream::kMaxVolume;
  26. LoopbackStream::LoopbackStream(
  27. CreatedCallback created_callback,
  28. BindingLostCallback binding_lost_callback,
  29. scoped_refptr<base::SequencedTaskRunner> flow_task_runner,
  30. mojo::PendingReceiver<media::mojom::AudioInputStream> receiver,
  31. mojo::PendingRemote<media::mojom::AudioInputStreamClient> client,
  32. mojo::PendingRemote<media::mojom::AudioInputStreamObserver> observer,
  33. const media::AudioParameters& params,
  34. uint32_t shared_memory_count,
  35. LoopbackCoordinator* coordinator,
  36. const base::UnguessableToken& group_id)
  37. : binding_lost_callback_(std::move(binding_lost_callback)),
  38. receiver_(this, std::move(receiver)),
  39. client_(std::move(client)),
  40. observer_(std::move(observer)),
  41. coordinator_(coordinator),
  42. group_id_(group_id),
  43. network_(nullptr, base::OnTaskRunnerDeleter(flow_task_runner)) {
  44. DCHECK(coordinator_);
  45. TRACE_EVENT1("audio", "LoopbackStream::LoopbackStream", "params",
  46. params.AsHumanReadableString());
  47. // Generate an error and shut down automatically whenever any of the mojo
  48. // bindings is closed.
  49. receiver_.set_disconnect_handler(
  50. base::BindOnce(&LoopbackStream::OnError, base::Unretained(this)));
  51. client_.set_disconnect_handler(
  52. base::BindOnce(&LoopbackStream::OnError, base::Unretained(this)));
  53. observer_.set_disconnect_handler(
  54. base::BindOnce(&LoopbackStream::OnError, base::Unretained(this)));
  55. // Construct the components of the AudioDataPipe, for delivering the data to
  56. // the consumer. If successful, create the FlowNetwork too.
  57. base::CancelableSyncSocket foreign_socket;
  58. std::unique_ptr<InputSyncWriter> writer = InputSyncWriter::Create(
  59. base::BindRepeating(
  60. [](const std::string& message) { VLOG(1) << message; }),
  61. shared_memory_count, params, &foreign_socket);
  62. if (writer) {
  63. base::ReadOnlySharedMemoryRegion shared_memory_region =
  64. writer->TakeSharedMemoryRegion();
  65. mojo::PlatformHandle socket_handle;
  66. if (shared_memory_region.IsValid()) {
  67. socket_handle = mojo::PlatformHandle(foreign_socket.Take());
  68. if (socket_handle.is_valid()) {
  69. std::move(created_callback)
  70. .Run({absl::in_place, std::move(shared_memory_region),
  71. std::move(socket_handle)});
  72. network_.reset(new FlowNetwork(std::move(flow_task_runner), params,
  73. std::move(writer)));
  74. return; // Success!
  75. }
  76. }
  77. }
  78. // If this point is reached, one or more AudioDataPipe components failed to
  79. // initialize. Report the error.
  80. std::move(created_callback).Run(nullptr);
  81. OnError();
  82. }
  83. LoopbackStream::~LoopbackStream() {
  84. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  85. TRACE_EVENT0("audio", "LoopbackStream::~LoopbackStream");
  86. if (network_) {
  87. if (network_->is_started()) {
  88. coordinator_->RemoveObserver(group_id_, this);
  89. while (!snoopers_.empty()) {
  90. OnMemberLeftGroup(snoopers_.begin()->first);
  91. }
  92. }
  93. DCHECK(snoopers_.empty());
  94. }
  95. }
  96. void LoopbackStream::Record() {
  97. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  98. if (!network_ || network_->is_started()) {
  99. return;
  100. }
  101. TRACE_EVENT0("audio", "LoopbackStream::Record");
  102. // Begin snooping on all group members. This will set up the mixer network
  103. // and begin accumulating audio data in the Snoopers' buffers.
  104. DCHECK(snoopers_.empty());
  105. coordinator_->ForEachMemberInGroup(
  106. group_id_, base::BindRepeating(&LoopbackStream::OnMemberJoinedGroup,
  107. base::Unretained(this)));
  108. coordinator_->AddObserver(group_id_, this);
  109. // Start the data flow.
  110. network_->Start();
  111. if (observer_) {
  112. observer_->DidStartRecording();
  113. }
  114. }
  115. void LoopbackStream::SetVolume(double volume) {
  116. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  117. TRACE_EVENT_INSTANT1("audio", "LoopbackStream::SetVolume",
  118. TRACE_EVENT_SCOPE_THREAD, "volume", volume);
  119. if (!std::isfinite(volume) || volume < 0.0) {
  120. receiver_.ReportBadMessage("Invalid volume");
  121. OnError();
  122. return;
  123. }
  124. if (network_) {
  125. network_->SetVolume(std::min(volume, kMaxVolume));
  126. }
  127. }
  128. void LoopbackStream::OnMemberJoinedGroup(LoopbackGroupMember* member) {
  129. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  130. if (!network_) {
  131. return;
  132. }
  133. if (!base::TimeTicks::IsHighResolution()) {
  134. // As of this writing, only machines manufactured before 2008 won't be able
  135. // to produce high-resolution timestamps. Since the buffer management logic
  136. // (to mitigate overruns/underruns) depends on them to function correctly,
  137. // simply return early (i.e., never start snooping on the |member|).
  138. TRACE_EVENT_INSTANT0("audio",
  139. "LoopbackStream::OnMemberJoinedGroup Rejected",
  140. TRACE_EVENT_SCOPE_THREAD);
  141. return;
  142. }
  143. TRACE_EVENT1("audio", "LoopbackStream::OnMemberJoinedGroup", "member",
  144. static_cast<void*>(member));
  145. const media::AudioParameters& input_params = member->GetAudioParameters();
  146. const auto emplace_result = snoopers_.emplace(
  147. std::piecewise_construct, std::forward_as_tuple(member),
  148. std::forward_as_tuple(input_params, network_->output_params()));
  149. DCHECK(emplace_result.second); // There was no pre-existing map entry.
  150. SnooperNode* const snooper = &(emplace_result.first->second);
  151. member->StartSnooping(snooper);
  152. network_->AddInput(snooper);
  153. }
  154. void LoopbackStream::OnMemberLeftGroup(LoopbackGroupMember* member) {
  155. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  156. if (!network_) {
  157. return;
  158. }
  159. const auto snoop_it = snoopers_.find(member);
  160. if (snoop_it == snoopers_.end()) {
  161. // See comments about "high-resolution timestamps" in OnMemberJoinedGroup().
  162. return;
  163. }
  164. TRACE_EVENT1("audio", "LoopbackStream::OnMemberLeftGroup", "member",
  165. static_cast<void*>(member));
  166. SnooperNode* const snooper = &(snoop_it->second);
  167. member->StopSnooping(snooper);
  168. network_->RemoveInput(snooper);
  169. snoopers_.erase(snoop_it);
  170. }
  171. void LoopbackStream::OnError() {
  172. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  173. if (!binding_lost_callback_) {
  174. return; // OnError() was already called.
  175. }
  176. TRACE_EVENT0("audio", "LoopbackStream::OnError");
  177. receiver_.reset();
  178. if (client_) {
  179. client_->OnError(media::mojom::InputStreamErrorCode::kUnknown);
  180. client_.reset();
  181. }
  182. observer_.reset();
  183. // Post a task to run the BindingLostCallback, since this method can be called
  184. // from the constructor.
  185. base::SequencedTaskRunnerHandle::Get()->PostTask(
  186. FROM_HERE,
  187. base::BindOnce(
  188. [](base::WeakPtr<LoopbackStream> weak_self,
  189. BindingLostCallback callback) {
  190. if (auto* self = weak_self.get()) {
  191. std::move(callback).Run(self);
  192. }
  193. },
  194. weak_factory_.GetWeakPtr(), std::move(binding_lost_callback_)));
  195. // No need to shut down anything else, as the destructor will run soon and
  196. // take care of the rest.
  197. }
  198. LoopbackStream::FlowNetwork::FlowNetwork(
  199. scoped_refptr<base::SequencedTaskRunner> flow_task_runner,
  200. const media::AudioParameters& output_params,
  201. std::unique_ptr<InputSyncWriter> writer)
  202. : clock_(base::DefaultTickClock::GetInstance()),
  203. flow_task_runner_(flow_task_runner),
  204. output_params_(output_params),
  205. writer_(std::move(writer)),
  206. mix_bus_(media::AudioBus::Create(output_params_)) {}
  207. void LoopbackStream::FlowNetwork::AddInput(SnooperNode* node) {
  208. DCHECK_CALLED_ON_VALID_SEQUENCE(control_sequence_);
  209. base::AutoLock scoped_lock(lock_);
  210. DCHECK(!base::Contains(inputs_, node));
  211. inputs_.push_back(node);
  212. }
  213. void LoopbackStream::FlowNetwork::RemoveInput(SnooperNode* node) {
  214. DCHECK_CALLED_ON_VALID_SEQUENCE(control_sequence_);
  215. base::AutoLock scoped_lock(lock_);
  216. const auto it = std::find(inputs_.begin(), inputs_.end(), node);
  217. DCHECK(it != inputs_.end());
  218. inputs_.erase(it);
  219. }
  220. void LoopbackStream::FlowNetwork::SetVolume(double volume) {
  221. DCHECK_CALLED_ON_VALID_SEQUENCE(control_sequence_);
  222. base::AutoLock scoped_lock(lock_);
  223. volume_ = volume;
  224. }
  225. void LoopbackStream::FlowNetwork::Start() {
  226. DCHECK_CALLED_ON_VALID_SEQUENCE(control_sequence_);
  227. DCHECK(!is_started());
  228. timer_.emplace();
  229. // Note: GenerateMoreAudio() will schedule the timer.
  230. first_generate_time_ = clock_->NowTicks();
  231. frames_elapsed_ = 0;
  232. next_generate_time_ = first_generate_time_;
  233. capture_delay_ = kInitialCaptureDelay;
  234. flow_task_runner_->PostTask(
  235. FROM_HERE,
  236. // Unretained is safe because the destructor will always be invoked from a
  237. // task that runs afterwards.
  238. base::BindOnce(&FlowNetwork::GenerateMoreAudio, base::Unretained(this)));
  239. }
  240. LoopbackStream::FlowNetwork::~FlowNetwork() {
  241. DCHECK(flow_task_runner_->RunsTasksInCurrentSequence());
  242. DCHECK(inputs_.empty());
  243. }
  244. void LoopbackStream::FlowNetwork::GenerateMoreAudio() {
  245. DCHECK(flow_task_runner_->RunsTasksInCurrentSequence());
  246. TRACE_EVENT_WITH_FLOW0("audio", "GenerateMoreAudio", this,
  247. TRACE_EVENT_FLAG_FLOW_IN | TRACE_EVENT_FLAG_FLOW_OUT);
  248. // Drive the audio flows from the SnooperNodes and produce the single result
  249. // stream. Hold the lock during this part of the process to prevent any of the
  250. // control methods from altering the configuration of the network.
  251. double output_volume;
  252. base::TimeTicks delayed_capture_time;
  253. {
  254. base::AutoLock scoped_lock(lock_);
  255. output_volume = volume_;
  256. // Compute the reference time to use for audio rendering. Query each input
  257. // node and update |capture_delay_|, if necessary. This is used to always
  258. // generate audio from a "safe point" in the recent past, to prevent buffer
  259. // underruns in the inputs. http://crbug.com/934770
  260. delayed_capture_time = next_generate_time_ - capture_delay_;
  261. for (SnooperNode* node : inputs_) {
  262. const absl::optional<base::TimeTicks> suggestion =
  263. node->SuggestLatestRenderTime(mix_bus_->frames());
  264. if (suggestion.value_or(delayed_capture_time) < delayed_capture_time) {
  265. const base::TimeDelta increase = delayed_capture_time - (*suggestion);
  266. TRACE_EVENT_INSTANT2("audio", "GenerateMoreAudio Capture Delay Change",
  267. TRACE_EVENT_SCOPE_THREAD, "old capture delay (µs)",
  268. capture_delay_.InMicroseconds(), "change (µs)",
  269. increase.InMicroseconds());
  270. delayed_capture_time = *suggestion;
  271. capture_delay_ += increase;
  272. }
  273. }
  274. TRACE_COUNTER_ID1("audio", "Loopback Capture Delay (µs)", this,
  275. capture_delay_.InMicroseconds());
  276. // Render the audio from each input, apply this stream's volume setting by
  277. // scaling the data, then mix it all together to form a single audio
  278. // signal. If there are no snoopers, just render silence.
  279. auto it = inputs_.begin();
  280. if (it == inputs_.end()) {
  281. mix_bus_->Zero();
  282. } else {
  283. // Render the first input's signal directly into |mix_bus_|.
  284. (*it)->Render(delayed_capture_time, mix_bus_.get());
  285. mix_bus_->Scale(volume_);
  286. // Render each successive input's signal into |transfer_bus_|, and then
  287. // mix it into |mix_bus_|.
  288. ++it;
  289. if (it != inputs_.end()) {
  290. if (!transfer_bus_) {
  291. transfer_bus_ = media::AudioBus::Create(output_params_);
  292. }
  293. do {
  294. (*it)->Render(delayed_capture_time, transfer_bus_.get());
  295. for (int ch = 0; ch < transfer_bus_->channels(); ++ch) {
  296. media::vector_math::FMAC(transfer_bus_->channel(ch), volume_,
  297. transfer_bus_->frames(),
  298. mix_bus_->channel(ch));
  299. }
  300. ++it;
  301. } while (it != inputs_.end());
  302. }
  303. }
  304. }
  305. // Insert the result into the AudioDataPipe.
  306. writer_->Write(mix_bus_.get(), output_volume, false, delayed_capture_time);
  307. // Determine when to generate more audio again. This is done by advancing the
  308. // frame count by one interval's worth, then computing the TimeTicks
  309. // corresponding to the new frame count. Also, check the clock to detect when
  310. // the user's machine is overloaded and the output needs to skip forward one
  311. // or more intervals.
  312. const int frames_per_buffer = mix_bus_->frames();
  313. frames_elapsed_ += frames_per_buffer;
  314. next_generate_time_ =
  315. first_generate_time_ +
  316. base::Microseconds(frames_elapsed_ * base::Time::kMicrosecondsPerSecond /
  317. output_params_.sample_rate());
  318. const base::TimeTicks now = clock_->NowTicks();
  319. if (next_generate_time_ < now) {
  320. TRACE_EVENT_INSTANT1("audio", "GenerateMoreAudio Is Behind",
  321. TRACE_EVENT_SCOPE_THREAD, "µsec_behind",
  322. (now - next_generate_time_).InMicroseconds());
  323. // Audio generation has fallen behind. Skip-ahead the frame counter so that
  324. // audio generation will resume for the next buffer after the one that
  325. // should be generating right now. http://crbug.com/847487
  326. const int64_t target_frame_count =
  327. (now - first_generate_time_).InMicroseconds() *
  328. output_params_.sample_rate() / base::Time::kMicrosecondsPerSecond;
  329. frames_elapsed_ =
  330. (target_frame_count / frames_per_buffer + 1) * frames_per_buffer;
  331. next_generate_time_ =
  332. first_generate_time_ +
  333. base::Microseconds(frames_elapsed_ *
  334. base::Time::kMicrosecondsPerSecond /
  335. output_params_.sample_rate());
  336. }
  337. // Note: It's acceptable for |next_generate_time_| to be slightly before |now|
  338. // due to integer truncation behaviors in the math above. The timer task
  339. // started below will just run immediately and there will be no harmful
  340. // effects in the next GenerateMoreAudio() call. http://crbug.com/847487
  341. timer_->Start(FROM_HERE, next_generate_time_, this,
  342. &FlowNetwork::GenerateMoreAudio, base::ExactDeadline(true));
  343. }
  344. } // namespace audio