audio_pump.cc 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "remoting/protocol/audio_pump.h"
  5. #include <memory>
  6. #include <utility>
  7. #include "base/bind.h"
  8. #include "base/check_op.h"
  9. #include "base/location.h"
  10. #include "base/notreached.h"
  11. #include "base/task/single_thread_task_runner.h"
  12. #include "base/threading/thread_task_runner_handle.h"
  13. #include "media/base/audio_bus.h"
  14. #include "media/base/audio_sample_types.h"
  15. #include "media/base/channel_layout.h"
  16. #include "media/base/channel_mixer.h"
  17. #include "remoting/codec/audio_encoder.h"
  18. #include "remoting/proto/audio.pb.h"
  19. #include "remoting/protocol/audio_source.h"
  20. #include "remoting/protocol/audio_stub.h"
  21. namespace {
  22. int CalculateFrameCount(const remoting::AudioPacket& packet) {
  23. return packet.data(0).size() / packet.channels() / packet.bytes_per_sample();
  24. }
  25. std::unique_ptr<media::AudioBus> AudioPacketToAudioBus(
  26. const remoting::AudioPacket& packet) {
  27. const int frame_count = CalculateFrameCount(packet);
  28. DCHECK_GT(frame_count, 0);
  29. std::unique_ptr<media::AudioBus> result =
  30. media::AudioBus::Create(packet.channels(), frame_count);
  31. result->FromInterleaved<media::SignedInt16SampleTypeTraits>(
  32. reinterpret_cast<const int16_t*>(packet.data(0).data()), frame_count);
  33. return result;
  34. }
  35. std::unique_ptr<remoting::AudioPacket> AudioBusToAudioPacket(
  36. const media::AudioBus& packet) {
  37. std::unique_ptr<remoting::AudioPacket> result =
  38. std::make_unique<remoting::AudioPacket>();
  39. result->add_data()->resize(
  40. packet.channels() * packet.frames() * sizeof(int16_t));
  41. packet.ToInterleaved<media::SignedInt16SampleTypeTraits>(
  42. packet.frames(),
  43. reinterpret_cast<int16_t*>(&(result->mutable_data(0)->at(0))));
  44. result->set_encoding(remoting::AudioPacket::ENCODING_RAW);
  45. result->set_channels(
  46. static_cast<remoting::AudioPacket::Channels>(packet.channels()));
  47. result->set_bytes_per_sample(remoting::AudioPacket::BYTES_PER_SAMPLE_2);
  48. return result;
  49. }
  50. media::ChannelLayout RetrieveLayout(const remoting::AudioPacket& packet) {
  51. // This switch should match AudioPacket::Channels enum in audio.proto.
  52. switch (packet.channels()) {
  53. case remoting::AudioPacket::CHANNELS_INVALID:
  54. return media::CHANNEL_LAYOUT_UNSUPPORTED;
  55. case remoting::AudioPacket::CHANNELS_MONO:
  56. return media::CHANNEL_LAYOUT_MONO;
  57. case remoting::AudioPacket::CHANNELS_STEREO:
  58. return media::CHANNEL_LAYOUT_STEREO;
  59. case remoting::AudioPacket::CHANNELS_SURROUND:
  60. return media::CHANNEL_LAYOUT_SURROUND;
  61. case remoting::AudioPacket::CHANNELS_4_0:
  62. return media::CHANNEL_LAYOUT_4_0;
  63. case remoting::AudioPacket::CHANNELS_4_1:
  64. return media::CHANNEL_LAYOUT_4_1;
  65. case remoting::AudioPacket::CHANNELS_5_1:
  66. return media::CHANNEL_LAYOUT_5_1;
  67. case remoting::AudioPacket::CHANNELS_6_1:
  68. return media::CHANNEL_LAYOUT_6_1;
  69. case remoting::AudioPacket::CHANNELS_7_1:
  70. return media::CHANNEL_LAYOUT_7_1;
  71. }
  72. NOTREACHED() << "Invalid AudioPacket::Channels";
  73. return media::CHANNEL_LAYOUT_UNSUPPORTED;
  74. }
  75. } // namespace
  76. namespace remoting {
  77. namespace protocol {
  78. // Limit the data stored in the pending send buffers to 250ms.
  79. const int kMaxBufferedIntervalMs = 250;
  80. class AudioPump::Core {
  81. public:
  82. Core(base::WeakPtr<AudioPump> pump,
  83. std::unique_ptr<AudioSource> audio_source,
  84. std::unique_ptr<AudioEncoder> audio_encoder);
  85. Core(const Core&) = delete;
  86. Core& operator=(const Core&) = delete;
  87. ~Core();
  88. void Start();
  89. void Pause(bool pause);
  90. void OnPacketSent(int size);
  91. private:
  92. std::unique_ptr<AudioPacket> Downmix(std::unique_ptr<AudioPacket> packet);
  93. void EncodeAudioPacket(std::unique_ptr<AudioPacket> packet);
  94. base::ThreadChecker thread_checker_;
  95. base::WeakPtr<AudioPump> pump_;
  96. scoped_refptr<base::SingleThreadTaskRunner> pump_task_runner_;
  97. std::unique_ptr<AudioSource> audio_source_;
  98. std::unique_ptr<AudioEncoder> audio_encoder_;
  99. bool enabled_;
  100. // Number of bytes in the queue that have been encoded but haven't been sent
  101. // yet.
  102. int bytes_pending_;
  103. std::unique_ptr<media::ChannelMixer> mixer_;
  104. media::ChannelLayout mixer_input_layout_ = media::CHANNEL_LAYOUT_NONE;
  105. };
  106. AudioPump::Core::Core(base::WeakPtr<AudioPump> pump,
  107. std::unique_ptr<AudioSource> audio_source,
  108. std::unique_ptr<AudioEncoder> audio_encoder)
  109. : pump_(pump),
  110. pump_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  111. audio_source_(std::move(audio_source)),
  112. audio_encoder_(std::move(audio_encoder)),
  113. enabled_(true),
  114. bytes_pending_(0) {
  115. thread_checker_.DetachFromThread();
  116. }
  117. AudioPump::Core::~Core() {
  118. DCHECK(thread_checker_.CalledOnValidThread());
  119. }
  120. void AudioPump::Core::Start() {
  121. DCHECK(thread_checker_.CalledOnValidThread());
  122. audio_source_->Start(
  123. base::BindRepeating(&Core::EncodeAudioPacket, base::Unretained(this)));
  124. }
  125. void AudioPump::Core::Pause(bool pause) {
  126. DCHECK(thread_checker_.CalledOnValidThread());
  127. enabled_ = !pause;
  128. }
  129. void AudioPump::Core::OnPacketSent(int size) {
  130. DCHECK(thread_checker_.CalledOnValidThread());
  131. bytes_pending_ -= size;
  132. DCHECK_GE(bytes_pending_, 0);
  133. }
  134. void AudioPump::Core::EncodeAudioPacket(std::unique_ptr<AudioPacket> packet) {
  135. DCHECK(thread_checker_.CalledOnValidThread());
  136. DCHECK(packet);
  137. int max_buffered_bytes =
  138. audio_encoder_->GetBitrate() * kMaxBufferedIntervalMs / 1000 / 8;
  139. if (!enabled_ || bytes_pending_ > max_buffered_bytes) {
  140. return;
  141. }
  142. if (packet->channels() > AudioPacket::CHANNELS_STEREO) {
  143. packet = Downmix(std::move(packet));
  144. }
  145. std::unique_ptr<AudioPacket> encoded_packet =
  146. audio_encoder_->Encode(std::move(packet));
  147. // The audio encoder returns a null audio packet if there's no audio to send.
  148. if (!encoded_packet) {
  149. return;
  150. }
  151. int packet_size = encoded_packet->ByteSize();
  152. bytes_pending_ += packet_size;
  153. pump_task_runner_->PostTask(
  154. FROM_HERE, base::BindOnce(&AudioPump::SendAudioPacket, pump_,
  155. std::move(encoded_packet), packet_size));
  156. }
  157. std::unique_ptr<AudioPacket> AudioPump::Core::Downmix(
  158. std::unique_ptr<AudioPacket> packet) {
  159. DCHECK(thread_checker_.CalledOnValidThread());
  160. DCHECK(packet);
  161. DCHECK_EQ(packet->data_size(), 1);
  162. DCHECK_EQ(packet->bytes_per_sample(), AudioPacket::BYTES_PER_SAMPLE_2);
  163. const media::ChannelLayout input_layout = RetrieveLayout(*packet);
  164. DCHECK_NE(input_layout, media::CHANNEL_LAYOUT_UNSUPPORTED);
  165. DCHECK_NE(input_layout, media::CHANNEL_LAYOUT_MONO);
  166. DCHECK_NE(input_layout, media::CHANNEL_LAYOUT_STEREO);
  167. if (!mixer_ || mixer_input_layout_ != input_layout) {
  168. mixer_input_layout_ = input_layout;
  169. mixer_ = std::make_unique<media::ChannelMixer>(
  170. input_layout, media::CHANNEL_LAYOUT_STEREO);
  171. }
  172. std::unique_ptr<media::AudioBus> input = AudioPacketToAudioBus(*packet);
  173. DCHECK(input);
  174. std::unique_ptr<media::AudioBus> output =
  175. media::AudioBus::Create(AudioPacket::CHANNELS_STEREO, input->frames());
  176. mixer_->Transform(input.get(), output.get());
  177. std::unique_ptr<AudioPacket> result = AudioBusToAudioPacket(*output);
  178. result->set_sampling_rate(packet->sampling_rate());
  179. return result;
  180. }
  181. AudioPump::AudioPump(
  182. scoped_refptr<base::SingleThreadTaskRunner> audio_task_runner,
  183. std::unique_ptr<AudioSource> audio_source,
  184. std::unique_ptr<AudioEncoder> audio_encoder,
  185. AudioStub* audio_stub)
  186. : audio_task_runner_(audio_task_runner), audio_stub_(audio_stub) {
  187. DCHECK(audio_stub_);
  188. core_ =
  189. std::make_unique<Core>(weak_factory_.GetWeakPtr(),
  190. std::move(audio_source), std::move(audio_encoder));
  191. audio_task_runner_->PostTask(
  192. FROM_HERE, base::BindOnce(&Core::Start, base::Unretained(core_.get())));
  193. }
  194. AudioPump::~AudioPump() {
  195. DCHECK(thread_checker_.CalledOnValidThread());
  196. audio_task_runner_->DeleteSoon(FROM_HERE, core_.release());
  197. }
  198. void AudioPump::Pause(bool pause) {
  199. DCHECK(thread_checker_.CalledOnValidThread());
  200. audio_task_runner_->PostTask(
  201. FROM_HERE,
  202. base::BindOnce(&Core::Pause, base::Unretained(core_.get()), pause));
  203. }
  204. void AudioPump::SendAudioPacket(std::unique_ptr<AudioPacket> packet, int size) {
  205. DCHECK(thread_checker_.CalledOnValidThread());
  206. DCHECK(packet);
  207. audio_stub_->ProcessAudioPacket(
  208. std::move(packet), base::BindOnce(&AudioPump::OnPacketSent,
  209. weak_factory_.GetWeakPtr(), size));
  210. }
  211. void AudioPump::OnPacketSent(int size) {
  212. audio_task_runner_->PostTask(
  213. FROM_HERE,
  214. base::BindOnce(&Core::OnPacketSent, base::Unretained(core_.get()), size));
  215. }
  216. } // namespace protocol
  217. } // namespace remoting