audio_converter.cc 10.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. //
  5. // AudioConverter implementation. Uses MultiChannelSincResampler for resampling
  6. // audio, ChannelMixer for channel mixing, and AudioPullFifo for buffering.
  7. //
  8. // Delay estimates are provided to InputCallbacks based on the frame delay
  9. // information reported via the resampler and FIFO units.
  10. #include "media/base/audio_converter.h"
  11. #include <algorithm>
  12. #include <memory>
  13. #include "base/bind.h"
  14. #include "base/callback_helpers.h"
  15. #include "base/logging.h"
  16. #include "base/trace_event/trace_event.h"
  17. #include "media/base/audio_bus.h"
  18. #include "media/base/audio_pull_fifo.h"
  19. #include "media/base/channel_mixer.h"
  20. #include "media/base/multi_channel_resampler.h"
  21. #include "media/base/vector_math.h"
  22. namespace media {
  23. AudioConverter::AudioConverter(const AudioParameters& input_params,
  24. const AudioParameters& output_params,
  25. bool disable_fifo)
  26. : chunk_size_(input_params.frames_per_buffer()),
  27. downmix_early_(false),
  28. initial_frames_delayed_(0),
  29. resampler_frames_delayed_(0),
  30. io_sample_rate_ratio_(input_params.sample_rate() /
  31. static_cast<double>(output_params.sample_rate())),
  32. input_channel_count_(input_params.channels()) {
  33. CHECK(input_params.IsValid());
  34. CHECK(output_params.IsValid());
  35. // Handle different input and output channel layouts.
  36. if (input_params.channel_layout() != output_params.channel_layout() ||
  37. input_params.channels() != output_params.channels()) {
  38. DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout()
  39. << " to " << output_params.channel_layout() << "; from "
  40. << input_params.channels() << " channels to "
  41. << output_params.channels() << " channels.";
  42. channel_mixer_ =
  43. std::make_unique<ChannelMixer>(input_params, output_params);
  44. // Pare off data as early as we can for efficiency.
  45. downmix_early_ = input_params.channels() > output_params.channels();
  46. }
  47. // Only resample if necessary since it's expensive.
  48. if (input_params.sample_rate() != output_params.sample_rate()) {
  49. DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to "
  50. << output_params.sample_rate();
  51. const int request_size = disable_fifo ? SincResampler::kDefaultRequestSize :
  52. input_params.frames_per_buffer();
  53. resampler_ = std::make_unique<MultiChannelResampler>(
  54. downmix_early_ ? output_params.channels() : input_params.channels(),
  55. io_sample_rate_ratio_, request_size,
  56. base::BindRepeating(&AudioConverter::ProvideInput,
  57. base::Unretained(this)));
  58. }
  59. // The resampler can be configured to work with a specific request size, so a
  60. // FIFO is not necessary when resampling.
  61. if (disable_fifo || resampler_)
  62. return;
  63. // Since the output device may want a different buffer size than the caller
  64. // asked for, we need to use a FIFO to ensure that both sides read in chunk
  65. // sizes they're configured for.
  66. if (input_params.frames_per_buffer() != output_params.frames_per_buffer()) {
  67. DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer()
  68. << " to " << output_params.frames_per_buffer();
  69. chunk_size_ = input_params.frames_per_buffer();
  70. audio_fifo_ = std::make_unique<AudioPullFifo>(
  71. downmix_early_ ? output_params.channels() : input_params.channels(),
  72. chunk_size_,
  73. base::BindRepeating(&AudioConverter::SourceCallback,
  74. base::Unretained(this)));
  75. }
  76. }
  77. AudioConverter::~AudioConverter() = default;
  78. void AudioConverter::AddInput(InputCallback* input) {
  79. DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) ==
  80. transform_inputs_.end());
  81. transform_inputs_.push_back(input);
  82. }
  83. void AudioConverter::RemoveInput(InputCallback* input) {
  84. DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) !=
  85. transform_inputs_.end());
  86. transform_inputs_.remove(input);
  87. if (transform_inputs_.empty())
  88. Reset();
  89. }
  90. void AudioConverter::Reset() {
  91. if (audio_fifo_)
  92. audio_fifo_->Clear();
  93. if (resampler_)
  94. resampler_->Flush();
  95. }
  96. int AudioConverter::ChunkSize() const {
  97. if (!resampler_)
  98. return chunk_size_;
  99. return resampler_->ChunkSize();
  100. }
  101. void AudioConverter::PrimeWithSilence() {
  102. if (resampler_) {
  103. resampler_->PrimeWithSilence();
  104. }
  105. }
  106. int AudioConverter::GetMaxInputFramesRequested(int output_frames_requested) {
  107. return resampler_
  108. ? resampler_->GetMaxInputFramesRequested(output_frames_requested)
  109. : output_frames_requested;
  110. }
  111. void AudioConverter::ConvertWithDelay(uint32_t initial_frames_delayed,
  112. AudioBus* dest) {
  113. initial_frames_delayed_ = initial_frames_delayed;
  114. if (transform_inputs_.empty()) {
  115. dest->Zero();
  116. return;
  117. }
  118. // Determine if channel mixing should be done and if it should be done before
  119. // or after resampling. If it's possible to reduce the channel count prior to
  120. // resampling we can save a lot of processing time. Vice versa, we don't want
  121. // to increase the channel count prior to resampling for the same reason.
  122. bool needs_mixing = channel_mixer_ && !downmix_early_;
  123. if (needs_mixing)
  124. CreateUnmixedAudioIfNecessary(dest->frames());
  125. AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest;
  126. DCHECK(temp_dest);
  127. // Figure out which method to call based on whether we're resampling and
  128. // rebuffering, just resampling, or just mixing. We want to avoid any extra
  129. // steps when possible since we may be converting audio data in real time.
  130. if (!resampler_ && !audio_fifo_) {
  131. SourceCallback(0, temp_dest);
  132. } else {
  133. if (resampler_)
  134. resampler_->Resample(temp_dest->frames(), temp_dest);
  135. else
  136. ProvideInput(0, temp_dest);
  137. }
  138. // Finally upmix the channels if we didn't do so earlier.
  139. if (needs_mixing) {
  140. DCHECK_EQ(temp_dest->frames(), dest->frames());
  141. channel_mixer_->Transform(temp_dest, dest);
  142. }
  143. }
  144. void AudioConverter::Convert(AudioBus* dest) {
  145. TRACE_EVENT1("audio", "AudioConverter::Convert", "sample rate ratio",
  146. io_sample_rate_ratio_);
  147. ConvertWithDelay(0, dest);
  148. }
  149. void AudioConverter::SourceCallback(int fifo_frame_delay, AudioBus* dest) {
  150. TRACE_EVENT1("audio", "AudioConverter::SourceCallback", "fifo frame delay",
  151. fifo_frame_delay);
  152. const bool needs_downmix = channel_mixer_ && downmix_early_;
  153. if (!mixer_input_audio_bus_ ||
  154. mixer_input_audio_bus_->frames() != dest->frames()) {
  155. mixer_input_audio_bus_ =
  156. AudioBus::Create(input_channel_count_, dest->frames());
  157. }
  158. // If we're downmixing early we need a temporary AudioBus which matches
  159. // the the input channel count and input frame size since we're passing
  160. // |unmixed_audio_| directly to the |source_callback_|.
  161. if (needs_downmix)
  162. CreateUnmixedAudioIfNecessary(dest->frames());
  163. AudioBus* const temp_dest = needs_downmix ? unmixed_audio_.get() : dest;
  164. // Sanity check our inputs.
  165. DCHECK_EQ(temp_dest->frames(), mixer_input_audio_bus_->frames());
  166. DCHECK_EQ(temp_dest->channels(), mixer_input_audio_bus_->channels());
  167. // |total_frames_delayed| is reported to the *input* source in terms of the
  168. // *input* sample rate. |initial_frames_delayed_| is given in terms of the
  169. // output sample rate, so we scale by sample rate ratio (in/out).
  170. uint32_t total_frames_delayed =
  171. std::round(initial_frames_delayed_ * io_sample_rate_ratio_);
  172. if (resampler_) {
  173. // |resampler_frames_delayed_| tallies frames queued up inside the resampler
  174. // that are already converted to the output format. Scale by ratio to get
  175. // delay in terms of input sample rate.
  176. total_frames_delayed +=
  177. std::round(resampler_frames_delayed_ * io_sample_rate_ratio_);
  178. }
  179. if (audio_fifo_) {
  180. total_frames_delayed += fifo_frame_delay;
  181. }
  182. // If we only have a single input, avoid an extra copy.
  183. AudioBus* const provide_input_dest =
  184. transform_inputs_.size() == 1 ? temp_dest : mixer_input_audio_bus_.get();
  185. // Have each mixer render its data into an output buffer then mix the result.
  186. for (auto* input : transform_inputs_) {
  187. const float volume =
  188. input->ProvideInput(provide_input_dest, total_frames_delayed);
  189. // Optimize the most common single input, full volume case.
  190. if (input == transform_inputs_.front()) {
  191. if (volume == 1.0f) {
  192. if (temp_dest != provide_input_dest)
  193. provide_input_dest->CopyTo(temp_dest);
  194. } else if (volume > 0) {
  195. for (int i = 0; i < provide_input_dest->channels(); ++i) {
  196. vector_math::FMUL(
  197. provide_input_dest->channel(i), volume,
  198. provide_input_dest->frames(), temp_dest->channel(i));
  199. }
  200. } else {
  201. // Zero |temp_dest| otherwise, so we're mixing into a clean buffer.
  202. temp_dest->Zero();
  203. }
  204. continue;
  205. }
  206. // Volume adjust and mix each mixer input into |temp_dest| after rendering.
  207. if (volume > 0) {
  208. for (int i = 0; i < mixer_input_audio_bus_->channels(); ++i) {
  209. vector_math::FMAC(
  210. mixer_input_audio_bus_->channel(i), volume,
  211. mixer_input_audio_bus_->frames(), temp_dest->channel(i));
  212. }
  213. }
  214. }
  215. if (needs_downmix) {
  216. DCHECK_EQ(temp_dest->frames(), dest->frames());
  217. channel_mixer_->Transform(temp_dest, dest);
  218. }
  219. }
  220. void AudioConverter::ProvideInput(int resampler_frame_delay, AudioBus* dest) {
  221. TRACE_EVENT1("audio", "AudioConverter::ProvideInput", "resampler frame delay",
  222. resampler_frame_delay);
  223. resampler_frames_delayed_ = resampler_frame_delay;
  224. if (audio_fifo_)
  225. audio_fifo_->Consume(dest, dest->frames());
  226. else
  227. SourceCallback(0, dest);
  228. }
  229. void AudioConverter::CreateUnmixedAudioIfNecessary(int frames) {
  230. if (!unmixed_audio_ || unmixed_audio_->frames() != frames)
  231. unmixed_audio_ = AudioBus::Create(input_channel_count_, frames);
  232. }
  233. } // namespace media