audio_renderer_algorithm.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. // AudioRendererAlgorithm buffers and transforms audio data. The owner of
  5. // this object provides audio data to the object through EnqueueBuffer() and
  6. // requests data from the buffer via FillBuffer().
  7. //
  8. // This class is *not* thread-safe. Calls to enqueue and retrieve data must be
  9. // locked if called from multiple threads.
  10. //
  11. // AudioRendererAlgorithm uses the Waveform Similarity Overlap and Add (WSOLA)
  12. // algorithm to stretch or compress audio data to meet playback speeds less than
  13. // or greater than the natural playback of the audio stream. The algorithm
  14. // preserves local properties of the audio, therefore, pitch and harmonics are
  15. // are preserved. See audio_renderer_algorith.cc for a more elaborate
  16. // description of the algorithm.
  17. //
  18. // Audio at very low or very high playback rates are muted to preserve quality.
  19. #ifndef MEDIA_FILTERS_AUDIO_RENDERER_ALGORITHM_H_
  20. #define MEDIA_FILTERS_AUDIO_RENDERER_ALGORITHM_H_
  21. #include <stdint.h>
  22. #include <memory>
  23. #include <vector>
  24. #include "base/memory/raw_ptr.h"
  25. #include "base/memory/ref_counted.h"
  26. #include "base/time/time.h"
  27. #include "media/base/audio_buffer.h"
  28. #include "media/base/audio_buffer_queue.h"
  29. #include "media/base/audio_parameters.h"
  30. #include "media/base/media_log.h"
  31. #include "media/base/multi_channel_resampler.h"
  32. #include "third_party/abseil-cpp/absl/types/optional.h"
  33. namespace media {
  34. class AudioBus;
  35. class MEDIA_EXPORT AudioRendererAlgorithm {
  36. public:
  37. AudioRendererAlgorithm(MediaLog* media_log);
  38. AudioRendererAlgorithm(MediaLog* media_log,
  39. AudioRendererAlgorithmParameters params);
  40. AudioRendererAlgorithm(const AudioRendererAlgorithm&) = delete;
  41. AudioRendererAlgorithm& operator=(const AudioRendererAlgorithm&) = delete;
  42. ~AudioRendererAlgorithm();
  43. // Initializes this object with information about the audio stream.
  44. void Initialize(const AudioParameters& params, bool is_encrypted);
  45. // Allows clients to specify which channels will be considered by the
  46. // algorithm when adapting for playback rate, other channels will be muted.
  47. // Useful to avoid performance overhead of the adapatation algorithm. Must
  48. // only be called after Initialize(); may be called multiple times if the
  49. // mask changes.
  50. //
  51. // E.g., If |channel_mask| is [true, false] only the first channel will be
  52. // used to construct the playback rate adapated signal. This is useful if
  53. // channel upmixing has been performed prior to this point.
  54. void SetChannelMask(std::vector<bool> channel_mask);
  55. // Tries to fill |requested_frames| frames into |dest| with possibly scaled
  56. // data from our |audio_buffer_|. Data is scaled based on |playback_rate|,
  57. // using a variation of the Overlap-Add method to combine sample windows.
  58. //
  59. // Data from |audio_buffer_| is consumed in proportion to the playback rate.
  60. //
  61. // |dest_offset| is the offset in frames for writing into |dest|.
  62. //
  63. // Returns the number of frames copied into |dest|.
  64. int FillBuffer(AudioBus* dest,
  65. int dest_offset,
  66. int requested_frames,
  67. double playback_rate);
  68. // Clears |audio_buffer_|.
  69. void FlushBuffers();
  70. // Enqueues a buffer. It is called from the owner of the algorithm after a
  71. // read completes.
  72. void EnqueueBuffer(scoped_refptr<AudioBuffer> buffer_in);
  73. // Sets a target queue latency. This target will be clamped and stored in
  74. // |playback_threshold_|. It may also cause an increase in |capacity_|. A
  75. // value of nullopt indicates the algorithm should restore the default value.
  76. void SetLatencyHint(absl::optional<base::TimeDelta> latency_hint);
  77. // Sets a flag indicating whether apply pitch adjustments when playing back
  78. // at rates other than 1.0. Concretely, we use WSOLA when this is true, and
  79. // resampling when this is false.
  80. void SetPreservesPitch(bool preserves_pitch);
  81. // Returns true if the |audio_buffer_| is >= |playback_threshold_|.
  82. bool IsQueueAdequateForPlayback();
  83. // Returns the required size for |audio_buffer_| to be "adequate for
  84. // playback". See IsQueueAdequateForPlayback().
  85. int QueuePlaybackThreshold() const { return playback_threshold_; }
  86. // Returns true if |audio_buffer_| is >= |capacity_|.
  87. bool IsQueueFull();
  88. // Returns the capacity of |audio_buffer_| in frames.
  89. int QueueCapacity() const { return capacity_; }
  90. // Increase the |playback_threshold_| and |capacity_| of |audio_buffer_| if
  91. // possible. Should not be called if a custom |playback_threshold_| was
  92. // specified.
  93. void IncreasePlaybackThreshold();
  94. // Sets a flag to bypass underflow detection, to read out all remaining data.
  95. void MarkEndOfStream();
  96. // Returns an estimate of the amount of memory (in bytes) used for frames.
  97. int64_t GetMemoryUsage() const;
  98. // Returns the total number of frames in |audio_buffer_| as well as
  99. // unconsumed input frames in the |resampler_|. The returned value may be
  100. // larger than QueueCapacity() in the event that EnqueueBuffer() delivered
  101. // more data than |audio_buffer_| was intending to hold.
  102. int BufferedFrames() const;
  103. // Returns the effective delay in output frames at the given |playback rate|.
  104. // Effectively this tells the caller, if new audio is enqueued via
  105. // EnqueueBuffer(), how many frames must be read via FillBuffer() at the
  106. // |playback_rate| before the new audio is read out. Note that this is
  107. // approximate, since due to WSOLA the audio output doesn't always directly
  108. // correspond to the audio input (some samples may be duplicated or skipped).
  109. double DelayInFrames(double playback_rate) const;
  110. // Returns the samples per second for this audio stream.
  111. int samples_per_second() const { return samples_per_second_; }
  112. std::vector<bool> channel_mask_for_testing() { return channel_mask_; }
  113. private:
  114. enum class FillBufferMode {
  115. kPassthrough,
  116. kResampler,
  117. kWSOLA,
  118. };
  119. // Remove buffered data that will be outdated if we switch fill mode.
  120. void SetFillBufferMode(FillBufferMode mode);
  121. // Within |search_block_|, find the block of data that is most similar to
  122. // |target_block_|, and write it in |optimal_block_|. This method assumes that
  123. // there is enough data to perform a search, i.e. |search_block_| and
  124. // |target_block_| can be extracted from the available frames.
  125. void GetOptimalBlock();
  126. // Read a maximum of |requested_frames| frames from |wsola_output_|. Returns
  127. // number of frames actually read.
  128. int WriteCompletedFramesTo(
  129. int requested_frames, int output_offset, AudioBus* dest);
  130. // Fill |dest| with frames from |audio_buffer_| starting from frame
  131. // |read_offset_frames|. |dest| is expected to have the same number of
  132. // channels as |audio_buffer_|. A negative offset, i.e.
  133. // |read_offset_frames| < 0, is accepted assuming that |audio_buffer| is zero
  134. // for negative indices. This might happen for few first frames. This method
  135. // assumes there is enough frames to fill |dest|, i.e. |read_offset_frames| +
  136. // |dest->frames()| does not extend to future.
  137. void PeekAudioWithZeroPrepend(int read_offset_frames, AudioBus* dest);
  138. // Run one iteration of WSOLA, if there are sufficient frames. This will
  139. // overlap-and-add one block to |wsola_output_|, hence, |num_complete_frames_|
  140. // is incremented by |ola_hop_size_|.
  141. bool RunOneWsolaIteration(double playback_rate);
  142. // Seek |audio_buffer_| forward to remove frames from input that are not used
  143. // any more. State of the WSOLA will be updated accordingly.
  144. void RemoveOldInputFrames(double playback_rate);
  145. // Update |output_time_| by |time_change|. In turn |search_block_index_| is
  146. // updated.
  147. void UpdateOutputTime(double playback_rate, double time_change);
  148. // Is |target_block_| fully within |search_block_|? If so, we don't need to
  149. // perform the search.
  150. bool TargetIsWithinSearchRegion() const;
  151. // Do we have enough data to perform one round of WSOLA?
  152. bool CanPerformWsola() const;
  153. // Creates or recreates |target_block_wrapper_| and |search_block_wrapper_|
  154. // after a |channel_mask_| change. May be called at anytime after a channel
  155. // mask has been specified.
  156. void CreateSearchWrappers();
  157. // Uses |resampler_| to speed up or slowdown audio, by using a resampling
  158. // ratio of |playback_rate|.
  159. int ResampleAndFill(AudioBus* dest,
  160. int dest_offset,
  161. int requested_frames,
  162. double playback_rate);
  163. // Called by |resampler_| to get more audio data.
  164. void OnResamplerRead(int frame_delay, AudioBus* audio_bus);
  165. raw_ptr<MediaLog> media_log_;
  166. // Parameters.
  167. AudioRendererAlgorithmParameters audio_renderer_algorithm_params_;
  168. // Number of channels in audio stream.
  169. int channels_;
  170. // Sample rate of audio stream.
  171. int samples_per_second_;
  172. // Is compressed audio output
  173. bool is_bitstream_format_;
  174. // Buffered audio data.
  175. AudioBufferQueue audio_buffer_;
  176. // Hint to adjust |playback_threshold_| as a means of controlling playback
  177. // start latency. See SetLatencyHint();
  178. absl::optional<base::TimeDelta> latency_hint_;
  179. // Whether to apply pitch adjusments or not when playing back at rates other
  180. // than 1.0. In other words, we use WSOLA to preserve pitch when this is on,
  181. // and resampling when this
  182. bool preserves_pitch_ = true;
  183. // How many frames to have in queue before beginning playback.
  184. int64_t playback_threshold_;
  185. // Minimum allowed value for |plabyack_threshold_| calculated by Initialize().
  186. int64_t min_playback_threshold_;
  187. // How many frames to have in the queue before we report the queue is full.
  188. int64_t capacity_;
  189. // Book keeping of the current time of generated audio, in frames. This
  190. // should be appropriately updated when out samples are generated, regardless
  191. // of whether we push samples out when FillBuffer() is called or we store
  192. // audio in |wsola_output_| for the subsequent calls to FillBuffer().
  193. // Furthermore, if samples from |audio_buffer_| are evicted then this
  194. // member variable should be updated based on |playback_rate_|.
  195. // Note that this member should be updated ONLY by calling UpdateOutputTime(),
  196. // so that |search_block_index_| is update accordingly.
  197. double output_time_;
  198. // The offset of the center frame of |search_block_| w.r.t. its first frame.
  199. int search_block_center_offset_;
  200. // Index of the beginning of the |search_block_|, in frames.
  201. int search_block_index_;
  202. // Number of Blocks to search to find the most similar one to the target
  203. // frame.
  204. int num_candidate_blocks_;
  205. // Index of the beginning of the target block, counted in frames.
  206. int target_block_index_;
  207. // Overlap-and-add window size in frames.
  208. int ola_window_size_;
  209. // The hop size of overlap-and-add in frames. This implementation assumes 50%
  210. // overlap-and-add.
  211. int ola_hop_size_;
  212. // Number of frames in |wsola_output_| that overlap-and-add is completed for
  213. // them and can be copied to output if FillBuffer() is called. It also
  214. // specifies the index where the next WSOLA window has to overlap-and-add.
  215. int num_complete_frames_;
  216. bool reached_end_of_stream_ = false;
  217. // Used to replace WSOLA algorithm at playback speeds close to 1.0. This is to
  218. // prevent noticeable audio artifacts introduced by WSOLA, at the expense of
  219. // changing the pitch of the audio.
  220. std::unique_ptr<MultiChannelResampler> resampler_;
  221. // True when the last call to OnResamplerRead() only gave silence to
  222. // |resampler_|. Used to determine whether or not we have played out all the
  223. // valid audio from |resampler.BufferedFrames()|.
  224. bool resampler_only_has_silence_ = false;
  225. // This stores a part of the output that is created but couldn't be rendered.
  226. // Output is generated frame-by-frame which at some point might exceed the
  227. // number of requested samples. Furthermore, due to overlap-and-add,
  228. // the last half-window of the output is incomplete, which is stored in this
  229. // buffer.
  230. std::unique_ptr<AudioBus> wsola_output_;
  231. // Overlap-and-add window.
  232. std::unique_ptr<float[]> ola_window_;
  233. // Transition window, used to update |optimal_block_| by a weighted sum of
  234. // |optimal_block_| and |target_block_|.
  235. std::unique_ptr<float[]> transition_window_;
  236. // Auxiliary variables to avoid allocation in every iteration.
  237. // Stores the optimal block in every iteration. This is the most
  238. // similar block to |target_block_| within |search_block_| and it is
  239. // overlap-and-added to |wsola_output_|.
  240. std::unique_ptr<AudioBus> optimal_block_;
  241. // A block of data that search is performed over to find the |optimal_block_|.
  242. std::unique_ptr<AudioBus> search_block_;
  243. // Stores the target block, denoted as |target| above. |search_block_| is
  244. // searched for a block (|optimal_block_|) that is most similar to
  245. // |target_block_|.
  246. std::unique_ptr<AudioBus> target_block_;
  247. // Active channels to consider while searching. Used to speed up WSOLA
  248. // processing by ignoring always muted channels. Wrappers are always
  249. // constructed during Initialize() and have <= |channels_|.
  250. std::vector<bool> channel_mask_;
  251. std::unique_ptr<AudioBus> search_block_wrapper_;
  252. std::unique_ptr<AudioBus> target_block_wrapper_;
  253. // The initial and maximum capacity calculated by Initialize().
  254. int64_t initial_capacity_;
  255. int64_t max_capacity_;
  256. FillBufferMode last_mode_ = FillBufferMode::kPassthrough;
  257. };
  258. } // namespace media
  259. #endif // MEDIA_FILTERS_AUDIO_RENDERER_ALGORITHM_H_