snooper_node.h 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef SERVICES_AUDIO_SNOOPER_NODE_H_
  5. #define SERVICES_AUDIO_SNOOPER_NODE_H_
  6. #include <limits>
  7. #include <memory>
  8. #include "base/synchronization/lock.h"
  9. #include "base/time/time.h"
  10. #include "media/base/audio_parameters.h"
  11. #include "media/base/channel_mixer.h"
  12. #include "media/base/multi_channel_resampler.h"
  13. #include "services/audio/delay_buffer.h"
  14. #include "services/audio/loopback_group_member.h"
  15. #include "third_party/abseil-cpp/absl/types/optional.h"
  16. namespace media {
  17. class AudioBus;
  18. } // namespace media
  19. namespace audio {
  20. // Thread-safe implementation of Snooper that records the audio from a
  21. // GroupMember on one thread, and re-renders it to the desired output format on
  22. // another thread. Since the data flow rates are known to be driven by different
  23. // clocks (audio hardware clock versus system clock), the base::TimeTicks
  24. // reference clock is used to detect drift and automatically correct for it to
  25. // maintain proper synchronization.
  26. //
  27. // Throughout this class, there are sample counters (in terms of the input
  28. // audio's sample rate) that are tracked/computed. They refer to the media
  29. // timestamp of the audio flowing through specific parts of the processing
  30. // pipeline: inbound from OnData() calls → through the delay buffer → through
  31. // the resampler → and outbound via Render() calls:
  32. //
  33. // write position: The position of audio about to be written into the delay
  34. // buffer. This is managed by OnData().
  35. // read position: The position of audio about to be read from the delay
  36. // buffer and pushed into the resampler. This is managed by
  37. // ReadFromDelayBuffer().
  38. // output position: The position of the audio about to come out of the
  39. // resampler. This is computed within Render(). Note that
  40. // this is a "virtual" position since it is in terms of the
  41. // input audio's sample count, but refers to audio about to
  42. // be generated in the output format (with a possibly
  43. // different sample rate).
  44. //
  45. // Note that the media timestamps represented by the "positions," as well as the
  46. // surrounding math operations, might seem backwards; but they are not. This is
  47. // because the inbound audio is from a source that pre-renders audio for playout
  48. // in the near future, while the outbound audio is audio that would have been
  49. // played-out in the recent past.
  50. class SnooperNode final : public LoopbackGroupMember::Snooper {
  51. public:
  52. // Use sample counts as a precise measure of audio signal position and time
  53. // duration.
  54. using FrameTicks = int64_t;
  55. // Contruct a SnooperNode that buffers input of one format and renders output
  56. // in [possibly] another format.
  57. SnooperNode(const media::AudioParameters& input_params,
  58. const media::AudioParameters& output_params);
  59. SnooperNode(const SnooperNode&) = delete;
  60. SnooperNode& operator=(const SnooperNode&) = delete;
  61. ~SnooperNode() final;
  62. // GroupMember::Snooper implementation. Inserts more data into the delay
  63. // buffer.
  64. void OnData(const media::AudioBus& input_bus,
  65. base::TimeTicks reference_time,
  66. double volume) final;
  67. // Given the timing of recent OnData() calls and the |duration| of output that
  68. // would be requested in a call to Render(), determine the latest possible
  69. // |reference_time| for a Render() call that won't result in an underrun.
  70. // Returns absl::nullopt while current conditions prohibit making a reliable
  71. // suggestion.
  72. absl::optional<base::TimeTicks> SuggestLatestRenderTime(FrameTicks duration);
  73. // Renders more audio that was recorded from the GroupMember until
  74. // |output_bus| is filled, resampling and remixing the channels if necessary.
  75. // |reference_time| is used for detecting skip-ahead (i.e., a significant
  76. // forward jump in the reference time) and also to maintain synchronization
  77. // with the input.
  78. void Render(base::TimeTicks reference_time, media::AudioBus* output_bus);
  79. private:
  80. // Helper to store the new |correction_fps|, recompute the resampling I/O
  81. // ratio, and reconfigure the resampler with the new ratio.
  82. void UpdateCorrectionRate(int correction_fps);
  83. // Called by the MultiChannelResampler to acquire more data from the delay
  84. // buffer. This is invoked in the same call stack (and thread) as Render(),
  85. // zero or more times as data is needed by the resampler.
  86. void ReadFromDelayBuffer(int ignored, media::AudioBus* resampler_bus);
  87. // Input and output audio parameters.
  88. const media::AudioParameters input_params_;
  89. const media::AudioParameters output_params_;
  90. // Input and output AudioBus time durations, pre-computed from the input and
  91. // output AudioParameters.
  92. const base::TimeDelta input_bus_duration_;
  93. const base::TimeDelta output_bus_duration_;
  94. // The ratio between the input sampling rate and the output sampling rate. It
  95. // is "perfect" because it assumes no clock skew. Corrections are applied to
  96. // this to determine the actual resampler I/O ratio.
  97. const double perfect_io_ratio_;
  98. // Protects concurrent access to |buffer_| and the |write_position_| and
  99. // |write_reference_time_|. All other members are either read-only, or are not
  100. // accessed by multiple threads.
  101. base::Lock lock_;
  102. // Allows input data to be recorded and then read-back from any position
  103. // later (by the resampler).
  104. DelayBuffer buffer_; // Guarded by |lock_|.
  105. // The next frame position at which to write into the delay buffer, and the
  106. // TimeTicks representing its corresponding system clock timestamp.
  107. FrameTicks write_position_; // Guarded by |lock_|.
  108. base::TimeTicks write_reference_time_; // Guarded by |lock_|.
  109. // Used by SuggestLatestRenderTime() to track whether OnData() has been called
  110. // recently, and as a basis for its suggestion. Other methods should not
  111. // depend on this value for anything.
  112. base::TimeTicks checkpoint_time_;
  113. // The next frame position from which to read from the delay buffer. This is
  114. // the position of the frames about to be pushed into the resampler, not the
  115. // position of frames about to be Render()'ed.
  116. FrameTicks read_position_;
  117. // The expected |reference_time| to be provided in the next call to Render().
  118. // This is used to detect skip-ahead in the output, and compensate when
  119. // necessary.
  120. base::TimeTicks render_reference_time_;
  121. // The additional number of frames currently being consumed by the resampler
  122. // each second to correct for drift.
  123. int correction_fps_;
  124. // Resamples input audio that is read from the delay buffer. Even if the input
  125. // and output have the same sampling rate, this is used to subtly stretch the
  126. // audio signal to correct for drift.
  127. media::MultiChannelResampler resampler_;
  128. // Specifies whether channel mixing should occur before or after resampling,
  129. // or is not needed. The strategy is chosen such that the minimal number of
  130. // channels are resampled, as resampling is the more-expensive operation.
  131. enum { kBefore, kAfter, kNone } const channel_mix_strategy_;
  132. // Only used when the input channel layout differs from the output.
  133. media::ChannelMixer channel_mixer_;
  134. // Only allocated when using the channel mixer. When using the kAfter
  135. // strategy, it is allocated just once, in the constructor, since its frame
  136. // length is constant. When using the kBefore strategy, it is re-allocated
  137. // whenever a larger one is needed and is reused thereafter.
  138. std::unique_ptr<media::AudioBus> mix_bus_;
  139. // An impossible value re-purposed to represent the "null" or "not set yet"
  140. // condition for |read_position_| and |write_position_|.
  141. static constexpr FrameTicks kNullPosition =
  142. std::numeric_limits<FrameTicks>::min();
  143. // The frame position where recording into the delay buffer always starts.
  144. static constexpr FrameTicks kWriteStartPosition = 0;
  145. };
  146. } // namespace audio
  147. #endif // SERVICES_AUDIO_SNOOPER_NODE_H_