webm_muxer.h 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_MUXERS_WEBM_MUXER_H_
  5. #define MEDIA_MUXERS_WEBM_MUXER_H_
  6. #include <stdint.h>
  7. #include <memory>
  8. #include "base/callback.h"
  9. #include "base/containers/circular_deque.h"
  10. #include "base/numerics/safe_math.h"
  11. #include "base/sequence_checker.h"
  12. #include "base/strings/string_piece.h"
  13. #include "base/time/time.h"
  14. #include "base/timer/elapsed_timer.h"
  15. #include "media/base/audio_codecs.h"
  16. #include "media/base/media_export.h"
  17. #include "media/base/video_codecs.h"
  18. #include "media/base/video_color_space.h"
  19. #include "third_party/abseil-cpp/absl/types/optional.h"
  20. #include "third_party/libwebm/source/mkvmuxer.hpp"
  21. #include "ui/gfx/geometry/size.h"
  22. namespace gfx {
  23. class Size;
  24. } // namespace gfx
  25. namespace media {
  26. class VideoFrame;
  27. class AudioParameters;
  28. // Adapter class to manage a WebM container [1], a simplified version of a
  29. // Matroska container [2], composed of an EBML header, and a single Segment
  30. // including at least a Track Section and a number of SimpleBlocks each
  31. // containing a single encoded video or audio frame. WebM container has no
  32. // Trailer.
  33. // Clients will push encoded VPx or AV1 video frames and Opus or PCM audio
  34. // frames one by one via OnEncoded{Video|Audio}(). libwebm will eventually ping
  35. // the WriteDataCB passed on contructor with the wrapped encoded data.
  36. // WebmMuxer is designed for use on a single thread.
  37. // [1] http://www.webmproject.org/docs/container/
  38. // [2] http://www.matroska.org/technical/specs/index.html
  39. class MEDIA_EXPORT WebmMuxer {
  40. public:
  41. // Defines an interface for delegates of WebmMuxer which should define how to
  42. // implement the |mkvmuxer::IMkvWriter| APIs (e.g. whether to support
  43. // non-seekable live mode writing, or seekable file mode writing).
  44. class MEDIA_EXPORT Delegate : public mkvmuxer::IMkvWriter {
  45. public:
  46. Delegate();
  47. ~Delegate() override;
  48. base::TimeTicks last_data_output_timestamp() const {
  49. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  50. return last_data_output_timestamp_;
  51. }
  52. // Initializes the given |segment| according to the mode desired by the
  53. // concrete implementation of this delegate.
  54. virtual void InitSegment(mkvmuxer::Segment* segment) = 0;
  55. // mkvmuxer::IMkvWriter:
  56. mkvmuxer::int32 Write(const void* buf, mkvmuxer::uint32 len) final;
  57. protected:
  58. // Does the actual writing of |len| bytes from the given |buf| depending on
  59. // the mode desired by the concrete implementation of this delegate.
  60. // Returns 0 on success, -1 otherwise.
  61. virtual mkvmuxer::int32 DoWrite(const void* buf, mkvmuxer::uint32 len) = 0;
  62. SEQUENCE_CHECKER(sequence_checker_);
  63. // The current writing position as set by libwebm.
  64. base::CheckedNumeric<mkvmuxer::int64> position_
  65. GUARDED_BY_CONTEXT(sequence_checker_) = 0;
  66. // Last time data was written via Write().
  67. base::TimeTicks last_data_output_timestamp_
  68. GUARDED_BY_CONTEXT(sequence_checker_);
  69. };
  70. // Container for the parameters that muxer uses that is extracted from
  71. // media::VideoFrame.
  72. struct MEDIA_EXPORT VideoParameters {
  73. explicit VideoParameters(scoped_refptr<media::VideoFrame> frame);
  74. VideoParameters(gfx::Size visible_rect_size,
  75. double frame_rate,
  76. VideoCodec codec,
  77. absl::optional<gfx::ColorSpace> color_space);
  78. VideoParameters(const VideoParameters&);
  79. ~VideoParameters();
  80. gfx::Size visible_rect_size;
  81. double frame_rate;
  82. VideoCodec codec;
  83. absl::optional<gfx::ColorSpace> color_space;
  84. };
  85. // |audio_codec| should coincide with whatever is sent in OnEncodedAudio(),
  86. WebmMuxer(AudioCodec audio_codec,
  87. bool has_video_,
  88. bool has_audio_,
  89. std::unique_ptr<Delegate> delegate);
  90. WebmMuxer(const WebmMuxer&) = delete;
  91. WebmMuxer& operator=(const WebmMuxer&) = delete;
  92. ~WebmMuxer();
  93. // Sets the maximum duration interval to cause data output on
  94. // |write_data_callback|, provided frames are delivered. The WebM muxer can
  95. // hold on to audio frames almost indefinitely in the case video is recorded
  96. // and video frames are temporarily not delivered. When this method is used, a
  97. // new WebM cluster is forced when the next frame arrives |duration| after the
  98. // last write.
  99. // The maximum duration between forced clusters is internally limited to not
  100. // go below 100 ms.
  101. void SetMaximumDurationToForceDataOutput(base::TimeDelta interval);
  102. // Functions to add video and audio frames with |encoded_data.data()|
  103. // to WebM Segment. Either one returns true on success.
  104. // |encoded_alpha| represents the encode output of alpha channel when
  105. // available, can be nullptr otherwise.
  106. bool OnEncodedVideo(const VideoParameters& params,
  107. std::string encoded_data,
  108. std::string encoded_alpha,
  109. base::TimeTicks timestamp,
  110. bool is_key_frame);
  111. bool OnEncodedAudio(const media::AudioParameters& params,
  112. std::string encoded_data,
  113. base::TimeTicks timestamp);
  114. // WebmMuxer may hold on to data. Make sure it gets out on the next frame.
  115. void ForceDataOutputOnNextFrame();
  116. // Call to handle mute and tracks getting disabled.
  117. void SetLiveAndEnabled(bool track_live_and_enabled, bool is_video);
  118. void Pause();
  119. void Resume();
  120. // Drains and writes out all buffered frames and finalizes the segment.
  121. // Returns true on success, false otherwise.
  122. bool Flush();
  123. void ForceOneLibWebmErrorForTesting() { force_one_libwebm_error_ = true; }
  124. private:
  125. friend class WebmMuxerTest;
  126. // Methods for creating and adding video and audio tracks, called upon
  127. // receiving the first frame of a given Track.
  128. // AddVideoTrack adds |frame_size| and |frame_rate| to the Segment
  129. // info, although individual frames passed to OnEncodedVideo() can have any
  130. // frame size.
  131. void AddVideoTrack(const gfx::Size& frame_size,
  132. double frame_rate,
  133. const absl::optional<gfx::ColorSpace>& color_space);
  134. void AddAudioTrack(const media::AudioParameters& params);
  135. // Adds all currently buffered frames to the mkvmuxer in timestamp order,
  136. // until the queues are depleted.
  137. void FlushQueues();
  138. // Flushes out frames to the mkvmuxer while ensuring monotonically increasing
  139. // timestamps as per the WebM specification,
  140. // https://www.webmproject.org/docs/container/. Returns true on success and
  141. // false on mkvmuxer failure.
  142. //
  143. // Note that frames may still be around in the queues after this call. The
  144. // method stops flushing when timestamp monotonicity can't be guaranteed
  145. // anymore.
  146. bool PartiallyFlushQueues();
  147. // Flushes out the next frame in timestamp order from the queues. Returns true
  148. // on success and false on mkvmuxer failure.
  149. //
  150. // Note: it's assumed that at least one video or audio frame is queued.
  151. bool FlushNextFrame();
  152. // Calculates a monotonically increasing timestamp from an input |timestamp|
  153. // and a pointer to a previously stored |last_timestamp| by taking the maximum
  154. // of |timestamp| and *|last_timestamp|. Updates *|last_timestamp| if
  155. // |timestamp| is greater.
  156. base::TimeTicks UpdateLastTimestampMonotonically(
  157. base::TimeTicks timestamp,
  158. base::TimeTicks* last_timestamp);
  159. // Forces data output from |segment_| on the next frame if recording video,
  160. // and |min_data_output_interval_| was configured and has passed since the
  161. // last received video frame.
  162. void MaybeForceNewCluster();
  163. // Audio codec configured on construction. Video codec is taken from first
  164. // received frame.
  165. const AudioCodec audio_codec_;
  166. VideoCodec video_codec_;
  167. // Caller-side identifiers to interact with |segment_|, initialised upon
  168. // first frame arrival to Add{Video, Audio}Track().
  169. uint8_t video_track_index_;
  170. uint8_t audio_track_index_;
  171. // Origin of times for frame timestamps.
  172. base::TimeTicks first_frame_timestamp_video_;
  173. base::TimeTicks last_frame_timestamp_video_;
  174. base::TimeTicks first_frame_timestamp_audio_;
  175. base::TimeTicks last_frame_timestamp_audio_;
  176. // Variables to measure and accumulate, respectively, the time in pause state.
  177. std::unique_ptr<base::ElapsedTimer> elapsed_time_in_pause_;
  178. base::TimeDelta total_time_in_pause_;
  179. // TODO(ajose): Change these when support is added for multiple tracks.
  180. // http://crbug.com/528523
  181. const bool has_video_;
  182. const bool has_audio_;
  183. // Variables to track live and enabled state of audio and video.
  184. bool video_track_live_and_enabled_ = true;
  185. bool audio_track_live_and_enabled_ = true;
  186. // Maximum interval between data output callbacks (given frames arriving)
  187. base::TimeDelta max_data_output_interval_;
  188. // Last timestamp written into the segment.
  189. base::TimeDelta last_timestamp_written_;
  190. std::unique_ptr<Delegate> delegate_;
  191. // The MkvMuxer active element.
  192. mkvmuxer::Segment segment_;
  193. // Flag to force the next call to a |segment_| method to return false.
  194. bool force_one_libwebm_error_;
  195. struct EncodedFrame {
  196. std::string data;
  197. std::string alpha_data;
  198. base::TimeDelta
  199. relative_timestamp; // relative to first_frame_timestamp_xxx_
  200. bool is_keyframe;
  201. };
  202. // The following two queues hold frames to ensure that monotonically
  203. // increasing timestamps are stored in the resulting webm file without
  204. // modifying the timestamps.
  205. base::circular_deque<EncodedFrame> audio_frames_;
  206. // If muxing audio and video, this queue holds frames until the first audio
  207. // frame appears.
  208. base::circular_deque<EncodedFrame> video_frames_;
  209. SEQUENCE_CHECKER(sequence_checker_);
  210. };
  211. } // namespace media
  212. #endif // MEDIA_MUXERS_WEBM_MUXER_H_