recording_encoder_muxer.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef ASH_SERVICES_RECORDING_RECORDING_ENCODER_MUXER_H_
  5. #define ASH_SERVICES_RECORDING_RECORDING_ENCODER_MUXER_H_
  6. #include <memory>
  7. #include "base/callback_forward.h"
  8. #include "base/containers/circular_deque.h"
  9. #include "base/containers/queue.h"
  10. #include "base/memory/scoped_refptr.h"
  11. #include "base/memory/weak_ptr.h"
  12. #include "base/sequence_checker.h"
  13. #include "base/task/sequenced_task_runner.h"
  14. #include "base/thread_annotations.h"
  15. #include "base/threading/sequence_bound.h"
  16. #include "base/time/time.h"
  17. #include "media/audio/audio_opus_encoder.h"
  18. #include "media/base/audio_bus.h"
  19. #include "media/base/audio_parameters.h"
  20. #include "media/muxers/webm_muxer.h"
  21. #include "media/video/vpx_video_encoder.h"
  22. #include "mojo/public/cpp/bindings/pending_remote.h"
  23. #include "third_party/abseil-cpp/absl/types/optional.h"
  24. #include "ui/gfx/geometry/size.h"
  25. namespace base {
  26. class FilePath;
  27. }
  28. namespace media {
  29. class VideoFrame;
  30. } // namespace media
  31. namespace recording {
  32. namespace mojom {
  33. class DriveFsQuotaDelegate;
  34. enum class RecordingStatus;
  35. } // namespace mojom
  36. // Defines a callback type to notify the user of RecordingEncoderMuxer of a
  37. // failure while encoding audio or video frames.
  38. using OnFailureCallback =
  39. base::OnceCallback<void(mojom::RecordingStatus status)>;
  40. // Encapsulates encoding and muxing audio and video frame. An instance of this
  41. // object can only be interacted with via a |base::SequenceBound| wrapper, which
  42. // guarantees all encoding and muxing operations as well as destruction of the
  43. // instance are done on the sequenced |blocking_task_runner| given to Create().
  44. // This prevents expensive encoding and muxing operations from blocking the main
  45. // thread of the recording service, on which the video frames are delivered.
  46. //
  47. // This object performs VP8 video encoding and Opus audio encoding, and mux the
  48. // audio and video encoded frames into a Webm container.
  49. class RecordingEncoderMuxer {
  50. public:
  51. RecordingEncoderMuxer(const RecordingEncoderMuxer&) = delete;
  52. RecordingEncoderMuxer& operator=(const RecordingEncoderMuxer&) = delete;
  53. // Creates an instance of this class that is bound to the given sequenced
  54. // |blocking_task_runner| on which all operations as well as destruction will
  55. // happen. |video_encoder_options| and |audio_input_params| will be used to
  56. // initialize the video and audio encoders respectively.
  57. // If |audio_input_params| is nullptr, then the service is not recording
  58. // audio, and the muxer will be initialized accordingly.
  59. // the webm muxer chunks will be written directly to a file at the given
  60. // |webm_file_path|.
  61. // |on_failure_callback| will be called to inform the owner of this object of
  62. // a failure, after which all subsequent calls to EncodeVideo() and
  63. // EncodeAudio() will be ignored.
  64. //
  65. // By default, |on_failure_callback| will be called on the same sequence of
  66. // |blocking_task_runner| (unless the caller binds the given callbacks to a
  67. // different sequence by means of base::BindPostTask()).
  68. static base::SequenceBound<RecordingEncoderMuxer> Create(
  69. scoped_refptr<base::SequencedTaskRunner> blocking_task_runner,
  70. const media::VideoEncoder::Options& video_encoder_options,
  71. const media::AudioParameters* audio_input_params,
  72. mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate,
  73. const base::FilePath& webm_file_path,
  74. OnFailureCallback on_failure_callback);
  75. bool did_failure_occur() const {
  76. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  77. return !on_failure_callback_;
  78. }
  79. // Creates and initializes the video encoder if none exists, or recreates and
  80. // reinitializes it otherwise. This is useful when the video frame dimensions
  81. // may need to change dynamically (such as when a recorded window gets moved
  82. // to a display with different bounds).
  83. void InitializeVideoEncoder(
  84. const media::VideoEncoder::Options& video_encoder_options);
  85. // Encodes and muxes the given video |frame|.
  86. void EncodeVideo(scoped_refptr<media::VideoFrame> frame);
  87. // Encodes and muxes the given audio frame in |audio_bus| captured at
  88. // |capture_time|.
  89. void EncodeAudio(std::unique_ptr<media::AudioBus> audio_bus,
  90. base::TimeTicks capture_time);
  91. // Audio and video encoders as well as the WebmMuxer may buffer several frames
  92. // before they're processed. It is important to flush all those buffers before
  93. // releasing this object so as not to drop the final portion of the recording.
  94. // |on_done| will be called when all remaining buffered frames have been
  95. // processed and written to the webm file.
  96. // By default, |on_done| will be called on the same sequence of
  97. // |blocking_task_runner| unless the caller binds it to another sequence by
  98. // means of base::BindPostTask().
  99. void FlushAndFinalize(base::OnceClosure on_done);
  100. private:
  101. class RecordingMuxerDelegate;
  102. struct AudioFrame {
  103. AudioFrame(std::unique_ptr<media::AudioBus>, base::TimeTicks);
  104. AudioFrame(AudioFrame&&);
  105. ~AudioFrame();
  106. std::unique_ptr<media::AudioBus> bus;
  107. base::TimeTicks capture_time;
  108. };
  109. friend class base::SequenceBound<RecordingEncoderMuxer>;
  110. RecordingEncoderMuxer(
  111. const media::VideoEncoder::Options& video_encoder_options,
  112. const media::AudioParameters* audio_input_params,
  113. mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate,
  114. const base::FilePath& webm_file_path,
  115. OnFailureCallback on_failure_callback);
  116. ~RecordingEncoderMuxer();
  117. // Creates and initializes the audio encoder.
  118. void InitializeAudioEncoder(const media::AudioEncoder::Options& options);
  119. // Called when the audio encoder is initialized to provide the |status| of
  120. // the initialization.
  121. void OnAudioEncoderInitialized(media::EncoderStatus status);
  122. // Called when the video |encoder| is initialized to provide the |status| of
  123. // the initialization. If initialization failed, |on_failure_callback_| will
  124. // be triggered.
  125. void OnVideoEncoderInitialized(media::VpxVideoEncoder* encoder,
  126. media::EncoderStatus status);
  127. // Performs the actual encoding of the given audio |frame|. It should never be
  128. // called before the audio encoder is initialized. Audio frames received
  129. // before initialization should be added to |pending_audio_frames_| and
  130. // handled once initialization is complete.
  131. void EncodeAudioImpl(AudioFrame frame);
  132. // Performs the actual encoding of the given video |frame|. It should never be
  133. // called before the video encoder is initialized. Video frames received
  134. // before initialization should be added to |pending_video_frames_| and
  135. // handled once initialization is complete.
  136. void EncodeVideoImpl(scoped_refptr<media::VideoFrame> frame);
  137. // Called by the video encoder to provided the encoded video frame |output|,
  138. // which will then by sent to muxer.
  139. void OnVideoEncoderOutput(
  140. media::VideoEncoderOutput output,
  141. absl::optional<media::VideoEncoder::CodecDescription> codec_description);
  142. // Called by the audio encoder to provide the |encoded_audio|.
  143. void OnAudioEncoded(
  144. media::EncodedAudioBuffer encoded_audio,
  145. absl::optional<media::AudioEncoder::CodecDescription> codec_description);
  146. // Called when the audio encoder flushes all its buffered frames, at which
  147. // point we can flush the video encoder. |on_done| will be passed to
  148. // OnVideoEncoderFlushed()
  149. void OnAudioEncoderFlushed(base::OnceClosure on_done,
  150. media::EncoderStatus status);
  151. // Called when the video encoder flushes all its buffered frames, at which
  152. // point we can flush the muxer. |on_done| will be called to signal that
  153. // flushing is complete.
  154. void OnVideoEncoderFlushed(base::OnceClosure on_done,
  155. media::EncoderStatus status);
  156. // Called by both the audio and video encoders to provide the |status| of
  157. // encoding tasks.
  158. void OnEncoderStatus(bool for_video, media::EncoderStatus status);
  159. // Notifies the owner of this object (via |on_failure_callback_|) that a
  160. // failure noted by |status| has occurred during audio or video encoding, or
  161. // muxing.
  162. void NotifyFailure(mojom::RecordingStatus status);
  163. SEQUENCE_CHECKER(sequence_checker_);
  164. // A callback triggered when a failure happens during encoding. Once
  165. // triggered, this callback is null, and therefore indicates that a failure
  166. // occurred (See did_failure_occur() above).
  167. // This has to be the first thing created, so it's the last thing that gets
  168. // destroyed, since any failure in the encoders or muxer rely on this callback
  169. // to notify the service about the failure.
  170. // See https://crbug.com/1255090.
  171. OnFailureCallback on_failure_callback_ GUARDED_BY_CONTEXT(sequence_checker_);
  172. std::unique_ptr<media::VpxVideoEncoder> video_encoder_
  173. GUARDED_BY_CONTEXT(sequence_checker_);
  174. std::unique_ptr<media::AudioOpusEncoder> audio_encoder_
  175. GUARDED_BY_CONTEXT(sequence_checker_);
  176. media::WebmMuxer webm_muxer_ GUARDED_BY_CONTEXT(sequence_checker_);
  177. // Holds video frames that were received before the video encoder is
  178. // initialized, so that they can be processed once initialization is complete.
  179. base::circular_deque<scoped_refptr<media::VideoFrame>> pending_video_frames_
  180. GUARDED_BY_CONTEXT(sequence_checker_);
  181. // Holds audio frames that were received before the audio encoder is
  182. // initialized, so that they can be processed once initialization is complete.
  183. base::circular_deque<AudioFrame> pending_audio_frames_
  184. GUARDED_BY_CONTEXT(sequence_checker_);
  185. // The total number of frames that we dropped to keep the size of
  186. // |pending_video_frames_| limited to |kMaxPendingFrames| to avoid consuming
  187. // too much memory, or stalling the capturer since it has a maximum number of
  188. // in-flight frames at a time.
  189. size_t num_dropped_frames_ GUARDED_BY_CONTEXT(sequence_checker_) = 0;
  190. // A queue containing the sizes of the visible region of the received video
  191. // frame in the same order of their encoding. Note that the visible rect sizes
  192. // may change from frame to frame (e.g. when recording a window, and the
  193. // window gets resized).
  194. base::queue<gfx::Size> video_visible_rect_sizes_
  195. GUARDED_BY_CONTEXT(sequence_checker_);
  196. // True once video encoder is initialized successfully.
  197. bool is_video_encoder_initialized_ GUARDED_BY_CONTEXT(sequence_checker_) =
  198. false;
  199. // True once audio encoder is initialized successfully.
  200. bool is_audio_encoder_initialized_ GUARDED_BY_CONTEXT(sequence_checker_) =
  201. false;
  202. base::WeakPtrFactory<RecordingEncoderMuxer> weak_ptr_factory_
  203. GUARDED_BY_CONTEXT(sequence_checker_){this};
  204. };
  205. } // namespace recording
  206. #endif // ASH_SERVICES_RECORDING_RECORDING_ENCODER_MUXER_H_