123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251 |
- // Copyright 2020 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #ifndef ASH_SERVICES_RECORDING_RECORDING_ENCODER_MUXER_H_
- #define ASH_SERVICES_RECORDING_RECORDING_ENCODER_MUXER_H_
- #include <memory>
- #include "base/callback_forward.h"
- #include "base/containers/circular_deque.h"
- #include "base/containers/queue.h"
- #include "base/memory/scoped_refptr.h"
- #include "base/memory/weak_ptr.h"
- #include "base/sequence_checker.h"
- #include "base/task/sequenced_task_runner.h"
- #include "base/thread_annotations.h"
- #include "base/threading/sequence_bound.h"
- #include "base/time/time.h"
- #include "media/audio/audio_opus_encoder.h"
- #include "media/base/audio_bus.h"
- #include "media/base/audio_parameters.h"
- #include "media/muxers/webm_muxer.h"
- #include "media/video/vpx_video_encoder.h"
- #include "mojo/public/cpp/bindings/pending_remote.h"
- #include "third_party/abseil-cpp/absl/types/optional.h"
- #include "ui/gfx/geometry/size.h"
- namespace base {
- class FilePath;
- }
- namespace media {
- class VideoFrame;
- } // namespace media
- namespace recording {
- namespace mojom {
- class DriveFsQuotaDelegate;
- enum class RecordingStatus;
- } // namespace mojom
- // Defines a callback type to notify the user of RecordingEncoderMuxer of a
- // failure while encoding audio or video frames.
- using OnFailureCallback =
- base::OnceCallback<void(mojom::RecordingStatus status)>;
- // Encapsulates encoding and muxing audio and video frame. An instance of this
- // object can only be interacted with via a |base::SequenceBound| wrapper, which
- // guarantees all encoding and muxing operations as well as destruction of the
- // instance are done on the sequenced |blocking_task_runner| given to Create().
- // This prevents expensive encoding and muxing operations from blocking the main
- // thread of the recording service, on which the video frames are delivered.
- //
- // This object performs VP8 video encoding and Opus audio encoding, and mux the
- // audio and video encoded frames into a Webm container.
- class RecordingEncoderMuxer {
- public:
- RecordingEncoderMuxer(const RecordingEncoderMuxer&) = delete;
- RecordingEncoderMuxer& operator=(const RecordingEncoderMuxer&) = delete;
- // Creates an instance of this class that is bound to the given sequenced
- // |blocking_task_runner| on which all operations as well as destruction will
- // happen. |video_encoder_options| and |audio_input_params| will be used to
- // initialize the video and audio encoders respectively.
- // If |audio_input_params| is nullptr, then the service is not recording
- // audio, and the muxer will be initialized accordingly.
- // the webm muxer chunks will be written directly to a file at the given
- // |webm_file_path|.
- // |on_failure_callback| will be called to inform the owner of this object of
- // a failure, after which all subsequent calls to EncodeVideo() and
- // EncodeAudio() will be ignored.
- //
- // By default, |on_failure_callback| will be called on the same sequence of
- // |blocking_task_runner| (unless the caller binds the given callbacks to a
- // different sequence by means of base::BindPostTask()).
- static base::SequenceBound<RecordingEncoderMuxer> Create(
- scoped_refptr<base::SequencedTaskRunner> blocking_task_runner,
- const media::VideoEncoder::Options& video_encoder_options,
- const media::AudioParameters* audio_input_params,
- mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate,
- const base::FilePath& webm_file_path,
- OnFailureCallback on_failure_callback);
- bool did_failure_occur() const {
- DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
- return !on_failure_callback_;
- }
- // Creates and initializes the video encoder if none exists, or recreates and
- // reinitializes it otherwise. This is useful when the video frame dimensions
- // may need to change dynamically (such as when a recorded window gets moved
- // to a display with different bounds).
- void InitializeVideoEncoder(
- const media::VideoEncoder::Options& video_encoder_options);
- // Encodes and muxes the given video |frame|.
- void EncodeVideo(scoped_refptr<media::VideoFrame> frame);
- // Encodes and muxes the given audio frame in |audio_bus| captured at
- // |capture_time|.
- void EncodeAudio(std::unique_ptr<media::AudioBus> audio_bus,
- base::TimeTicks capture_time);
- // Audio and video encoders as well as the WebmMuxer may buffer several frames
- // before they're processed. It is important to flush all those buffers before
- // releasing this object so as not to drop the final portion of the recording.
- // |on_done| will be called when all remaining buffered frames have been
- // processed and written to the webm file.
- // By default, |on_done| will be called on the same sequence of
- // |blocking_task_runner| unless the caller binds it to another sequence by
- // means of base::BindPostTask().
- void FlushAndFinalize(base::OnceClosure on_done);
- private:
- class RecordingMuxerDelegate;
- struct AudioFrame {
- AudioFrame(std::unique_ptr<media::AudioBus>, base::TimeTicks);
- AudioFrame(AudioFrame&&);
- ~AudioFrame();
- std::unique_ptr<media::AudioBus> bus;
- base::TimeTicks capture_time;
- };
- friend class base::SequenceBound<RecordingEncoderMuxer>;
- RecordingEncoderMuxer(
- const media::VideoEncoder::Options& video_encoder_options,
- const media::AudioParameters* audio_input_params,
- mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate,
- const base::FilePath& webm_file_path,
- OnFailureCallback on_failure_callback);
- ~RecordingEncoderMuxer();
- // Creates and initializes the audio encoder.
- void InitializeAudioEncoder(const media::AudioEncoder::Options& options);
- // Called when the audio encoder is initialized to provide the |status| of
- // the initialization.
- void OnAudioEncoderInitialized(media::EncoderStatus status);
- // Called when the video |encoder| is initialized to provide the |status| of
- // the initialization. If initialization failed, |on_failure_callback_| will
- // be triggered.
- void OnVideoEncoderInitialized(media::VpxVideoEncoder* encoder,
- media::EncoderStatus status);
- // Performs the actual encoding of the given audio |frame|. It should never be
- // called before the audio encoder is initialized. Audio frames received
- // before initialization should be added to |pending_audio_frames_| and
- // handled once initialization is complete.
- void EncodeAudioImpl(AudioFrame frame);
- // Performs the actual encoding of the given video |frame|. It should never be
- // called before the video encoder is initialized. Video frames received
- // before initialization should be added to |pending_video_frames_| and
- // handled once initialization is complete.
- void EncodeVideoImpl(scoped_refptr<media::VideoFrame> frame);
- // Called by the video encoder to provided the encoded video frame |output|,
- // which will then by sent to muxer.
- void OnVideoEncoderOutput(
- media::VideoEncoderOutput output,
- absl::optional<media::VideoEncoder::CodecDescription> codec_description);
- // Called by the audio encoder to provide the |encoded_audio|.
- void OnAudioEncoded(
- media::EncodedAudioBuffer encoded_audio,
- absl::optional<media::AudioEncoder::CodecDescription> codec_description);
- // Called when the audio encoder flushes all its buffered frames, at which
- // point we can flush the video encoder. |on_done| will be passed to
- // OnVideoEncoderFlushed()
- void OnAudioEncoderFlushed(base::OnceClosure on_done,
- media::EncoderStatus status);
- // Called when the video encoder flushes all its buffered frames, at which
- // point we can flush the muxer. |on_done| will be called to signal that
- // flushing is complete.
- void OnVideoEncoderFlushed(base::OnceClosure on_done,
- media::EncoderStatus status);
- // Called by both the audio and video encoders to provide the |status| of
- // encoding tasks.
- void OnEncoderStatus(bool for_video, media::EncoderStatus status);
- // Notifies the owner of this object (via |on_failure_callback_|) that a
- // failure noted by |status| has occurred during audio or video encoding, or
- // muxing.
- void NotifyFailure(mojom::RecordingStatus status);
- SEQUENCE_CHECKER(sequence_checker_);
- // A callback triggered when a failure happens during encoding. Once
- // triggered, this callback is null, and therefore indicates that a failure
- // occurred (See did_failure_occur() above).
- // This has to be the first thing created, so it's the last thing that gets
- // destroyed, since any failure in the encoders or muxer rely on this callback
- // to notify the service about the failure.
- // See https://crbug.com/1255090.
- OnFailureCallback on_failure_callback_ GUARDED_BY_CONTEXT(sequence_checker_);
- std::unique_ptr<media::VpxVideoEncoder> video_encoder_
- GUARDED_BY_CONTEXT(sequence_checker_);
- std::unique_ptr<media::AudioOpusEncoder> audio_encoder_
- GUARDED_BY_CONTEXT(sequence_checker_);
- media::WebmMuxer webm_muxer_ GUARDED_BY_CONTEXT(sequence_checker_);
- // Holds video frames that were received before the video encoder is
- // initialized, so that they can be processed once initialization is complete.
- base::circular_deque<scoped_refptr<media::VideoFrame>> pending_video_frames_
- GUARDED_BY_CONTEXT(sequence_checker_);
- // Holds audio frames that were received before the audio encoder is
- // initialized, so that they can be processed once initialization is complete.
- base::circular_deque<AudioFrame> pending_audio_frames_
- GUARDED_BY_CONTEXT(sequence_checker_);
- // The total number of frames that we dropped to keep the size of
- // |pending_video_frames_| limited to |kMaxPendingFrames| to avoid consuming
- // too much memory, or stalling the capturer since it has a maximum number of
- // in-flight frames at a time.
- size_t num_dropped_frames_ GUARDED_BY_CONTEXT(sequence_checker_) = 0;
- // A queue containing the sizes of the visible region of the received video
- // frame in the same order of their encoding. Note that the visible rect sizes
- // may change from frame to frame (e.g. when recording a window, and the
- // window gets resized).
- base::queue<gfx::Size> video_visible_rect_sizes_
- GUARDED_BY_CONTEXT(sequence_checker_);
- // True once video encoder is initialized successfully.
- bool is_video_encoder_initialized_ GUARDED_BY_CONTEXT(sequence_checker_) =
- false;
- // True once audio encoder is initialized successfully.
- bool is_audio_encoder_initialized_ GUARDED_BY_CONTEXT(sequence_checker_) =
- false;
- base::WeakPtrFactory<RecordingEncoderMuxer> weak_ptr_factory_
- GUARDED_BY_CONTEXT(sequence_checker_){this};
- };
- } // namespace recording
- #endif // ASH_SERVICES_RECORDING_RECORDING_ENCODER_MUXER_H_
|