recording_encoder_muxer.cc 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ash/services/recording/recording_encoder_muxer.h"
  5. #include "ash/services/recording/public/mojom/recording_service.mojom.h"
  6. #include "ash/services/recording/recording_service_constants.h"
  7. #include "base/bind.h"
  8. #include "base/check_op.h"
  9. #include "base/files/file_path.h"
  10. #include "base/logging.h"
  11. #include "base/memory/weak_ptr.h"
  12. #include "base/strings/string_util.h"
  13. #include "base/system/sys_info.h"
  14. #include "media/base/audio_codecs.h"
  15. #include "media/base/video_codecs.h"
  16. #include "media/base/video_frame.h"
  17. #include "media/muxers/file_webm_muxer_delegate.h"
  18. #include "mojo/public/cpp/bindings/remote.h"
  19. namespace recording {
  20. namespace {
  21. // The audio and video encoders are initialized asynchronously, and until that
  22. // happens, all received audio and video frames are added to
  23. // |pending_video_frames_| and |pending_audio_frames_|. However, in order
  24. // to avoid an OOM situation if the encoder takes too long to initialize or it
  25. // never does, we impose an upper-bound to the number of pending frames. The
  26. // below value is equal to the maximum number of in-flight frames that the
  27. // capturer uses (See |viz::FrameSinkVideoCapturerImpl::kDesignLimitMaxFrames|)
  28. // before it stops sending frames. Once we hit that limit in
  29. // |pending_video_frames_|, we will start dropping frames to let the capturer
  30. // proceed, with an upper limit of how many frames we can drop that is
  31. // equivalent to 4 seconds, after which we'll declare an encoder initialization
  32. // failure. For convenience the same limit is used for as a cap on number of
  33. // audio frames stored in |pending_audio_frames_|.
  34. constexpr size_t kMaxPendingFrames = 10;
  35. constexpr size_t kMaxDroppedFrames = 4 * kMaxFrameRate;
  36. // We use a threshold of 512 MB to end the video recording due to low disk
  37. // space, which is the same threshold as that used by the low disk space
  38. // notification (See low_disk_notification.cc).
  39. constexpr int64_t kLowDiskSpaceThresholdInBytes = 512 * 1024 * 1024;
  40. // To avoid checking the remaining desk space after every write operation, we do
  41. // it only once every 10 MB written of webm data.
  42. constexpr int64_t kMinNumBytesBetweenDiskSpaceChecks = 10 * 1024 * 1024;
  43. } // namespace
  44. // -----------------------------------------------------------------------------
  45. // RecordingEncoderMuxer::RecordingMuxerDelegate:
  46. // Defines a delegate for the WebmMuxer which extends the capability of
  47. // |media::FileWebmMuxerDelegate| (which writes seekable webm chunks directly to
  48. // a file), by adding recording specific behavior such as ending the recording
  49. // when an IO file write fails, or when a critical disk space threshold is
  50. // reached. An instance of this object is owned by the WebmMuxer, which in turn
  51. // is owned by the RecordingEncoderMuxer instance.
  52. class RecordingEncoderMuxer::RecordingMuxerDelegate
  53. : public media::FileWebmMuxerDelegate {
  54. public:
  55. RecordingMuxerDelegate(
  56. const base::FilePath& webm_file_path,
  57. RecordingEncoderMuxer* muxer_owner,
  58. mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate)
  59. : FileWebmMuxerDelegate(base::File(
  60. webm_file_path,
  61. base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE)),
  62. muxer_owner_(muxer_owner),
  63. drive_fs_quota_delegate_remote_(std::move(drive_fs_quota_delegate)),
  64. webm_file_path_(webm_file_path) {
  65. DCHECK(muxer_owner_);
  66. }
  67. RecordingMuxerDelegate(const RecordingMuxerDelegate&) = delete;
  68. RecordingMuxerDelegate& operator=(const RecordingMuxerDelegate&) = delete;
  69. ~RecordingMuxerDelegate() override = default;
  70. protected:
  71. // media::FileWebmMuxerDelegate:
  72. mkvmuxer::int32 DoWrite(const void* buf, mkvmuxer::uint32 len) override {
  73. const auto result = FileWebmMuxerDelegate::DoWrite(buf, len);
  74. num_bytes_till_next_disk_space_check_ -= len;
  75. if (result != 0) {
  76. muxer_owner_->NotifyFailure(mojom::RecordingStatus::kIoError);
  77. return result;
  78. }
  79. MaybeCheckRemainingSpace();
  80. return result;
  81. }
  82. private:
  83. // Returns true if the video file is being written to a path `webm_file_path_`
  84. // that exists in DriveFS, false if it's a local file.
  85. bool IsDriveFsFile() const {
  86. return drive_fs_quota_delegate_remote_.is_bound();
  87. }
  88. // Checks the remaining free space (whether for a local file, or a DriveFS
  89. // file) once `num_bytes_till_next_disk_space_check_` goes below zero.
  90. void MaybeCheckRemainingSpace() {
  91. if (num_bytes_till_next_disk_space_check_ > 0)
  92. return;
  93. if (!IsDriveFsFile()) {
  94. OnGotRemainingFreeSpace(
  95. mojom::RecordingStatus::kLowDiskSpace,
  96. base::SysInfo::AmountOfFreeDiskSpace(webm_file_path_));
  97. return;
  98. }
  99. if (waiting_for_drive_fs_delegate_)
  100. return;
  101. DCHECK(drive_fs_quota_delegate_remote_);
  102. waiting_for_drive_fs_delegate_ = true;
  103. drive_fs_quota_delegate_remote_->GetDriveFsFreeSpaceBytes(
  104. base::BindOnce(&RecordingMuxerDelegate::OnGotRemainingFreeSpace,
  105. weak_ptr_factory_.GetWeakPtr(),
  106. mojom::RecordingStatus::kLowDriveFsQuota));
  107. }
  108. // Called to test the `remaining_free_space_bytes` against the minimum
  109. // threshold below which we end the recording with a failure. The failure type
  110. // that will be propagated to the client is the given `status`.
  111. void OnGotRemainingFreeSpace(mojom::RecordingStatus status,
  112. int64_t remaining_free_space_bytes) {
  113. waiting_for_drive_fs_delegate_ = false;
  114. num_bytes_till_next_disk_space_check_ = kMinNumBytesBetweenDiskSpaceChecks;
  115. if (remaining_free_space_bytes < 0) {
  116. // A negative value (e.g. -1) indicates a failure in computing the free
  117. // space.
  118. return;
  119. }
  120. if (remaining_free_space_bytes < kLowDiskSpaceThresholdInBytes) {
  121. LOG(WARNING) << "Ending recording due to " << status
  122. << ", and remaining free space of "
  123. << base::FormatBytesUnlocalized(remaining_free_space_bytes);
  124. muxer_owner_->NotifyFailure(status);
  125. }
  126. }
  127. // A reference to the owner of the WebmMuxer instance that owns |this|. It is
  128. // used to notify with any IO or disk space errors while writing the webm
  129. // chunks.
  130. RecordingEncoderMuxer* const muxer_owner_; // Not owned.
  131. // A remote end to the DriveFS delegate that can calculate the remaining free
  132. // space in Drive. This is bound only when the `webm_file_path_` points to a
  133. // file in DriveFS. Being unbound means the file is a local disk file.
  134. mojo::Remote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate_remote_;
  135. // The path of the webm file to which the muxer output will be written.
  136. const base::FilePath webm_file_path_;
  137. // Once this value becomes <= 0, we trigger a remaining disk space poll.
  138. // Initialized to 0, so that we poll the disk space on the very first write
  139. // operation.
  140. int64_t num_bytes_till_next_disk_space_check_ = 0;
  141. // True when we're waiting for a reply from the remote DriveFS quota delegate.
  142. bool waiting_for_drive_fs_delegate_ = false;
  143. base::WeakPtrFactory<RecordingMuxerDelegate> weak_ptr_factory_{this};
  144. };
  145. // -----------------------------------------------------------------------------
  146. // RecordingEncoderMuxer::AudioFrame:
  147. RecordingEncoderMuxer::AudioFrame::AudioFrame(
  148. std::unique_ptr<media::AudioBus> audio_bus,
  149. base::TimeTicks time)
  150. : bus(std::move(audio_bus)), capture_time(time) {}
  151. RecordingEncoderMuxer::AudioFrame::AudioFrame(AudioFrame&&) = default;
  152. RecordingEncoderMuxer::AudioFrame::~AudioFrame() = default;
  153. // -----------------------------------------------------------------------------
  154. // RecordingEncoderMuxer:
  155. // static
  156. base::SequenceBound<RecordingEncoderMuxer> RecordingEncoderMuxer::Create(
  157. scoped_refptr<base::SequencedTaskRunner> blocking_task_runner,
  158. const media::VideoEncoder::Options& video_encoder_options,
  159. const media::AudioParameters* audio_input_params,
  160. mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate,
  161. const base::FilePath& webm_file_path,
  162. OnFailureCallback on_failure_callback) {
  163. return base::SequenceBound<RecordingEncoderMuxer>(
  164. std::move(blocking_task_runner), video_encoder_options,
  165. audio_input_params, std::move(drive_fs_quota_delegate), webm_file_path,
  166. std::move(on_failure_callback));
  167. }
  168. void RecordingEncoderMuxer::InitializeVideoEncoder(
  169. const media::VideoEncoder::Options& video_encoder_options) {
  170. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  171. // Note: The VpxVideoEncoder supports changing the encoding options
  172. // dynamically, but it won't work for all frame size changes and may cause
  173. // encoding failures. Therefore, it's better to recreate and reinitialize a
  174. // new encoder. See media::VpxVideoEncoder::ChangeOptions() for more details.
  175. if (video_encoder_ && is_video_encoder_initialized_) {
  176. auto* encoder_ptr = video_encoder_.get();
  177. encoder_ptr->Flush(base::BindOnce(
  178. // Holds on to the old encoder until it flushes its buffers, then
  179. // destroys it.
  180. [](std::unique_ptr<media::VpxVideoEncoder> old_encoder,
  181. media::EncoderStatus status) {},
  182. std::move(video_encoder_)));
  183. }
  184. is_video_encoder_initialized_ = false;
  185. video_encoder_ = std::make_unique<media::VpxVideoEncoder>();
  186. video_encoder_->Initialize(
  187. media::VP8PROFILE_ANY, video_encoder_options,
  188. base::BindRepeating(&RecordingEncoderMuxer::OnVideoEncoderOutput,
  189. weak_ptr_factory_.GetWeakPtr()),
  190. base::BindOnce(&RecordingEncoderMuxer::OnVideoEncoderInitialized,
  191. weak_ptr_factory_.GetWeakPtr(), video_encoder_.get()));
  192. }
  193. void RecordingEncoderMuxer::EncodeVideo(
  194. scoped_refptr<media::VideoFrame> frame) {
  195. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  196. if (is_video_encoder_initialized_) {
  197. EncodeVideoImpl(std::move(frame));
  198. return;
  199. }
  200. pending_video_frames_.push_back(std::move(frame));
  201. if (pending_video_frames_.size() == kMaxPendingFrames) {
  202. pending_video_frames_.pop_front();
  203. DCHECK_LT(pending_video_frames_.size(), kMaxPendingFrames);
  204. if (++num_dropped_frames_ >= kMaxDroppedFrames) {
  205. LOG(ERROR) << "Video encoder took too long to initialize.";
  206. NotifyFailure(mojom::RecordingStatus::kVideoEncoderInitializationFailure);
  207. }
  208. }
  209. }
  210. void RecordingEncoderMuxer::EncodeAudio(
  211. std::unique_ptr<media::AudioBus> audio_bus,
  212. base::TimeTicks capture_time) {
  213. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  214. DCHECK(audio_encoder_);
  215. AudioFrame frame(std::move(audio_bus), capture_time);
  216. if (is_audio_encoder_initialized_) {
  217. EncodeAudioImpl(std::move(frame));
  218. return;
  219. }
  220. pending_audio_frames_.push_back(std::move(frame));
  221. if (pending_audio_frames_.size() == kMaxPendingFrames) {
  222. pending_audio_frames_.pop_front();
  223. DCHECK_LT(pending_audio_frames_.size(), kMaxPendingFrames);
  224. }
  225. }
  226. void RecordingEncoderMuxer::FlushAndFinalize(base::OnceClosure on_done) {
  227. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  228. if (audio_encoder_) {
  229. audio_encoder_->Flush(
  230. base::BindOnce(&RecordingEncoderMuxer::OnAudioEncoderFlushed,
  231. weak_ptr_factory_.GetWeakPtr(), std::move(on_done)));
  232. } else {
  233. OnAudioEncoderFlushed(std::move(on_done), media::EncoderStatus::Codes::kOk);
  234. }
  235. }
  236. RecordingEncoderMuxer::RecordingEncoderMuxer(
  237. const media::VideoEncoder::Options& video_encoder_options,
  238. const media::AudioParameters* audio_input_params,
  239. mojo::PendingRemote<mojom::DriveFsQuotaDelegate> drive_fs_quota_delegate,
  240. const base::FilePath& webm_file_path,
  241. OnFailureCallback on_failure_callback)
  242. : on_failure_callback_(std::move(on_failure_callback)),
  243. webm_muxer_(media::AudioCodec::kOpus,
  244. /*has_video_=*/true,
  245. /*has_audio_=*/!!audio_input_params,
  246. std::make_unique<RecordingMuxerDelegate>(
  247. webm_file_path,
  248. this,
  249. std::move(drive_fs_quota_delegate))) {
  250. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  251. if (audio_input_params) {
  252. media::AudioEncoder::Options audio_encoder_options;
  253. audio_encoder_options.codec = media::AudioCodec::kOpus;
  254. audio_encoder_options.channels = audio_input_params->channels();
  255. audio_encoder_options.sample_rate = audio_input_params->sample_rate();
  256. InitializeAudioEncoder(audio_encoder_options);
  257. }
  258. InitializeVideoEncoder(video_encoder_options);
  259. }
  260. RecordingEncoderMuxer::~RecordingEncoderMuxer() {
  261. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  262. }
  263. void RecordingEncoderMuxer::InitializeAudioEncoder(
  264. const media::AudioEncoder::Options& options) {
  265. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  266. is_audio_encoder_initialized_ = false;
  267. audio_encoder_ = std::make_unique<media::AudioOpusEncoder>();
  268. audio_encoder_->Initialize(
  269. options,
  270. base::BindRepeating(&RecordingEncoderMuxer::OnAudioEncoded,
  271. weak_ptr_factory_.GetWeakPtr()),
  272. base::BindOnce(&RecordingEncoderMuxer::OnAudioEncoderInitialized,
  273. weak_ptr_factory_.GetWeakPtr()));
  274. }
  275. void RecordingEncoderMuxer::OnAudioEncoderInitialized(
  276. media::EncoderStatus status) {
  277. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  278. if (!status.is_ok()) {
  279. LOG(ERROR) << "Could not initialize the audio encoder: "
  280. << status.message();
  281. NotifyFailure(mojom::RecordingStatus::kAudioEncoderInitializationFailure);
  282. return;
  283. }
  284. is_audio_encoder_initialized_ = true;
  285. for (auto& frame : pending_audio_frames_)
  286. EncodeAudioImpl(std::move(frame));
  287. pending_audio_frames_.clear();
  288. }
  289. void RecordingEncoderMuxer::OnVideoEncoderInitialized(
  290. media::VpxVideoEncoder* encoder,
  291. media::EncoderStatus status) {
  292. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  293. // Ignore initialization of encoders that were removed as part of
  294. // reinitialization.
  295. if (video_encoder_.get() != encoder)
  296. return;
  297. if (!status.is_ok()) {
  298. LOG(ERROR) << "Could not initialize the video encoder: "
  299. << status.message();
  300. NotifyFailure(mojom::RecordingStatus::kVideoEncoderInitializationFailure);
  301. return;
  302. }
  303. is_video_encoder_initialized_ = true;
  304. for (auto& frame : pending_video_frames_)
  305. EncodeVideoImpl(frame);
  306. pending_video_frames_.clear();
  307. }
  308. void RecordingEncoderMuxer::EncodeAudioImpl(AudioFrame frame) {
  309. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  310. DCHECK(is_audio_encoder_initialized_);
  311. if (did_failure_occur())
  312. return;
  313. audio_encoder_->Encode(
  314. std::move(frame.bus), frame.capture_time,
  315. base::BindOnce(&RecordingEncoderMuxer::OnEncoderStatus,
  316. weak_ptr_factory_.GetWeakPtr(), /*for_video=*/false));
  317. }
  318. void RecordingEncoderMuxer::EncodeVideoImpl(
  319. scoped_refptr<media::VideoFrame> frame) {
  320. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  321. DCHECK(is_video_encoder_initialized_);
  322. if (did_failure_occur())
  323. return;
  324. video_visible_rect_sizes_.push(frame->visible_rect().size());
  325. video_encoder_->Encode(
  326. frame, /*key_frame=*/false,
  327. base::BindOnce(&RecordingEncoderMuxer::OnEncoderStatus,
  328. weak_ptr_factory_.GetWeakPtr(), /*for_video=*/true));
  329. }
  330. void RecordingEncoderMuxer::OnVideoEncoderOutput(
  331. media::VideoEncoderOutput output,
  332. absl::optional<media::VideoEncoder::CodecDescription> codec_description) {
  333. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  334. media::WebmMuxer::VideoParameters params(
  335. video_visible_rect_sizes_.front(), kMaxFrameRate, media::VideoCodec::kVP8,
  336. kColorSpace);
  337. video_visible_rect_sizes_.pop();
  338. // TODO(crbug.com/1143798): Explore changing the WebmMuxer so it doesn't work
  339. // with strings, to avoid copying the encoded data.
  340. std::string data{reinterpret_cast<const char*>(output.data.get()),
  341. output.size};
  342. webm_muxer_.OnEncodedVideo(params, std::move(data), std::string(),
  343. base::TimeTicks() + output.timestamp,
  344. output.key_frame);
  345. }
  346. void RecordingEncoderMuxer::OnAudioEncoded(
  347. media::EncodedAudioBuffer encoded_audio,
  348. absl::optional<media::AudioEncoder::CodecDescription> codec_description) {
  349. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  350. DCHECK(audio_encoder_);
  351. // TODO(crbug.com/1143798): Explore changing the WebmMuxer so it doesn't work
  352. // with strings, to avoid copying the encoded data.
  353. std::string encoded_data{
  354. reinterpret_cast<const char*>(encoded_audio.encoded_data.get()),
  355. encoded_audio.encoded_data_size};
  356. webm_muxer_.OnEncodedAudio(encoded_audio.params, std::move(encoded_data),
  357. encoded_audio.timestamp);
  358. }
  359. void RecordingEncoderMuxer::OnAudioEncoderFlushed(base::OnceClosure on_done,
  360. media::EncoderStatus status) {
  361. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  362. if (!status.is_ok())
  363. LOG(ERROR) << "Could not flush audio encoder: " << status.message();
  364. DCHECK(video_encoder_);
  365. video_encoder_->Flush(
  366. base::BindOnce(&RecordingEncoderMuxer::OnVideoEncoderFlushed,
  367. weak_ptr_factory_.GetWeakPtr(), std::move(on_done)));
  368. }
  369. void RecordingEncoderMuxer::OnVideoEncoderFlushed(base::OnceClosure on_done,
  370. media::EncoderStatus status) {
  371. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  372. if (!status.is_ok()) {
  373. LOG(ERROR) << "Could not flush remaining video frames: "
  374. << status.message();
  375. }
  376. webm_muxer_.Flush();
  377. std::move(on_done).Run();
  378. }
  379. void RecordingEncoderMuxer::OnEncoderStatus(bool for_video,
  380. media::EncoderStatus status) {
  381. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  382. if (status.is_ok())
  383. return;
  384. LOG(ERROR) << "Failed to encode " << (for_video ? "video" : "audio")
  385. << " frame: " << status.message();
  386. NotifyFailure(for_video ? mojom::RecordingStatus::kVideoEncodingError
  387. : mojom::RecordingStatus::kAudioEncodingError);
  388. }
  389. void RecordingEncoderMuxer::NotifyFailure(mojom::RecordingStatus status) {
  390. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  391. if (on_failure_callback_)
  392. std::move(on_failure_callback_).Run(status);
  393. }
  394. } // namespace recording