gav1_video_decoder.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/gav1_video_decoder.h"
  5. #include <stdint.h>
  6. #include <numeric>
  7. #include "base/bits.h"
  8. #include "base/callback_helpers.h"
  9. #include "base/numerics/safe_conversions.h"
  10. #include "base/system/sys_info.h"
  11. #include "base/threading/sequenced_task_runner_handle.h"
  12. #include "media/base/bind_to_current_loop.h"
  13. #include "media/base/limits.h"
  14. #include "media/base/media_log.h"
  15. #include "media/base/video_frame.h"
  16. #include "media/base/video_util.h"
  17. #include "third_party/libgav1/src/src/gav1/decoder.h"
  18. #include "third_party/libgav1/src/src/gav1/decoder_settings.h"
  19. #include "third_party/libgav1/src/src/gav1/frame_buffer.h"
  20. namespace media {
  21. namespace {
  22. VideoPixelFormat Libgav1ImageFormatToVideoPixelFormat(
  23. const libgav1::ImageFormat libgav1_format,
  24. int bitdepth) {
  25. switch (libgav1_format) {
  26. // Single plane monochrome images will be converted to standard 3 plane ones
  27. // since Chromium doesn't support single Y plane images.
  28. case libgav1::kImageFormatMonochrome400:
  29. case libgav1::kImageFormatYuv420:
  30. switch (bitdepth) {
  31. case 8:
  32. return PIXEL_FORMAT_I420;
  33. case 10:
  34. return PIXEL_FORMAT_YUV420P10;
  35. case 12:
  36. return PIXEL_FORMAT_YUV420P12;
  37. default:
  38. DLOG(ERROR) << "Unsupported bit depth: " << bitdepth;
  39. return PIXEL_FORMAT_UNKNOWN;
  40. }
  41. case libgav1::kImageFormatYuv422:
  42. switch (bitdepth) {
  43. case 8:
  44. return PIXEL_FORMAT_I422;
  45. case 10:
  46. return PIXEL_FORMAT_YUV422P10;
  47. case 12:
  48. return PIXEL_FORMAT_YUV422P12;
  49. default:
  50. DLOG(ERROR) << "Unsupported bit depth: " << bitdepth;
  51. return PIXEL_FORMAT_UNKNOWN;
  52. }
  53. case libgav1::kImageFormatYuv444:
  54. switch (bitdepth) {
  55. case 8:
  56. return PIXEL_FORMAT_I444;
  57. case 10:
  58. return PIXEL_FORMAT_YUV444P10;
  59. case 12:
  60. return PIXEL_FORMAT_YUV444P12;
  61. default:
  62. DLOG(ERROR) << "Unsupported bit depth: " << bitdepth;
  63. return PIXEL_FORMAT_UNKNOWN;
  64. }
  65. }
  66. }
  67. int GetDecoderThreadCounts(int coded_height) {
  68. // Thread counts based on currently available content. We set the number of
  69. // threads to be equal to the general number of tiles for the given
  70. // resolution. As of now, YouTube content has the following tile settings:
  71. // 240p and below - 1 tile
  72. // 360p and 480p - 2 tiles
  73. // 720p - 4 tiles
  74. // 1080p - 8 tiles
  75. // libgav1 supports frame parallel decoding, but we do not use it (yet) since
  76. // the performance for this thread configuration is good enough without it.
  77. // Also, the memory usage is much lower in non-frame parallel mode. This can
  78. // be revisited as performance numbers change/evolve.
  79. static const int num_cores = base::SysInfo::NumberOfProcessors();
  80. auto threads_by_height = [](int coded_height) {
  81. if (coded_height >= 1000)
  82. return 8;
  83. if (coded_height >= 700)
  84. return 4;
  85. if (coded_height >= 300)
  86. return 2;
  87. return 1;
  88. };
  89. return std::min(threads_by_height(coded_height), num_cores);
  90. }
  91. libgav1::StatusCode GetFrameBufferImpl(void* callback_private_data,
  92. int bitdepth,
  93. libgav1::ImageFormat image_format,
  94. int width,
  95. int height,
  96. int left_border,
  97. int right_border,
  98. int top_border,
  99. int bottom_border,
  100. int stride_alignment,
  101. Libgav1FrameBuffer* frame_buffer) {
  102. DCHECK(callback_private_data);
  103. DCHECK(frame_buffer);
  104. DCHECK(base::bits::IsPowerOfTwo(stride_alignment));
  105. // VideoFramePool creates frames with a fixed alignment of
  106. // VideoFrame::kFrameAddressAlignment. If libgav1 requests a larger
  107. // alignment, it cannot be supported.
  108. CHECK_LE(static_cast<size_t>(stride_alignment),
  109. VideoFrame::kFrameAddressAlignment);
  110. const VideoPixelFormat format =
  111. Libgav1ImageFormatToVideoPixelFormat(image_format, bitdepth);
  112. if (format == PIXEL_FORMAT_UNKNOWN)
  113. return libgav1::kStatusUnimplemented;
  114. // VideoFramePool aligns video_frame->data(i), but libgav1 needs
  115. // video_frame->visible_data(i) to be aligned. To accomplish that, pad
  116. // left_border to be a multiple of stride_alignment.
  117. //
  118. // Here is an example:
  119. // width=6, height=4, left/right/top/bottom_border=2, stride_alignment=16
  120. //
  121. // X*|TTTTTT|**pppppp
  122. // **|TTTTTT|**pppppp
  123. // --+------+--------
  124. // LL|YFFFFF|RRpppppp
  125. // LL|FFFFFF|RRpppppp
  126. // LL|FFFFFF|RRpppppp
  127. // LL|FFFFFF|RRpppppp
  128. // --+------+--------
  129. // **|BBBBBB|**pppppp
  130. // **|BBBBBB|**pppppp
  131. //
  132. // F indicates the frame proper. L, R, T, B indicate the
  133. // left/right/top/bottom borders. Lowercase p indicates the padding at the
  134. // end of a row. The asterisk * indicates the borders at the four corners.
  135. //
  136. // Libgav1 requires that the callback align the first byte of the frame
  137. // proper, indicated by Y. VideoFramePool aligns the first byte of the
  138. // buffer, indicated by X. To make sure the byte indicated by Y is also
  139. // aligned, we need to pad left_border to be a multiple of stride_alignment.
  140. left_border = base::bits::AlignUp(left_border, stride_alignment);
  141. gfx::Size coded_size(left_border + width + right_border,
  142. top_border + height + bottom_border);
  143. gfx::Rect visible_rect(left_border, top_border, width, height);
  144. auto* decoder = static_cast<Gav1VideoDecoder*>(callback_private_data);
  145. auto video_frame =
  146. decoder->CreateVideoFrame(format, coded_size, visible_rect);
  147. if (!video_frame)
  148. return libgav1::kStatusInvalidArgument;
  149. for (int i = 0; i < 3; i++) {
  150. // frame_buffer->plane[i] points to the first byte of the frame proper,
  151. // not the first byte of the buffer.
  152. frame_buffer->plane[i] = video_frame->visible_data(i);
  153. frame_buffer->stride[i] = video_frame->stride(i);
  154. }
  155. if (image_format == libgav1::kImageFormatMonochrome400) {
  156. int uv_height = (height + 1) >> 1;
  157. const size_t size_needed = video_frame->stride(1) * uv_height;
  158. for (int i = 1; i < 3; i++) {
  159. frame_buffer->plane[i] = nullptr;
  160. frame_buffer->stride[i] = 0;
  161. // An AV1 monochrome (grayscale) frame has no U and V planes. Set all U
  162. // and V samples in video_frame to the blank value.
  163. if (bitdepth == 8) {
  164. constexpr uint8_t kBlankUV = 256 / 2;
  165. memset(video_frame->visible_data(i), kBlankUV, size_needed);
  166. } else {
  167. const uint16_t kBlankUV = (1 << bitdepth) / 2;
  168. uint16_t* data =
  169. reinterpret_cast<uint16_t*>(video_frame->visible_data(i));
  170. std::fill(data, data + size_needed / 2, kBlankUV);
  171. }
  172. }
  173. }
  174. frame_buffer->private_data = video_frame.get();
  175. video_frame->AddRef();
  176. return libgav1::kStatusOk;
  177. }
  178. void ReleaseFrameBufferImpl(void* callback_private_data,
  179. void* buffer_private_data) {
  180. DCHECK(callback_private_data);
  181. DCHECK(buffer_private_data);
  182. static_cast<VideoFrame*>(buffer_private_data)->Release();
  183. }
  184. void ReleaseInputBufferImpl(void* callback_private_data,
  185. void* buffer_private_data) {
  186. DCHECK(callback_private_data);
  187. DCHECK(buffer_private_data);
  188. static_cast<DecoderBuffer*>(buffer_private_data)->Release();
  189. }
  190. scoped_refptr<VideoFrame> FormatVideoFrame(
  191. const libgav1::DecoderBuffer& buffer,
  192. const VideoColorSpace& container_color_space) {
  193. scoped_refptr<VideoFrame> frame =
  194. static_cast<VideoFrame*>(buffer.buffer_private_data);
  195. frame->set_timestamp(base::Microseconds(buffer.user_private_data));
  196. // AV1 color space defines match ISO 23001-8:2016 via ISO/IEC 23091-4/ITU-T
  197. // H.273. https://aomediacodec.github.io/av1-spec/#color-config-semantics
  198. media::VideoColorSpace color_space(
  199. buffer.color_primary, buffer.transfer_characteristics,
  200. buffer.matrix_coefficients,
  201. buffer.color_range == libgav1::kColorRangeStudio
  202. ? gfx::ColorSpace::RangeID::LIMITED
  203. : gfx::ColorSpace::RangeID::FULL);
  204. // If the frame doesn't specify a color space, use the container's.
  205. if (!color_space.IsSpecified())
  206. color_space = container_color_space;
  207. frame->set_color_space(color_space.ToGfxColorSpace());
  208. frame->metadata().power_efficient = false;
  209. return frame;
  210. }
  211. } // namespace
  212. // static
  213. SupportedVideoDecoderConfigs Gav1VideoDecoder::SupportedConfigs() {
  214. return {{/*profile_min=*/AV1PROFILE_PROFILE_MAIN,
  215. /*profile_max=*/AV1PROFILE_PROFILE_HIGH,
  216. /*coded_size_min=*/kDefaultSwDecodeSizeMin,
  217. /*coded_size_max=*/kDefaultSwDecodeSizeMax,
  218. /*allow_encrypted=*/false,
  219. /*require_encrypted=*/false}};
  220. }
  221. Gav1VideoDecoder::Gav1VideoDecoder(MediaLog* media_log,
  222. OffloadState offload_state)
  223. : media_log_(media_log),
  224. bind_callbacks_(offload_state == OffloadState::kNormal) {
  225. DETACH_FROM_SEQUENCE(sequence_checker_);
  226. }
  227. Gav1VideoDecoder::~Gav1VideoDecoder() {
  228. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  229. CloseDecoder();
  230. }
  231. VideoDecoderType Gav1VideoDecoder::GetDecoderType() const {
  232. return VideoDecoderType::kGav1;
  233. }
  234. void Gav1VideoDecoder::Initialize(const VideoDecoderConfig& config,
  235. bool low_delay,
  236. CdmContext* /* cdm_context */,
  237. InitCB init_cb,
  238. const OutputCB& output_cb,
  239. const WaitingCB& /* waiting_cb */) {
  240. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  241. DCHECK(config.IsValidConfig());
  242. InitCB bound_init_cb = bind_callbacks_ ? BindToCurrentLoop(std::move(init_cb))
  243. : std::move(init_cb);
  244. if (config.is_encrypted() || config.codec() != VideoCodec::kAV1) {
  245. std::move(bound_init_cb)
  246. .Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
  247. return;
  248. }
  249. // Clear any previously initialized decoder.
  250. CloseDecoder();
  251. libgav1::DecoderSettings settings;
  252. settings.threads = VideoDecoder::GetRecommendedThreadCount(
  253. GetDecoderThreadCounts(config.coded_size().height()));
  254. settings.get_frame_buffer = GetFrameBufferImpl;
  255. settings.release_frame_buffer = ReleaseFrameBufferImpl;
  256. settings.release_input_buffer = ReleaseInputBufferImpl;
  257. settings.callback_private_data = this;
  258. if (low_delay || config.is_rtc()) {
  259. // The `frame_parallel` setting is false by default, so this serves more as
  260. // documentation that it should be false for low delay decoding.
  261. settings.frame_parallel = false;
  262. }
  263. libgav1_decoder_ = std::make_unique<libgav1::Decoder>();
  264. libgav1::StatusCode status = libgav1_decoder_->Init(&settings);
  265. if (status != kLibgav1StatusOk) {
  266. MEDIA_LOG(ERROR, media_log_) << "libgav1::Decoder::Init() failed, "
  267. << "status=" << status;
  268. std::move(bound_init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
  269. return;
  270. }
  271. output_cb_ = output_cb;
  272. state_ = DecoderState::kDecoding;
  273. color_space_ = config.color_space_info();
  274. aspect_ratio_ = config.aspect_ratio();
  275. std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
  276. }
  277. void Gav1VideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
  278. DecodeCB decode_cb) {
  279. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  280. DCHECK(buffer);
  281. DCHECK(decode_cb);
  282. DCHECK(libgav1_decoder_);
  283. DCHECK_NE(state_, DecoderState::kUninitialized)
  284. << "Called Decode() before successful Initialize()";
  285. DecodeCB bound_decode_cb = bind_callbacks_
  286. ? BindToCurrentLoop(std::move(decode_cb))
  287. : std::move(decode_cb);
  288. if (state_ == DecoderState::kError) {
  289. std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
  290. return;
  291. }
  292. if (!DecodeBuffer(std::move(buffer))) {
  293. state_ = DecoderState::kError;
  294. std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
  295. return;
  296. }
  297. // VideoDecoderShim expects |decode_cb| call after |output_cb_|.
  298. std::move(bound_decode_cb).Run(DecoderStatus::Codes::kOk);
  299. }
  300. bool Gav1VideoDecoder::DecodeBuffer(scoped_refptr<DecoderBuffer> buffer) {
  301. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  302. const bool is_end_of_stream = buffer->end_of_stream();
  303. // Used to ensure that EnqueueFrame() actually takes the packet. If we exit
  304. // this function without enqueuing |buffer|, that packet will be lost. We do
  305. // not have anything to enqueue at the end of stream.
  306. bool enqueued = is_end_of_stream;
  307. while (is_end_of_stream || !enqueued) {
  308. // Try to enqueue the packet if it has not been enqueued already.
  309. if (!enqueued) {
  310. libgav1::StatusCode status = libgav1_decoder_->EnqueueFrame(
  311. buffer->data(), buffer->data_size(),
  312. /* user_private_data = */ buffer->timestamp().InMicroseconds(),
  313. /* buffer_private_data = */ buffer.get());
  314. if (status == kLibgav1StatusOk) {
  315. buffer->AddRef();
  316. enqueued = true;
  317. } else if (status != kLibgav1StatusTryAgain) {
  318. MEDIA_LOG(ERROR, media_log_)
  319. << "libgav1::Decoder::EnqueueFrame failed, status=" << status
  320. << " on " << buffer->AsHumanReadableString();
  321. return false;
  322. }
  323. }
  324. // Try to dequeue a decoded frame.
  325. const libgav1::DecoderBuffer* decoded_buffer;
  326. libgav1::StatusCode status =
  327. libgav1_decoder_->DequeueFrame(&decoded_buffer);
  328. if (status != kLibgav1StatusOk) {
  329. if (status != kLibgav1StatusTryAgain &&
  330. status != kLibgav1StatusNothingToDequeue) {
  331. MEDIA_LOG(ERROR, media_log_)
  332. << "libgav1::Decoder::DequeueFrame failed, status=" << status;
  333. return false;
  334. }
  335. // We've reached end of stream and no frames are remaining to be dequeued.
  336. if (is_end_of_stream && status == kLibgav1StatusNothingToDequeue) {
  337. return true;
  338. }
  339. continue;
  340. }
  341. if (!decoded_buffer) {
  342. // The packet did not have any displayable frames. Not an error.
  343. continue;
  344. }
  345. scoped_refptr<VideoFrame> frame =
  346. FormatVideoFrame(*decoded_buffer, color_space_);
  347. if (!frame) {
  348. MEDIA_LOG(ERROR, media_log_) << "Failed formatting VideoFrame from "
  349. << "libgav1::DecoderBuffer";
  350. return false;
  351. }
  352. output_cb_.Run(std::move(frame));
  353. }
  354. DCHECK(enqueued);
  355. return true;
  356. }
  357. void Gav1VideoDecoder::Reset(base::OnceClosure reset_cb) {
  358. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  359. state_ = DecoderState::kDecoding;
  360. libgav1::StatusCode status = libgav1_decoder_->SignalEOS();
  361. if (status != kLibgav1StatusOk) {
  362. MEDIA_LOG(WARNING, media_log_) << "libgav1::Decoder::SignalEOS() failed, "
  363. << "status=" << status;
  364. }
  365. if (bind_callbacks_) {
  366. base::SequencedTaskRunnerHandle::Get()->PostTask(FROM_HERE,
  367. std::move(reset_cb));
  368. } else {
  369. std::move(reset_cb).Run();
  370. }
  371. }
  372. void Gav1VideoDecoder::Detach() {
  373. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  374. DCHECK(!bind_callbacks_);
  375. CloseDecoder();
  376. DETACH_FROM_SEQUENCE(sequence_checker_);
  377. }
  378. scoped_refptr<VideoFrame> Gav1VideoDecoder::CreateVideoFrame(
  379. VideoPixelFormat format,
  380. const gfx::Size& coded_size,
  381. const gfx::Rect& visible_rect) {
  382. // The comment for VideoFramePool::CreateFrame() says:
  383. // The buffer for the new frame will be zero initialized. Reused frames
  384. // will not be zero initialized.
  385. // The zero initialization is necessary for FFmpeg but not for libgav1.
  386. return frame_pool_.CreateFrame(format, coded_size, visible_rect,
  387. aspect_ratio_.GetNaturalSize(visible_rect),
  388. kNoTimestamp);
  389. }
  390. void Gav1VideoDecoder::CloseDecoder() {
  391. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  392. libgav1_decoder_.reset();
  393. state_ = DecoderState::kUninitialized;
  394. }
  395. } // namespace media