dav1d_video_decoder.cc 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/dav1d_video_decoder.h"
  5. #include <memory>
  6. #include <string>
  7. #include <utility>
  8. #include "base/bind.h"
  9. #include "base/bits.h"
  10. #include "base/callback.h"
  11. #include "base/logging.h"
  12. #include "base/strings/stringprintf.h"
  13. #include "base/threading/sequenced_task_runner_handle.h"
  14. #include "media/base/bind_to_current_loop.h"
  15. #include "media/base/decoder_buffer.h"
  16. #include "media/base/limits.h"
  17. #include "media/base/media_log.h"
  18. #include "media/base/video_aspect_ratio.h"
  19. #include "media/base/video_util.h"
  20. extern "C" {
  21. #include "third_party/dav1d/libdav1d/include/dav1d/dav1d.h"
  22. }
  23. namespace media {
  24. static void GetDecoderThreadCounts(const int coded_height,
  25. int* tile_threads,
  26. int* frame_threads) {
  27. // Tile thread counts based on currently available content. Recommended by
  28. // YouTube, while frame thread values fit within
  29. // limits::kMaxVideoDecodeThreads.
  30. if (coded_height >= 700) {
  31. *tile_threads =
  32. 4; // Current 720p content is encoded in 5 tiles and 1080p content with
  33. // 8 tiles, but we'll exceed limits::kMaxVideoDecodeThreads with 5+
  34. // tile threads with 3 frame threads (5 * 3 + 3 = 18 threads vs 16
  35. // max).
  36. //
  37. // Since 720p playback isn't smooth without 3 frame threads, we've
  38. // chosen a slightly lower tile thread count.
  39. *frame_threads = 3;
  40. } else if (coded_height >= 300) {
  41. *tile_threads = 3;
  42. *frame_threads = 2;
  43. } else {
  44. *tile_threads = 2;
  45. *frame_threads = 2;
  46. }
  47. }
  48. static VideoPixelFormat Dav1dImgFmtToVideoPixelFormat(
  49. const Dav1dPictureParameters* pic) {
  50. switch (pic->layout) {
  51. // Single plane monochrome images will be converted to standard 3 plane ones
  52. // since Chromium doesn't support single Y plane images.
  53. case DAV1D_PIXEL_LAYOUT_I400:
  54. case DAV1D_PIXEL_LAYOUT_I420:
  55. switch (pic->bpc) {
  56. case 8:
  57. return PIXEL_FORMAT_I420;
  58. case 10:
  59. return PIXEL_FORMAT_YUV420P10;
  60. case 12:
  61. return PIXEL_FORMAT_YUV420P12;
  62. default:
  63. DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  64. return PIXEL_FORMAT_UNKNOWN;
  65. }
  66. case DAV1D_PIXEL_LAYOUT_I422:
  67. switch (pic->bpc) {
  68. case 8:
  69. return PIXEL_FORMAT_I422;
  70. case 10:
  71. return PIXEL_FORMAT_YUV422P10;
  72. case 12:
  73. return PIXEL_FORMAT_YUV422P12;
  74. default:
  75. DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  76. return PIXEL_FORMAT_UNKNOWN;
  77. }
  78. case DAV1D_PIXEL_LAYOUT_I444:
  79. switch (pic->bpc) {
  80. case 8:
  81. return PIXEL_FORMAT_I444;
  82. case 10:
  83. return PIXEL_FORMAT_YUV444P10;
  84. case 12:
  85. return PIXEL_FORMAT_YUV444P12;
  86. default:
  87. DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  88. return PIXEL_FORMAT_UNKNOWN;
  89. }
  90. }
  91. }
  92. static void ReleaseDecoderBuffer(const uint8_t* buffer, void* opaque) {
  93. if (opaque)
  94. static_cast<DecoderBuffer*>(opaque)->Release();
  95. }
  96. static void LogDav1dMessage(void* cookie, const char* format, va_list ap) {
  97. auto log = base::StringPrintV(format, ap);
  98. if (log.empty())
  99. return;
  100. if (log.back() == '\n')
  101. log.pop_back();
  102. DLOG(ERROR) << log;
  103. }
  104. // std::unique_ptr release helpers. We need to release both the containing
  105. // structs as well as refs held within the structures.
  106. struct ScopedDav1dDataFree {
  107. void operator()(void* x) const {
  108. auto* data = static_cast<Dav1dData*>(x);
  109. dav1d_data_unref(data);
  110. delete data;
  111. }
  112. };
  113. struct ScopedDav1dPictureFree {
  114. void operator()(void* x) const {
  115. auto* pic = static_cast<Dav1dPicture*>(x);
  116. dav1d_picture_unref(pic);
  117. delete pic;
  118. }
  119. };
  120. // static
  121. SupportedVideoDecoderConfigs Dav1dVideoDecoder::SupportedConfigs() {
  122. return {{/*profile_min=*/AV1PROFILE_PROFILE_MAIN,
  123. /*profile_max=*/AV1PROFILE_PROFILE_HIGH,
  124. /*coded_size_min=*/kDefaultSwDecodeSizeMin,
  125. /*coded_size_max=*/kDefaultSwDecodeSizeMax,
  126. /*allow_encrypted=*/false,
  127. /*require_encrypted=*/false}};
  128. }
  129. Dav1dVideoDecoder::Dav1dVideoDecoder(MediaLog* media_log,
  130. OffloadState offload_state)
  131. : media_log_(media_log),
  132. bind_callbacks_(offload_state == OffloadState::kNormal) {
  133. DETACH_FROM_SEQUENCE(sequence_checker_);
  134. }
  135. Dav1dVideoDecoder::~Dav1dVideoDecoder() {
  136. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  137. CloseDecoder();
  138. }
  139. VideoDecoderType Dav1dVideoDecoder::GetDecoderType() const {
  140. return VideoDecoderType::kDav1d;
  141. }
  142. void Dav1dVideoDecoder::Initialize(const VideoDecoderConfig& config,
  143. bool low_delay,
  144. CdmContext* /* cdm_context */,
  145. InitCB init_cb,
  146. const OutputCB& output_cb,
  147. const WaitingCB& /* waiting_cb */) {
  148. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  149. DCHECK(config.IsValidConfig());
  150. InitCB bound_init_cb = bind_callbacks_ ? BindToCurrentLoop(std::move(init_cb))
  151. : std::move(init_cb);
  152. if (config.is_encrypted()) {
  153. std::move(bound_init_cb)
  154. .Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
  155. return;
  156. }
  157. if (config.codec() != VideoCodec::kAV1) {
  158. std::move(bound_init_cb)
  159. .Run(DecoderStatus(DecoderStatus::Codes::kUnsupportedCodec)
  160. .WithData("codec", config.codec()));
  161. return;
  162. }
  163. // Clear any previously initialized decoder.
  164. CloseDecoder();
  165. Dav1dSettings s;
  166. dav1d_default_settings(&s);
  167. // Compute the ideal thread count values. We'll then clamp these based on the
  168. // maximum number of recommended threads (using number of processors, etc).
  169. int tile_threads, frame_threads;
  170. GetDecoderThreadCounts(config.coded_size().height(), &tile_threads,
  171. &frame_threads);
  172. // While dav1d has switched to a thread pool, preserve the same thread counts
  173. // we used when tile and frame threads were configured distinctly. It may be
  174. // possible to lower this after some performance analysis of the new system.
  175. s.n_threads = VideoDecoder::GetRecommendedThreadCount(frame_threads *
  176. (tile_threads + 1));
  177. // We only want 1 frame thread in low delay mode, since otherwise we'll
  178. // require at least two buffers before the first frame can be output.
  179. if (low_delay || config.is_rtc())
  180. s.max_frame_delay = 1;
  181. // Route dav1d internal logs through Chrome's DLOG system.
  182. s.logger = {nullptr, &LogDav1dMessage};
  183. // Set a maximum frame size limit to avoid OOM'ing fuzzers.
  184. s.frame_size_limit = limits::kMaxCanvas;
  185. // TODO(tmathmeyer) write the dav1d error into the data for the media error.
  186. if (dav1d_open(&dav1d_decoder_, &s) < 0) {
  187. std::move(bound_init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
  188. return;
  189. }
  190. config_ = config;
  191. state_ = DecoderState::kNormal;
  192. output_cb_ = output_cb;
  193. std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
  194. }
  195. void Dav1dVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
  196. DecodeCB decode_cb) {
  197. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  198. DCHECK(buffer);
  199. DCHECK(decode_cb);
  200. DCHECK_NE(state_, DecoderState::kUninitialized)
  201. << "Called Decode() before successful Initialize()";
  202. DecodeCB bound_decode_cb = bind_callbacks_
  203. ? BindToCurrentLoop(std::move(decode_cb))
  204. : std::move(decode_cb);
  205. if (state_ == DecoderState::kError) {
  206. std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
  207. return;
  208. }
  209. if (!DecodeBuffer(std::move(buffer))) {
  210. state_ = DecoderState::kError;
  211. std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
  212. return;
  213. }
  214. // VideoDecoderShim expects |decode_cb| call after |output_cb_|.
  215. std::move(bound_decode_cb).Run(DecoderStatus::Codes::kOk);
  216. }
  217. void Dav1dVideoDecoder::Reset(base::OnceClosure reset_cb) {
  218. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  219. state_ = DecoderState::kNormal;
  220. dav1d_flush(dav1d_decoder_);
  221. if (bind_callbacks_)
  222. base::SequencedTaskRunnerHandle::Get()->PostTask(FROM_HERE,
  223. std::move(reset_cb));
  224. else
  225. std::move(reset_cb).Run();
  226. }
  227. void Dav1dVideoDecoder::Detach() {
  228. // Even though we offload all resolutions of AV1, this may be called in a
  229. // transition from clear to encrypted content. Which will subsequently fail
  230. // Initialize() since encrypted content isn't supported by this decoder.
  231. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  232. DCHECK(!bind_callbacks_);
  233. CloseDecoder();
  234. DETACH_FROM_SEQUENCE(sequence_checker_);
  235. }
  236. void Dav1dVideoDecoder::CloseDecoder() {
  237. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  238. if (!dav1d_decoder_)
  239. return;
  240. dav1d_close(&dav1d_decoder_);
  241. DCHECK(!dav1d_decoder_);
  242. }
  243. bool Dav1dVideoDecoder::DecodeBuffer(scoped_refptr<DecoderBuffer> buffer) {
  244. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  245. using ScopedPtrDav1dData = std::unique_ptr<Dav1dData, ScopedDav1dDataFree>;
  246. ScopedPtrDav1dData input_buffer;
  247. if (!buffer->end_of_stream()) {
  248. input_buffer.reset(new Dav1dData{0});
  249. if (dav1d_data_wrap(input_buffer.get(), buffer->data(), buffer->data_size(),
  250. &ReleaseDecoderBuffer, buffer.get()) < 0) {
  251. return false;
  252. }
  253. input_buffer->m.timestamp = buffer->timestamp().InMicroseconds();
  254. buffer->AddRef();
  255. }
  256. // Used to DCHECK that dav1d_send_data() actually takes the packet. If we exit
  257. // this function without sending |input_buffer| that packet will be lost. We
  258. // have no packet to send at end of stream.
  259. bool send_data_completed = buffer->end_of_stream();
  260. while (!input_buffer || input_buffer->sz) {
  261. if (input_buffer) {
  262. const int res = dav1d_send_data(dav1d_decoder_, input_buffer.get());
  263. if (res < 0 && res != -EAGAIN) {
  264. MEDIA_LOG(ERROR, media_log_) << "dav1d_send_data() failed on "
  265. << buffer->AsHumanReadableString();
  266. return false;
  267. }
  268. if (res != -EAGAIN)
  269. send_data_completed = true;
  270. // Even if dav1d_send_data() returned EAGAIN, try dav1d_get_picture().
  271. }
  272. using ScopedPtrDav1dPicture =
  273. std::unique_ptr<Dav1dPicture, ScopedDav1dPictureFree>;
  274. ScopedPtrDav1dPicture p(new Dav1dPicture{0});
  275. const int res = dav1d_get_picture(dav1d_decoder_, p.get());
  276. if (res < 0) {
  277. if (res != -EAGAIN) {
  278. MEDIA_LOG(ERROR, media_log_) << "dav1d_get_picture() failed on "
  279. << buffer->AsHumanReadableString();
  280. return false;
  281. }
  282. // We've reached end of stream and no frames remain to drain.
  283. if (!input_buffer) {
  284. DCHECK(send_data_completed);
  285. return true;
  286. }
  287. continue;
  288. }
  289. auto frame = BindImageToVideoFrame(p.get());
  290. if (!frame) {
  291. MEDIA_LOG(DEBUG, media_log_)
  292. << "Failed to produce video frame from Dav1dPicture.";
  293. return false;
  294. }
  295. // AV1 color space defines match ISO 23001-8:2016 via ISO/IEC 23091-4/ITU-T
  296. // H.273. https://aomediacodec.github.io/av1-spec/#color-config-semantics
  297. VideoColorSpace color_space(
  298. p->seq_hdr->pri, p->seq_hdr->trc, p->seq_hdr->mtrx,
  299. p->seq_hdr->color_range ? gfx::ColorSpace::RangeID::FULL
  300. : gfx::ColorSpace::RangeID::LIMITED);
  301. // If the frame doesn't specify a color space, use the container's.
  302. if (!color_space.IsSpecified())
  303. color_space = config_.color_space_info();
  304. frame->set_color_space(color_space.ToGfxColorSpace());
  305. frame->metadata().power_efficient = false;
  306. frame->set_hdr_metadata(config_.hdr_metadata());
  307. // When we use bind mode, our image data is dependent on the Dav1dPicture,
  308. // so we must ensure it stays alive along enough.
  309. frame->AddDestructionObserver(
  310. base::BindOnce([](ScopedPtrDav1dPicture) {}, std::move(p)));
  311. output_cb_.Run(std::move(frame));
  312. }
  313. DCHECK(send_data_completed);
  314. return true;
  315. }
  316. scoped_refptr<VideoFrame> Dav1dVideoDecoder::BindImageToVideoFrame(
  317. const Dav1dPicture* pic) {
  318. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  319. const gfx::Size visible_size(pic->p.w, pic->p.h);
  320. VideoPixelFormat pixel_format = Dav1dImgFmtToVideoPixelFormat(&pic->p);
  321. if (pixel_format == PIXEL_FORMAT_UNKNOWN)
  322. return nullptr;
  323. auto uv_plane_stride = pic->stride[1];
  324. auto* u_plane = static_cast<uint8_t*>(pic->data[1]);
  325. auto* v_plane = static_cast<uint8_t*>(pic->data[2]);
  326. const bool needs_fake_uv_planes = pic->p.layout == DAV1D_PIXEL_LAYOUT_I400;
  327. if (needs_fake_uv_planes) {
  328. // UV planes are half the size of the Y plane.
  329. uv_plane_stride = base::bits::AlignUp(pic->stride[0] / 2, ptrdiff_t{2});
  330. const auto uv_plane_height = (pic->p.h + 1) / 2;
  331. const size_t size_needed = uv_plane_stride * uv_plane_height;
  332. if (!fake_uv_data_ || fake_uv_data_->size() != size_needed) {
  333. if (pic->p.bpc == 8) {
  334. // Avoid having base::RefCountedBytes zero initialize the memory just to
  335. // fill it with a different value.
  336. constexpr uint8_t kBlankUV = 256 / 2;
  337. std::vector<unsigned char> empty_data(size_needed, kBlankUV);
  338. // When we resize, existing frames will keep their refs on the old data.
  339. fake_uv_data_ = base::RefCountedBytes::TakeVector(&empty_data);
  340. } else {
  341. DCHECK(pic->p.bpc == 10 || pic->p.bpc == 12);
  342. const uint16_t kBlankUV = (1 << pic->p.bpc) / 2;
  343. fake_uv_data_ =
  344. base::MakeRefCounted<base::RefCountedBytes>(size_needed);
  345. uint16_t* data = fake_uv_data_->front_as<uint16_t>();
  346. std::fill(data, data + size_needed / 2, kBlankUV);
  347. }
  348. }
  349. u_plane = v_plane = fake_uv_data_->front_as<uint8_t>();
  350. }
  351. auto frame = VideoFrame::WrapExternalYuvData(
  352. pixel_format, visible_size, gfx::Rect(visible_size),
  353. config_.aspect_ratio().GetNaturalSize(gfx::Rect(visible_size)),
  354. pic->stride[0], uv_plane_stride, uv_plane_stride,
  355. static_cast<uint8_t*>(pic->data[0]), u_plane, v_plane,
  356. base::Microseconds(pic->m.timestamp));
  357. if (!frame)
  358. return nullptr;
  359. // Each frame needs a ref on the fake UV data to keep it alive until done.
  360. if (needs_fake_uv_planes) {
  361. frame->AddDestructionObserver(base::BindOnce(
  362. [](scoped_refptr<base::RefCountedBytes>) {}, fake_uv_data_));
  363. }
  364. return frame;
  365. }
  366. } // namespace media