av1_decoder_unittest.cc 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/av1_decoder.h"
  5. #include <string.h>
  6. #include <string>
  7. #include <vector>
  8. #include "base/files/file_util.h"
  9. #include "base/logging.h"
  10. #include "base/memory/raw_ptr.h"
  11. #include "base/numerics/safe_conversions.h"
  12. #include "base/stl_util.h"
  13. #include "media/base/decoder_buffer.h"
  14. #include "media/base/test_data_util.h"
  15. #include "media/ffmpeg/ffmpeg_common.h"
  16. #include "media/filters/ffmpeg_demuxer.h"
  17. #include "media/filters/in_memory_url_protocol.h"
  18. #include "media/filters/ivf_parser.h"
  19. #include "media/gpu/av1_picture.h"
  20. #include "media/media_buildflags.h"
  21. #include "testing/gmock/include/gmock/gmock.h"
  22. #include "testing/gtest/include/gtest/gtest.h"
  23. #include "third_party/libgav1/src/src/obu_parser.h"
  24. #include "third_party/libgav1/src/src/utils/constants.h"
  25. #include "third_party/libgav1/src/src/utils/types.h"
  26. using ::testing::_;
  27. using ::testing::DoAll;
  28. using ::testing::Return;
  29. using ::testing::SaveArg;
  30. namespace media {
  31. namespace {
  32. class FakeAV1Picture : public AV1Picture {
  33. public:
  34. FakeAV1Picture() = default;
  35. protected:
  36. ~FakeAV1Picture() override = default;
  37. private:
  38. scoped_refptr<AV1Picture> CreateDuplicate() override {
  39. return base::MakeRefCounted<FakeAV1Picture>();
  40. }
  41. };
  42. bool IsYUV420(int8_t subsampling_x, int8_t subsampling_y, bool is_monochrome) {
  43. return subsampling_x == 1 && subsampling_y == 1 && !is_monochrome;
  44. }
  45. MATCHER_P(SameAV1PictureInstance, av1_picture, "") {
  46. return &arg == av1_picture.get();
  47. }
  48. MATCHER_P2(MatchesFrameSizeAndRenderSize, frame_size, render_size, "") {
  49. const auto& frame_header = arg.frame_header;
  50. return base::strict_cast<int>(frame_header.width) == frame_size.width() &&
  51. base::strict_cast<int>(frame_header.height) == frame_size.height() &&
  52. base::strict_cast<int>(frame_header.render_width) ==
  53. render_size.width() &&
  54. base::strict_cast<int>(frame_header.render_height) ==
  55. render_size.height();
  56. }
  57. MATCHER_P4(MatchesFrameHeader,
  58. frame_size,
  59. render_size,
  60. show_existing_frame,
  61. show_frame,
  62. "") {
  63. const auto& frame_header = arg.frame_header;
  64. return base::strict_cast<int>(frame_header.width) == frame_size.width() &&
  65. base::strict_cast<int>(frame_header.height) == frame_size.height() &&
  66. base::strict_cast<int>(frame_header.render_width) ==
  67. render_size.width() &&
  68. base::strict_cast<int>(frame_header.render_height) ==
  69. render_size.height() &&
  70. frame_header.show_existing_frame == show_existing_frame &&
  71. frame_header.show_frame == show_frame;
  72. }
  73. MATCHER_P4(MatchesYUV420SequenceHeader,
  74. profile,
  75. bitdepth,
  76. max_frame_size,
  77. film_grain_params_present,
  78. "") {
  79. return arg.profile == profile && arg.color_config.bitdepth == bitdepth &&
  80. base::strict_cast<int>(arg.max_frame_width) ==
  81. max_frame_size.width() &&
  82. base::strict_cast<int>(arg.max_frame_height) ==
  83. max_frame_size.height() &&
  84. arg.film_grain_params_present == film_grain_params_present &&
  85. IsYUV420(arg.color_config.subsampling_x,
  86. arg.color_config.subsampling_y,
  87. arg.color_config.is_monochrome);
  88. }
  89. MATCHER(NonEmptyTileBuffers, "") {
  90. return !arg.empty();
  91. }
  92. MATCHER_P(MatchesFrameData, decoder_buffer, "") {
  93. return arg.data() == decoder_buffer->data() &&
  94. arg.size() == decoder_buffer->data_size();
  95. }
  96. class MockAV1Accelerator : public AV1Decoder::AV1Accelerator {
  97. public:
  98. MockAV1Accelerator() = default;
  99. ~MockAV1Accelerator() override = default;
  100. MOCK_METHOD1(CreateAV1Picture, scoped_refptr<AV1Picture>(bool));
  101. MOCK_METHOD5(SubmitDecode,
  102. Status(const AV1Picture&,
  103. const libgav1::ObuSequenceHeader&,
  104. const AV1ReferenceFrameVector&,
  105. const libgav1::Vector<libgav1::TileBuffer>&,
  106. base::span<const uint8_t>));
  107. MOCK_METHOD1(OutputPicture, bool(const AV1Picture&));
  108. };
  109. } // namespace
  110. class AV1DecoderTest : public ::testing::Test {
  111. public:
  112. using DecodeResult = AcceleratedVideoDecoder::DecodeResult;
  113. AV1DecoderTest() = default;
  114. ~AV1DecoderTest() override = default;
  115. void SetUp() override;
  116. std::vector<DecodeResult> Decode(scoped_refptr<DecoderBuffer> buffer);
  117. const libgav1::DecoderState* GetDecoderState() const;
  118. AV1ReferenceFrameVector& GetReferenceFrames() const;
  119. void Reset();
  120. scoped_refptr<DecoderBuffer> ReadDecoderBuffer(const std::string& fname);
  121. std::vector<scoped_refptr<DecoderBuffer>> ReadIVF(const std::string& fname);
  122. std::vector<scoped_refptr<DecoderBuffer>> ReadWebm(const std::string& fname);
  123. protected:
  124. base::FilePath GetTestFilePath(const std::string& fname) {
  125. base::FilePath file_path(
  126. base::FilePath(base::FilePath::kCurrentDirectory).AppendASCII(fname));
  127. if (base::PathExists(file_path)) {
  128. return file_path;
  129. }
  130. return GetTestDataFilePath(fname);
  131. }
  132. // Owned by |decoder_|.
  133. raw_ptr<MockAV1Accelerator> mock_accelerator_;
  134. std::unique_ptr<AV1Decoder> decoder_;
  135. int32_t bitstream_id_ = 0;
  136. };
  137. void AV1DecoderTest::SetUp() {
  138. auto accelerator = std::make_unique<MockAV1Accelerator>();
  139. mock_accelerator_ = accelerator.get();
  140. decoder_ = std::make_unique<AV1Decoder>(std::move(accelerator),
  141. VIDEO_CODEC_PROFILE_UNKNOWN);
  142. }
  143. std::vector<AcceleratedVideoDecoder::DecodeResult> AV1DecoderTest::Decode(
  144. scoped_refptr<DecoderBuffer> buffer) {
  145. if (buffer)
  146. decoder_->SetStream(bitstream_id_++, *buffer);
  147. std::vector<DecodeResult> results;
  148. DecodeResult res;
  149. do {
  150. res = decoder_->Decode();
  151. results.push_back(res);
  152. } while (res != DecodeResult::kDecodeError &&
  153. res != DecodeResult::kRanOutOfStreamData &&
  154. res != DecodeResult::kTryAgain);
  155. return results;
  156. }
  157. const libgav1::DecoderState* AV1DecoderTest::GetDecoderState() const {
  158. return decoder_->state_.get();
  159. }
  160. AV1ReferenceFrameVector& AV1DecoderTest::GetReferenceFrames() const {
  161. return decoder_->ref_frames_;
  162. }
  163. void AV1DecoderTest::Reset() {
  164. EXPECT_NE(decoder_->state_->current_frame_id, -1);
  165. EXPECT_TRUE(decoder_->parser_);
  166. EXPECT_EQ(decoder_->accelerator_.get(), mock_accelerator_);
  167. EXPECT_LT(base::checked_cast<AV1ReferenceFrameVector::size_type>(
  168. base::STLCount(decoder_->ref_frames_, nullptr)),
  169. decoder_->ref_frames_.size());
  170. EXPECT_FALSE(decoder_->current_frame_header_);
  171. EXPECT_FALSE(decoder_->current_frame_);
  172. EXPECT_NE(decoder_->stream_id_, 0);
  173. EXPECT_TRUE(decoder_->stream_);
  174. EXPECT_GT(decoder_->stream_size_, 0u);
  175. decoder_->Reset();
  176. EXPECT_EQ(decoder_->state_->current_frame_id, -1);
  177. EXPECT_FALSE(decoder_->parser_);
  178. EXPECT_EQ(decoder_->accelerator_.get(), mock_accelerator_);
  179. EXPECT_EQ(base::checked_cast<AV1ReferenceFrameVector::size_type>(
  180. base::STLCount(decoder_->ref_frames_, nullptr)),
  181. decoder_->ref_frames_.size());
  182. EXPECT_FALSE(decoder_->current_frame_header_);
  183. EXPECT_FALSE(decoder_->current_frame_);
  184. EXPECT_EQ(decoder_->stream_id_, 0);
  185. EXPECT_FALSE(decoder_->stream_);
  186. EXPECT_EQ(decoder_->stream_size_, 0u);
  187. }
  188. scoped_refptr<DecoderBuffer> AV1DecoderTest::ReadDecoderBuffer(
  189. const std::string& fname) {
  190. auto input_file = GetTestFilePath(fname);
  191. std::string bitstream;
  192. EXPECT_TRUE(base::ReadFileToString(input_file, &bitstream));
  193. auto buffer = DecoderBuffer::CopyFrom(
  194. reinterpret_cast<const uint8_t*>(bitstream.data()), bitstream.size());
  195. EXPECT_TRUE(!!buffer);
  196. return buffer;
  197. }
  198. std::vector<scoped_refptr<DecoderBuffer>> AV1DecoderTest::ReadIVF(
  199. const std::string& fname) {
  200. std::string ivf_data;
  201. auto input_file = GetTestFilePath(fname);
  202. EXPECT_TRUE(base::ReadFileToString(input_file, &ivf_data));
  203. IvfParser ivf_parser;
  204. IvfFileHeader ivf_header{};
  205. EXPECT_TRUE(
  206. ivf_parser.Initialize(reinterpret_cast<const uint8_t*>(ivf_data.data()),
  207. ivf_data.size(), &ivf_header));
  208. EXPECT_EQ(ivf_header.fourcc, /*AV01=*/0x31305641u);
  209. std::vector<scoped_refptr<DecoderBuffer>> buffers;
  210. IvfFrameHeader ivf_frame_header{};
  211. const uint8_t* data;
  212. while (ivf_parser.ParseNextFrame(&ivf_frame_header, &data)) {
  213. buffers.push_back(DecoderBuffer::CopyFrom(
  214. reinterpret_cast<const uint8_t*>(data), ivf_frame_header.frame_size));
  215. }
  216. return buffers;
  217. }
  218. std::vector<scoped_refptr<DecoderBuffer>> AV1DecoderTest::ReadWebm(
  219. const std::string& fname) {
  220. std::string webm_data;
  221. auto input_file = GetTestFilePath(fname);
  222. EXPECT_TRUE(base::ReadFileToString(input_file, &webm_data));
  223. InMemoryUrlProtocol protocol(
  224. reinterpret_cast<const uint8_t*>(webm_data.data()), webm_data.size(),
  225. false);
  226. FFmpegGlue glue(&protocol);
  227. LOG_ASSERT(glue.OpenContext());
  228. int stream_index = -1;
  229. for (unsigned int i = 0; i < glue.format_context()->nb_streams; ++i) {
  230. const AVStream* stream = glue.format_context()->streams[i];
  231. const AVCodecParameters* codec_parameters = stream->codecpar;
  232. const AVMediaType codec_type = codec_parameters->codec_type;
  233. const AVCodecID codec_id = codec_parameters->codec_id;
  234. if (codec_type == AVMEDIA_TYPE_VIDEO && codec_id == AV_CODEC_ID_AV1) {
  235. stream_index = i;
  236. break;
  237. }
  238. }
  239. EXPECT_NE(stream_index, -1) << "No AV1 data found in " << input_file;
  240. std::vector<scoped_refptr<DecoderBuffer>> buffers;
  241. AVPacket packet{};
  242. while (av_read_frame(glue.format_context(), &packet) >= 0) {
  243. if (packet.stream_index == stream_index)
  244. buffers.push_back(DecoderBuffer::CopyFrom(packet.data, packet.size));
  245. av_packet_unref(&packet);
  246. }
  247. return buffers;
  248. }
  249. TEST_F(AV1DecoderTest, DecodeInvalidOBU) {
  250. std::string kInvalidData = "ThisIsInvalidData";
  251. auto kInvalidBuffer = DecoderBuffer::CopyFrom(
  252. reinterpret_cast<const uint8_t*>(kInvalidData.data()),
  253. kInvalidData.size());
  254. std::vector<DecodeResult> results = Decode(kInvalidBuffer);
  255. std::vector<DecodeResult> expected = {DecodeResult::kDecodeError};
  256. EXPECT_EQ(results, expected);
  257. }
  258. TEST_F(AV1DecoderTest, DecodeEmptyOBU) {
  259. auto kEmptyBuffer = base::MakeRefCounted<DecoderBuffer>(0);
  260. std::vector<DecodeResult> results = Decode(kEmptyBuffer);
  261. std::vector<DecodeResult> expected = {DecodeResult::kRanOutOfStreamData};
  262. EXPECT_EQ(results, expected);
  263. }
  264. TEST_F(AV1DecoderTest, DecodeOneIFrame) {
  265. constexpr gfx::Size kFrameSize(320, 240);
  266. constexpr gfx::Size kRenderSize(320, 240);
  267. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  268. const std::string kIFrame("av1-I-frame-320x240");
  269. scoped_refptr<DecoderBuffer> i_frame_buffer = ReadDecoderBuffer(kIFrame);
  270. ASSERT_TRUE(!!i_frame_buffer);
  271. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  272. ::testing::InSequence s;
  273. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  274. .WillOnce(Return(av1_picture));
  275. EXPECT_CALL(
  276. *mock_accelerator_,
  277. SubmitDecode(
  278. MatchesFrameHeader(kFrameSize, kRenderSize,
  279. /*show_existing_frame=*/false,
  280. /*show_frame=*/true),
  281. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  282. /*film_grain_params_present=*/false),
  283. _, NonEmptyTileBuffers(), MatchesFrameData(i_frame_buffer)))
  284. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  285. EXPECT_CALL(*mock_accelerator_,
  286. OutputPicture(SameAV1PictureInstance(av1_picture)))
  287. .WillOnce(Return(true));
  288. std::vector<DecodeResult> results = Decode(i_frame_buffer);
  289. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange,
  290. DecodeResult::kRanOutOfStreamData};
  291. EXPECT_EQ(results, expected);
  292. }
  293. TEST_F(AV1DecoderTest, DecodeSimpleStream) {
  294. constexpr gfx::Size kFrameSize(320, 240);
  295. constexpr gfx::Size kRenderSize(320, 240);
  296. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  297. const std::string kSimpleStream("bear-av1.webm");
  298. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadWebm(kSimpleStream);
  299. ASSERT_FALSE(buffers.empty());
  300. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange};
  301. std::vector<DecodeResult> results;
  302. for (auto buffer : buffers) {
  303. ::testing::InSequence sequence;
  304. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  305. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  306. .WillOnce(Return(av1_picture));
  307. EXPECT_CALL(
  308. *mock_accelerator_,
  309. SubmitDecode(
  310. MatchesFrameHeader(kFrameSize, kRenderSize,
  311. /*show_existing_frame=*/false,
  312. /*show_frame=*/true),
  313. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  314. /*film_grain_params_present=*/false),
  315. _, NonEmptyTileBuffers(), MatchesFrameData(buffer)))
  316. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  317. EXPECT_CALL(*mock_accelerator_,
  318. OutputPicture(SameAV1PictureInstance(av1_picture)))
  319. .WillOnce(Return(true));
  320. for (DecodeResult r : Decode(buffer))
  321. results.push_back(r);
  322. expected.push_back(DecodeResult::kRanOutOfStreamData);
  323. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  324. }
  325. EXPECT_EQ(results, expected);
  326. }
  327. TEST_F(AV1DecoderTest, DecodeShowExistingPictureStream) {
  328. constexpr gfx::Size kFrameSize(208, 144);
  329. constexpr gfx::Size kRenderSize(208, 144);
  330. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  331. constexpr size_t kDecodedFrames = 10;
  332. constexpr size_t kOutputFrames = 10;
  333. const std::string kShowExistingFrameStream("av1-show_existing_frame.ivf");
  334. std::vector<scoped_refptr<DecoderBuffer>> buffers =
  335. ReadIVF(kShowExistingFrameStream);
  336. ASSERT_FALSE(buffers.empty());
  337. // TODO(hiroh): Test what's unique about the show_existing_frame path.
  338. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange};
  339. std::vector<DecodeResult> results;
  340. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  341. .Times(kDecodedFrames)
  342. .WillRepeatedly(Return(base::MakeRefCounted<FakeAV1Picture>()));
  343. EXPECT_CALL(
  344. *mock_accelerator_,
  345. SubmitDecode(
  346. MatchesFrameSizeAndRenderSize(kFrameSize, kRenderSize),
  347. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  348. /*film_grain_params_present=*/false),
  349. _, NonEmptyTileBuffers(), _))
  350. .Times(kDecodedFrames)
  351. .WillRepeatedly(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  352. EXPECT_CALL(*mock_accelerator_, OutputPicture(_))
  353. .Times(kOutputFrames)
  354. .WillRepeatedly(Return(true));
  355. for (auto buffer : buffers) {
  356. for (DecodeResult r : Decode(buffer))
  357. results.push_back(r);
  358. expected.push_back(DecodeResult::kRanOutOfStreamData);
  359. }
  360. EXPECT_EQ(results, expected);
  361. }
  362. TEST_F(AV1DecoderTest, Decode10bitStream) {
  363. const std::string k10bitStream("bear-av1-320x180-10bit.webm");
  364. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadWebm(k10bitStream);
  365. ASSERT_FALSE(buffers.empty());
  366. constexpr gfx::Size kFrameSize(320, 180);
  367. constexpr gfx::Size kRenderSize(320, 180);
  368. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  369. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange};
  370. std::vector<DecodeResult> results;
  371. for (auto buffer : buffers) {
  372. ::testing::InSequence sequence;
  373. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  374. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  375. .WillOnce(Return(av1_picture));
  376. EXPECT_CALL(
  377. *mock_accelerator_,
  378. SubmitDecode(
  379. MatchesFrameHeader(kFrameSize, kRenderSize,
  380. /*show_existing_frame=*/false,
  381. /*show_frame=*/true),
  382. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/10, kFrameSize,
  383. /*film_grain_params_present=*/false),
  384. _, NonEmptyTileBuffers(), MatchesFrameData(buffer)))
  385. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  386. EXPECT_CALL(*mock_accelerator_,
  387. OutputPicture(SameAV1PictureInstance(av1_picture)))
  388. .WillOnce(Return(true));
  389. for (DecodeResult r : Decode(buffer))
  390. results.push_back(r);
  391. expected.push_back(DecodeResult::kRanOutOfStreamData);
  392. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  393. }
  394. EXPECT_EQ(results, expected);
  395. }
  396. TEST_F(AV1DecoderTest, DecodeSVCStream) {
  397. const std::string kSVCStream("av1-svc-L2T2.ivf");
  398. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadIVF(kSVCStream);
  399. ASSERT_FALSE(buffers.empty());
  400. std::vector<DecodeResult> expected = {DecodeResult::kDecodeError};
  401. EXPECT_EQ(Decode(buffers[0]), expected);
  402. // Once AV1Decoder gets into an error state, Decode() returns kDecodeError
  403. // until Reset().
  404. EXPECT_EQ(Decode(buffers[1]), expected);
  405. }
  406. TEST_F(AV1DecoderTest, DenyDecodeNonYUV420) {
  407. const std::string kYUV444Stream("blackwhite_yuv444p-frame.av1.ivf");
  408. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadIVF(kYUV444Stream);
  409. ASSERT_EQ(buffers.size(), 1u);
  410. std::vector<DecodeResult> expected = {DecodeResult::kDecodeError};
  411. EXPECT_EQ(Decode(buffers[0]), expected);
  412. // Once AV1Decoder gets into an error state, Decode() returns kDecodeError
  413. // until Reset().
  414. EXPECT_EQ(Decode(buffers[0]), expected);
  415. }
  416. TEST_F(AV1DecoderTest, DecodeFilmGrain) {
  417. // Note: This video also contains show_existing_frame.
  418. const std::string kFilmGrainStream("av1-film_grain.ivf");
  419. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadIVF(kFilmGrainStream);
  420. ASSERT_FALSE(buffers.empty());
  421. constexpr size_t kDecodedFrames = 11;
  422. constexpr size_t kOutputFrames = 10;
  423. constexpr gfx::Size kFrameSize(352, 288);
  424. constexpr gfx::Size kRenderSize(352, 288);
  425. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  426. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange};
  427. std::vector<DecodeResult> results;
  428. // TODO(hiroh): test that CreateAV1Picture is called with the right parameter
  429. // which depends on the frame
  430. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(_))
  431. .Times(kDecodedFrames)
  432. .WillRepeatedly(Return(base::MakeRefCounted<FakeAV1Picture>()));
  433. EXPECT_CALL(
  434. *mock_accelerator_,
  435. SubmitDecode(
  436. MatchesFrameSizeAndRenderSize(kFrameSize, kRenderSize),
  437. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  438. /*film_grain_params_present=*/true),
  439. _, NonEmptyTileBuffers(), _))
  440. .Times(kDecodedFrames)
  441. .WillRepeatedly(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  442. EXPECT_CALL(*mock_accelerator_, OutputPicture(_))
  443. .Times(kOutputFrames)
  444. .WillRepeatedly(Return(true));
  445. for (auto buffer : buffers) {
  446. for (DecodeResult r : Decode(buffer))
  447. results.push_back(r);
  448. expected.push_back(DecodeResult::kRanOutOfStreamData);
  449. }
  450. EXPECT_EQ(results, expected);
  451. }
  452. // TODO(b/175895249): Test in isolation each of the conditions that trigger a
  453. // kConfigChange event.
  454. TEST_F(AV1DecoderTest, ConfigChange) {
  455. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  456. constexpr auto kMediaProfile = VideoCodecProfile::AV1PROFILE_PROFILE_MAIN;
  457. const std::string kSimpleStreams[] = {"bear-av1.webm",
  458. "bear-av1-480x360.webm"};
  459. constexpr gfx::Size kFrameSizes[] = {{320, 240}, {480, 360}};
  460. constexpr gfx::Size kRenderSizes[] = {{320, 240}, {480, 360}};
  461. std::vector<DecodeResult> expected;
  462. std::vector<DecodeResult> results;
  463. for (size_t i = 0; i < std::size(kSimpleStreams); ++i) {
  464. std::vector<scoped_refptr<DecoderBuffer>> buffers =
  465. ReadWebm(kSimpleStreams[i]);
  466. ASSERT_FALSE(buffers.empty());
  467. expected.push_back(DecodeResult::kConfigChange);
  468. for (auto buffer : buffers) {
  469. ::testing::InSequence sequence;
  470. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  471. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  472. .WillOnce(Return(av1_picture));
  473. EXPECT_CALL(
  474. *mock_accelerator_,
  475. SubmitDecode(MatchesFrameHeader(kFrameSizes[i], kRenderSizes[i],
  476. /*show_existing_frame=*/false,
  477. /*show_frame=*/true),
  478. MatchesYUV420SequenceHeader(
  479. kProfile, /*bitdepth=*/8, kFrameSizes[i],
  480. /*film_grain_params_present=*/false),
  481. _, NonEmptyTileBuffers(), MatchesFrameData(buffer)))
  482. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  483. EXPECT_CALL(*mock_accelerator_,
  484. OutputPicture(SameAV1PictureInstance(av1_picture)))
  485. .WillOnce(Return(true));
  486. for (DecodeResult r : Decode(buffer))
  487. results.push_back(r);
  488. expected.push_back(DecodeResult::kRanOutOfStreamData);
  489. EXPECT_EQ(decoder_->GetProfile(), kMediaProfile);
  490. EXPECT_EQ(decoder_->GetPicSize(), kFrameSizes[i]);
  491. EXPECT_EQ(decoder_->GetVisibleRect(), gfx::Rect(kRenderSizes[i]));
  492. EXPECT_EQ(decoder_->GetBitDepth(), 8u);
  493. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  494. }
  495. }
  496. EXPECT_EQ(results, expected);
  497. }
  498. TEST_F(AV1DecoderTest, Reset) {
  499. constexpr gfx::Size kFrameSize(320, 240);
  500. constexpr gfx::Size kRenderSize(320, 240);
  501. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  502. constexpr auto kMediaProfile = VideoCodecProfile::AV1PROFILE_PROFILE_MAIN;
  503. constexpr uint8_t kBitDepth = 8u;
  504. const std::string kSimpleStream("bear-av1.webm");
  505. std::vector<DecodeResult> expected;
  506. std::vector<DecodeResult> results;
  507. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadWebm(kSimpleStream);
  508. ASSERT_FALSE(buffers.empty());
  509. expected.push_back(DecodeResult::kConfigChange);
  510. for (int k = 0; k < 2; k++) {
  511. for (auto buffer : buffers) {
  512. ::testing::InSequence sequence;
  513. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  514. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  515. .WillOnce(Return(av1_picture));
  516. EXPECT_CALL(
  517. *mock_accelerator_,
  518. SubmitDecode(
  519. MatchesFrameHeader(kFrameSize, kRenderSize,
  520. /*show_existing_frame=*/false,
  521. /*show_frame=*/true),
  522. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  523. /*film_grain_params_present=*/false),
  524. _, NonEmptyTileBuffers(), MatchesFrameData(buffer)))
  525. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  526. EXPECT_CALL(*mock_accelerator_,
  527. OutputPicture(SameAV1PictureInstance(av1_picture)))
  528. .WillOnce(Return(true));
  529. for (DecodeResult r : Decode(buffer))
  530. results.push_back(r);
  531. expected.push_back(DecodeResult::kRanOutOfStreamData);
  532. EXPECT_EQ(decoder_->GetProfile(), kMediaProfile);
  533. EXPECT_EQ(decoder_->GetPicSize(), kFrameSize);
  534. EXPECT_EQ(decoder_->GetVisibleRect(), gfx::Rect(kRenderSize));
  535. EXPECT_EQ(decoder_->GetBitDepth(), kBitDepth);
  536. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  537. }
  538. Reset();
  539. // Ensures Reset() doesn't clear the stored stream states.
  540. EXPECT_EQ(decoder_->GetProfile(), kMediaProfile);
  541. EXPECT_EQ(decoder_->GetPicSize(), kFrameSize);
  542. EXPECT_EQ(decoder_->GetVisibleRect(), gfx::Rect(kRenderSize));
  543. EXPECT_EQ(decoder_->GetBitDepth(), kBitDepth);
  544. }
  545. EXPECT_EQ(results, expected);
  546. }
  547. TEST_F(AV1DecoderTest, ResetAndConfigChange) {
  548. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  549. constexpr auto kMediaProfile = VideoCodecProfile::AV1PROFILE_PROFILE_MAIN;
  550. const std::string kSimpleStreams[] = {"bear-av1.webm",
  551. "bear-av1-480x360.webm"};
  552. constexpr gfx::Size kFrameSizes[] = {{320, 240}, {480, 360}};
  553. constexpr gfx::Size kRenderSizes[] = {{320, 240}, {480, 360}};
  554. constexpr uint8_t kBitDepth = 8u;
  555. std::vector<DecodeResult> expected;
  556. std::vector<DecodeResult> results;
  557. for (size_t i = 0; i < std::size(kSimpleStreams); ++i) {
  558. std::vector<scoped_refptr<DecoderBuffer>> buffers =
  559. ReadWebm(kSimpleStreams[i]);
  560. ASSERT_FALSE(buffers.empty());
  561. expected.push_back(DecodeResult::kConfigChange);
  562. for (auto buffer : buffers) {
  563. ::testing::InSequence sequence;
  564. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  565. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  566. .WillOnce(Return(av1_picture));
  567. EXPECT_CALL(
  568. *mock_accelerator_,
  569. SubmitDecode(MatchesFrameHeader(kFrameSizes[i], kRenderSizes[i],
  570. /*show_existing_frame=*/false,
  571. /*show_frame=*/true),
  572. MatchesYUV420SequenceHeader(
  573. kProfile, /*bitdepth=*/8, kFrameSizes[i],
  574. /*film_grain_params_present=*/false),
  575. _, NonEmptyTileBuffers(), MatchesFrameData(buffer)))
  576. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  577. EXPECT_CALL(*mock_accelerator_,
  578. OutputPicture(SameAV1PictureInstance(av1_picture)))
  579. .WillOnce(Return(true));
  580. for (DecodeResult r : Decode(buffer))
  581. results.push_back(r);
  582. expected.push_back(DecodeResult::kRanOutOfStreamData);
  583. EXPECT_EQ(decoder_->GetProfile(), kMediaProfile);
  584. EXPECT_EQ(decoder_->GetPicSize(), kFrameSizes[i]);
  585. EXPECT_EQ(decoder_->GetVisibleRect(), gfx::Rect(kRenderSizes[i]));
  586. EXPECT_EQ(decoder_->GetBitDepth(), kBitDepth);
  587. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  588. }
  589. Reset();
  590. // Ensures Reset() doesn't clear the stored stream states.
  591. EXPECT_EQ(decoder_->GetProfile(), kMediaProfile);
  592. EXPECT_EQ(decoder_->GetPicSize(), kFrameSizes[i]);
  593. EXPECT_EQ(decoder_->GetVisibleRect(), gfx::Rect(kRenderSizes[i]));
  594. EXPECT_EQ(decoder_->GetBitDepth(), kBitDepth);
  595. }
  596. EXPECT_EQ(results, expected);
  597. }
  598. // This test ensures that the AV1Decoder fails gracefully if for some reason,
  599. // the reference frame state tracked by AV1Decoder becomes inconsistent with the
  600. // state tracked by libgav1.
  601. TEST_F(AV1DecoderTest, InconsistentReferenceFrameState) {
  602. const std::string kSimpleStream("bear-av1.webm");
  603. std::vector<scoped_refptr<DecoderBuffer>> buffers = ReadWebm(kSimpleStream);
  604. ASSERT_GE(buffers.size(), 2u);
  605. // In this test stream, the first frame is an intra frame and the second one
  606. // is not. Let's start by decoding the first frame and inspecting the
  607. // reference frame state.
  608. {
  609. ::testing::InSequence sequence;
  610. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  611. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  612. .WillOnce(Return(av1_picture));
  613. AV1ReferenceFrameVector ref_frames;
  614. EXPECT_CALL(*mock_accelerator_,
  615. SubmitDecode(SameAV1PictureInstance(av1_picture), _, _, _, _))
  616. .WillOnce(DoAll(SaveArg<2>(&ref_frames),
  617. Return(AV1Decoder::AV1Accelerator::Status::kOk)));
  618. EXPECT_CALL(*mock_accelerator_,
  619. OutputPicture(SameAV1PictureInstance(av1_picture)))
  620. .WillOnce(Return(true));
  621. // Before decoding, let's make sure that libgav1 doesn't think any reference
  622. // frames are valid.
  623. const libgav1::DecoderState* decoder_state = GetDecoderState();
  624. ASSERT_TRUE(decoder_state);
  625. EXPECT_EQ(base::STLCount(decoder_state->reference_frame, nullptr),
  626. base::checked_cast<long>(decoder_state->reference_frame.size()));
  627. // And to be consistent, AV1Decoder should not be tracking any reference
  628. // frames yet.
  629. const AV1ReferenceFrameVector& internal_ref_frames = GetReferenceFrames();
  630. EXPECT_EQ(base::STLCount(internal_ref_frames, nullptr),
  631. base::checked_cast<long>(internal_ref_frames.size()));
  632. // Now try to decode one frame and make sure that the frame is intra.
  633. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange,
  634. DecodeResult::kRanOutOfStreamData};
  635. std::vector<DecodeResult> results = Decode(buffers[0]);
  636. EXPECT_EQ(results, expected);
  637. EXPECT_TRUE(libgav1::IsIntraFrame(av1_picture->frame_header.frame_type));
  638. // SubmitDecode() should have received the reference frames before they were
  639. // updated. That means that it should have received no reference frames
  640. // since this SubmitDecode() refers to the first frame.
  641. EXPECT_EQ(base::STLCount(ref_frames, nullptr),
  642. base::checked_cast<long>(ref_frames.size()));
  643. // Now let's inspect the current state of things (which is after the
  644. // reference frames have been updated): libgav1 should have decided that all
  645. // reference frames are valid.
  646. ASSERT_TRUE(decoder_state);
  647. EXPECT_EQ(base::STLCount(decoder_state->reference_frame, nullptr), 0);
  648. // And to be consistent, all the reference frames tracked by the AV1Decoder
  649. // should also be valid and they should be pointing to the only AV1Picture
  650. // so far.
  651. EXPECT_TRUE(
  652. std::all_of(internal_ref_frames.begin(), internal_ref_frames.end(),
  653. [&av1_picture](const scoped_refptr<AV1Picture>& ref_frame) {
  654. return ref_frame.get() == av1_picture.get();
  655. }));
  656. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  657. }
  658. // Now we will purposefully mess up the reference frame state tracked by the
  659. // AV1Decoder by removing one of the reference frames. This should cause the
  660. // decode of the second frame to fail because the AV1Decoder should detect the
  661. // inconsistency.
  662. GetReferenceFrames()[1] = nullptr;
  663. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  664. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  665. .WillOnce(Return(av1_picture));
  666. std::vector<DecodeResult> expected = {DecodeResult::kDecodeError};
  667. std::vector<DecodeResult> results = Decode(buffers[1]);
  668. EXPECT_EQ(results, expected);
  669. // Just for rigor, let's check the state at the moment of failure. First, the
  670. // current frame should be an inter frame (and its header should have been
  671. // stored in the AV1Picture).
  672. EXPECT_EQ(av1_picture->frame_header.frame_type, libgav1::kFrameInter);
  673. // Next, let's check the reference frames that frame needs.
  674. for (int8_t i = 0; i < libgav1::kNumInterReferenceFrameTypes; ++i)
  675. EXPECT_EQ(av1_picture->frame_header.reference_frame_index[i], i);
  676. // Finally, let's check that libgav1 thought that all the reference frames
  677. // were valid.
  678. const libgav1::DecoderState* decoder_state = GetDecoderState();
  679. ASSERT_TRUE(decoder_state);
  680. EXPECT_EQ(base::STLCount(decoder_state->reference_frame, nullptr), 0);
  681. }
  682. TEST_F(AV1DecoderTest, TryAgainSubmitDecode) {
  683. constexpr gfx::Size kFrameSize(320, 240);
  684. constexpr gfx::Size kRenderSize(320, 240);
  685. constexpr auto kProfile = libgav1::BitstreamProfile::kProfile0;
  686. const std::string kIFrame("av1-I-frame-320x240");
  687. scoped_refptr<DecoderBuffer> i_frame_buffer = ReadDecoderBuffer(kIFrame);
  688. ASSERT_TRUE(!!i_frame_buffer);
  689. auto av1_picture = base::MakeRefCounted<AV1Picture>();
  690. ::testing::InSequence s;
  691. EXPECT_CALL(*mock_accelerator_, CreateAV1Picture(/*apply_grain=*/false))
  692. .WillOnce(Return(av1_picture));
  693. EXPECT_CALL(
  694. *mock_accelerator_,
  695. SubmitDecode(
  696. MatchesFrameHeader(kFrameSize, kRenderSize,
  697. /*show_existing_frame=*/false,
  698. /*show_frame=*/true),
  699. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  700. /*film_grain_params_present=*/false),
  701. _, NonEmptyTileBuffers(), MatchesFrameData(i_frame_buffer)))
  702. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kTryAgain));
  703. EXPECT_CALL(*mock_accelerator_, OutputPicture(_)).Times(0);
  704. std::vector<DecodeResult> results = Decode(i_frame_buffer);
  705. std::vector<DecodeResult> expected = {DecodeResult::kConfigChange,
  706. DecodeResult::kTryAgain};
  707. EXPECT_EQ(results, expected);
  708. testing::Mock::VerifyAndClearExpectations(mock_accelerator_);
  709. // Now try again and have it succeed.
  710. EXPECT_CALL(
  711. *mock_accelerator_,
  712. SubmitDecode(
  713. MatchesFrameHeader(kFrameSize, kRenderSize,
  714. /*show_existing_frame=*/false,
  715. /*show_frame=*/true),
  716. MatchesYUV420SequenceHeader(kProfile, /*bitdepth=*/8, kFrameSize,
  717. /*film_grain_params_present=*/false),
  718. _, NonEmptyTileBuffers(), MatchesFrameData(i_frame_buffer)))
  719. .WillOnce(Return(AV1Decoder::AV1Accelerator::Status::kOk));
  720. EXPECT_CALL(*mock_accelerator_,
  721. OutputPicture(SameAV1PictureInstance(av1_picture)))
  722. .WillOnce(Return(true));
  723. results = Decode(nullptr);
  724. expected = {DecodeResult::kRanOutOfStreamData};
  725. EXPECT_EQ(results, expected);
  726. }
  727. // TODO(hiroh): Add more tests: reference frame tracking, render size change,
  728. // profile change, bit depth change, render size different than the frame size,
  729. // visible rectangle change in the middle of video sequence, reset while waiting
  730. // for buffers, flushing.
  731. } // namespace media