video_frame_unittest.cc 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/base/video_frame.h"
  5. #include <stddef.h>
  6. #include <stdint.h>
  7. #include <memory>
  8. #include "base/bind.h"
  9. #include "base/callback_helpers.h"
  10. #include "base/format_macros.h"
  11. #include "base/memory/aligned_memory.h"
  12. #include "base/memory/unsafe_shared_memory_region.h"
  13. #include "base/strings/stringprintf.h"
  14. #include "build/build_config.h"
  15. #include "gpu/command_buffer/common/mailbox_holder.h"
  16. #include "media/base/color_plane_layout.h"
  17. #include "media/base/simple_sync_token_client.h"
  18. #include "media/video/fake_gpu_memory_buffer.h"
  19. #include "testing/gtest/include/gtest/gtest.h"
  20. #include "third_party/libyuv/include/libyuv.h"
  21. namespace {
  22. // Creates the backing storage for a frame suitable for WrapExternalData. Note
  23. // that this is currently used only to test frame creation and tear-down, and so
  24. // may not have alignment or other properties correct further video processing.
  25. // |memory| must be at least 2 * coded_size.width() * coded_size.height() in
  26. // bytes.
  27. void CreateTestY16Frame(const gfx::Size& coded_size,
  28. const gfx::Rect& visible_rect,
  29. void* memory) {
  30. const int offset_x = visible_rect.x();
  31. const int offset_y = visible_rect.y();
  32. const int stride = coded_size.width();
  33. // In the visible rect, fill upper byte with [0-255] and lower with [255-0].
  34. uint16_t* data = static_cast<uint16_t*>(memory);
  35. for (int j = 0; j < visible_rect.height(); j++) {
  36. for (int i = 0; i < visible_rect.width(); i++) {
  37. const int value = i + j * visible_rect.width();
  38. data[(stride * (j + offset_y)) + i + offset_x] =
  39. ((value & 0xFF) << 8) | (~value & 0xFF);
  40. }
  41. }
  42. }
  43. // Returns a VideoFrameMetadata object with a value for each field.
  44. media::VideoFrameMetadata GetFullVideoFrameMetadata() {
  45. // Assign a non-default, distinct (when possible), value to all fields, and
  46. // make sure values are preserved across serialization.
  47. media::VideoFrameMetadata metadata;
  48. // ints
  49. metadata.capture_counter = 123;
  50. // gfx::Rects
  51. metadata.capture_update_rect = gfx::Rect(12, 34, 360, 480);
  52. // media::VideoTransformation
  53. metadata.transformation = media::VIDEO_ROTATION_90;
  54. // bools
  55. metadata.allow_overlay = true;
  56. metadata.copy_required = true;
  57. metadata.end_of_stream = true;
  58. metadata.texture_owner = true;
  59. metadata.wants_promotion_hint = true;
  60. metadata.protected_video = true;
  61. metadata.hw_protected = true;
  62. metadata.power_efficient = true;
  63. metadata.read_lock_fences_enabled = true;
  64. metadata.interactive_content = true;
  65. // base::UnguessableTokens
  66. metadata.overlay_plane_id = base::UnguessableToken::Create();
  67. // doubles
  68. metadata.device_scale_factor = 2.0;
  69. metadata.page_scale_factor = 2.1;
  70. metadata.root_scroll_offset_x = 100.2;
  71. metadata.root_scroll_offset_y = 200.1;
  72. metadata.top_controls_visible_height = 25.5;
  73. metadata.frame_rate = 29.94;
  74. metadata.rtp_timestamp = 1.0;
  75. // base::TimeTicks
  76. base::TimeTicks now = base::TimeTicks::Now();
  77. metadata.receive_time = now + base::Milliseconds(10);
  78. metadata.capture_begin_time = now + base::Milliseconds(20);
  79. metadata.capture_end_time = now + base::Milliseconds(30);
  80. metadata.decode_begin_time = now + base::Milliseconds(40);
  81. metadata.decode_end_time = now + base::Milliseconds(50);
  82. metadata.reference_time = now + base::Milliseconds(60);
  83. // base::TimeDeltas
  84. metadata.processing_time = base::Milliseconds(500);
  85. metadata.frame_duration = base::Milliseconds(16);
  86. metadata.wallclock_frame_duration = base::Milliseconds(17);
  87. return metadata;
  88. }
  89. void VerifyVideoFrameMetadataEquality(const media::VideoFrameMetadata& a,
  90. const media::VideoFrameMetadata& b) {
  91. EXPECT_EQ(a.allow_overlay, b.allow_overlay);
  92. EXPECT_EQ(a.capture_begin_time, b.capture_begin_time);
  93. EXPECT_EQ(a.capture_end_time, b.capture_end_time);
  94. EXPECT_EQ(a.capture_counter, b.capture_counter);
  95. EXPECT_EQ(a.capture_update_rect, b.capture_update_rect);
  96. EXPECT_EQ(a.copy_required, b.copy_required);
  97. EXPECT_EQ(a.end_of_stream, b.end_of_stream);
  98. EXPECT_EQ(a.frame_duration, b.frame_duration);
  99. EXPECT_EQ(a.frame_rate, b.frame_rate);
  100. EXPECT_EQ(a.interactive_content, b.interactive_content);
  101. EXPECT_EQ(a.reference_time, b.reference_time);
  102. EXPECT_EQ(a.read_lock_fences_enabled, b.read_lock_fences_enabled);
  103. EXPECT_EQ(a.transformation, b.transformation);
  104. EXPECT_EQ(a.texture_owner, b.texture_owner);
  105. EXPECT_EQ(a.wants_promotion_hint, b.wants_promotion_hint);
  106. EXPECT_EQ(a.protected_video, b.protected_video);
  107. EXPECT_EQ(a.hw_protected, b.hw_protected);
  108. EXPECT_EQ(a.overlay_plane_id, b.overlay_plane_id);
  109. EXPECT_EQ(a.power_efficient, b.power_efficient);
  110. EXPECT_EQ(a.device_scale_factor, b.device_scale_factor);
  111. EXPECT_EQ(a.page_scale_factor, b.page_scale_factor);
  112. EXPECT_EQ(a.root_scroll_offset_x, b.root_scroll_offset_x);
  113. EXPECT_EQ(a.root_scroll_offset_y, b.root_scroll_offset_y);
  114. EXPECT_EQ(a.top_controls_visible_height, b.top_controls_visible_height);
  115. EXPECT_EQ(a.decode_begin_time, b.decode_begin_time);
  116. EXPECT_EQ(a.decode_end_time, b.decode_end_time);
  117. EXPECT_EQ(a.processing_time, b.processing_time);
  118. EXPECT_EQ(a.rtp_timestamp, b.rtp_timestamp);
  119. EXPECT_EQ(a.receive_time, b.receive_time);
  120. EXPECT_EQ(a.wallclock_frame_duration, b.wallclock_frame_duration);
  121. }
  122. } // namespace
  123. namespace media {
  124. using base::MD5DigestToBase16;
  125. // Helper function that initializes a YV12 frame with white and black scan
  126. // lines based on the |white_to_black| parameter. If 0, then the entire
  127. // frame will be black, if 1 then the entire frame will be white.
  128. void InitializeYV12Frame(VideoFrame* frame, double white_to_black) {
  129. EXPECT_EQ(PIXEL_FORMAT_YV12, frame->format());
  130. const int first_black_row =
  131. static_cast<int>(frame->coded_size().height() * white_to_black);
  132. uint8_t* y_plane = frame->data(VideoFrame::kYPlane);
  133. for (int row = 0; row < frame->coded_size().height(); ++row) {
  134. int color = (row < first_black_row) ? 0xFF : 0x00;
  135. memset(y_plane, color, frame->stride(VideoFrame::kYPlane));
  136. y_plane += frame->stride(VideoFrame::kYPlane);
  137. }
  138. uint8_t* u_plane = frame->data(VideoFrame::kUPlane);
  139. uint8_t* v_plane = frame->data(VideoFrame::kVPlane);
  140. for (int row = 0; row < frame->coded_size().height(); row += 2) {
  141. memset(u_plane, 0x80, frame->stride(VideoFrame::kUPlane));
  142. memset(v_plane, 0x80, frame->stride(VideoFrame::kVPlane));
  143. u_plane += frame->stride(VideoFrame::kUPlane);
  144. v_plane += frame->stride(VideoFrame::kVPlane);
  145. }
  146. }
  147. // Given a |yv12_frame| this method converts the YV12 frame to RGBA and
  148. // makes sure that all the pixels of the RBG frame equal |expect_rgb_color|.
  149. void ExpectFrameColor(VideoFrame* yv12_frame, uint32_t expect_rgb_color) {
  150. ASSERT_EQ(PIXEL_FORMAT_YV12, yv12_frame->format());
  151. ASSERT_EQ(yv12_frame->stride(VideoFrame::kUPlane),
  152. yv12_frame->stride(VideoFrame::kVPlane));
  153. ASSERT_EQ(
  154. yv12_frame->coded_size().width() & (VideoFrame::kFrameSizeAlignment - 1),
  155. 0u);
  156. ASSERT_EQ(
  157. yv12_frame->coded_size().height() & (VideoFrame::kFrameSizeAlignment - 1),
  158. 0u);
  159. size_t bytes_per_row = yv12_frame->coded_size().width() * 4u;
  160. uint8_t* rgb_data = reinterpret_cast<uint8_t*>(
  161. base::AlignedAlloc(bytes_per_row * yv12_frame->coded_size().height() +
  162. VideoFrame::kFrameSizePadding,
  163. VideoFrame::kFrameAddressAlignment));
  164. libyuv::I420ToARGB(yv12_frame->data(VideoFrame::kYPlane),
  165. yv12_frame->stride(VideoFrame::kYPlane),
  166. yv12_frame->data(VideoFrame::kUPlane),
  167. yv12_frame->stride(VideoFrame::kUPlane),
  168. yv12_frame->data(VideoFrame::kVPlane),
  169. yv12_frame->stride(VideoFrame::kVPlane), rgb_data,
  170. bytes_per_row, yv12_frame->coded_size().width(),
  171. yv12_frame->coded_size().height());
  172. for (int row = 0; row < yv12_frame->coded_size().height(); ++row) {
  173. uint32_t* rgb_row_data =
  174. reinterpret_cast<uint32_t*>(rgb_data + (bytes_per_row * row));
  175. for (int col = 0; col < yv12_frame->coded_size().width(); ++col) {
  176. SCOPED_TRACE(base::StringPrintf("Checking (%d, %d)", row, col));
  177. EXPECT_EQ(expect_rgb_color, rgb_row_data[col]);
  178. }
  179. }
  180. base::AlignedFree(rgb_data);
  181. }
  182. // Fill each plane to its reported extents and verify accessors report non
  183. // zero values. Additionally, for the first plane verify the rows, row_bytes,
  184. // and columns values are correct.
  185. void ExpectFrameExtents(VideoPixelFormat format, const char* expected_hash) {
  186. const unsigned char kFillByte = 0x80;
  187. const int kWidth = 61;
  188. const int kHeight = 31;
  189. const base::TimeDelta kTimestamp = base::Microseconds(1337);
  190. gfx::Size size(kWidth, kHeight);
  191. scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
  192. format, size, gfx::Rect(size), size, kTimestamp);
  193. ASSERT_TRUE(frame.get());
  194. int planes = VideoFrame::NumPlanes(format);
  195. for (int plane = 0; plane < planes; plane++) {
  196. SCOPED_TRACE(base::StringPrintf("Checking plane %d", plane));
  197. EXPECT_TRUE(frame->data(plane));
  198. EXPECT_TRUE(frame->stride(plane));
  199. EXPECT_TRUE(frame->rows(plane));
  200. EXPECT_TRUE(frame->row_bytes(plane));
  201. EXPECT_TRUE(frame->columns(plane));
  202. memset(frame->data(plane), kFillByte,
  203. frame->stride(plane) * frame->rows(plane));
  204. }
  205. base::MD5Context context;
  206. base::MD5Init(&context);
  207. VideoFrame::HashFrameForTesting(&context, *frame.get());
  208. base::MD5Digest digest;
  209. base::MD5Final(&digest, &context);
  210. EXPECT_EQ(MD5DigestToBase16(digest), expected_hash);
  211. }
  212. TEST(VideoFrame, CreateFrame) {
  213. const int kWidth = 64;
  214. const int kHeight = 48;
  215. const base::TimeDelta kTimestamp = base::Microseconds(1337);
  216. // Create a YV12 Video Frame.
  217. gfx::Size size(kWidth, kHeight);
  218. scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
  219. PIXEL_FORMAT_YV12, size, gfx::Rect(size), size, kTimestamp);
  220. ASSERT_TRUE(frame.get());
  221. // Test VideoFrame implementation.
  222. EXPECT_EQ(PIXEL_FORMAT_YV12, frame->format());
  223. {
  224. SCOPED_TRACE("");
  225. InitializeYV12Frame(frame.get(), 0.0f);
  226. ExpectFrameColor(frame.get(), 0xFF000000);
  227. }
  228. base::MD5Digest digest;
  229. base::MD5Context context;
  230. base::MD5Init(&context);
  231. VideoFrame::HashFrameForTesting(&context, *frame.get());
  232. base::MD5Final(&digest, &context);
  233. EXPECT_EQ(MD5DigestToBase16(digest), "9065c841d9fca49186ef8b4ef547e79b");
  234. {
  235. SCOPED_TRACE("");
  236. InitializeYV12Frame(frame.get(), 1.0f);
  237. ExpectFrameColor(frame.get(), 0xFFFFFFFF);
  238. }
  239. base::MD5Init(&context);
  240. VideoFrame::HashFrameForTesting(&context, *frame.get());
  241. base::MD5Final(&digest, &context);
  242. EXPECT_EQ(MD5DigestToBase16(digest), "911991d51438ad2e1a40ed5f6fc7c796");
  243. // Test single planar frame.
  244. frame = VideoFrame::CreateFrame(PIXEL_FORMAT_ARGB, size, gfx::Rect(size),
  245. size, kTimestamp);
  246. EXPECT_EQ(PIXEL_FORMAT_ARGB, frame->format());
  247. EXPECT_GE(frame->stride(VideoFrame::kARGBPlane), frame->coded_size().width());
  248. // Test double planar frame.
  249. frame = VideoFrame::CreateFrame(PIXEL_FORMAT_NV12, size, gfx::Rect(size),
  250. size, kTimestamp);
  251. EXPECT_EQ(PIXEL_FORMAT_NV12, frame->format());
  252. // Test an empty frame.
  253. frame = VideoFrame::CreateEOSFrame();
  254. EXPECT_TRUE(frame->metadata().end_of_stream);
  255. // Test an video hole frame.
  256. frame = VideoFrame::CreateVideoHoleFrame(base::UnguessableToken::Create(),
  257. size, kTimestamp);
  258. ASSERT_TRUE(frame);
  259. }
  260. TEST(VideoFrame, CreateZeroInitializedFrame) {
  261. const int kWidth = 2;
  262. const int kHeight = 2;
  263. const base::TimeDelta kTimestamp = base::Microseconds(1337);
  264. // Create a YV12 Video Frame.
  265. gfx::Size size(kWidth, kHeight);
  266. scoped_refptr<VideoFrame> frame = VideoFrame::CreateZeroInitializedFrame(
  267. PIXEL_FORMAT_YV12, size, gfx::Rect(size), size, kTimestamp);
  268. ASSERT_TRUE(frame.get());
  269. EXPECT_TRUE(frame->IsMappable());
  270. // Verify that frame is initialized with zeros.
  271. // TODO(emircan): Check all the contents when we know the exact size of the
  272. // allocated buffer.
  273. for (size_t i = 0; i < VideoFrame::NumPlanes(frame->format()); ++i)
  274. EXPECT_EQ(0, frame->data(i)[0]);
  275. }
  276. TEST(VideoFrame, CreateBlackFrame) {
  277. const int kWidth = 2;
  278. const int kHeight = 2;
  279. const uint8_t kExpectedYRow[] = {0, 0};
  280. const uint8_t kExpectedUVRow[] = {128};
  281. scoped_refptr<VideoFrame> frame =
  282. VideoFrame::CreateBlackFrame(gfx::Size(kWidth, kHeight));
  283. ASSERT_TRUE(frame.get());
  284. EXPECT_TRUE(frame->IsMappable());
  285. // Test basic properties.
  286. EXPECT_EQ(0, frame->timestamp().InMicroseconds());
  287. EXPECT_FALSE(frame->metadata().end_of_stream);
  288. // Test |frame| properties.
  289. EXPECT_EQ(PIXEL_FORMAT_I420, frame->format());
  290. EXPECT_EQ(kWidth, frame->coded_size().width());
  291. EXPECT_EQ(kHeight, frame->coded_size().height());
  292. // Test frames themselves.
  293. uint8_t* y_plane = frame->data(VideoFrame::kYPlane);
  294. for (int y = 0; y < frame->coded_size().height(); ++y) {
  295. EXPECT_EQ(0, memcmp(kExpectedYRow, y_plane, std::size(kExpectedYRow)));
  296. y_plane += frame->stride(VideoFrame::kYPlane);
  297. }
  298. uint8_t* u_plane = frame->data(VideoFrame::kUPlane);
  299. uint8_t* v_plane = frame->data(VideoFrame::kVPlane);
  300. for (int y = 0; y < frame->coded_size().height() / 2; ++y) {
  301. EXPECT_EQ(0, memcmp(kExpectedUVRow, u_plane, std::size(kExpectedUVRow)));
  302. EXPECT_EQ(0, memcmp(kExpectedUVRow, v_plane, std::size(kExpectedUVRow)));
  303. u_plane += frame->stride(VideoFrame::kUPlane);
  304. v_plane += frame->stride(VideoFrame::kVPlane);
  305. }
  306. }
  307. static void FrameNoLongerNeededCallback(bool* triggered) {
  308. *triggered = true;
  309. }
  310. TEST(VideoFrame, WrapVideoFrame) {
  311. const int kWidth = 4;
  312. const int kHeight = 4;
  313. const base::TimeDelta kFrameDuration = base::Microseconds(42);
  314. scoped_refptr<VideoFrame> frame, frame2;
  315. bool base_frame_done_callback_was_run = false;
  316. bool wrapped_frame_done_callback_was_run = false;
  317. {
  318. auto base_frame = VideoFrame::CreateBlackFrame(gfx::Size(kWidth, kHeight));
  319. ASSERT_TRUE(base_frame);
  320. gfx::Rect visible_rect(0, 0, 2, 2);
  321. gfx::Size natural_size = visible_rect.size();
  322. base_frame->metadata().frame_duration = kFrameDuration;
  323. frame = VideoFrame::WrapVideoFrame(base_frame, base_frame->format(),
  324. visible_rect, natural_size);
  325. base_frame->AddDestructionObserver(base::BindOnce(
  326. &FrameNoLongerNeededCallback, &base_frame_done_callback_was_run));
  327. EXPECT_EQ(base_frame->coded_size(), frame->coded_size());
  328. EXPECT_EQ(base_frame->data(VideoFrame::kYPlane),
  329. frame->data(VideoFrame::kYPlane));
  330. EXPECT_NE(base_frame->visible_rect(), frame->visible_rect());
  331. EXPECT_EQ(visible_rect, frame->visible_rect());
  332. EXPECT_NE(base_frame->natural_size(), frame->natural_size());
  333. EXPECT_EQ(natural_size, frame->natural_size());
  334. // Verify metadata was copied to the wrapped frame.
  335. EXPECT_EQ(*frame->metadata().frame_duration, kFrameDuration);
  336. // Verify the metadata copy was a deep copy.
  337. base_frame->clear_metadata();
  338. EXPECT_NE(base_frame->metadata().frame_duration.has_value(),
  339. frame->metadata().frame_duration.has_value());
  340. frame->AddDestructionObserver(base::BindOnce(
  341. &FrameNoLongerNeededCallback, &wrapped_frame_done_callback_was_run));
  342. visible_rect = gfx::Rect(0, 0, 1, 1);
  343. natural_size = visible_rect.size();
  344. frame2 = VideoFrame::WrapVideoFrame(frame, frame->format(), visible_rect,
  345. natural_size);
  346. EXPECT_EQ(base_frame->coded_size(), frame2->coded_size());
  347. EXPECT_EQ(base_frame->data(VideoFrame::kYPlane),
  348. frame2->data(VideoFrame::kYPlane));
  349. EXPECT_NE(base_frame->visible_rect(), frame2->visible_rect());
  350. EXPECT_EQ(visible_rect, frame2->visible_rect());
  351. EXPECT_NE(base_frame->natural_size(), frame2->natural_size());
  352. EXPECT_EQ(natural_size, frame2->natural_size());
  353. }
  354. // At this point |base_frame| is held by |frame|, |frame2|.
  355. EXPECT_FALSE(base_frame_done_callback_was_run);
  356. EXPECT_FALSE(wrapped_frame_done_callback_was_run);
  357. // At this point |base_frame| is held by |frame2|, which also holds |frame|.
  358. frame.reset();
  359. EXPECT_FALSE(base_frame_done_callback_was_run);
  360. EXPECT_FALSE(wrapped_frame_done_callback_was_run);
  361. // Now all |base_frame| references should be released.
  362. frame2.reset();
  363. EXPECT_TRUE(base_frame_done_callback_was_run);
  364. }
  365. // Create a frame that wraps unowned memory.
  366. TEST(VideoFrame, WrapExternalData) {
  367. uint8_t memory[2 * 256 * 256];
  368. gfx::Size coded_size(256, 256);
  369. gfx::Rect visible_rect(coded_size);
  370. CreateTestY16Frame(coded_size, visible_rect, memory);
  371. auto timestamp = base::Milliseconds(1);
  372. auto frame = VideoFrame::WrapExternalData(PIXEL_FORMAT_Y16, coded_size,
  373. visible_rect, visible_rect.size(),
  374. memory, sizeof(memory), timestamp);
  375. EXPECT_EQ(frame->coded_size(), coded_size);
  376. EXPECT_EQ(frame->visible_rect(), visible_rect);
  377. EXPECT_EQ(frame->timestamp(), timestamp);
  378. EXPECT_EQ(frame->data(VideoFrame::kYPlane)[0], 0xff);
  379. }
  380. // Create a frame that wraps read-only shared memory.
  381. TEST(VideoFrame, WrapSharedMemory) {
  382. const size_t kDataSize = 2 * 256 * 256;
  383. base::UnsafeSharedMemoryRegion region =
  384. base::UnsafeSharedMemoryRegion::Create(kDataSize);
  385. ASSERT_TRUE(region.IsValid());
  386. base::WritableSharedMemoryMapping mapping = region.Map();
  387. ASSERT_TRUE(mapping.IsValid());
  388. gfx::Size coded_size(256, 256);
  389. gfx::Rect visible_rect(coded_size);
  390. CreateTestY16Frame(coded_size, visible_rect, mapping.memory());
  391. auto timestamp = base::Milliseconds(1);
  392. auto frame = VideoFrame::WrapExternalData(
  393. PIXEL_FORMAT_Y16, coded_size, visible_rect, visible_rect.size(),
  394. mapping.GetMemoryAsSpan<uint8_t>().data(), kDataSize, timestamp);
  395. frame->BackWithSharedMemory(&region);
  396. EXPECT_EQ(frame->coded_size(), coded_size);
  397. EXPECT_EQ(frame->visible_rect(), visible_rect);
  398. EXPECT_EQ(frame->timestamp(), timestamp);
  399. EXPECT_EQ(frame->data(VideoFrame::kYPlane)[0], 0xff);
  400. }
  401. TEST(VideoFrame, WrapExternalGpuMemoryBuffer) {
  402. gfx::Size coded_size = gfx::Size(256, 256);
  403. gfx::Rect visible_rect(coded_size);
  404. auto timestamp = base::Milliseconds(1);
  405. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  406. const uint64_t modifier = 0x001234567890abcdULL;
  407. #else
  408. const uint64_t modifier = gfx::NativePixmapHandle::kNoModifier;
  409. #endif
  410. std::unique_ptr<gfx::GpuMemoryBuffer> gmb =
  411. std::make_unique<FakeGpuMemoryBuffer>(
  412. coded_size, gfx::BufferFormat::YUV_420_BIPLANAR, modifier);
  413. gfx::GpuMemoryBuffer* gmb_raw_ptr = gmb.get();
  414. gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes] = {
  415. gpu::MailboxHolder(gpu::Mailbox::Generate(), gpu::SyncToken(), 5),
  416. gpu::MailboxHolder(gpu::Mailbox::Generate(), gpu::SyncToken(), 10)};
  417. auto frame = VideoFrame::WrapExternalGpuMemoryBuffer(
  418. visible_rect, coded_size, std::move(gmb), mailbox_holders,
  419. base::DoNothing(), timestamp);
  420. EXPECT_EQ(frame->layout().format(), PIXEL_FORMAT_NV12);
  421. EXPECT_EQ(frame->layout().coded_size(), coded_size);
  422. EXPECT_EQ(frame->layout().num_planes(), 2u);
  423. EXPECT_EQ(frame->layout().is_multi_planar(), false);
  424. for (size_t i = 0; i < 2; ++i) {
  425. EXPECT_EQ(frame->layout().planes()[i].stride, coded_size.width());
  426. }
  427. EXPECT_EQ(frame->layout().modifier(), modifier);
  428. EXPECT_EQ(frame->storage_type(), VideoFrame::STORAGE_GPU_MEMORY_BUFFER);
  429. EXPECT_TRUE(frame->HasGpuMemoryBuffer());
  430. EXPECT_EQ(frame->GetGpuMemoryBuffer(), gmb_raw_ptr);
  431. EXPECT_EQ(frame->coded_size(), coded_size);
  432. EXPECT_EQ(frame->visible_rect(), visible_rect);
  433. EXPECT_EQ(frame->timestamp(), timestamp);
  434. EXPECT_EQ(frame->HasTextures(), true);
  435. EXPECT_EQ(frame->HasReleaseMailboxCB(), true);
  436. EXPECT_EQ(frame->mailbox_holder(0).mailbox, mailbox_holders[0].mailbox);
  437. EXPECT_EQ(frame->mailbox_holder(1).mailbox, mailbox_holders[1].mailbox);
  438. }
  439. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  440. TEST(VideoFrame, WrapExternalDmabufs) {
  441. gfx::Size coded_size = gfx::Size(256, 256);
  442. gfx::Rect visible_rect(coded_size);
  443. std::vector<int32_t> strides = {384, 192, 192};
  444. std::vector<size_t> offsets = {0, 100, 200};
  445. std::vector<size_t> sizes = {100, 50, 50};
  446. std::vector<ColorPlaneLayout> planes(strides.size());
  447. for (size_t i = 0; i < planes.size(); i++) {
  448. planes[i].stride = strides[i];
  449. planes[i].offset = offsets[i];
  450. planes[i].size = sizes[i];
  451. }
  452. auto timestamp = base::Milliseconds(1);
  453. auto layout =
  454. VideoFrameLayout::CreateWithPlanes(PIXEL_FORMAT_I420, coded_size, planes);
  455. ASSERT_TRUE(layout);
  456. std::vector<base::ScopedFD> dmabuf_fds(3u);
  457. auto frame = VideoFrame::WrapExternalDmabufs(
  458. *layout, visible_rect, visible_rect.size(), std::move(dmabuf_fds),
  459. timestamp);
  460. EXPECT_EQ(frame->layout().format(), PIXEL_FORMAT_I420);
  461. EXPECT_EQ(frame->layout().coded_size(), coded_size);
  462. EXPECT_EQ(frame->layout().num_planes(), 3u);
  463. EXPECT_EQ(frame->layout().is_multi_planar(), false);
  464. for (size_t i = 0; i < 3; ++i) {
  465. EXPECT_EQ(frame->layout().planes()[i].stride, strides[i]);
  466. EXPECT_EQ(frame->layout().planes()[i].offset, offsets[i]);
  467. EXPECT_EQ(frame->layout().planes()[i].size, sizes[i]);
  468. }
  469. EXPECT_TRUE(frame->HasDmaBufs());
  470. EXPECT_EQ(frame->DmabufFds().size(), 3u);
  471. EXPECT_EQ(frame->coded_size(), coded_size);
  472. EXPECT_EQ(frame->visible_rect(), visible_rect);
  473. EXPECT_EQ(frame->timestamp(), timestamp);
  474. // Wrapped DMABUF frames must share the same memory as their wrappee.
  475. auto wrapped_frame = VideoFrame::WrapVideoFrame(
  476. frame, frame->format(), visible_rect, visible_rect.size());
  477. ASSERT_NE(wrapped_frame, nullptr);
  478. ASSERT_EQ(wrapped_frame->IsSameDmaBufsAs(*frame), true);
  479. // Multi-level wrapping should share same memory as well.
  480. auto wrapped_frame2 = VideoFrame::WrapVideoFrame(
  481. wrapped_frame, frame->format(), visible_rect, visible_rect.size());
  482. ASSERT_NE(wrapped_frame2, nullptr);
  483. ASSERT_EQ(wrapped_frame2->IsSameDmaBufsAs(*wrapped_frame), true);
  484. ASSERT_EQ(wrapped_frame2->IsSameDmaBufsAs(*frame), true);
  485. }
  486. #endif
  487. // Ensure each frame is properly sized and allocated. Will trigger OOB reads
  488. // and writes as well as incorrect frame hashes otherwise.
  489. TEST(VideoFrame, CheckFrameExtents) {
  490. // Each call consists of a Format and the expected hash of all
  491. // planes if filled with kFillByte (defined in ExpectFrameExtents).
  492. ExpectFrameExtents(PIXEL_FORMAT_YV12, "8e5d54cb23cd0edca111dd35ffb6ff05");
  493. ExpectFrameExtents(PIXEL_FORMAT_I422, "cce408a044b212db42a10dfec304b3ef");
  494. }
  495. static void TextureCallback(gpu::SyncToken* called_sync_token,
  496. const gpu::SyncToken& release_sync_token) {
  497. *called_sync_token = release_sync_token;
  498. }
  499. // Verify the gpu::MailboxHolder::ReleaseCallback is called when VideoFrame is
  500. // destroyed with the default release sync point.
  501. TEST(VideoFrame, TextureNoLongerNeededCallbackIsCalled) {
  502. gpu::SyncToken called_sync_token(gpu::CommandBufferNamespace::GPU_IO,
  503. gpu::CommandBufferId::FromUnsafeValue(1), 1);
  504. {
  505. gpu::MailboxHolder holders[VideoFrame::kMaxPlanes] = {
  506. gpu::MailboxHolder(gpu::Mailbox::Generate(), gpu::SyncToken(), 5)};
  507. scoped_refptr<VideoFrame> frame = VideoFrame::WrapNativeTextures(
  508. PIXEL_FORMAT_ARGB, holders,
  509. base::BindOnce(&TextureCallback, &called_sync_token),
  510. gfx::Size(10, 10), // coded_size
  511. gfx::Rect(10, 10), // visible_rect
  512. gfx::Size(10, 10), // natural_size
  513. base::TimeDelta()); // timestamp
  514. EXPECT_EQ(PIXEL_FORMAT_ARGB, frame->format());
  515. EXPECT_EQ(VideoFrame::STORAGE_OPAQUE, frame->storage_type());
  516. EXPECT_TRUE(frame->HasTextures());
  517. }
  518. // Nobody set a sync point to |frame|, so |frame| set |called_sync_token|
  519. // cleared to default value.
  520. EXPECT_FALSE(called_sync_token.HasData());
  521. }
  522. // Verify the gpu::MailboxHolder::ReleaseCallback is called when VideoFrame is
  523. // destroyed with the release sync point, which was updated by clients.
  524. // (i.e. the compositor, webgl).
  525. TEST(VideoFrame,
  526. TexturesNoLongerNeededCallbackAfterTakingAndReleasingMailboxes) {
  527. const int kPlanesNum = 3;
  528. const gpu::CommandBufferNamespace kNamespace =
  529. gpu::CommandBufferNamespace::GPU_IO;
  530. const gpu::CommandBufferId kCommandBufferId =
  531. gpu::CommandBufferId::FromUnsafeValue(0x123);
  532. gpu::Mailbox mailbox[kPlanesNum];
  533. for (int i = 0; i < kPlanesNum; ++i) {
  534. mailbox[i].name[0] = 50 + 1;
  535. }
  536. gpu::SyncToken sync_token(kNamespace, kCommandBufferId, 7);
  537. sync_token.SetVerifyFlush();
  538. uint32_t target = 9;
  539. gpu::SyncToken release_sync_token(kNamespace, kCommandBufferId, 111);
  540. release_sync_token.SetVerifyFlush();
  541. gpu::SyncToken called_sync_token;
  542. {
  543. gpu::MailboxHolder holders[VideoFrame::kMaxPlanes] = {
  544. gpu::MailboxHolder(mailbox[VideoFrame::kYPlane], sync_token, target),
  545. gpu::MailboxHolder(mailbox[VideoFrame::kUPlane], sync_token, target),
  546. gpu::MailboxHolder(mailbox[VideoFrame::kVPlane], sync_token, target),
  547. };
  548. scoped_refptr<VideoFrame> frame = VideoFrame::WrapNativeTextures(
  549. PIXEL_FORMAT_I420, holders,
  550. base::BindOnce(&TextureCallback, &called_sync_token),
  551. gfx::Size(10, 10), // coded_size
  552. gfx::Rect(10, 10), // visible_rect
  553. gfx::Size(10, 10), // natural_size
  554. base::TimeDelta()); // timestamp
  555. EXPECT_EQ(VideoFrame::STORAGE_OPAQUE, frame->storage_type());
  556. EXPECT_EQ(PIXEL_FORMAT_I420, frame->format());
  557. EXPECT_EQ(3u, VideoFrame::NumPlanes(frame->format()));
  558. EXPECT_TRUE(frame->HasTextures());
  559. for (size_t i = 0; i < VideoFrame::NumPlanes(frame->format()); ++i) {
  560. const gpu::MailboxHolder& mailbox_holder = frame->mailbox_holder(i);
  561. EXPECT_EQ(mailbox[i].name[0], mailbox_holder.mailbox.name[0]);
  562. EXPECT_EQ(target, mailbox_holder.texture_target);
  563. EXPECT_EQ(sync_token, mailbox_holder.sync_token);
  564. }
  565. SimpleSyncTokenClient client(release_sync_token);
  566. frame->UpdateReleaseSyncToken(&client);
  567. EXPECT_EQ(sync_token,
  568. frame->mailbox_holder(VideoFrame::kYPlane).sync_token);
  569. }
  570. EXPECT_EQ(release_sync_token, called_sync_token);
  571. }
  572. TEST(VideoFrame, IsValidConfig_OddCodedSize) {
  573. // Odd sizes are valid for all formats. Odd formats may be internally rounded
  574. // in VideoFrame::CreateFrame because VideoFrame owns the allocation and can
  575. // pad the requested coded_size to ensure the UV sample boundaries line up
  576. // with the Y plane after subsample scaling. See CreateFrame_OddWidth.
  577. gfx::Size odd_size(677, 288);
  578. // First choosing a format with sub-sampling for UV.
  579. EXPECT_TRUE(VideoFrame::IsValidConfig(
  580. PIXEL_FORMAT_I420, VideoFrame::STORAGE_OWNED_MEMORY, odd_size,
  581. gfx::Rect(odd_size), odd_size));
  582. // Next try a format with no sub-sampling for UV.
  583. EXPECT_TRUE(VideoFrame::IsValidConfig(
  584. PIXEL_FORMAT_I444, VideoFrame::STORAGE_OWNED_MEMORY, odd_size,
  585. gfx::Rect(odd_size), odd_size));
  586. }
  587. TEST(VideoFrame, CreateFrame_OddWidth) {
  588. // Odd sizes are non-standard for YUV formats that subsample the UV, but they
  589. // do exist in the wild and should be gracefully handled by VideoFrame in
  590. // situations where VideoFrame allocates the YUV memory. See discussion in
  591. // crrev.com/1240833003
  592. const gfx::Size odd_size(677, 288);
  593. const base::TimeDelta kTimestamp = base::TimeDelta();
  594. // First create a frame that sub-samples UV.
  595. scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
  596. PIXEL_FORMAT_I420, odd_size, gfx::Rect(odd_size), odd_size, kTimestamp);
  597. ASSERT_TRUE(frame.get());
  598. // I420 aligns UV to every 2 Y pixels. Hence, 677 should be rounded to 678
  599. // which is the nearest value such that width % 2 == 0
  600. EXPECT_EQ(678, frame->coded_size().width());
  601. // Next create a frame that does not sub-sample UV.
  602. frame = VideoFrame::CreateFrame(PIXEL_FORMAT_I444, odd_size,
  603. gfx::Rect(odd_size), odd_size, kTimestamp);
  604. ASSERT_TRUE(frame.get());
  605. // No sub-sampling for YV24 will mean odd width can remain odd since any pixel
  606. // in the Y plane has a a corresponding pixel in the UV planes at the same
  607. // index.
  608. EXPECT_EQ(677, frame->coded_size().width());
  609. }
  610. TEST(VideoFrame, AllocationSize_OddSize) {
  611. const gfx::Size size(3, 5);
  612. for (unsigned int i = 1u; i <= PIXEL_FORMAT_MAX; ++i) {
  613. const VideoPixelFormat format = static_cast<VideoPixelFormat>(i);
  614. switch (format) {
  615. case PIXEL_FORMAT_YUV444P9:
  616. case PIXEL_FORMAT_YUV444P10:
  617. case PIXEL_FORMAT_YUV444P12:
  618. case PIXEL_FORMAT_YUV422AP10:
  619. EXPECT_EQ(144u, VideoFrame::AllocationSize(format, size))
  620. << VideoPixelFormatToString(format);
  621. break;
  622. case PIXEL_FORMAT_YUV422P9:
  623. case PIXEL_FORMAT_YUV422P10:
  624. case PIXEL_FORMAT_YUV422P12:
  625. case PIXEL_FORMAT_I444A:
  626. EXPECT_EQ(96u, VideoFrame::AllocationSize(format, size))
  627. << VideoPixelFormatToString(format);
  628. break;
  629. case PIXEL_FORMAT_I444:
  630. case PIXEL_FORMAT_YUV420P9:
  631. case PIXEL_FORMAT_YUV420P10:
  632. case PIXEL_FORMAT_YUV420P12:
  633. case PIXEL_FORMAT_P016LE:
  634. case PIXEL_FORMAT_I422A:
  635. EXPECT_EQ(72u, VideoFrame::AllocationSize(format, size))
  636. << VideoPixelFormatToString(format);
  637. break;
  638. case PIXEL_FORMAT_UYVY:
  639. case PIXEL_FORMAT_YUY2:
  640. case PIXEL_FORMAT_I422:
  641. EXPECT_EQ(48u, VideoFrame::AllocationSize(format, size))
  642. << VideoPixelFormatToString(format);
  643. break;
  644. case PIXEL_FORMAT_YV12:
  645. case PIXEL_FORMAT_I420:
  646. case PIXEL_FORMAT_NV12:
  647. case PIXEL_FORMAT_NV21:
  648. EXPECT_EQ(36u, VideoFrame::AllocationSize(format, size))
  649. << VideoPixelFormatToString(format);
  650. break;
  651. case PIXEL_FORMAT_ARGB:
  652. case PIXEL_FORMAT_BGRA:
  653. case PIXEL_FORMAT_XRGB:
  654. case PIXEL_FORMAT_I420A:
  655. case PIXEL_FORMAT_ABGR:
  656. case PIXEL_FORMAT_XBGR:
  657. case PIXEL_FORMAT_XR30:
  658. case PIXEL_FORMAT_XB30:
  659. EXPECT_EQ(60u, VideoFrame::AllocationSize(format, size))
  660. << VideoPixelFormatToString(format);
  661. break;
  662. case PIXEL_FORMAT_RGB24:
  663. EXPECT_EQ(45u, VideoFrame::AllocationSize(format, size))
  664. << VideoPixelFormatToString(format);
  665. break;
  666. case PIXEL_FORMAT_Y16:
  667. EXPECT_EQ(30u, VideoFrame::AllocationSize(format, size))
  668. << VideoPixelFormatToString(format);
  669. break;
  670. case PIXEL_FORMAT_RGBAF16:
  671. case PIXEL_FORMAT_YUV420AP10:
  672. EXPECT_EQ(120u, VideoFrame::AllocationSize(format, size))
  673. << VideoPixelFormatToString(format);
  674. break;
  675. case PIXEL_FORMAT_YUV444AP10:
  676. EXPECT_EQ(192u, VideoFrame::AllocationSize(format, size))
  677. << VideoPixelFormatToString(format);
  678. break;
  679. case PIXEL_FORMAT_MJPEG:
  680. case PIXEL_FORMAT_UNKNOWN:
  681. continue;
  682. }
  683. }
  684. }
  685. TEST(VideoFrameMetadata, MergeMetadata) {
  686. VideoFrameMetadata reference_metadata = GetFullVideoFrameMetadata();
  687. VideoFrameMetadata full_metadata = reference_metadata;
  688. VideoFrameMetadata empty_metadata;
  689. // Merging empty metadata into full metadata should be a no-op.
  690. full_metadata.MergeMetadataFrom(empty_metadata);
  691. VerifyVideoFrameMetadataEquality(full_metadata, reference_metadata);
  692. // Merging full metadata into empty metadata should fill it up.
  693. empty_metadata.MergeMetadataFrom(full_metadata);
  694. VerifyVideoFrameMetadataEquality(empty_metadata, reference_metadata);
  695. }
  696. TEST(VideoFrameMetadata, PartialMergeMetadata) {
  697. VideoFrameMetadata full_metadata = GetFullVideoFrameMetadata();
  698. const gfx::Rect kTempRect{100, 200, 300, 400};
  699. const base::TimeTicks kTempTicks = base::TimeTicks::Now() + base::Seconds(2);
  700. const base::TimeDelta kTempDelta = base::Milliseconds(31415);
  701. VideoFrameMetadata partial_metadata;
  702. partial_metadata.capture_update_rect = kTempRect;
  703. partial_metadata.reference_time = kTempTicks;
  704. partial_metadata.processing_time = kTempDelta;
  705. partial_metadata.allow_overlay = false;
  706. partial_metadata.texture_origin_is_top_left = false;
  707. // Merging partial metadata into full metadata partially override it.
  708. full_metadata.MergeMetadataFrom(partial_metadata);
  709. EXPECT_EQ(partial_metadata.capture_update_rect, kTempRect);
  710. EXPECT_EQ(partial_metadata.reference_time, kTempTicks);
  711. EXPECT_EQ(partial_metadata.processing_time, kTempDelta);
  712. EXPECT_EQ(partial_metadata.allow_overlay, false);
  713. EXPECT_EQ(partial_metadata.texture_origin_is_top_left, false);
  714. }
  715. } // namespace media