image_processor_test.cc 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include <memory>
  5. #include <string>
  6. #include <tuple>
  7. #include "base/command_line.h"
  8. #include "base/files/file_path.h"
  9. #include "base/files/file_util.h"
  10. #include "base/hash/md5.h"
  11. #include "base/test/launcher/unit_test_launcher.h"
  12. #include "base/test/test_suite.h"
  13. #include "build/build_config.h"
  14. #include "build/chromeos_buildflags.h"
  15. #include "media/base/video_frame.h"
  16. #include "media/base/video_frame_layout.h"
  17. #include "media/base/video_types.h"
  18. #include "media/gpu/chromeos/fourcc.h"
  19. #include "media/gpu/chromeos/image_processor.h"
  20. #include "media/gpu/test/image.h"
  21. #include "media/gpu/test/image_processor/image_processor_client.h"
  22. #include "media/gpu/test/video_frame_file_writer.h"
  23. #include "media/gpu/test/video_frame_helpers.h"
  24. #include "media/gpu/test/video_frame_validator.h"
  25. #include "media/gpu/test/video_test_environment.h"
  26. #include "mojo/core/embedder/embedder.h"
  27. #include "testing/gtest/include/gtest/gtest.h"
  28. #include "ui/gfx/geometry/size.h"
  29. namespace media {
  30. namespace {
  31. const char* usage_msg =
  32. "usage: image_processor_test\n"
  33. "[--gtest_help] [--help] [-v=<level>] [--vmodule=<config>] "
  34. "[--save_images] [--source_directory=<dir>]\n";
  35. const char* help_msg =
  36. "Run the image processor tests.\n\n"
  37. "The following arguments are supported:\n"
  38. " --gtest_help display the gtest help and exit.\n"
  39. " --help display this help and exit.\n"
  40. " -v enable verbose mode, e.g. -v=2.\n"
  41. " --vmodule enable verbose mode for the specified module.\n"
  42. " --save_images write images processed by a image processor to\n"
  43. " the \"<testname>\" folder.\n"
  44. " --source_directory specify the directory that contains test source\n"
  45. " files. Defaults to the current directory.\n";
  46. bool g_save_images = false;
  47. base::FilePath g_source_directory =
  48. base::FilePath(base::FilePath::kCurrentDirectory);
  49. base::FilePath BuildSourceFilePath(const base::FilePath& filename) {
  50. return media::g_source_directory.Append(filename);
  51. }
  52. media::test::VideoTestEnvironment* g_env;
  53. // Files for pixel format conversion test.
  54. const base::FilePath::CharType* kNV12Image =
  55. FILE_PATH_LITERAL("bear_320x192.nv12.yuv");
  56. const base::FilePath::CharType* kYV12Image =
  57. FILE_PATH_LITERAL("bear_320x192.yv12.yuv");
  58. const base::FilePath::CharType* kI420Image =
  59. FILE_PATH_LITERAL("bear_320x192.i420.yuv");
  60. const base::FilePath::CharType* kI422Image =
  61. FILE_PATH_LITERAL("bear_320x192.i422.yuv");
  62. const base::FilePath::CharType* kYUYVImage =
  63. FILE_PATH_LITERAL("bear_320x192.yuyv.yuv");
  64. // Files for scaling test.
  65. const base::FilePath::CharType* kNV12Image720P =
  66. FILE_PATH_LITERAL("puppets-1280x720.nv12.yuv");
  67. const base::FilePath::CharType* kNV12Image360P =
  68. FILE_PATH_LITERAL("puppets-640x360.nv12.yuv");
  69. const base::FilePath::CharType* kNV12Image270P =
  70. FILE_PATH_LITERAL("puppets-480x270.nv12.yuv");
  71. const base::FilePath::CharType* kNV12Image180P =
  72. FILE_PATH_LITERAL("puppets-320x180.nv12.yuv");
  73. const base::FilePath::CharType* kNV12Image360PIn480P =
  74. FILE_PATH_LITERAL("puppets-640x360_in_640x480.nv12.yuv");
  75. const base::FilePath::CharType* kI422Image360P =
  76. FILE_PATH_LITERAL("puppets-640x360.i422.yuv");
  77. const base::FilePath::CharType* kYUYVImage360P =
  78. FILE_PATH_LITERAL("puppets-640x360.yuyv.yuv");
  79. const base::FilePath::CharType* kI420Image360P =
  80. FILE_PATH_LITERAL("puppets-640x360.i420.yuv");
  81. const base::FilePath::CharType* kI420Image270P =
  82. FILE_PATH_LITERAL("puppets-480x270.i420.yuv");
  83. // Files for rotation test.
  84. const base::FilePath::CharType* kNV12Image90 =
  85. FILE_PATH_LITERAL("bear_192x320_90.nv12.yuv");
  86. const base::FilePath::CharType* kNV12Image180 =
  87. FILE_PATH_LITERAL("bear_320x192_180.nv12.yuv");
  88. const base::FilePath::CharType* kNV12Image270 =
  89. FILE_PATH_LITERAL("bear_192x320_270.nv12.yuv");
  90. enum class YuvSubsampling {
  91. kYuv420,
  92. kYuv422,
  93. kYuv444,
  94. };
  95. YuvSubsampling ToYuvSubsampling(VideoPixelFormat format) {
  96. switch (format) {
  97. case PIXEL_FORMAT_I420:
  98. case PIXEL_FORMAT_NV12:
  99. case PIXEL_FORMAT_YV12:
  100. return YuvSubsampling::kYuv420;
  101. case PIXEL_FORMAT_I422:
  102. case PIXEL_FORMAT_YUY2:
  103. return YuvSubsampling::kYuv422;
  104. default:
  105. NOTREACHED() << "Invalid format " << format;
  106. return YuvSubsampling::kYuv444;
  107. }
  108. }
  109. #if BUILDFLAG(IS_CHROMEOS_ASH)
  110. bool IsFormatTestedForDmabufAndGbm(VideoPixelFormat format) {
  111. switch (format) {
  112. case PIXEL_FORMAT_NV12:
  113. case PIXEL_FORMAT_YV12:
  114. return true;
  115. default:
  116. return false;
  117. }
  118. }
  119. #endif // BUILDFLAG(IS_CHROMEOS_ASH)
  120. class ImageProcessorParamTest
  121. : public ::testing::Test,
  122. public ::testing::WithParamInterface<
  123. std::tuple<base::FilePath, base::FilePath>> {
  124. public:
  125. void SetUp() override {}
  126. void TearDown() override { model_frames_.clear(); }
  127. std::unique_ptr<test::ImageProcessorClient> CreateImageProcessorClient(
  128. const test::Image& input_image,
  129. const std::vector<VideoFrame::StorageType>& input_storage_types,
  130. test::Image* const output_image,
  131. const std::vector<VideoFrame::StorageType>& output_storage_types) {
  132. bool is_single_planar_input = true;
  133. bool is_single_planar_output = true;
  134. #if defined(ARCH_CPU_ARM_FAMILY)
  135. // [Ugly Hack] In the use scenario of V4L2ImageProcessor in
  136. // V4L2VideoEncodeAccelerator. ImageProcessor needs to read and write
  137. // contents with arbitrary offsets. The format needs to be multi planar.
  138. is_single_planar_input = !IsYuvPlanar(input_image.PixelFormat());
  139. is_single_planar_output = !IsYuvPlanar(output_image->PixelFormat());
  140. #endif
  141. Fourcc input_fourcc = *Fourcc::FromVideoPixelFormat(
  142. input_image.PixelFormat(), is_single_planar_input);
  143. Fourcc output_fourcc = *Fourcc::FromVideoPixelFormat(
  144. output_image->PixelFormat(), is_single_planar_output);
  145. auto input_layout = test::CreateVideoFrameLayout(input_image.PixelFormat(),
  146. input_image.Size());
  147. auto output_layout = test::CreateVideoFrameLayout(
  148. output_image->PixelFormat(), output_image->Size());
  149. LOG_ASSERT(input_layout && output_layout);
  150. ImageProcessor::PortConfig input_config(
  151. input_fourcc, input_image.Size(), input_layout->planes(),
  152. input_image.VisibleRect(), input_storage_types);
  153. ImageProcessor::PortConfig output_config(
  154. output_fourcc, output_image->Size(), output_layout->planes(),
  155. output_image->VisibleRect(), output_storage_types);
  156. int rotation = (output_image->Rotation() - input_image.Rotation()) % 360;
  157. if (rotation < 0)
  158. rotation += 360;
  159. VideoRotation relative_rotation = VIDEO_ROTATION_0;
  160. switch (rotation) {
  161. case 0:
  162. relative_rotation = VIDEO_ROTATION_0;
  163. break;
  164. case 90:
  165. relative_rotation = VIDEO_ROTATION_90;
  166. break;
  167. case 180:
  168. relative_rotation = VIDEO_ROTATION_180;
  169. break;
  170. case 270:
  171. relative_rotation = VIDEO_ROTATION_270;
  172. break;
  173. default:
  174. NOTREACHED() << "Invalid rotation: " << rotation;
  175. return nullptr;
  176. }
  177. // TODO(crbug.com/917951): Select more appropriate number of buffers.
  178. constexpr size_t kNumBuffers = 1;
  179. LOG_ASSERT(output_image->IsMetadataLoaded());
  180. std::vector<std::unique_ptr<test::VideoFrameProcessor>> frame_processors;
  181. // TODO(crbug.com/944823): Use VideoFrameValidator for RGB formats.
  182. // Validating processed frames is currently not supported when a format is
  183. // not YUV or when scaling images.
  184. if (IsYuvPlanar(input_fourcc.ToVideoPixelFormat()) &&
  185. IsYuvPlanar(output_fourcc.ToVideoPixelFormat()) &&
  186. ToYuvSubsampling(input_fourcc.ToVideoPixelFormat()) ==
  187. ToYuvSubsampling(output_fourcc.ToVideoPixelFormat())) {
  188. if (input_image.Size() == output_image->Size()) {
  189. auto vf_validator = test::MD5VideoFrameValidator::Create(
  190. {output_image->Checksum()}, output_image->PixelFormat());
  191. LOG_ASSERT(vf_validator);
  192. frame_processors.push_back(std::move(vf_validator));
  193. } else if (input_fourcc == output_fourcc) {
  194. // Scaling case.
  195. LOG_ASSERT(output_image->Load());
  196. scoped_refptr<const VideoFrame> model_frame =
  197. CreateVideoFrameFromImage(*output_image);
  198. LOG_ASSERT(model_frame) << "Failed to create from image";
  199. model_frames_ = {model_frame};
  200. auto vf_validator = test::SSIMVideoFrameValidator::Create(
  201. base::BindRepeating(&ImageProcessorParamTest::GetModelFrame,
  202. base::Unretained(this)));
  203. frame_processors.push_back(std::move(vf_validator));
  204. }
  205. }
  206. if (g_save_images) {
  207. base::FilePath output_dir =
  208. base::FilePath(base::FilePath::kCurrentDirectory)
  209. .Append(g_env->GetTestOutputFilePath());
  210. test::VideoFrameFileWriter::OutputFormat saved_file_format =
  211. IsYuvPlanar(output_fourcc.ToVideoPixelFormat())
  212. ? test::VideoFrameFileWriter::OutputFormat::kYUV
  213. : test::VideoFrameFileWriter::OutputFormat::kPNG;
  214. frame_processors.push_back(
  215. test::VideoFrameFileWriter::Create(output_dir, saved_file_format));
  216. }
  217. auto ip_client = test::ImageProcessorClient::Create(
  218. input_config, output_config, kNumBuffers, relative_rotation,
  219. std::move(frame_processors));
  220. return ip_client;
  221. }
  222. private:
  223. scoped_refptr<const VideoFrame> GetModelFrame(size_t frame_index) const {
  224. if (frame_index >= model_frames_.size()) {
  225. LOG(ERROR) << "Failed to get model frame with index=" << frame_index;
  226. ADD_FAILURE();
  227. return nullptr;
  228. }
  229. return model_frames_[frame_index];
  230. }
  231. std::vector<scoped_refptr<const VideoFrame>> model_frames_;
  232. };
  233. TEST_P(ImageProcessorParamTest, ConvertOneTime_MemToMem) {
  234. // Load the test input image. We only need the output image's metadata so we
  235. // can compare checksums.
  236. test::Image input_image(BuildSourceFilePath(std::get<0>(GetParam())));
  237. test::Image output_image(BuildSourceFilePath(std::get<1>(GetParam())));
  238. ASSERT_TRUE(input_image.Load());
  239. ASSERT_TRUE(output_image.LoadMetadata());
  240. auto ip_client = CreateImageProcessorClient(
  241. input_image, {VideoFrame::STORAGE_OWNED_MEMORY}, &output_image,
  242. {VideoFrame::STORAGE_OWNED_MEMORY});
  243. ASSERT_TRUE(ip_client);
  244. ip_client->Process(input_image, output_image);
  245. EXPECT_TRUE(ip_client->WaitUntilNumImageProcessed(1u));
  246. EXPECT_EQ(ip_client->GetErrorCount(), 0u);
  247. EXPECT_EQ(ip_client->GetNumOfProcessedImages(), 1u);
  248. EXPECT_TRUE(ip_client->WaitForFrameProcessors());
  249. }
  250. #if BUILDFLAG(IS_CHROMEOS_ASH)
  251. // We don't yet have the function to create Dmabuf-backed VideoFrame on
  252. // platforms except ChromeOS. So MemToDmabuf test is limited on ChromeOS.
  253. TEST_P(ImageProcessorParamTest, ConvertOneTime_DmabufToMem) {
  254. // Load the test input image. We only need the output image's metadata so we
  255. // can compare checksums.
  256. test::Image input_image(BuildSourceFilePath(std::get<0>(GetParam())));
  257. test::Image output_image(BuildSourceFilePath(std::get<1>(GetParam())));
  258. ASSERT_TRUE(input_image.Load());
  259. ASSERT_TRUE(output_image.LoadMetadata());
  260. if (!IsFormatTestedForDmabufAndGbm(input_image.PixelFormat()))
  261. GTEST_SKIP() << "Skipping Dmabuf format " << input_image.PixelFormat();
  262. auto ip_client = CreateImageProcessorClient(
  263. input_image, {VideoFrame::STORAGE_DMABUFS}, &output_image,
  264. {VideoFrame::STORAGE_OWNED_MEMORY});
  265. ASSERT_TRUE(ip_client);
  266. ip_client->Process(input_image, output_image);
  267. EXPECT_TRUE(ip_client->WaitUntilNumImageProcessed(1u));
  268. EXPECT_EQ(ip_client->GetErrorCount(), 0u);
  269. EXPECT_EQ(ip_client->GetNumOfProcessedImages(), 1u);
  270. EXPECT_TRUE(ip_client->WaitForFrameProcessors());
  271. }
  272. TEST_P(ImageProcessorParamTest, ConvertOneTime_DmabufToDmabuf) {
  273. // Load the test input image. We only need the output image's metadata so we
  274. // can compare checksums.
  275. test::Image input_image(BuildSourceFilePath(std::get<0>(GetParam())));
  276. test::Image output_image(BuildSourceFilePath(std::get<1>(GetParam())));
  277. ASSERT_TRUE(input_image.Load());
  278. ASSERT_TRUE(output_image.LoadMetadata());
  279. if (!IsFormatTestedForDmabufAndGbm(input_image.PixelFormat()))
  280. GTEST_SKIP() << "Skipping Dmabuf format " << input_image.PixelFormat();
  281. if (!IsFormatTestedForDmabufAndGbm(output_image.PixelFormat()))
  282. GTEST_SKIP() << "Skipping Dmabuf format " << output_image.PixelFormat();
  283. auto ip_client =
  284. CreateImageProcessorClient(input_image, {VideoFrame::STORAGE_DMABUFS},
  285. &output_image, {VideoFrame::STORAGE_DMABUFS});
  286. ASSERT_TRUE(ip_client);
  287. ip_client->Process(input_image, output_image);
  288. EXPECT_TRUE(ip_client->WaitUntilNumImageProcessed(1u));
  289. EXPECT_EQ(ip_client->GetErrorCount(), 0u);
  290. EXPECT_EQ(ip_client->GetNumOfProcessedImages(), 1u);
  291. EXPECT_TRUE(ip_client->WaitForFrameProcessors());
  292. }
  293. // Although GpuMemoryBuffer is a cross platform class, code for image processor
  294. // test is designed only for ChromeOS. So this test runs on ChromeOS only.
  295. TEST_P(ImageProcessorParamTest, ConvertOneTime_GmbToGmb) {
  296. // Load the test input image. We only need the output image's metadata so we
  297. // can compare checksums.
  298. test::Image input_image(BuildSourceFilePath(std::get<0>(GetParam())));
  299. test::Image output_image(BuildSourceFilePath(std::get<1>(GetParam())));
  300. ASSERT_TRUE(input_image.Load());
  301. ASSERT_TRUE(output_image.LoadMetadata());
  302. if (!IsFormatTestedForDmabufAndGbm(input_image.PixelFormat())) {
  303. GTEST_SKIP() << "Skipping GpuMemoryBuffer format "
  304. << input_image.PixelFormat();
  305. }
  306. if (!IsFormatTestedForDmabufAndGbm(output_image.PixelFormat())) {
  307. GTEST_SKIP() << "Skipping GpuMemoryBuffer format "
  308. << output_image.PixelFormat();
  309. }
  310. auto ip_client = CreateImageProcessorClient(
  311. input_image, {VideoFrame::STORAGE_GPU_MEMORY_BUFFER}, &output_image,
  312. {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  313. ASSERT_TRUE(ip_client);
  314. ip_client->Process(input_image, output_image);
  315. EXPECT_TRUE(ip_client->WaitUntilNumImageProcessed(1u));
  316. EXPECT_EQ(ip_client->GetErrorCount(), 0u);
  317. EXPECT_EQ(ip_client->GetNumOfProcessedImages(), 1u);
  318. EXPECT_TRUE(ip_client->WaitForFrameProcessors());
  319. }
  320. #endif // BUILDFLAG(IS_CHROMEOS_ASH)
  321. INSTANTIATE_TEST_SUITE_P(
  322. PixelFormatConversionToNV12,
  323. ImageProcessorParamTest,
  324. ::testing::Values(std::make_tuple(kI420Image, kNV12Image),
  325. std::make_tuple(kYV12Image, kNV12Image),
  326. std::make_tuple(kI422Image, kNV12Image),
  327. std::make_tuple(kYUYVImage, kNV12Image)));
  328. INSTANTIATE_TEST_SUITE_P(
  329. PixelFormatConversionToI420,
  330. ImageProcessorParamTest,
  331. ::testing::Values(std::make_tuple(kI420Image, kI420Image),
  332. std::make_tuple(kI422Image, kI420Image),
  333. std::make_tuple(kYUYVImage, kI420Image)));
  334. INSTANTIATE_TEST_SUITE_P(
  335. NV12DownScaling,
  336. ImageProcessorParamTest,
  337. ::testing::Values(std::make_tuple(kNV12Image720P, kNV12Image360P),
  338. std::make_tuple(kNV12Image720P, kNV12Image270P),
  339. std::make_tuple(kNV12Image720P, kNV12Image180P),
  340. std::make_tuple(kNV12Image360P, kNV12Image270P),
  341. std::make_tuple(kNV12Image360P, kNV12Image180P)));
  342. INSTANTIATE_TEST_SUITE_P(I420DownScaling,
  343. ImageProcessorParamTest,
  344. ::testing::Values(std::make_tuple(kI420Image360P,
  345. kI420Image270P)));
  346. INSTANTIATE_TEST_SUITE_P(
  347. DownScalingConversionToNV12,
  348. ImageProcessorParamTest,
  349. ::testing::Values(std::make_tuple(kI422Image360P, kNV12Image270P),
  350. std::make_tuple(kYUYVImage360P, kNV12Image270P)));
  351. INSTANTIATE_TEST_SUITE_P(
  352. DownScalingConversionToI420,
  353. ImageProcessorParamTest,
  354. ::testing::Values(std::make_tuple(kI420Image360P, kI420Image270P),
  355. std::make_tuple(kI422Image360P, kI420Image270P),
  356. std::make_tuple(kYUYVImage360P, kI420Image270P)));
  357. // Crop 360P frame from 480P.
  358. INSTANTIATE_TEST_SUITE_P(NV12Cropping,
  359. ImageProcessorParamTest,
  360. ::testing::Values(std::make_tuple(kNV12Image360PIn480P,
  361. kNV12Image360P)));
  362. // Crop 360p frame from 480P and scale the area to 270P.
  363. INSTANTIATE_TEST_SUITE_P(NV12CroppingAndScaling,
  364. ImageProcessorParamTest,
  365. ::testing::Values(std::make_tuple(kNV12Image360PIn480P,
  366. kNV12Image270P)));
  367. // Rotate frame to specified rotation.
  368. // Now only VaapiIP maybe support rotaion.
  369. INSTANTIATE_TEST_SUITE_P(
  370. NV12Rotation,
  371. ImageProcessorParamTest,
  372. ::testing::Values(std::make_tuple(kNV12Image, kNV12Image90),
  373. std::make_tuple(kNV12Image, kNV12Image180),
  374. std::make_tuple(kNV12Image, kNV12Image270),
  375. std::make_tuple(kNV12Image180, kNV12Image90),
  376. std::make_tuple(kNV12Image180, kNV12Image)));
  377. #if BUILDFLAG(IS_CHROMEOS_ASH)
  378. // TODO(hiroh): Add more tests.
  379. // MEM->DMABUF (V4L2VideoEncodeAccelerator),
  380. #endif
  381. } // namespace
  382. } // namespace media
  383. int main(int argc, char** argv) {
  384. base::CommandLine::Init(argc, argv);
  385. // Print the help message if requested. This needs to be done before
  386. // initializing gtest, to overwrite the default gtest help message.
  387. const base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
  388. LOG_ASSERT(cmd_line);
  389. if (cmd_line->HasSwitch("help")) {
  390. std::cout << media::usage_msg << "\n" << media::help_msg;
  391. return 0;
  392. }
  393. base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
  394. for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
  395. it != switches.end(); ++it) {
  396. if (it->first.find("gtest_") == 0 || // Handled by GoogleTest
  397. it->first == "v" || it->first == "vmodule") { // Handled by Chrome
  398. continue;
  399. }
  400. if (it->first == "save_images") {
  401. media::g_save_images = true;
  402. } else if (it->first == "source_directory") {
  403. media::g_source_directory = base::FilePath(it->second);
  404. } else {
  405. std::cout << "unknown option: --" << it->first << "\n"
  406. << media::usage_msg;
  407. return EXIT_FAILURE;
  408. }
  409. }
  410. testing::InitGoogleTest(&argc, argv);
  411. auto* const test_environment = new media::test::VideoTestEnvironment;
  412. media::g_env = reinterpret_cast<media::test::VideoTestEnvironment*>(
  413. testing::AddGlobalTestEnvironment(test_environment));
  414. return RUN_ALL_TESTS();
  415. }