mjpeg_decode_accelerator_unittest.cc 52 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/chromeos_camera/mjpeg_decode_accelerator.h"
  5. #include <stddef.h>
  6. #include <stdint.h>
  7. #include <string.h>
  8. #include <unistd.h>
  9. #include <memory>
  10. #include <numeric>
  11. #include <string>
  12. #include <utility>
  13. #include <vector>
  14. #include "base/at_exit.h"
  15. #include "base/bind.h"
  16. #include "base/command_line.h"
  17. #include "base/files/file_util.h"
  18. #include "base/files/scoped_file.h"
  19. #include "base/gtest_prod_util.h"
  20. #include "base/json/json_writer.h"
  21. #include "base/logging.h"
  22. #include "base/memory/shared_memory_mapping.h"
  23. #include "base/memory/unsafe_shared_memory_region.h"
  24. #include "base/numerics/safe_conversions.h"
  25. #include "base/path_service.h"
  26. #include "base/posix/eintr_wrapper.h"
  27. #include "base/strings/string_split.h"
  28. #include "base/strings/stringprintf.h"
  29. #include "base/threading/thread.h"
  30. #include "base/threading/thread_task_runner_handle.h"
  31. #include "base/time/time.h"
  32. #include "base/timer/elapsed_timer.h"
  33. #include "base/values.h"
  34. #include "build/build_config.h"
  35. #include "components/chromeos_camera/gpu_mjpeg_decode_accelerator_factory.h"
  36. #include "media/base/color_plane_layout.h"
  37. #include "media/base/format_utils.h"
  38. #include "media/base/test_data_util.h"
  39. #include "media/base/video_frame_layout.h"
  40. #include "media/base/video_util.h"
  41. #include "media/gpu/buildflags.h"
  42. #include "media/gpu/test/local_gpu_memory_buffer_manager.h"
  43. #include "media/gpu/test/video_frame_helpers.h"
  44. #include "media/gpu/test/video_test_helpers.h"
  45. #include "media/parsers/jpeg_parser.h"
  46. #include "mojo/core/embedder/embedder.h"
  47. #include "testing/gtest/include/gtest/gtest.h"
  48. #include "third_party/abseil-cpp/absl/types/optional.h"
  49. #include "third_party/libyuv/include/libyuv.h"
  50. #include "ui/gfx/buffer_types.h"
  51. #include "ui/gfx/codec/jpeg_codec.h"
  52. #include "ui/gfx/codec/png_codec.h"
  53. #include "ui/gfx/geometry/rect.h"
  54. #include "ui/gfx/geometry/size.h"
  55. #include "ui/gfx/gpu_memory_buffer.h"
  56. #include "ui/gfx/native_pixmap_handle.h"
  57. #if BUILDFLAG(USE_VAAPI)
  58. #include "media/gpu/vaapi/vaapi_wrapper.h"
  59. #endif
  60. namespace chromeos_camera {
  61. namespace {
  62. // Default test image file.
  63. const base::FilePath::CharType* kDefaultJpegFilename =
  64. FILE_PATH_LITERAL("peach_pi-1280x720.jpg");
  65. constexpr int kDefaultPerfDecodeTimes = 600;
  66. // Decide to save decode results to files or not. Output files will be saved
  67. // in the same directory with unittest. File name is like input file but
  68. // changing the extension to "yuv".
  69. bool g_save_to_file = false;
  70. // Threshold for mean absolute difference of hardware and software decode.
  71. // Absolute difference is to calculate the difference between each pixel in two
  72. // images. This is used for measuring of the similarity of two images.
  73. constexpr double kDecodeSimilarityThreshold = 1.25;
  74. // The buffer usage used to create GpuMemoryBuffer for testing.
  75. constexpr gfx::BufferUsage kBufferUsage =
  76. gfx::BufferUsage::SCANOUT_CPU_READ_WRITE;
  77. // Environment to create test data for all test cases.
  78. class MjpegDecodeAcceleratorTestEnvironment;
  79. MjpegDecodeAcceleratorTestEnvironment* g_env;
  80. // This struct holds a parsed, complete JPEG blob. It can be created from a
  81. // FilePath or can be simply a black image.
  82. struct ParsedJpegImage {
  83. static std::unique_ptr<ParsedJpegImage> CreateFromFile(
  84. const base::FilePath& file_path) {
  85. auto image = std::make_unique<ParsedJpegImage>(file_path);
  86. LOG_ASSERT(base::ReadFileToString(file_path, &image->data_str))
  87. << file_path;
  88. media::JpegParseResult parse_result;
  89. LOG_ASSERT(ParseJpegPicture(
  90. reinterpret_cast<const uint8_t*>(image->data_str.data()),
  91. image->data_str.size(), &parse_result));
  92. image->InitializeSizes(parse_result.frame_header.visible_width,
  93. parse_result.frame_header.visible_height);
  94. return image;
  95. }
  96. static std::unique_ptr<ParsedJpegImage> CreateBlackImage(
  97. int width,
  98. int height,
  99. SkJpegEncoder::Downsample downsample = SkJpegEncoder::Downsample::k420) {
  100. // Generate a black image with the specified resolution.
  101. constexpr size_t kBytesPerPixel = 4;
  102. const std::vector<unsigned char> input_buffer(width * height *
  103. kBytesPerPixel);
  104. const SkImageInfo info = SkImageInfo::Make(
  105. width, height, kRGBA_8888_SkColorType, kOpaque_SkAlphaType);
  106. const SkPixmap src(info, input_buffer.data(), width * kBytesPerPixel);
  107. // Encode the generated image in the JPEG format, the output buffer will be
  108. // automatically resized while encoding.
  109. constexpr int kJpegQuality = 100;
  110. std::vector<unsigned char> encoded;
  111. LOG_ASSERT(gfx::JPEGCodec::Encode(src, kJpegQuality, downsample, &encoded));
  112. base::FilePath filename;
  113. LOG_ASSERT(base::GetTempDir(&filename));
  114. filename =
  115. filename.Append(base::StringPrintf("black-%dx%d.jpg", width, height));
  116. auto image = std::make_unique<ParsedJpegImage>(filename);
  117. image->data_str.append(encoded.begin(), encoded.end());
  118. image->InitializeSizes(width, height);
  119. return image;
  120. }
  121. explicit ParsedJpegImage(const base::FilePath& path) : file_path(path) {}
  122. void InitializeSizes(int width, int height) {
  123. visible_size.SetSize(width, height);
  124. // We don't expect odd dimensions for camera captures.
  125. ASSERT_EQ(0, width % 2);
  126. ASSERT_EQ(0, height % 2);
  127. coded_size.SetSize((visible_size.width() + 1) & ~1,
  128. (visible_size.height() + 1) & ~1);
  129. }
  130. const base::FilePath::StringType& filename() const {
  131. return file_path.value();
  132. }
  133. const base::FilePath file_path;
  134. std::string data_str;
  135. gfx::Size visible_size;
  136. gfx::Size coded_size;
  137. };
  138. // Global singleton to hold on to common data and other user-defined options.
  139. class MjpegDecodeAcceleratorTestEnvironment : public ::testing::Environment {
  140. public:
  141. MjpegDecodeAcceleratorTestEnvironment(
  142. const base::FilePath::CharType* jpeg_filenames,
  143. const base::FilePath::CharType* test_data_path,
  144. const base::FilePath::CharType* perf_output_path,
  145. int perf_decode_times)
  146. : perf_decode_times_(perf_decode_times ? perf_decode_times
  147. : kDefaultPerfDecodeTimes),
  148. user_jpeg_filenames_(jpeg_filenames ? jpeg_filenames
  149. : kDefaultJpegFilename),
  150. test_data_path_(test_data_path),
  151. perf_output_path_(perf_output_path) {}
  152. void SetUp() override;
  153. void TearDown() override;
  154. // Resolve the specified file path. The file path can be either an absolute
  155. // path, relative to the current directory, or relative to the test data path.
  156. // This is either a custom test data path provided by --test_data_path, or the
  157. // default test data path (//media/test/data).
  158. base::FilePath GetOriginalOrTestDataFilePath(const std::string& file_path) {
  159. const base::FilePath original_file_path = base::FilePath(file_path);
  160. if (base::PathExists(original_file_path))
  161. return original_file_path;
  162. if (test_data_path_)
  163. return base::FilePath(test_data_path_).Append(original_file_path);
  164. return media::GetTestDataFilePath(file_path);
  165. }
  166. // Creates a zero-initialized shared memory backed VideoFrame.
  167. scoped_refptr<media::VideoFrame> CreateShmVideoFrame(
  168. media::VideoPixelFormat format,
  169. const gfx::Size& coded_size,
  170. const gfx::Size& visible_size);
  171. // Creates a zero-initialized DMA-buf backed VideoFrame. Also returns the
  172. // backing GpuMemoryBuffer in |backing_gmb| if it is not null.
  173. scoped_refptr<media::VideoFrame> CreateDmaBufVideoFrame(
  174. media::VideoPixelFormat format,
  175. const gfx::Size& coded_size,
  176. const gfx::Size& visible_size,
  177. std::unique_ptr<gfx::GpuMemoryBuffer>* backing_gmb = nullptr);
  178. // Maps |gmb| into a VideoFrame containing the data pointers. |gmb| should
  179. // outlive the returned Videoframe.
  180. scoped_refptr<media::VideoFrame> MapToVideoFrame(
  181. gfx::GpuMemoryBuffer* gmb,
  182. const media::VideoFrameLayout& layout,
  183. const gfx::Rect& visible_rect);
  184. // Creates a DMA buffer file descriptor that contains |size| bytes of linear
  185. // data initialized with |data|.
  186. base::ScopedFD CreateDmaBufFd(const void* data, size_t size);
  187. // Gets a list of supported DMA-buf frame formats for
  188. // CreateDmaBufVideoFrame().
  189. std::vector<media::VideoPixelFormat> GetSupportedDmaBufFormats();
  190. void AddMetric(const std::string& name, const base::TimeDelta& time);
  191. // Used for InputSizeChange test case. The image size should be smaller than
  192. // |kDefaultJpegFilename|.
  193. std::unique_ptr<ParsedJpegImage> image_data_1280x720_black_;
  194. // Used for ResolutionChange test case.
  195. std::unique_ptr<ParsedJpegImage> image_data_640x368_black_;
  196. // Used for testing some drivers which will align the output resolution to a
  197. // multiple of 16. 640x360 will be aligned to 640x368.
  198. std::unique_ptr<ParsedJpegImage> image_data_640x360_black_;
  199. // Generated black image used to test different JPEG sampling formats.
  200. std::unique_ptr<ParsedJpegImage> image_data_640x368_422_black_;
  201. // Parsed data of "peach_pi-1280x720.jpg".
  202. std::unique_ptr<ParsedJpegImage> image_data_1280x720_default_;
  203. // Parsed data of failure image.
  204. std::unique_ptr<ParsedJpegImage> image_data_invalid_;
  205. // Parsed data from command line.
  206. std::vector<std::unique_ptr<ParsedJpegImage>> image_data_user_;
  207. // Decode times for performance measurement.
  208. int perf_decode_times_;
  209. private:
  210. const base::FilePath::CharType* user_jpeg_filenames_;
  211. const base::FilePath::CharType* test_data_path_;
  212. const base::FilePath::CharType* perf_output_path_;
  213. base::Value metrics_;
  214. std::unique_ptr<media::LocalGpuMemoryBufferManager>
  215. gpu_memory_buffer_manager_;
  216. };
  217. void MjpegDecodeAcceleratorTestEnvironment::SetUp() {
  218. image_data_1280x720_black_ = ParsedJpegImage::CreateBlackImage(1280, 720);
  219. image_data_640x368_black_ = ParsedJpegImage::CreateBlackImage(640, 368);
  220. image_data_640x360_black_ = ParsedJpegImage::CreateBlackImage(640, 360);
  221. image_data_640x368_422_black_ = ParsedJpegImage::CreateBlackImage(
  222. 640, 368, SkJpegEncoder::Downsample::k422);
  223. image_data_1280x720_default_ = ParsedJpegImage::CreateFromFile(
  224. GetOriginalOrTestDataFilePath(kDefaultJpegFilename));
  225. image_data_invalid_ =
  226. std::make_unique<ParsedJpegImage>(base::FilePath("failure.jpg"));
  227. image_data_invalid_->data_str.resize(100, 0);
  228. image_data_invalid_->InitializeSizes(1280, 720);
  229. // |user_jpeg_filenames_| may include many files and use ';' as delimiter.
  230. std::vector<base::FilePath::StringType> filenames = base::SplitString(
  231. user_jpeg_filenames_, base::FilePath::StringType(1, ';'),
  232. base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
  233. for (const auto& filename : filenames) {
  234. const base::FilePath input_file = GetOriginalOrTestDataFilePath(filename);
  235. auto image_data = ParsedJpegImage::CreateFromFile(input_file);
  236. image_data_user_.push_back(std::move(image_data));
  237. }
  238. gpu_memory_buffer_manager_ =
  239. std::make_unique<media::LocalGpuMemoryBufferManager>();
  240. metrics_ = base::Value(base::Value::Type::DICTIONARY);
  241. }
  242. void MjpegDecodeAcceleratorTestEnvironment::TearDown() {
  243. // Write recorded metrics to file in JSON format.
  244. if (perf_output_path_ != nullptr) {
  245. std::string json;
  246. ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
  247. metrics_, base::JSONWriter::OPTIONS_PRETTY_PRINT, &json));
  248. ASSERT_TRUE(base::WriteFile(base::FilePath(perf_output_path_), json));
  249. }
  250. }
  251. scoped_refptr<media::VideoFrame>
  252. MjpegDecodeAcceleratorTestEnvironment::CreateShmVideoFrame(
  253. media::VideoPixelFormat format,
  254. const gfx::Size& coded_size,
  255. const gfx::Size& visible_size) {
  256. const size_t data_size =
  257. media::VideoFrame::AllocationSize(format, coded_size);
  258. auto shm_region = base::UnsafeSharedMemoryRegion::Create(data_size);
  259. if (!shm_region.IsValid()) {
  260. LOG(ERROR) << "Failed to create shared memory";
  261. return nullptr;
  262. }
  263. base::WritableSharedMemoryMapping shm_mapping = shm_region.Map();
  264. if (!shm_mapping.IsValid()) {
  265. LOG(ERROR) << "Failed to map shared memory region";
  266. return nullptr;
  267. }
  268. memset(shm_mapping.memory(), 0, data_size);
  269. scoped_refptr<media::VideoFrame> frame = media::VideoFrame::WrapExternalData(
  270. format, coded_size, gfx::Rect(visible_size), visible_size,
  271. shm_mapping.GetMemoryAsSpan<uint8_t>().data(), data_size,
  272. base::TimeDelta());
  273. if (!frame) {
  274. LOG(ERROR) << "Failed to create video frame";
  275. return nullptr;
  276. }
  277. frame->BackWithOwnedSharedMemory(std::move(shm_region),
  278. std::move(shm_mapping));
  279. return frame;
  280. }
  281. scoped_refptr<media::VideoFrame>
  282. MjpegDecodeAcceleratorTestEnvironment::CreateDmaBufVideoFrame(
  283. media::VideoPixelFormat format,
  284. const gfx::Size& coded_size,
  285. const gfx::Size& visible_size,
  286. std::unique_ptr<gfx::GpuMemoryBuffer>* backing_gmb) {
  287. DCHECK(gpu_memory_buffer_manager_);
  288. // Create a GpuMemoryBuffer and get a NativePixmapHandle from it.
  289. const absl::optional<gfx::BufferFormat> gfx_format =
  290. media::VideoPixelFormatToGfxBufferFormat(format);
  291. if (!gfx_format) {
  292. LOG(ERROR) << "Unsupported pixel format: " << format;
  293. return nullptr;
  294. }
  295. std::unique_ptr<gfx::GpuMemoryBuffer> gmb =
  296. gpu_memory_buffer_manager_->CreateGpuMemoryBuffer(
  297. coded_size, *gfx_format, kBufferUsage, gpu::kNullSurfaceHandle,
  298. nullptr);
  299. if (!gmb) {
  300. LOG(ERROR) << "Failed to create GpuMemoryBuffer";
  301. return nullptr;
  302. }
  303. gfx::GpuMemoryBufferHandle gmb_handle = gmb->CloneHandle();
  304. if (gmb_handle.type != gfx::NATIVE_PIXMAP) {
  305. LOG(ERROR) << "The GpuMemoryBufferHandle doesn't have type NATIVE_PIXMAP";
  306. return nullptr;
  307. }
  308. const size_t num_planes = media::VideoFrame::NumPlanes(format);
  309. if (gmb_handle.native_pixmap_handle.planes.size() != num_planes) {
  310. LOG(ERROR) << "The number of planes of NativePixmapHandle doesn't match "
  311. "the pixel format";
  312. return nullptr;
  313. }
  314. // Fill in the memory with zeros.
  315. if (!gmb->Map()) {
  316. LOG(ERROR) << "Failed to map GpuMemoryBuffer";
  317. return nullptr;
  318. }
  319. for (size_t i = 0; i < num_planes; i++) {
  320. gfx::NativePixmapPlane& plane = gmb_handle.native_pixmap_handle.planes[i];
  321. memset(gmb->memory(i), 0, plane.size);
  322. }
  323. gmb->Unmap();
  324. // Create a VideoFrame from the NativePixmapHandle.
  325. std::vector<media::ColorPlaneLayout> planes;
  326. std::vector<base::ScopedFD> dmabuf_fds;
  327. for (size_t i = 0; i < num_planes; i++) {
  328. gfx::NativePixmapPlane& plane = gmb_handle.native_pixmap_handle.planes[i];
  329. planes.emplace_back(base::checked_cast<int32_t>(plane.stride),
  330. base::checked_cast<size_t>(plane.offset),
  331. base::checked_cast<size_t>(plane.size));
  332. dmabuf_fds.push_back(std::move(plane.fd));
  333. }
  334. const absl::optional<media::VideoFrameLayout> layout =
  335. media::VideoFrameLayout::CreateWithPlanes(format, coded_size,
  336. std::move(planes));
  337. if (!layout) {
  338. LOG(ERROR) << "Failed to create VideoFrameLayout";
  339. return nullptr;
  340. }
  341. if (backing_gmb)
  342. *backing_gmb = std::move(gmb);
  343. return media::VideoFrame::WrapExternalDmabufs(
  344. *layout, gfx::Rect(visible_size), visible_size, std::move(dmabuf_fds),
  345. base::TimeDelta());
  346. }
  347. scoped_refptr<media::VideoFrame>
  348. MjpegDecodeAcceleratorTestEnvironment::MapToVideoFrame(
  349. gfx::GpuMemoryBuffer* gmb,
  350. const media::VideoFrameLayout& layout,
  351. const gfx::Rect& visible_rect) {
  352. DCHECK(gmb);
  353. if (!gmb->Map()) {
  354. LOG(ERROR) << "Failed to map GpuMemoryBuffer";
  355. return nullptr;
  356. }
  357. std::array<uint8_t*, 3> data{};
  358. for (size_t i = 0; i < layout.num_planes(); i++)
  359. data[i] = static_cast<uint8_t*>(gmb->memory(i));
  360. scoped_refptr<media::VideoFrame> frame =
  361. media::VideoFrame::WrapExternalYuvDataWithLayout(
  362. layout, visible_rect, visible_rect.size(), data[0], data[1], data[2],
  363. base::TimeDelta());
  364. if (!frame) {
  365. LOG(ERROR) << "Failed to create VideoFrame";
  366. return nullptr;
  367. }
  368. frame->AddDestructionObserver(
  369. base::BindOnce(&gfx::GpuMemoryBuffer::Unmap, base::Unretained(gmb)));
  370. return frame;
  371. }
  372. base::ScopedFD MjpegDecodeAcceleratorTestEnvironment::CreateDmaBufFd(
  373. const void* data,
  374. size_t size) {
  375. DCHECK(data);
  376. DCHECK_GT(size, 0u);
  377. DCHECK(gpu_memory_buffer_manager_);
  378. // The DMA-buf FD is intended to allow importing into hardware accelerators,
  379. // so we allocate the buffer by GMB manager instead of simply memfd_create().
  380. // The GMB has R_8 format and dimensions (|size|, 1).
  381. std::unique_ptr<gfx::GpuMemoryBuffer> gmb =
  382. gpu_memory_buffer_manager_->CreateGpuMemoryBuffer(
  383. gfx::Size(base::checked_cast<int>(size), 1), gfx::BufferFormat::R_8,
  384. kBufferUsage, gpu::kNullSurfaceHandle, nullptr);
  385. if (!gmb) {
  386. LOG(ERROR) << "Failed to create GpuMemoryBuffer";
  387. return base::ScopedFD();
  388. }
  389. gfx::GpuMemoryBufferHandle gmb_handle = gmb->CloneHandle();
  390. if (gmb_handle.type != gfx::NATIVE_PIXMAP) {
  391. LOG(ERROR) << "The GpuMemoryBufferHandle doesn't have type NATIVE_PIXMAP";
  392. return base::ScopedFD();
  393. }
  394. if (gmb_handle.native_pixmap_handle.planes.size() != 1) {
  395. LOG(ERROR) << "The number of planes of NativePixmapHandle is not 1 for R_8 "
  396. "format";
  397. return base::ScopedFD();
  398. }
  399. if (gmb_handle.native_pixmap_handle.planes[0].offset != 0) {
  400. LOG(ERROR) << "The memory offset is not zero";
  401. return base::ScopedFD();
  402. }
  403. // Fill in the memory with |data|.
  404. if (!gmb->Map()) {
  405. LOG(ERROR) << "Failed to map GpuMemoryBuffer";
  406. return base::ScopedFD();
  407. }
  408. memcpy(gmb->memory(0), data, size);
  409. gmb->Unmap();
  410. return std::move(gmb_handle.native_pixmap_handle.planes[0].fd);
  411. }
  412. std::vector<media::VideoPixelFormat>
  413. MjpegDecodeAcceleratorTestEnvironment::GetSupportedDmaBufFormats() {
  414. constexpr media::VideoPixelFormat kPreferredFormats[] = {
  415. media::PIXEL_FORMAT_NV12,
  416. media::PIXEL_FORMAT_YV12,
  417. };
  418. std::vector<media::VideoPixelFormat> supported_formats;
  419. for (const media::VideoPixelFormat format : kPreferredFormats) {
  420. const absl::optional<gfx::BufferFormat> gfx_format =
  421. media::VideoPixelFormatToGfxBufferFormat(format);
  422. if (gfx_format && gpu_memory_buffer_manager_->IsFormatAndUsageSupported(
  423. *gfx_format, kBufferUsage))
  424. supported_formats.push_back(format);
  425. }
  426. return supported_formats;
  427. }
  428. void MjpegDecodeAcceleratorTestEnvironment::AddMetric(
  429. const std::string& name,
  430. const base::TimeDelta& time) {
  431. metrics_.SetDoubleKey(name, time.InMillisecondsF());
  432. }
  433. enum ClientState {
  434. CS_CREATED,
  435. CS_INITIALIZED,
  436. CS_DECODE_PASS,
  437. CS_ERROR,
  438. };
  439. struct DecodeTask {
  440. const ParsedJpegImage* image;
  441. gfx::Size target_size;
  442. DecodeTask(const ParsedJpegImage* im)
  443. : image(im), target_size(im->visible_size) {}
  444. DecodeTask(const ParsedJpegImage* im, const gfx::Size ts)
  445. : image(im), target_size(ts) {}
  446. };
  447. struct PerfMetrics {
  448. size_t num_frames_decoded;
  449. base::TimeDelta total_decode_time;
  450. base::TimeDelta total_decode_map_time;
  451. };
  452. class JpegClient : public MjpegDecodeAccelerator::Client {
  453. public:
  454. // JpegClient takes ownership of |note|.
  455. JpegClient(
  456. const std::vector<DecodeTask>& tasks,
  457. std::unique_ptr<media::test::ClientStateNotification<ClientState>> note,
  458. bool use_dmabuf,
  459. bool skip_result_checking);
  460. JpegClient(const JpegClient&) = delete;
  461. JpegClient& operator=(const JpegClient&) = delete;
  462. ~JpegClient() override;
  463. void CreateJpegDecoder();
  464. void StartDecode(int32_t task_id, bool do_prepare_memory);
  465. void PrepareMemory(int32_t task_id);
  466. bool GetSoftwareDecodeResult(int32_t task_id);
  467. PerfMetrics GetPerfMetrics() const;
  468. // MjpegDecodeAccelerator::Client implementation.
  469. void VideoFrameReady(int32_t task_id) override;
  470. void NotifyError(int32_t task_id,
  471. MjpegDecodeAccelerator::Error error) override;
  472. // Accessors.
  473. media::test::ClientStateNotification<ClientState>* note() const {
  474. return note_.get();
  475. }
  476. private:
  477. FRIEND_TEST_ALL_PREFIXES(JpegClientTest, GetMeanAbsoluteDifference);
  478. void SetState(ClientState new_state);
  479. void OnInitialize(bool initialize_result);
  480. // Save a video frame that contains a decoded JPEG. The output is a PNG file.
  481. // The suffix will be added before the .png extension.
  482. void SaveToFile(int32_t task_id,
  483. scoped_refptr<media::VideoFrame> in_frame,
  484. const std::string& suffix = "");
  485. // Calculate mean absolute difference of hardware and software decode results
  486. // to check the similarity.
  487. double GetMeanAbsoluteDifference();
  488. // JpegClient doesn't own |tasks_|.
  489. const std::vector<DecodeTask>& tasks_;
  490. ClientState state_;
  491. // Used to notify another thread about the state. JpegClient owns this.
  492. std::unique_ptr<media::test::ClientStateNotification<ClientState>> note_;
  493. // Use DMA-buf backed output buffer for hardware decoder.
  494. bool use_dmabuf_;
  495. // Skip JDA decode result. Used for testing performance.
  496. bool skip_result_checking_;
  497. // Input shared memory and mapping.
  498. base::UnsafeSharedMemoryRegion in_shm_;
  499. base::WritableSharedMemoryMapping in_shm_mapping_;
  500. // Input DMA buffer file descriptor.
  501. base::ScopedFD in_dmabuf_fd_;
  502. // Output video frame from the hardware decoder.
  503. std::unique_ptr<gfx::GpuMemoryBuffer> hw_out_gmb_;
  504. scoped_refptr<media::VideoFrame> hw_out_dmabuf_frame_;
  505. scoped_refptr<media::VideoFrame> hw_out_frame_;
  506. // Output and intermediate frame for the software decoder.
  507. scoped_refptr<media::VideoFrame> sw_out_frame_;
  508. scoped_refptr<media::VideoFrame> sw_tmp_frame_;
  509. // This should be the first member to get destroyed because |decoder_|
  510. // potentially uses other members in the JpegClient instance. For example,
  511. // as decode tasks finish in a new thread spawned by |decoder_|,
  512. // |hw_out_frame_| can be accessed.
  513. std::unique_ptr<MjpegDecodeAccelerator> decoder_;
  514. // Timers for individual decoding calls indexed by |task_id|.
  515. std::map<int32_t, base::ElapsedTimer> timers_;
  516. // Recorded performance metrics.
  517. std::vector<base::TimeDelta> decode_times_;
  518. std::vector<base::TimeDelta> decode_map_times_;
  519. base::WeakPtrFactory<JpegClient> weak_factory_{this};
  520. };
  521. JpegClient::JpegClient(
  522. const std::vector<DecodeTask>& tasks,
  523. std::unique_ptr<media::test::ClientStateNotification<ClientState>> note,
  524. bool use_dmabuf,
  525. bool skip_result_checking)
  526. : tasks_(tasks),
  527. state_(CS_CREATED),
  528. note_(std::move(note)),
  529. use_dmabuf_(use_dmabuf),
  530. skip_result_checking_(skip_result_checking) {}
  531. JpegClient::~JpegClient() {}
  532. void JpegClient::CreateJpegDecoder() {
  533. decoder_ = nullptr;
  534. auto jda_factories =
  535. GpuMjpegDecodeAcceleratorFactory::GetAcceleratorFactories();
  536. if (jda_factories.empty()) {
  537. LOG(ERROR) << "MjpegDecodeAccelerator not supported on this platform.";
  538. SetState(CS_ERROR);
  539. return;
  540. }
  541. for (auto& create_jda_func : jda_factories) {
  542. decoder_ =
  543. std::move(create_jda_func).Run(base::ThreadTaskRunnerHandle::Get());
  544. if (decoder_)
  545. break;
  546. }
  547. if (!decoder_) {
  548. LOG(ERROR) << "Failed to create MjpegDecodeAccelerator.";
  549. SetState(CS_ERROR);
  550. return;
  551. }
  552. decoder_->InitializeAsync(this, base::BindOnce(&JpegClient::OnInitialize,
  553. weak_factory_.GetWeakPtr()));
  554. }
  555. void JpegClient::OnInitialize(bool initialize_result) {
  556. if (initialize_result) {
  557. SetState(CS_INITIALIZED);
  558. return;
  559. }
  560. LOG(ERROR) << "MjpegDecodeAccelerator::InitializeAsync() failed";
  561. SetState(CS_ERROR);
  562. }
  563. void JpegClient::VideoFrameReady(int32_t task_id) {
  564. const auto timer_iter = timers_.find(task_id);
  565. ASSERT_TRUE(timer_iter != timers_.end());
  566. base::ElapsedTimer& timer = timer_iter->second;
  567. decode_times_.push_back(timer.Elapsed());
  568. scoped_refptr<media::VideoFrame> mapped_dmabuf_frame;
  569. if (use_dmabuf_) {
  570. // Map and convert the output frame to I420.
  571. mapped_dmabuf_frame = g_env->MapToVideoFrame(
  572. hw_out_gmb_.get(), hw_out_dmabuf_frame_->layout(),
  573. hw_out_dmabuf_frame_->visible_rect());
  574. ASSERT_TRUE(mapped_dmabuf_frame);
  575. decode_map_times_.push_back(timer.Elapsed());
  576. }
  577. timers_.erase(timer_iter);
  578. if (skip_result_checking_) {
  579. SetState(CS_DECODE_PASS);
  580. return;
  581. }
  582. if (use_dmabuf_) {
  583. hw_out_frame_ = media::test::ConvertVideoFrame(mapped_dmabuf_frame.get(),
  584. media::PIXEL_FORMAT_I420);
  585. ASSERT_TRUE(hw_out_frame_);
  586. }
  587. if (!GetSoftwareDecodeResult(task_id)) {
  588. SetState(CS_ERROR);
  589. return;
  590. }
  591. if (g_save_to_file) {
  592. SaveToFile(task_id, hw_out_frame_, "_hw");
  593. SaveToFile(task_id, sw_out_frame_, "_sw");
  594. }
  595. double difference = GetMeanAbsoluteDifference();
  596. if (difference <= kDecodeSimilarityThreshold) {
  597. SetState(CS_DECODE_PASS);
  598. } else {
  599. LOG(ERROR) << "The mean absolute difference between software and hardware "
  600. << "decode is " << difference;
  601. SetState(CS_ERROR);
  602. }
  603. }
  604. void JpegClient::NotifyError(int32_t task_id,
  605. MjpegDecodeAccelerator::Error error) {
  606. LOG(ERROR) << "Notifying of error " << error << " for task id " << task_id;
  607. SetState(CS_ERROR);
  608. }
  609. void JpegClient::PrepareMemory(int32_t task_id) {
  610. const DecodeTask& task = tasks_[task_id];
  611. if (use_dmabuf_) {
  612. in_dmabuf_fd_ = g_env->CreateDmaBufFd(task.image->data_str.data(),
  613. task.image->data_str.size());
  614. ASSERT_TRUE(in_dmabuf_fd_.is_valid());
  615. // TODO(kamesan): create test cases for more formats when they're used.
  616. std::vector<media::VideoPixelFormat> supported_formats =
  617. g_env->GetSupportedDmaBufFormats();
  618. ASSERT_FALSE(supported_formats.empty());
  619. hw_out_dmabuf_frame_ = g_env->CreateDmaBufVideoFrame(
  620. supported_formats[0], task.target_size, task.target_size, &hw_out_gmb_);
  621. ASSERT_TRUE(hw_out_dmabuf_frame_);
  622. ASSERT_TRUE(hw_out_gmb_);
  623. } else {
  624. in_shm_mapping_ = base::WritableSharedMemoryMapping();
  625. in_shm_ =
  626. base::UnsafeSharedMemoryRegion::Create(task.image->data_str.size());
  627. ASSERT_TRUE(in_shm_.IsValid());
  628. in_shm_mapping_ = in_shm_.Map();
  629. ASSERT_TRUE(in_shm_mapping_.IsValid());
  630. memcpy(in_shm_mapping_.memory(), task.image->data_str.data(),
  631. task.image->data_str.size());
  632. // Only I420 output buffer is used in the shared memory path.
  633. hw_out_frame_ = g_env->CreateShmVideoFrame(
  634. media::PIXEL_FORMAT_I420, task.target_size, task.target_size);
  635. ASSERT_TRUE(hw_out_frame_);
  636. }
  637. if (task.image->visible_size != task.target_size) {
  638. // Needs an intermediate buffer for cropping/scaling.
  639. sw_tmp_frame_ = g_env->CreateShmVideoFrame(media::PIXEL_FORMAT_I420,
  640. task.image->coded_size,
  641. task.image->visible_size);
  642. ASSERT_TRUE(sw_tmp_frame_);
  643. }
  644. sw_out_frame_ = g_env->CreateShmVideoFrame(
  645. media::PIXEL_FORMAT_I420, task.target_size, task.target_size);
  646. ASSERT_TRUE(sw_out_frame_);
  647. }
  648. void JpegClient::SetState(ClientState new_state) {
  649. DVLOG(2) << "Changing state " << state_ << "->" << new_state;
  650. note_->Notify(new_state);
  651. state_ = new_state;
  652. }
  653. void JpegClient::SaveToFile(int32_t task_id,
  654. scoped_refptr<media::VideoFrame> in_frame,
  655. const std::string& suffix) {
  656. LOG_ASSERT(in_frame);
  657. const DecodeTask& task = tasks_[task_id];
  658. // First convert to ARGB format. Note that in our case, the coded size and the
  659. // visible size will be the same.
  660. scoped_refptr<media::VideoFrame> argb_out_frame =
  661. media::VideoFrame::CreateFrame(
  662. media::VideoPixelFormat::PIXEL_FORMAT_ARGB, task.target_size,
  663. gfx::Rect(task.target_size), task.target_size, base::TimeDelta());
  664. LOG_ASSERT(argb_out_frame);
  665. LOG_ASSERT(in_frame->visible_rect() == argb_out_frame->visible_rect());
  666. // Note that we use J420ToARGB instead of I420ToARGB so that the
  667. // kYuvJPEGConstants YUV-to-RGB conversion matrix is used.
  668. const int conversion_status = libyuv::J420ToARGB(
  669. in_frame->visible_data(media::VideoFrame::kYPlane),
  670. in_frame->stride(media::VideoFrame::kYPlane),
  671. in_frame->visible_data(media::VideoFrame::kUPlane),
  672. in_frame->stride(media::VideoFrame::kUPlane),
  673. in_frame->visible_data(media::VideoFrame::kVPlane),
  674. in_frame->stride(media::VideoFrame::kVPlane),
  675. argb_out_frame->visible_data(media::VideoFrame::kARGBPlane),
  676. argb_out_frame->stride(media::VideoFrame::kARGBPlane),
  677. argb_out_frame->visible_rect().width(),
  678. argb_out_frame->visible_rect().height());
  679. LOG_ASSERT(conversion_status == 0);
  680. // Save as a PNG.
  681. std::vector<uint8_t> png_output;
  682. const bool png_encode_status = gfx::PNGCodec::Encode(
  683. argb_out_frame->visible_data(media::VideoFrame::kARGBPlane),
  684. gfx::PNGCodec::FORMAT_BGRA, argb_out_frame->visible_rect().size(),
  685. argb_out_frame->stride(media::VideoFrame::kARGBPlane),
  686. true, /* discard_transparency */
  687. std::vector<gfx::PNGCodec::Comment>(), &png_output);
  688. LOG_ASSERT(png_encode_status);
  689. const base::FilePath in_filename(task.image->filename());
  690. const base::FilePath out_filename =
  691. in_filename.ReplaceExtension(".png").InsertBeforeExtension(suffix);
  692. const int size = base::checked_cast<int>(png_output.size());
  693. const int file_written_bytes = base::WriteFile(
  694. out_filename, reinterpret_cast<char*>(png_output.data()), size);
  695. LOG_ASSERT(file_written_bytes == size);
  696. }
  697. double JpegClient::GetMeanAbsoluteDifference() {
  698. double mean_abs_difference = 0;
  699. size_t num_samples = 0;
  700. const size_t planes[] = {media::VideoFrame::kYPlane,
  701. media::VideoFrame::kUPlane,
  702. media::VideoFrame::kVPlane};
  703. for (size_t plane : planes) {
  704. const uint8_t* hw_data = hw_out_frame_->data(plane);
  705. const uint8_t* sw_data = sw_out_frame_->data(plane);
  706. LOG_ASSERT(hw_out_frame_->visible_rect() == sw_out_frame_->visible_rect());
  707. const size_t rows =
  708. media::VideoFrame::Rows(plane, media::PIXEL_FORMAT_I420,
  709. hw_out_frame_->visible_rect().height());
  710. const size_t columns = media::VideoFrame::Columns(
  711. plane, media::PIXEL_FORMAT_I420, hw_out_frame_->visible_rect().width());
  712. const int hw_stride = hw_out_frame_->stride(plane);
  713. const int sw_stride = sw_out_frame_->stride(plane);
  714. for (size_t row = 0; row < rows; ++row) {
  715. for (size_t col = 0; col < columns; ++col)
  716. mean_abs_difference += std::abs(hw_data[col] - sw_data[col]);
  717. hw_data += hw_stride;
  718. sw_data += sw_stride;
  719. }
  720. num_samples += rows * columns;
  721. }
  722. LOG_ASSERT(num_samples > 0);
  723. mean_abs_difference /= num_samples;
  724. return mean_abs_difference;
  725. }
  726. void JpegClient::StartDecode(int32_t task_id, bool do_prepare_memory) {
  727. ASSERT_LT(base::checked_cast<size_t>(task_id), tasks_.size());
  728. const DecodeTask& task = tasks_[task_id];
  729. if (do_prepare_memory)
  730. PrepareMemory(task_id);
  731. timers_[task_id] = base::ElapsedTimer();
  732. if (use_dmabuf_) {
  733. base::ScopedFD duped_in_dmabuf_fd(HANDLE_EINTR(dup(in_dmabuf_fd_.get())));
  734. ASSERT_TRUE(duped_in_dmabuf_fd.is_valid());
  735. decoder_->Decode(task_id, std::move(duped_in_dmabuf_fd),
  736. task.image->data_str.size(), 0 /* src_offset */,
  737. hw_out_dmabuf_frame_);
  738. } else {
  739. ASSERT_EQ(in_shm_.GetSize(), task.image->data_str.size());
  740. media::BitstreamBuffer bitstream_buffer(task_id, in_shm_.Duplicate(),
  741. task.image->data_str.size());
  742. decoder_->Decode(std::move(bitstream_buffer), hw_out_frame_);
  743. }
  744. }
  745. bool JpegClient::GetSoftwareDecodeResult(int32_t task_id) {
  746. const DecodeTask& task = tasks_[task_id];
  747. const bool do_crop_scale = task.target_size != task.image->visible_size;
  748. DCHECK(sw_out_frame_->IsMappable());
  749. DCHECK_EQ(sw_out_frame_->format(), media::PIXEL_FORMAT_I420);
  750. if (do_crop_scale) {
  751. DCHECK(sw_tmp_frame_->IsMappable());
  752. DCHECK_EQ(sw_tmp_frame_->format(), media::PIXEL_FORMAT_I420);
  753. }
  754. media::VideoFrame* decode_frame =
  755. do_crop_scale ? sw_tmp_frame_.get() : sw_out_frame_.get();
  756. if (libyuv::ConvertToI420(
  757. reinterpret_cast<const uint8_t*>(task.image->data_str.data()),
  758. task.image->data_str.size(),
  759. decode_frame->visible_data(media::VideoFrame::kYPlane),
  760. decode_frame->stride(media::VideoFrame::kYPlane),
  761. decode_frame->visible_data(media::VideoFrame::kUPlane),
  762. decode_frame->stride(media::VideoFrame::kUPlane),
  763. decode_frame->visible_data(media::VideoFrame::kVPlane),
  764. decode_frame->stride(media::VideoFrame::kVPlane), 0, 0,
  765. decode_frame->visible_rect().width(),
  766. decode_frame->visible_rect().height(),
  767. decode_frame->visible_rect().width(),
  768. decode_frame->visible_rect().height(), libyuv::kRotate0,
  769. libyuv::FOURCC_MJPG) != 0) {
  770. LOG(ERROR) << "Software decode " << task.image->filename() << " failed.";
  771. return false;
  772. }
  773. if (do_crop_scale) {
  774. const gfx::Rect crop = media::CropSizeForScalingToTarget(
  775. sw_tmp_frame_->visible_rect().size(),
  776. sw_out_frame_->visible_rect().size(), /*alignment=*/2u);
  777. if (crop.IsEmpty()) {
  778. LOG(ERROR) << "Failed to calculate crop rectangle for "
  779. << sw_tmp_frame_->visible_rect().size().ToString() << " to "
  780. << sw_out_frame_->visible_rect().size().ToString();
  781. return false;
  782. }
  783. if (libyuv::I420Scale(
  784. sw_tmp_frame_->visible_data(media::VideoFrame::kYPlane) +
  785. crop.y() * sw_tmp_frame_->stride(media::VideoFrame::kYPlane) +
  786. crop.x(),
  787. sw_tmp_frame_->stride(media::VideoFrame::kYPlane),
  788. sw_tmp_frame_->visible_data(media::VideoFrame::kUPlane) +
  789. crop.y() / 2 *
  790. sw_tmp_frame_->stride(media::VideoFrame::kUPlane) +
  791. crop.x() / 2,
  792. sw_tmp_frame_->stride(media::VideoFrame::kUPlane),
  793. sw_tmp_frame_->visible_data(media::VideoFrame::kVPlane) +
  794. crop.y() / 2 *
  795. sw_tmp_frame_->stride(media::VideoFrame::kVPlane) +
  796. crop.x() / 2,
  797. sw_tmp_frame_->stride(media::VideoFrame::kVPlane), crop.width(),
  798. crop.height(),
  799. sw_out_frame_->visible_data(media::VideoFrame::kYPlane),
  800. sw_out_frame_->stride(media::VideoFrame::kYPlane),
  801. sw_out_frame_->visible_data(media::VideoFrame::kUPlane),
  802. sw_out_frame_->stride(media::VideoFrame::kUPlane),
  803. sw_out_frame_->visible_data(media::VideoFrame::kVPlane),
  804. sw_out_frame_->stride(media::VideoFrame::kVPlane),
  805. sw_out_frame_->visible_rect().width(),
  806. sw_out_frame_->visible_rect().height(),
  807. libyuv::kFilterBilinear) != 0) {
  808. LOG(ERROR) << "Software crop/scale failed.";
  809. return false;
  810. }
  811. }
  812. return true;
  813. }
  814. PerfMetrics JpegClient::GetPerfMetrics() const {
  815. return PerfMetrics{
  816. .num_frames_decoded = decode_times_.size(),
  817. .total_decode_time = std::accumulate(
  818. decode_times_.begin(), decode_times_.end(), base::TimeDelta()),
  819. .total_decode_map_time =
  820. std::accumulate(decode_map_times_.begin(), decode_map_times_.end(),
  821. base::TimeDelta()),
  822. };
  823. }
  824. // This class holds a |client| that will be deleted on |task_runner|. This is
  825. // necessary because |client->decoder_| expects to be destroyed on the thread on
  826. // which it was created.
  827. class ScopedJpegClient {
  828. public:
  829. ScopedJpegClient(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
  830. std::unique_ptr<JpegClient> client)
  831. : task_runner_(task_runner), client_(std::move(client)) {}
  832. ScopedJpegClient(const ScopedJpegClient&) = delete;
  833. ScopedJpegClient& operator=(const ScopedJpegClient&) = delete;
  834. ~ScopedJpegClient() {
  835. task_runner_->DeleteSoon(FROM_HERE, std::move(client_));
  836. }
  837. JpegClient* client() const { return client_.get(); }
  838. private:
  839. scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  840. std::unique_ptr<JpegClient> client_;
  841. };
  842. class MjpegDecodeAcceleratorTest : public ::testing::TestWithParam<bool> {
  843. public:
  844. MjpegDecodeAcceleratorTest(const MjpegDecodeAcceleratorTest&) = delete;
  845. MjpegDecodeAcceleratorTest& operator=(const MjpegDecodeAcceleratorTest&) =
  846. delete;
  847. protected:
  848. MjpegDecodeAcceleratorTest() = default;
  849. void TestDecode(const std::vector<DecodeTask>& tasks,
  850. const std::vector<ClientState>& expected_status,
  851. size_t num_concurrent_decoders = 1);
  852. void PerfDecodeByJDA(int decode_times, const std::vector<DecodeTask>& tasks);
  853. void PerfDecodeBySW(int decode_times, const std::vector<DecodeTask>& tasks);
  854. };
  855. void MjpegDecodeAcceleratorTest::TestDecode(
  856. const std::vector<DecodeTask>& tasks,
  857. const std::vector<ClientState>& expected_status,
  858. size_t num_concurrent_decoders) {
  859. LOG_ASSERT(tasks.size() >= expected_status.size());
  860. base::Thread decoder_thread("DecoderThread");
  861. ASSERT_TRUE(decoder_thread.Start());
  862. std::vector<std::unique_ptr<ScopedJpegClient>> scoped_clients;
  863. for (size_t i = 0; i < num_concurrent_decoders; i++) {
  864. auto client = std::make_unique<JpegClient>(
  865. tasks,
  866. std::make_unique<media::test::ClientStateNotification<ClientState>>(),
  867. GetParam() /* use_dmabuf */, false /* skip_result_checking */);
  868. scoped_clients.emplace_back(
  869. new ScopedJpegClient(decoder_thread.task_runner(), std::move(client)));
  870. decoder_thread.task_runner()->PostTask(
  871. FROM_HERE,
  872. base::BindOnce(&JpegClient::CreateJpegDecoder,
  873. base::Unretained(scoped_clients.back()->client())));
  874. ASSERT_EQ(scoped_clients.back()->client()->note()->Wait(), CS_INITIALIZED);
  875. }
  876. for (size_t index = 0; index < tasks.size(); index++) {
  877. for (const auto& scoped_client : scoped_clients) {
  878. decoder_thread.task_runner()->PostTask(
  879. FROM_HERE, base::BindOnce(&JpegClient::StartDecode,
  880. base::Unretained(scoped_client->client()),
  881. index, true /* do_prepare_memory */));
  882. }
  883. if (index < expected_status.size()) {
  884. for (const auto& scoped_client : scoped_clients) {
  885. ASSERT_EQ(scoped_client->client()->note()->Wait(),
  886. expected_status[index]);
  887. }
  888. }
  889. }
  890. }
  891. void MjpegDecodeAcceleratorTest::PerfDecodeByJDA(
  892. int decode_times,
  893. const std::vector<DecodeTask>& tasks) {
  894. LOG_ASSERT(tasks.size() == 1);
  895. base::Thread decoder_thread("DecoderThread");
  896. ASSERT_TRUE(decoder_thread.Start());
  897. const bool use_dmabuf = GetParam();
  898. auto client = std::make_unique<JpegClient>(
  899. tasks,
  900. std::make_unique<media::test::ClientStateNotification<ClientState>>(),
  901. use_dmabuf, true /* skip_result_checking */);
  902. auto scoped_client = std::make_unique<ScopedJpegClient>(
  903. decoder_thread.task_runner(), std::move(client));
  904. decoder_thread.task_runner()->PostTask(
  905. FROM_HERE, base::BindOnce(&JpegClient::CreateJpegDecoder,
  906. base::Unretained(scoped_client->client())));
  907. ASSERT_EQ(scoped_client->client()->note()->Wait(), CS_INITIALIZED);
  908. const int32_t task_id = 0;
  909. scoped_client->client()->PrepareMemory(task_id);
  910. for (int index = 0; index < decode_times; index++) {
  911. decoder_thread.task_runner()->PostTask(
  912. FROM_HERE, base::BindOnce(&JpegClient::StartDecode,
  913. base::Unretained(scoped_client->client()),
  914. task_id, false /* do_prepare_memory */));
  915. ASSERT_EQ(scoped_client->client()->note()->Wait(), CS_DECODE_PASS);
  916. }
  917. const PerfMetrics metrics = scoped_client->client()->GetPerfMetrics();
  918. const base::TimeDelta avg_decode_time =
  919. metrics.total_decode_time / metrics.num_frames_decoded;
  920. LOG(INFO) << "Decode: " << metrics.total_decode_time << " for "
  921. << metrics.num_frames_decoded
  922. << " iterations (avg: " << avg_decode_time << ")";
  923. g_env->AddMetric(
  924. use_dmabuf ? "hw_jpeg_decode_latency" : "hw_shm_jpeg_decode_latency",
  925. avg_decode_time);
  926. if (use_dmabuf) {
  927. const base::TimeDelta avg_decode_map_time =
  928. metrics.total_decode_map_time / metrics.num_frames_decoded;
  929. LOG(INFO) << "Decode + map: " << metrics.total_decode_map_time << " for "
  930. << metrics.num_frames_decoded
  931. << " iterations (avg: " << avg_decode_map_time << ")";
  932. g_env->AddMetric("hw_jpeg_decode_map_latency", avg_decode_map_time);
  933. }
  934. LOG(INFO) << "-- " << tasks[0].image->visible_size.ToString() << " ("
  935. << tasks[0].image->visible_size.GetArea() << " pixels), "
  936. << tasks[0].image->filename();
  937. }
  938. void MjpegDecodeAcceleratorTest::PerfDecodeBySW(
  939. int decode_times,
  940. const std::vector<DecodeTask>& tasks) {
  941. LOG_ASSERT(tasks.size() == 1);
  942. std::unique_ptr<JpegClient> client = std::make_unique<JpegClient>(
  943. tasks,
  944. std::make_unique<media::test::ClientStateNotification<ClientState>>(),
  945. false /* use_dmabuf */, true /* skip_result_checking */);
  946. const int32_t task_id = 0;
  947. client->PrepareMemory(task_id);
  948. const base::ElapsedTimer timer;
  949. for (int index = 0; index < decode_times; index++)
  950. ASSERT_TRUE(client->GetSoftwareDecodeResult(task_id));
  951. const base::TimeDelta elapsed_time = timer.Elapsed();
  952. const base::TimeDelta avg_decode_time = elapsed_time / decode_times;
  953. LOG(INFO) << "Decode: " << elapsed_time << " for " << decode_times
  954. << " iterations (avg: " << avg_decode_time << ")";
  955. LOG(INFO) << "-- " << tasks[0].image->visible_size.ToString() << ", ("
  956. << tasks[0].image->visible_size.GetArea() << " pixels) "
  957. << tasks[0].image->filename();
  958. g_env->AddMetric("sw_jpeg_decode_latency", avg_decode_time);
  959. }
  960. // Returns a media::VideoFrame that contains YUV data using 4:2:0 subsampling.
  961. // The visible size is 3x3, and the coded size is 4x4 which is 3x3 rounded up to
  962. // the next even dimensions.
  963. scoped_refptr<media::VideoFrame> GetTestDecodedData() {
  964. scoped_refptr<media::VideoFrame> frame =
  965. media::VideoFrame::CreateZeroInitializedFrame(
  966. media::PIXEL_FORMAT_I420, gfx::Size(4, 4) /* coded_size */,
  967. gfx::Rect(3, 3) /* visible_rect */,
  968. gfx::Size(3, 3) /* natural_size */, base::TimeDelta());
  969. LOG_ASSERT(frame.get());
  970. uint8_t* y_data = frame->data(media::VideoFrame::kYPlane);
  971. int y_stride = frame->stride(media::VideoFrame::kYPlane);
  972. uint8_t* u_data = frame->data(media::VideoFrame::kUPlane);
  973. int u_stride = frame->stride(media::VideoFrame::kUPlane);
  974. uint8_t* v_data = frame->data(media::VideoFrame::kVPlane);
  975. int v_stride = frame->stride(media::VideoFrame::kVPlane);
  976. // Data for the Y plane.
  977. memcpy(&y_data[0 * y_stride], "\x01\x02\x03", 3);
  978. memcpy(&y_data[1 * y_stride], "\x04\x05\x06", 3);
  979. memcpy(&y_data[2 * y_stride], "\x07\x08\x09", 3);
  980. // Data for the U plane.
  981. memcpy(&u_data[0 * u_stride], "\x0A\x0B", 2);
  982. memcpy(&u_data[1 * u_stride], "\x0C\x0D", 2);
  983. // Data for the V plane.
  984. memcpy(&v_data[0 * v_stride], "\x0E\x0F", 2);
  985. memcpy(&v_data[1 * v_stride], "\x10\x11", 2);
  986. return frame;
  987. }
  988. TEST(JpegClientTest, GetMeanAbsoluteDifference) {
  989. JpegClient client(std::vector<DecodeTask>{}, nullptr, false, false);
  990. client.hw_out_frame_ = GetTestDecodedData();
  991. client.sw_out_frame_ = GetTestDecodedData();
  992. uint8_t* y_data = client.sw_out_frame_->data(media::VideoFrame::kYPlane);
  993. const int y_stride = client.sw_out_frame_->stride(media::VideoFrame::kYPlane);
  994. uint8_t* u_data = client.sw_out_frame_->data(media::VideoFrame::kUPlane);
  995. const int u_stride = client.sw_out_frame_->stride(media::VideoFrame::kUPlane);
  996. uint8_t* v_data = client.sw_out_frame_->data(media::VideoFrame::kVPlane);
  997. const int v_stride = client.sw_out_frame_->stride(media::VideoFrame::kVPlane);
  998. // Change some visible data in the software decoding result.
  999. double expected_abs_mean_diff = 0;
  1000. y_data[0] = 0xF0; // Previously 0x01.
  1001. expected_abs_mean_diff += 0xF0 - 0x01;
  1002. y_data[y_stride + 1] = 0x8A; // Previously 0x05.
  1003. expected_abs_mean_diff += 0x8A - 0x05;
  1004. u_data[u_stride] = 0x02; // Previously 0x0C.
  1005. expected_abs_mean_diff += 0x0C - 0x02;
  1006. v_data[v_stride + 1] = 0x54; // Previously 0x11.
  1007. expected_abs_mean_diff += 0x54 - 0x11;
  1008. expected_abs_mean_diff /= 3 * 3 + 2 * 2 * 2;
  1009. constexpr double kMaxAllowedDifference = 1e-7;
  1010. EXPECT_NEAR(expected_abs_mean_diff, client.GetMeanAbsoluteDifference(),
  1011. kMaxAllowedDifference);
  1012. // Change some non-visible data in the software decoding result, i.e., part of
  1013. // the stride padding. This should not affect the absolute mean difference.
  1014. y_data[3] = 0xAB;
  1015. EXPECT_NEAR(expected_abs_mean_diff, client.GetMeanAbsoluteDifference(),
  1016. kMaxAllowedDifference);
  1017. }
  1018. TEST_P(MjpegDecodeAcceleratorTest, SimpleDecode) {
  1019. std::vector<DecodeTask> tasks;
  1020. for (auto& image : g_env->image_data_user_)
  1021. tasks.emplace_back(image.get());
  1022. const std::vector<ClientState> expected_status(tasks.size(), CS_DECODE_PASS);
  1023. TestDecode(tasks, expected_status);
  1024. }
  1025. #if BUILDFLAG(USE_VAAPI)
  1026. TEST_P(MjpegDecodeAcceleratorTest, DecodeBlit) {
  1027. std::vector<DecodeTask> tasks;
  1028. for (auto& image : g_env->image_data_user_) {
  1029. tasks.emplace_back(image.get(),
  1030. gfx::Size((image->visible_size.width() / 2) & ~1,
  1031. (image->visible_size.height() / 2) & ~1));
  1032. tasks.emplace_back(image.get(),
  1033. gfx::Size((image->visible_size.width() / 2) & ~1,
  1034. (image->visible_size.height() * 2 / 3) & ~1));
  1035. tasks.emplace_back(image.get(),
  1036. gfx::Size((image->visible_size.width() * 2 / 3) & ~1,
  1037. (image->visible_size.height() / 2) & ~1));
  1038. }
  1039. const std::vector<ClientState> expected_status(tasks.size(), CS_DECODE_PASS);
  1040. TestDecode(tasks, expected_status);
  1041. }
  1042. #endif
  1043. TEST_P(MjpegDecodeAcceleratorTest, InvalidTargetSize) {
  1044. std::vector<DecodeTask> tasks;
  1045. for (auto& image : g_env->image_data_user_) {
  1046. // Upscaling is not supported.
  1047. tasks.emplace_back(image.get(), gfx::Size(image->visible_size.width() * 2,
  1048. image->visible_size.height()));
  1049. tasks.emplace_back(image.get(),
  1050. gfx::Size(image->visible_size.width(),
  1051. image->visible_size.height() * 2));
  1052. // Odd dimensions are not supported.
  1053. tasks.emplace_back(image.get(),
  1054. gfx::Size((image->visible_size.width() / 2) | 1,
  1055. image->visible_size.height() / 2));
  1056. tasks.emplace_back(image.get(),
  1057. gfx::Size(image->visible_size.width() / 2,
  1058. (image->visible_size.height() / 2) | 1));
  1059. }
  1060. const std::vector<ClientState> expected_status(tasks.size(), CS_ERROR);
  1061. TestDecode(tasks, expected_status);
  1062. }
  1063. TEST_P(MjpegDecodeAcceleratorTest, MultipleDecoders) {
  1064. std::vector<DecodeTask> tasks;
  1065. for (auto& image : g_env->image_data_user_)
  1066. tasks.emplace_back(image.get());
  1067. const std::vector<ClientState> expected_status(tasks.size(), CS_DECODE_PASS);
  1068. TestDecode(tasks, expected_status, 3 /* num_concurrent_decoders */);
  1069. }
  1070. TEST_P(MjpegDecodeAcceleratorTest, InputSizeChange) {
  1071. // The size of |image_data_1280x720_black_| is smaller than
  1072. // |image_data_1280x720_default_|.
  1073. const std::vector<DecodeTask> tasks = {
  1074. DecodeTask(g_env->image_data_1280x720_black_.get()),
  1075. DecodeTask(g_env->image_data_1280x720_default_.get()),
  1076. DecodeTask(g_env->image_data_1280x720_black_.get()),
  1077. };
  1078. const std::vector<ClientState> expected_status(tasks.size(), CS_DECODE_PASS);
  1079. TestDecode(tasks, expected_status);
  1080. }
  1081. TEST_P(MjpegDecodeAcceleratorTest, ResolutionChange) {
  1082. const std::vector<DecodeTask> tasks = {
  1083. DecodeTask(g_env->image_data_640x368_black_.get()),
  1084. DecodeTask(g_env->image_data_1280x720_default_.get()),
  1085. DecodeTask(g_env->image_data_640x368_black_.get()),
  1086. };
  1087. const std::vector<ClientState> expected_status(tasks.size(), CS_DECODE_PASS);
  1088. TestDecode(tasks, expected_status);
  1089. }
  1090. TEST_P(MjpegDecodeAcceleratorTest, CodedSizeAlignment) {
  1091. const std::vector<DecodeTask> tasks = {
  1092. DecodeTask(g_env->image_data_640x360_black_.get()),
  1093. };
  1094. const std::vector<ClientState> expected_status = {CS_DECODE_PASS};
  1095. TestDecode(tasks, expected_status);
  1096. }
  1097. // Tests whether different JPEG sampling formats will be decoded correctly.
  1098. TEST_P(MjpegDecodeAcceleratorTest, SamplingFormatChange) {
  1099. const std::vector<DecodeTask> tasks = {
  1100. DecodeTask(g_env->image_data_640x368_black_.get()),
  1101. DecodeTask(g_env->image_data_640x368_422_black_.get()),
  1102. };
  1103. const std::vector<ClientState> expected_status(tasks.size(), CS_DECODE_PASS);
  1104. TestDecode(tasks, expected_status);
  1105. }
  1106. TEST_P(MjpegDecodeAcceleratorTest, FailureJpeg) {
  1107. const std::vector<DecodeTask> tasks = {
  1108. DecodeTask(g_env->image_data_invalid_.get()),
  1109. };
  1110. const std::vector<ClientState> expected_status = {CS_ERROR};
  1111. TestDecode(tasks, expected_status);
  1112. }
  1113. TEST_P(MjpegDecodeAcceleratorTest, KeepDecodeAfterFailure) {
  1114. const std::vector<DecodeTask> tasks = {
  1115. DecodeTask(g_env->image_data_invalid_.get()),
  1116. DecodeTask(g_env->image_data_1280x720_default_.get()),
  1117. };
  1118. const std::vector<ClientState> expected_status = {CS_ERROR, CS_DECODE_PASS};
  1119. TestDecode(tasks, expected_status);
  1120. }
  1121. TEST_P(MjpegDecodeAcceleratorTest, Abort) {
  1122. constexpr size_t kNumOfJpegToDecode = 5;
  1123. const std::vector<DecodeTask> tasks(
  1124. kNumOfJpegToDecode,
  1125. DecodeTask(g_env->image_data_1280x720_default_.get()));
  1126. // Verify only one decode success to ensure both decoders have started the
  1127. // decoding. Then destroy the first decoder when it is still decoding. The
  1128. // kernel should not crash during this test.
  1129. const std::vector<ClientState> expected_status = {CS_DECODE_PASS};
  1130. TestDecode(tasks, expected_status, 2 /* num_concurrent_decoders */);
  1131. }
  1132. TEST_P(MjpegDecodeAcceleratorTest, PerfJDA) {
  1133. // Only the first image will be used for perf testing.
  1134. ASSERT_GE(g_env->image_data_user_.size(), 1u);
  1135. const std::vector<DecodeTask> tasks = {
  1136. DecodeTask(g_env->image_data_user_[0].get()),
  1137. };
  1138. PerfDecodeByJDA(g_env->perf_decode_times_, tasks);
  1139. }
  1140. TEST_F(MjpegDecodeAcceleratorTest, PerfSW) {
  1141. // Only the first image will be used for perf testing.
  1142. ASSERT_GE(g_env->image_data_user_.size(), 1u);
  1143. const std::vector<DecodeTask> tasks = {
  1144. DecodeTask(g_env->image_data_user_[0].get()),
  1145. };
  1146. PerfDecodeBySW(g_env->perf_decode_times_, tasks);
  1147. }
  1148. std::string TestParamToString(::testing::TestParamInfo<bool> param_info) {
  1149. return param_info.param ? "DMABUF" : "SHMEM";
  1150. }
  1151. INSTANTIATE_TEST_SUITE_P(All,
  1152. MjpegDecodeAcceleratorTest,
  1153. ::testing::Bool(),
  1154. TestParamToString);
  1155. } // namespace
  1156. } // namespace chromeos_camera
  1157. int main(int argc, char** argv) {
  1158. testing::InitGoogleTest(&argc, argv);
  1159. base::CommandLine::Init(argc, argv);
  1160. mojo::core::Init();
  1161. base::ShadowingAtExitManager at_exit_manager;
  1162. // Needed to enable DVLOG through --vmodule.
  1163. logging::LoggingSettings settings;
  1164. settings.logging_dest =
  1165. logging::LOG_TO_SYSTEM_DEBUG_LOG | logging::LOG_TO_STDERR;
  1166. LOG_ASSERT(logging::InitLogging(settings));
  1167. const base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
  1168. LOG_ASSERT(cmd_line);
  1169. const base::FilePath::CharType* jpeg_filenames = nullptr;
  1170. const base::FilePath::CharType* test_data_path = nullptr;
  1171. const base::FilePath::CharType* perf_output_path = nullptr;
  1172. int perf_decode_times = 0;
  1173. base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
  1174. for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
  1175. it != switches.end(); ++it) {
  1176. // jpeg_filenames can include one or many files and use ';' as delimiter.
  1177. if (it->first == "jpeg_filenames") {
  1178. jpeg_filenames = it->second.c_str();
  1179. continue;
  1180. }
  1181. if (it->first == "test_data_path") {
  1182. test_data_path = it->second.c_str();
  1183. continue;
  1184. }
  1185. if (it->first == "perf_output_path") {
  1186. perf_output_path = it->second.c_str();
  1187. continue;
  1188. }
  1189. if (it->first == "perf_decode_times") {
  1190. perf_decode_times = std::stoi(it->second);
  1191. continue;
  1192. }
  1193. if (it->first == "save_to_file") {
  1194. chromeos_camera::g_save_to_file = true;
  1195. continue;
  1196. }
  1197. if (it->first == "v" || it->first == "vmodule")
  1198. continue;
  1199. if (it->first == "h" || it->first == "help")
  1200. continue;
  1201. LOG(ERROR) << "Unexpected switch: " << it->first << ":" << it->second;
  1202. return -EINVAL;
  1203. }
  1204. #if BUILDFLAG(USE_VAAPI)
  1205. media::VaapiWrapper::PreSandboxInitialization();
  1206. #endif
  1207. chromeos_camera::g_env =
  1208. reinterpret_cast<chromeos_camera::MjpegDecodeAcceleratorTestEnvironment*>(
  1209. testing::AddGlobalTestEnvironment(
  1210. new chromeos_camera::MjpegDecodeAcceleratorTestEnvironment(
  1211. jpeg_filenames, test_data_path, perf_output_path,
  1212. perf_decode_times)));
  1213. return RUN_ALL_TESTS();
  1214. }