video_encode_accelerator_perf_tests.cc 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include <algorithm>
  5. #include <iomanip>
  6. #include <iostream>
  7. #include <map>
  8. #include <numeric>
  9. #include <vector>
  10. #include "base/command_line.h"
  11. #include "base/files/file_util.h"
  12. #include "base/json/json_writer.h"
  13. #include "base/memory/raw_ptr.h"
  14. #include "base/strings/string_number_conversions.h"
  15. #include "base/time/time.h"
  16. #include "media/base/bitstream_buffer.h"
  17. #include "media/base/media_switches.h"
  18. #include "media/base/media_util.h"
  19. #include "media/base/test_data_util.h"
  20. #include "media/base/video_decoder_config.h"
  21. #include "media/gpu/test/video.h"
  22. #include "media/gpu/test/video_encoder/bitstream_validator.h"
  23. #include "media/gpu/test/video_encoder/video_encoder.h"
  24. #include "media/gpu/test/video_encoder/video_encoder_client.h"
  25. #include "media/gpu/test/video_encoder/video_encoder_test_environment.h"
  26. #include "media/gpu/test/video_frame_validator.h"
  27. #include "media/gpu/test/video_test_helpers.h"
  28. #include "testing/gtest/include/gtest/gtest.h"
  29. #include "third_party/abseil-cpp/absl/types/optional.h"
  30. namespace media {
  31. namespace test {
  32. namespace {
  33. // Video encoder perf tests usage message. Make sure to also update the
  34. // documentation under docs/media/gpu/video_encoder_perf_test_usage.md when
  35. // making changes here.
  36. // TODO(b/211783271): Add video_encoder_perf_test_usage.md
  37. constexpr const char* usage_msg =
  38. R"(usage: video_encode_accelerator_perf_tests
  39. [--codec=<codec>] [--num_spatial_layers=<number>]
  40. [--num_temporal_layers=<number>] [--bitrate_mode=(cbr|vbr)]
  41. [--reverse] [--bitrate=<bitrate>]
  42. [-v=<level>] [--vmodule=<config>] [--output_folder]
  43. [--disable_vaapi_lock]
  44. [--gtest_help] [--help]
  45. [<video path>] [<video metadata path>]
  46. )";
  47. // Video encoder performance tests help message.
  48. constexpr const char* help_msg =
  49. R"""(Run the video encode accelerator performance tests on the video
  50. specified by <video path>. If no <video path> is given the default
  51. "bear_320x192_40frames.yuv.webm" video will be used.
  52. The <video metadata path> should specify the location of a json file
  53. containing the video's metadata. By default <video path>.json will be
  54. used.
  55. The following arguments are supported:
  56. -v enable verbose mode, e.g. -v=2.
  57. --vmodule enable verbose mode for the specified module,
  58. --codec codec profile to encode, "h264 (baseline)",
  59. "h264main, "h264high", "vp8" and "vp9"
  60. --num_spatial_layers the number of spatial layers of the encoded
  61. bitstream. A default value is 1. Only affected
  62. if --codec=vp9 currently.
  63. --num_temporal_layers the number of temporal layers of the encoded
  64. bitstream. A default value is 1.
  65. --bitrate_mode The rate control mode for encoding, one of "cbr"
  66. (default) or "vbr".
  67. --reverse the stream plays backwards if the stream reaches
  68. end of stream. So the input stream to be encoded
  69. is consecutive. By default this is false.
  70. --bitrate bitrate (bits in second) of a produced bitstram.
  71. If not specified, a proper value for the video
  72. resolution is selected by the test.
  73. --output_folder overwrite the output folder used to store
  74. performance metrics, if not specified results
  75. will be stored in the current working directory.
  76. --disable_vaapi_lock disable the global VA-API lock if applicable,
  77. i.e., only on devices that use the VA-API with a libva
  78. backend that's known to be thread-safe and only in
  79. portions of the Chrome stack that should be able to
  80. deal with the absence of the lock
  81. (not the VaapiVideoDecodeAccelerator).
  82. --gtest_help display the gtest help and exit.
  83. --help display this help and exit.
  84. )""";
  85. // Default video to be used if no test video was specified.
  86. constexpr base::FilePath::CharType kDefaultTestVideoPath[] =
  87. FILE_PATH_LITERAL("bear_320x192_40frames.yuv.webm");
  88. media::test::VideoEncoderTestEnvironment* g_env;
  89. constexpr size_t kNumFramesToEncodeForPerformance = 300;
  90. constexpr size_t kMaxTemporalLayers = 3;
  91. constexpr size_t kMaxSpatialLayers = 3;
  92. // The event timeout used in perf tests because encoding 2160p
  93. // |kNumFramesToEncodeForPerformance| frames take much time.
  94. constexpr base::TimeDelta kPerfEventTimeout = base::Seconds(180);
  95. // Default output folder used to store performance metrics.
  96. constexpr const base::FilePath::CharType* kDefaultOutputFolder =
  97. FILE_PATH_LITERAL("perf_metrics");
  98. // Struct storing various time-related statistics.
  99. struct PerformanceTimeStats {
  100. PerformanceTimeStats() {}
  101. explicit PerformanceTimeStats(const std::vector<double>& times);
  102. double avg_ms_ = 0.0;
  103. double percentile_25_ms_ = 0.0;
  104. double percentile_50_ms_ = 0.0;
  105. double percentile_75_ms_ = 0.0;
  106. };
  107. PerformanceTimeStats::PerformanceTimeStats(const std::vector<double>& times) {
  108. if (times.empty())
  109. return;
  110. avg_ms_ = std::accumulate(times.begin(), times.end(), 0.0) / times.size();
  111. std::vector<double> sorted_times = times;
  112. std::sort(sorted_times.begin(), sorted_times.end());
  113. percentile_25_ms_ = sorted_times[sorted_times.size() / 4];
  114. percentile_50_ms_ = sorted_times[sorted_times.size() / 2];
  115. percentile_75_ms_ = sorted_times[(sorted_times.size() * 3) / 4];
  116. }
  117. // TODO(dstaessens): Investigate using more appropriate metrics for encoding.
  118. struct PerformanceMetrics {
  119. // Write the collected performance metrics to the console.
  120. void WriteToConsole() const;
  121. // Write the collected performance metrics to file.
  122. void WriteToFile() const;
  123. // Total measurement duration.
  124. base::TimeDelta total_duration_;
  125. // The number of bitstreams encoded.
  126. size_t bitstreams_encoded_ = 0;
  127. // The overall number of bitstreams encoded per second.
  128. double bitstreams_per_second_ = 0.0;
  129. // List of times between subsequent bitstream buffer deliveries. This is
  130. // important in real-time encoding scenarios, where the delivery time should
  131. // be less than the frame rate used.
  132. std::vector<double> bitstream_delivery_times_;
  133. // Statistics related to the time between bitstream buffer deliveries.
  134. PerformanceTimeStats bitstream_delivery_stats_;
  135. // List of times between queuing an encode operation and getting back the
  136. // encoded bitstream buffer.
  137. std::vector<double> bitstream_encode_times_;
  138. // Statistics related to the encode times.
  139. PerformanceTimeStats bitstream_encode_stats_;
  140. };
  141. // The performance evaluator can be plugged into the video encoder to collect
  142. // various performance metrics.
  143. class PerformanceEvaluator : public BitstreamProcessor {
  144. public:
  145. // Create a new performance evaluator.
  146. PerformanceEvaluator() {}
  147. void ProcessBitstream(scoped_refptr<BitstreamRef> bitstream,
  148. size_t frame_index) override;
  149. bool WaitUntilDone() override { return true; }
  150. // Start/Stop collecting performance metrics.
  151. void StartMeasuring();
  152. void StopMeasuring();
  153. // Get the collected performance metrics.
  154. const PerformanceMetrics& Metrics() const { return perf_metrics_; }
  155. private:
  156. // Start/end time of the measurement period.
  157. base::TimeTicks start_time_;
  158. base::TimeTicks end_time_;
  159. // Time at which the previous bitstream was delivered.
  160. base::TimeTicks prev_bitstream_delivery_time_;
  161. // Collection of various performance metrics.
  162. PerformanceMetrics perf_metrics_;
  163. };
  164. void PerformanceEvaluator::ProcessBitstream(
  165. scoped_refptr<BitstreamRef> bitstream,
  166. size_t frame_index) {
  167. base::TimeTicks now = base::TimeTicks::Now();
  168. base::TimeDelta delivery_time = (now - prev_bitstream_delivery_time_);
  169. perf_metrics_.bitstream_delivery_times_.push_back(
  170. delivery_time.InMillisecondsF());
  171. prev_bitstream_delivery_time_ = now;
  172. // TODO(hiroh): |encode_time| on upper spatial layer in SVC encoding becomes
  173. // larger because the bitstram is produced after lower spatial layers are
  174. // produced. |encode_time| should be aggregated per spatial layer.
  175. base::TimeDelta encode_time =
  176. base::TimeTicks::Now() - bitstream->source_timestamp;
  177. perf_metrics_.bitstream_encode_times_.push_back(
  178. encode_time.InMillisecondsF());
  179. }
  180. void PerformanceEvaluator::StartMeasuring() {
  181. start_time_ = base::TimeTicks::Now();
  182. prev_bitstream_delivery_time_ = start_time_;
  183. }
  184. void PerformanceEvaluator::StopMeasuring() {
  185. DCHECK_EQ(perf_metrics_.bitstream_delivery_times_.size(),
  186. perf_metrics_.bitstream_encode_times_.size());
  187. end_time_ = base::TimeTicks::Now();
  188. perf_metrics_.total_duration_ = end_time_ - start_time_;
  189. perf_metrics_.bitstreams_encoded_ =
  190. perf_metrics_.bitstream_encode_times_.size();
  191. perf_metrics_.bitstreams_per_second_ =
  192. perf_metrics_.bitstreams_encoded_ /
  193. perf_metrics_.total_duration_.InSecondsF();
  194. // Calculate delivery and encode time metrics.
  195. perf_metrics_.bitstream_delivery_stats_ =
  196. PerformanceTimeStats(perf_metrics_.bitstream_delivery_times_);
  197. perf_metrics_.bitstream_encode_stats_ =
  198. PerformanceTimeStats(perf_metrics_.bitstream_encode_times_);
  199. }
  200. void PerformanceMetrics::WriteToConsole() const {
  201. std::cout << "Bitstreams encoded: " << bitstreams_encoded_ << std::endl;
  202. std::cout << "Total duration: " << total_duration_.InMillisecondsF()
  203. << "ms" << std::endl;
  204. std::cout << "FPS: " << bitstreams_per_second_
  205. << std::endl;
  206. std::cout << "Bitstream delivery time - average: "
  207. << bitstream_delivery_stats_.avg_ms_ << "ms" << std::endl;
  208. std::cout << "Bitstream delivery time - percentile 25: "
  209. << bitstream_delivery_stats_.percentile_25_ms_ << "ms" << std::endl;
  210. std::cout << "Bitstream delivery time - percentile 50: "
  211. << bitstream_delivery_stats_.percentile_50_ms_ << "ms" << std::endl;
  212. std::cout << "Bitstream delivery time - percentile 75: "
  213. << bitstream_delivery_stats_.percentile_75_ms_ << "ms" << std::endl;
  214. std::cout << "Bitstream encode time - average: "
  215. << bitstream_encode_stats_.avg_ms_ << "ms" << std::endl;
  216. std::cout << "Bitstream encode time - percentile 25: "
  217. << bitstream_encode_stats_.percentile_25_ms_ << "ms" << std::endl;
  218. std::cout << "Bitstream encode time - percentile 50: "
  219. << bitstream_encode_stats_.percentile_50_ms_ << "ms" << std::endl;
  220. std::cout << "Bitstream encode time - percentile 75: "
  221. << bitstream_encode_stats_.percentile_75_ms_ << "ms" << std::endl;
  222. }
  223. void PerformanceMetrics::WriteToFile() const {
  224. base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
  225. if (!DirectoryExists(output_folder_path))
  226. base::CreateDirectory(output_folder_path);
  227. output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);
  228. // Write performance metrics to json.
  229. base::Value metrics(base::Value::Type::DICTIONARY);
  230. metrics.SetKey("BitstreamsEncoded",
  231. base::Value(base::checked_cast<int>(bitstreams_encoded_)));
  232. metrics.SetKey("TotalDurationMs",
  233. base::Value(total_duration_.InMillisecondsF()));
  234. metrics.SetKey("FPS", base::Value(bitstreams_per_second_));
  235. metrics.SetKey("BitstreamDeliveryTimeAverage",
  236. base::Value(bitstream_delivery_stats_.avg_ms_));
  237. metrics.SetKey("BitstreamDeliveryTimePercentile25",
  238. base::Value(bitstream_delivery_stats_.percentile_25_ms_));
  239. metrics.SetKey("BitstreamDeliveryTimePercentile50",
  240. base::Value(bitstream_delivery_stats_.percentile_50_ms_));
  241. metrics.SetKey("BitstreamDeliveryTimePercentile75",
  242. base::Value(bitstream_delivery_stats_.percentile_75_ms_));
  243. metrics.SetKey("BitstreamEncodeTimeAverage",
  244. base::Value(bitstream_encode_stats_.avg_ms_));
  245. metrics.SetKey("BitstreamEncodeTimePercentile25",
  246. base::Value(bitstream_encode_stats_.percentile_25_ms_));
  247. metrics.SetKey("BitstreamEncodeTimePercentile50",
  248. base::Value(bitstream_encode_stats_.percentile_50_ms_));
  249. metrics.SetKey("BitstreamEncodeTimePercentile75",
  250. base::Value(bitstream_encode_stats_.percentile_75_ms_));
  251. // Write bitstream delivery times to json.
  252. base::Value delivery_times(base::Value::Type::LIST);
  253. for (double bitstream_delivery_time : bitstream_delivery_times_) {
  254. delivery_times.Append(bitstream_delivery_time);
  255. }
  256. metrics.SetKey("BitstreamDeliveryTimes", std::move(delivery_times));
  257. // Write bitstream encodes times to json.
  258. base::Value encode_times(base::Value::Type::LIST);
  259. for (double bitstream_encode_time : bitstream_encode_times_) {
  260. encode_times.Append(bitstream_encode_time);
  261. }
  262. metrics.SetKey("BitstreamEncodeTimes", std::move(encode_times));
  263. // Write json to file.
  264. std::string metrics_str;
  265. ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
  266. metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
  267. base::FilePath metrics_file_path = output_folder_path.Append(
  268. g_env->GetTestOutputFilePath().AddExtension(FILE_PATH_LITERAL(".json")));
  269. // Make sure that the directory into which json is saved is created.
  270. LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
  271. base::File metrics_output_file(
  272. base::FilePath(metrics_file_path),
  273. base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
  274. int bytes_written = metrics_output_file.WriteAtCurrentPos(
  275. metrics_str.data(), metrics_str.length());
  276. ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
  277. VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
  278. }
  279. struct BitstreamQualityMetrics {
  280. BitstreamQualityMetrics(const PSNRVideoFrameValidator* const psnr_validator,
  281. const SSIMVideoFrameValidator* const ssim_validator,
  282. const absl::optional<size_t>& spatial_idx,
  283. const absl::optional<size_t>& temporal_idx);
  284. void Output(uint32_t target_bitrate, uint32_t actual_bitrate);
  285. absl::optional<size_t> spatial_idx;
  286. absl::optional<size_t> temporal_idx;
  287. private:
  288. struct QualityStats {
  289. QualityStats() = default;
  290. QualityStats(const QualityStats&) = default;
  291. QualityStats& operator=(const QualityStats&) = default;
  292. double avg = 0;
  293. double percentile_25 = 0;
  294. double percentile_50 = 0;
  295. double percentile_75 = 0;
  296. std::vector<double> values_in_order;
  297. };
  298. static QualityStats ComputeQualityStats(
  299. const std::map<size_t, double>& values);
  300. void WriteToConsole(const std::string& svc_text,
  301. const BitstreamQualityMetrics::QualityStats& psnr_stats,
  302. const BitstreamQualityMetrics::QualityStats& ssim_stats,
  303. uint32_t target_bitrate,
  304. uint32_t actual_bitrate) const;
  305. void WriteToFile(const std::string& svc_text,
  306. const BitstreamQualityMetrics::QualityStats& psnr_stats,
  307. const BitstreamQualityMetrics::QualityStats& ssim_stats,
  308. uint32_t target_bitrate,
  309. uint32_t actual_bitrate) const;
  310. const PSNRVideoFrameValidator* const psnr_validator;
  311. const SSIMVideoFrameValidator* const ssim_validator;
  312. };
  313. BitstreamQualityMetrics::BitstreamQualityMetrics(
  314. const PSNRVideoFrameValidator* const psnr_validator,
  315. const SSIMVideoFrameValidator* const ssim_validator,
  316. const absl::optional<size_t>& spatial_idx,
  317. const absl::optional<size_t>& temporal_idx)
  318. : spatial_idx(spatial_idx),
  319. temporal_idx(temporal_idx),
  320. psnr_validator(psnr_validator),
  321. ssim_validator(ssim_validator) {}
  322. // static
  323. BitstreamQualityMetrics::QualityStats
  324. BitstreamQualityMetrics::ComputeQualityStats(
  325. const std::map<size_t, double>& values) {
  326. if (values.empty())
  327. return QualityStats();
  328. std::vector<double> sorted_values;
  329. std::vector<std::pair<size_t, double>> index_and_value;
  330. sorted_values.reserve(values.size());
  331. index_and_value.reserve(values.size());
  332. for (const auto& v : values) {
  333. sorted_values.push_back(v.second);
  334. index_and_value.emplace_back(v.first, v.second);
  335. }
  336. std::sort(sorted_values.begin(), sorted_values.end());
  337. std::sort(index_and_value.begin(), index_and_value.end());
  338. QualityStats stats;
  339. stats.avg = std::accumulate(sorted_values.begin(), sorted_values.end(), 0.0) /
  340. sorted_values.size();
  341. stats.percentile_25 = sorted_values[sorted_values.size() / 4];
  342. stats.percentile_50 = sorted_values[sorted_values.size() / 2];
  343. stats.percentile_75 = sorted_values[(sorted_values.size() * 3) / 4];
  344. stats.values_in_order.resize(index_and_value.size());
  345. for (size_t i = 0; i < index_and_value.size(); ++i)
  346. stats.values_in_order[i] = index_and_value[i].second;
  347. return stats;
  348. }
  349. void BitstreamQualityMetrics::Output(uint32_t target_bitrate,
  350. uint32_t actual_bitrate) {
  351. std::string svc_text;
  352. if (spatial_idx)
  353. svc_text += "L" + base::NumberToString(*spatial_idx + 1);
  354. if (temporal_idx)
  355. svc_text += "T" + base::NumberToString(*temporal_idx + 1);
  356. auto psnr_stats = ComputeQualityStats(psnr_validator->GetPSNRValues());
  357. auto ssim_stats = ComputeQualityStats(ssim_validator->GetSSIMValues());
  358. WriteToConsole(svc_text, psnr_stats, ssim_stats, target_bitrate,
  359. actual_bitrate);
  360. WriteToFile(svc_text, psnr_stats, ssim_stats, target_bitrate, actual_bitrate);
  361. }
  362. void BitstreamQualityMetrics::WriteToConsole(
  363. const std::string& svc_text,
  364. const BitstreamQualityMetrics::QualityStats& psnr_stats,
  365. const BitstreamQualityMetrics::QualityStats& ssim_stats,
  366. uint32_t target_bitrate,
  367. uint32_t actual_bitrate) const {
  368. const auto default_ssize = std::cout.precision();
  369. std::cout << "[ Result " << svc_text << "]" << std::endl;
  370. std::cout << "Bitrate: " << actual_bitrate << " (target: " << target_bitrate
  371. << ")" << std::endl;
  372. std::cout << "Bitrate deviation: " << std::fixed << std::setprecision(2)
  373. << (actual_bitrate * 100.0 / target_bitrate) - 100.0 << " %"
  374. << std::endl;
  375. std::cout << std::fixed << std::setprecision(4);
  376. std::cout << "SSIM - average: " << ssim_stats.avg << std::endl;
  377. std::cout << "SSIM - percentile 25: " << ssim_stats.percentile_25
  378. << std::endl;
  379. std::cout << "SSIM - percentile 50: " << ssim_stats.percentile_50
  380. << std::endl;
  381. std::cout << "SSIM - percentile 75: " << ssim_stats.percentile_75
  382. << std::endl;
  383. std::cout << "PSNR - average: " << psnr_stats.avg << std::endl;
  384. std::cout << "PSNR - percentile 25: " << psnr_stats.percentile_25
  385. << std::endl;
  386. std::cout << "PSNR - percentile 50: " << psnr_stats.percentile_50
  387. << std::endl;
  388. std::cout << "PSNR - percentile 75: " << psnr_stats.percentile_75
  389. << std::endl;
  390. std::cout.precision(default_ssize);
  391. }
  392. void BitstreamQualityMetrics::WriteToFile(
  393. const std::string& svc_text,
  394. const BitstreamQualityMetrics::QualityStats& psnr_stats,
  395. const BitstreamQualityMetrics::QualityStats& ssim_stats,
  396. uint32_t target_bitrate,
  397. uint32_t actual_bitrate) const {
  398. base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
  399. if (!DirectoryExists(output_folder_path))
  400. base::CreateDirectory(output_folder_path);
  401. output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);
  402. // Write quality metrics to json.
  403. base::Value metrics(base::Value::Type::DICTIONARY);
  404. if (!svc_text.empty())
  405. metrics.SetKey("SVC", base::Value(svc_text));
  406. metrics.SetKey("Bitrate",
  407. base::Value(base::checked_cast<int>(actual_bitrate)));
  408. metrics.SetKey(
  409. "BitrateDeviation",
  410. base::Value((actual_bitrate * 100.0 / target_bitrate) - 100.0));
  411. metrics.SetKey("SSIMAverage", base::Value(ssim_stats.avg));
  412. metrics.SetKey("PSNRAverage", base::Value(psnr_stats.avg));
  413. // Write ssim values bitstream delivery times to json.
  414. base::Value ssim_values(base::Value::Type::LIST);
  415. for (double value : ssim_stats.values_in_order)
  416. ssim_values.Append(value);
  417. metrics.SetKey("SSIMValues", std::move(ssim_values));
  418. // Write psnr values to json.
  419. base::Value psnr_values(base::Value::Type::LIST);
  420. for (double value : psnr_stats.values_in_order)
  421. psnr_values.Append(value);
  422. metrics.SetKey("PSNRValues", std::move(psnr_values));
  423. // Write json to file.
  424. std::string metrics_str;
  425. ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
  426. metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
  427. base::FilePath metrics_file_path = output_folder_path.Append(
  428. g_env->GetTestOutputFilePath()
  429. .AddExtension(svc_text.empty() ? "" : "." + svc_text)
  430. .AddExtension(FILE_PATH_LITERAL(".json")));
  431. // Make sure that the directory into which json is saved is created.
  432. LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
  433. base::File metrics_output_file(
  434. base::FilePath(metrics_file_path),
  435. base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
  436. int bytes_written = metrics_output_file.WriteAtCurrentPos(
  437. metrics_str.data(), metrics_str.length());
  438. ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
  439. VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
  440. }
  441. // Video encode test class. Performs setup and teardown for each single test.
  442. // It measures the performance in encoding NV12 GpuMemoryBuffer based
  443. // VideoFrame.
  444. class VideoEncoderTest : public ::testing::Test {
  445. public:
  446. // Creates VideoEncoder for encoding NV12 GpuMemoryBuffer based VideoFrames.
  447. // The input VideoFrames are provided every 1 / |encoder_rate| seconds if it
  448. // is specified. Or they are provided as soon as the previous input VideoFrame
  449. // is consumed by VideoEncoder. |measure_quality| measures SSIM and PSNR
  450. // values of encoded bitstream comparing the original input VideoFrames.
  451. std::unique_ptr<VideoEncoder> CreateVideoEncoder(
  452. absl::optional<uint32_t> encode_rate,
  453. bool measure_quality) {
  454. Video* video = g_env->GenerateNV12Video();
  455. VideoCodecProfile profile = g_env->Profile();
  456. const media::VideoBitrateAllocation& bitrate = g_env->BitrateAllocation();
  457. const std::vector<VideoEncodeAccelerator::Config::SpatialLayer>&
  458. spatial_layers = g_env->SpatialLayers();
  459. std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
  460. if (measure_quality) {
  461. bitstream_processors = CreateBitstreamProcessorsForQualityPerformance(
  462. video, profile, spatial_layers);
  463. } else {
  464. auto performance_evaluator = std::make_unique<PerformanceEvaluator>();
  465. performance_evaluator_ = performance_evaluator.get();
  466. bitstream_processors.push_back(std::move(performance_evaluator));
  467. }
  468. LOG_ASSERT(!bitstream_processors.empty())
  469. << "Failed to create bitstream processors";
  470. VideoEncoderClientConfig config(video, profile, spatial_layers, bitrate,
  471. g_env->Reverse());
  472. config.input_storage_type =
  473. VideoEncodeAccelerator::Config::StorageType::kGpuMemoryBuffer;
  474. config.num_frames_to_encode = kNumFramesToEncodeForPerformance;
  475. if (encode_rate) {
  476. config.encode_interval = base::Seconds(1u) / encode_rate.value();
  477. }
  478. auto video_encoder =
  479. VideoEncoder::Create(config, std::move(bitstream_processors));
  480. LOG_ASSERT(video_encoder);
  481. LOG_ASSERT(video_encoder->Initialize(video));
  482. return video_encoder;
  483. }
  484. protected:
  485. raw_ptr<PerformanceEvaluator> performance_evaluator_;
  486. std::vector<BitstreamQualityMetrics> quality_metrics_;
  487. private:
  488. std::unique_ptr<BitstreamValidator> CreateBitstreamValidator(
  489. const VideoCodecProfile profile,
  490. const gfx::Rect& visible_rect,
  491. const absl::optional<size_t>& spatial_layer_index_to_decode,
  492. const absl::optional<size_t>& temporal_layer_index_to_decode,
  493. const std::vector<gfx::Size>& spatial_layer_resolutions) {
  494. std::vector<std::unique_ptr<VideoFrameProcessor>> video_frame_processors;
  495. VideoFrameValidator::GetModelFrameCB get_model_frame_cb =
  496. base::BindRepeating(&VideoEncoderTest::GetModelFrame,
  497. base::Unretained(this), visible_rect);
  498. auto ssim_validator = SSIMVideoFrameValidator::Create(
  499. get_model_frame_cb, /*corrupt_frame_processor=*/nullptr,
  500. VideoFrameValidator::ValidationMode::kAverage,
  501. /*tolerance=*/0.0);
  502. LOG_ASSERT(ssim_validator);
  503. auto psnr_validator = PSNRVideoFrameValidator::Create(
  504. get_model_frame_cb, /*corrupt_frame_processor=*/nullptr,
  505. VideoFrameValidator::ValidationMode::kAverage,
  506. /*tolerance=*/0.0);
  507. LOG_ASSERT(psnr_validator);
  508. quality_metrics_.push_back(BitstreamQualityMetrics(
  509. psnr_validator.get(), ssim_validator.get(),
  510. spatial_layer_index_to_decode, temporal_layer_index_to_decode));
  511. video_frame_processors.push_back(std::move(ssim_validator));
  512. video_frame_processors.push_back(std::move(psnr_validator));
  513. VideoDecoderConfig decoder_config(
  514. VideoCodecProfileToVideoCodec(profile), profile,
  515. VideoDecoderConfig::AlphaMode::kIsOpaque, VideoColorSpace(),
  516. kNoTransformation, visible_rect.size(), visible_rect,
  517. visible_rect.size(), EmptyExtraData(), EncryptionScheme::kUnencrypted);
  518. return BitstreamValidator::Create(
  519. decoder_config, kNumFramesToEncodeForPerformance - 1,
  520. std::move(video_frame_processors), spatial_layer_index_to_decode,
  521. temporal_layer_index_to_decode, spatial_layer_resolutions);
  522. }
  523. // Create bitstream processors for quality performance tests.
  524. std::vector<std::unique_ptr<BitstreamProcessor>>
  525. CreateBitstreamProcessorsForQualityPerformance(
  526. Video* video,
  527. VideoCodecProfile profile,
  528. const std::vector<VideoEncodeAccelerator::Config::SpatialLayer>&
  529. spatial_layers) {
  530. std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
  531. raw_data_helper_ = RawDataHelper::Create(video, g_env->Reverse());
  532. if (!raw_data_helper_) {
  533. LOG(ERROR) << "Failed to create raw data helper";
  534. return bitstream_processors;
  535. }
  536. if (spatial_layers.empty()) {
  537. // Simple stream encoding.
  538. bitstream_processors.push_back(CreateBitstreamValidator(
  539. profile, gfx::Rect(video->Resolution()),
  540. /*spatial_layer_index_to_decode=*/absl::nullopt,
  541. /*temporal_layer_index_to_decode=*/absl::nullopt,
  542. /*spatial_layer_resolutions=*/{}));
  543. LOG_ASSERT(!!bitstream_processors.back());
  544. } else {
  545. // Temporal/Spatial layer encoding.
  546. std::vector<gfx::Size> spatial_layer_resolutions;
  547. for (const auto& sl : spatial_layers)
  548. spatial_layer_resolutions.emplace_back(sl.width, sl.height);
  549. for (size_t sid = 0; sid < spatial_layers.size(); ++sid) {
  550. for (size_t tid = 0; tid < spatial_layers[sid].num_of_temporal_layers;
  551. ++tid) {
  552. bitstream_processors.push_back(CreateBitstreamValidator(
  553. profile, gfx::Rect(spatial_layer_resolutions[sid]), sid, tid,
  554. spatial_layer_resolutions));
  555. LOG_ASSERT(!!bitstream_processors.back());
  556. }
  557. }
  558. }
  559. return bitstream_processors;
  560. }
  561. scoped_refptr<const VideoFrame> GetModelFrame(const gfx::Rect& visible_rect,
  562. size_t frame_index) {
  563. LOG_ASSERT(raw_data_helper_);
  564. auto frame = raw_data_helper_->GetFrame(frame_index);
  565. if (!frame)
  566. return nullptr;
  567. if (visible_rect.size() == frame->visible_rect().size())
  568. return frame;
  569. return ScaleVideoFrame(frame.get(), visible_rect.size());
  570. }
  571. std::unique_ptr<RawDataHelper> raw_data_helper_;
  572. };
  573. } // namespace
  574. // Encode |kNumFramesToEncodeForPerformance| frames while measuring uncapped
  575. // performance. This test will encode a video as fast as possible, and gives an
  576. // idea about the maximum output of the encoder.
  577. TEST_F(VideoEncoderTest, MeasureUncappedPerformance) {
  578. auto encoder = CreateVideoEncoder(/*encode_rate=*/absl::nullopt,
  579. /*measure_quality=*/false);
  580. encoder->SetEventWaitTimeout(kPerfEventTimeout);
  581. performance_evaluator_->StartMeasuring();
  582. encoder->Encode();
  583. EXPECT_TRUE(encoder->WaitForFlushDone());
  584. performance_evaluator_->StopMeasuring();
  585. auto metrics = performance_evaluator_->Metrics();
  586. metrics.WriteToConsole();
  587. metrics.WriteToFile();
  588. EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
  589. EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
  590. }
  591. // Encode |kNumFramesToEncodeForPerformance| frames while measuring capped
  592. // performance. This test will encode a video at a fixed ratio, 30fps.
  593. // This test can be used to measure the cpu metrics during encoding.
  594. TEST_F(VideoEncoderTest, MeasureCappedPerformance) {
  595. const uint32_t kEncodeRate = 30;
  596. auto encoder = CreateVideoEncoder(/*encode_rate=*/kEncodeRate,
  597. /*measure_quality=*/false);
  598. encoder->SetEventWaitTimeout(kPerfEventTimeout);
  599. performance_evaluator_->StartMeasuring();
  600. encoder->Encode();
  601. EXPECT_TRUE(encoder->WaitForFlushDone());
  602. performance_evaluator_->StopMeasuring();
  603. auto metrics = performance_evaluator_->Metrics();
  604. metrics.WriteToConsole();
  605. metrics.WriteToFile();
  606. EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
  607. EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
  608. }
  609. TEST_F(VideoEncoderTest, MeasureProducedBitstreamQuality) {
  610. auto encoder = CreateVideoEncoder(/*encode_rate=*/absl::nullopt,
  611. /*measure_quality=*/true);
  612. encoder->SetEventWaitTimeout(kPerfEventTimeout);
  613. encoder->Encode();
  614. EXPECT_TRUE(encoder->WaitForFlushDone());
  615. EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
  616. EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
  617. EXPECT_TRUE(encoder->WaitForBitstreamProcessors());
  618. const VideoEncoderStats stats = encoder->GetStats();
  619. for (auto& metrics : quality_metrics_) {
  620. absl::optional<size_t> spatial_idx = metrics.spatial_idx;
  621. absl::optional<size_t> temporal_idx = metrics.temporal_idx;
  622. uint32_t target_bitrate = 0;
  623. uint32_t actual_bitrate = 0;
  624. if (!spatial_idx && !temporal_idx) {
  625. target_bitrate = g_env->BitrateAllocation().GetSumBps();
  626. actual_bitrate = stats.Bitrate();
  627. } else {
  628. CHECK(spatial_idx && temporal_idx);
  629. // Target and actual bitrates in temporal layer encoding are the sum of
  630. // bitrates of the temporal layers in the spatial layer.
  631. for (size_t tid = 0; tid <= *temporal_idx; ++tid) {
  632. target_bitrate +=
  633. g_env->BitrateAllocation().GetBitrateBps(*spatial_idx, tid);
  634. actual_bitrate += stats.LayerBitrate(*spatial_idx, tid);
  635. }
  636. }
  637. metrics.Output(target_bitrate, actual_bitrate);
  638. }
  639. }
  640. // TODO(b/211783279) The |performance_evaluator_| only keeps track of the last
  641. // created encoder. We should instead keep track of multiple evaluators, and
  642. // then decide how to aggregate/report those metrics.
  643. TEST_F(VideoEncoderTest,
  644. MeasureUncappedPerformance_MultipleConcurrentEncoders) {
  645. // Run two encoders for larger resolutions to avoid creating shared memory
  646. // buffers during the test on lower end devices.
  647. constexpr gfx::Size k1080p(1920, 1080);
  648. const size_t kMinSupportedConcurrentEncoders =
  649. g_env->Video()->Resolution().GetArea() >= k1080p.GetArea() ? 2 : 3;
  650. std::vector<std::unique_ptr<VideoEncoder>> encoders(
  651. kMinSupportedConcurrentEncoders);
  652. for (size_t i = 0; i < kMinSupportedConcurrentEncoders; ++i) {
  653. encoders[i] = CreateVideoEncoder(/*encode_rate=*/absl::nullopt,
  654. /*measure_quality=*/false);
  655. encoders[i]->SetEventWaitTimeout(kPerfEventTimeout);
  656. }
  657. performance_evaluator_->StartMeasuring();
  658. for (auto&& encoder : encoders)
  659. encoder->Encode();
  660. for (auto&& encoder : encoders) {
  661. EXPECT_TRUE(encoder->WaitForFlushDone());
  662. EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
  663. EXPECT_EQ(encoder->GetFrameReleasedCount(),
  664. kNumFramesToEncodeForPerformance);
  665. }
  666. performance_evaluator_->StopMeasuring();
  667. auto metrics = performance_evaluator_->Metrics();
  668. metrics.WriteToConsole();
  669. metrics.WriteToFile();
  670. }
  671. } // namespace test
  672. } // namespace media
  673. int main(int argc, char** argv) {
  674. // Set the default test data path.
  675. media::test::Video::SetTestDataPath(media::GetTestDataPath());
  676. // Print the help message if requested. This needs to be done before
  677. // initializing gtest, to overwrite the default gtest help message.
  678. base::CommandLine::Init(argc, argv);
  679. const base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
  680. LOG_ASSERT(cmd_line);
  681. if (cmd_line->HasSwitch("help")) {
  682. std::cout << media::test::usage_msg << "\n" << media::test::help_msg;
  683. return 0;
  684. }
  685. // Check if a video was specified on the command line.
  686. base::CommandLine::StringVector args = cmd_line->GetArgs();
  687. base::FilePath video_path =
  688. (args.size() >= 1) ? base::FilePath(args[0])
  689. : base::FilePath(media::test::kDefaultTestVideoPath);
  690. base::FilePath video_metadata_path =
  691. (args.size() >= 2) ? base::FilePath(args[1]) : base::FilePath();
  692. std::string codec = "h264";
  693. size_t num_spatial_layers = 1u;
  694. size_t num_temporal_layers = 1u;
  695. media::Bitrate::Mode bitrate_mode = media::Bitrate::Mode::kConstant;
  696. bool reverse = false;
  697. absl::optional<uint32_t> encode_bitrate;
  698. std::vector<base::Feature> disabled_features;
  699. // Parse command line arguments.
  700. base::FilePath::StringType output_folder = media::test::kDefaultOutputFolder;
  701. base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
  702. for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
  703. it != switches.end(); ++it) {
  704. if (it->first.find("gtest_") == 0 || // Handled by GoogleTest
  705. it->first == "v" || it->first == "vmodule") { // Handled by Chrome
  706. continue;
  707. }
  708. if (it->first == "output_folder") {
  709. output_folder = it->second;
  710. } else if (it->first == "codec") {
  711. codec = it->second;
  712. } else if (it->first == "num_spatial_layers") {
  713. if (!base::StringToSizeT(it->second, &num_spatial_layers)) {
  714. std::cout << "invalid number of spatial layers: " << it->second << "\n";
  715. return EXIT_FAILURE;
  716. }
  717. if (num_spatial_layers > media::test::kMaxSpatialLayers) {
  718. std::cout << "unsupported number of spatial layers: " << it->second
  719. << "\n";
  720. return EXIT_FAILURE;
  721. }
  722. } else if (it->first == "num_temporal_layers") {
  723. if (!base::StringToSizeT(it->second, &num_temporal_layers)) {
  724. std::cout << "invalid number of temporal layers: " << it->second
  725. << "\n";
  726. return EXIT_FAILURE;
  727. }
  728. if (num_spatial_layers > media::test::kMaxTemporalLayers) {
  729. std::cout << "unsupported number of temporal layers: " << it->second
  730. << "\n";
  731. return EXIT_FAILURE;
  732. }
  733. } else if (it->first == "bitrate_mode") {
  734. if (it->second == "vbr") {
  735. bitrate_mode = media::Bitrate::Mode::kVariable;
  736. } else if (it->second != "cbr") {
  737. std::cout << "unknown bitrate mode \"" << it->second
  738. << "\", possible values are \"cbr|vbr\"\n";
  739. return EXIT_FAILURE;
  740. }
  741. } else if (it->first == "reverse") {
  742. reverse = true;
  743. } else if (it->first == "bitrate") {
  744. unsigned value;
  745. if (!base::StringToUint(it->second, &value)) {
  746. std::cout << "invalid bitrate " << it->second << "\n"
  747. << media::test::usage_msg;
  748. return EXIT_FAILURE;
  749. }
  750. encode_bitrate = base::checked_cast<uint32_t>(value);
  751. } else if (it->first == "disable_vaapi_lock") {
  752. disabled_features.push_back(media::kGlobalVaapiLock);
  753. } else {
  754. std::cout << "unknown option: --" << it->first << "\n"
  755. << media::test::usage_msg;
  756. return EXIT_FAILURE;
  757. }
  758. }
  759. testing::InitGoogleTest(&argc, argv);
  760. // Set up our test environment.
  761. media::test::VideoEncoderTestEnvironment* test_environment =
  762. media::test::VideoEncoderTestEnvironment::Create(
  763. video_path, video_metadata_path, false, base::FilePath(output_folder),
  764. codec, num_temporal_layers, num_spatial_layers,
  765. false /* output_bitstream */, encode_bitrate, bitrate_mode, reverse,
  766. media::test::FrameOutputConfig(),
  767. /*enabled_features=*/{}, disabled_features);
  768. if (!test_environment)
  769. return EXIT_FAILURE;
  770. media::test::g_env = static_cast<media::test::VideoEncoderTestEnvironment*>(
  771. testing::AddGlobalTestEnvironment(test_environment));
  772. return RUN_ALL_TESTS();
  773. }