123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875 |
- // Copyright 2020 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include <algorithm>
- #include <iomanip>
- #include <iostream>
- #include <map>
- #include <numeric>
- #include <vector>
- #include "base/command_line.h"
- #include "base/files/file_util.h"
- #include "base/json/json_writer.h"
- #include "base/memory/raw_ptr.h"
- #include "base/strings/string_number_conversions.h"
- #include "base/time/time.h"
- #include "media/base/bitstream_buffer.h"
- #include "media/base/media_switches.h"
- #include "media/base/media_util.h"
- #include "media/base/test_data_util.h"
- #include "media/base/video_decoder_config.h"
- #include "media/gpu/test/video.h"
- #include "media/gpu/test/video_encoder/bitstream_validator.h"
- #include "media/gpu/test/video_encoder/video_encoder.h"
- #include "media/gpu/test/video_encoder/video_encoder_client.h"
- #include "media/gpu/test/video_encoder/video_encoder_test_environment.h"
- #include "media/gpu/test/video_frame_validator.h"
- #include "media/gpu/test/video_test_helpers.h"
- #include "testing/gtest/include/gtest/gtest.h"
- #include "third_party/abseil-cpp/absl/types/optional.h"
- namespace media {
- namespace test {
- namespace {
- // Video encoder perf tests usage message. Make sure to also update the
- // documentation under docs/media/gpu/video_encoder_perf_test_usage.md when
- // making changes here.
- // TODO(b/211783271): Add video_encoder_perf_test_usage.md
- constexpr const char* usage_msg =
- R"(usage: video_encode_accelerator_perf_tests
- [--codec=<codec>] [--num_spatial_layers=<number>]
- [--num_temporal_layers=<number>] [--bitrate_mode=(cbr|vbr)]
- [--reverse] [--bitrate=<bitrate>]
- [-v=<level>] [--vmodule=<config>] [--output_folder]
- [--disable_vaapi_lock]
- [--gtest_help] [--help]
- [<video path>] [<video metadata path>]
- )";
- // Video encoder performance tests help message.
- constexpr const char* help_msg =
- R"""(Run the video encode accelerator performance tests on the video
- specified by <video path>. If no <video path> is given the default
- "bear_320x192_40frames.yuv.webm" video will be used.
- The <video metadata path> should specify the location of a json file
- containing the video's metadata. By default <video path>.json will be
- used.
- The following arguments are supported:
- -v enable verbose mode, e.g. -v=2.
- --vmodule enable verbose mode for the specified module,
- --codec codec profile to encode, "h264 (baseline)",
- "h264main, "h264high", "vp8" and "vp9"
- --num_spatial_layers the number of spatial layers of the encoded
- bitstream. A default value is 1. Only affected
- if --codec=vp9 currently.
- --num_temporal_layers the number of temporal layers of the encoded
- bitstream. A default value is 1.
- --bitrate_mode The rate control mode for encoding, one of "cbr"
- (default) or "vbr".
- --reverse the stream plays backwards if the stream reaches
- end of stream. So the input stream to be encoded
- is consecutive. By default this is false.
- --bitrate bitrate (bits in second) of a produced bitstram.
- If not specified, a proper value for the video
- resolution is selected by the test.
- --output_folder overwrite the output folder used to store
- performance metrics, if not specified results
- will be stored in the current working directory.
- --disable_vaapi_lock disable the global VA-API lock if applicable,
- i.e., only on devices that use the VA-API with a libva
- backend that's known to be thread-safe and only in
- portions of the Chrome stack that should be able to
- deal with the absence of the lock
- (not the VaapiVideoDecodeAccelerator).
- --gtest_help display the gtest help and exit.
- --help display this help and exit.
- )""";
- // Default video to be used if no test video was specified.
- constexpr base::FilePath::CharType kDefaultTestVideoPath[] =
- FILE_PATH_LITERAL("bear_320x192_40frames.yuv.webm");
- media::test::VideoEncoderTestEnvironment* g_env;
- constexpr size_t kNumFramesToEncodeForPerformance = 300;
- constexpr size_t kMaxTemporalLayers = 3;
- constexpr size_t kMaxSpatialLayers = 3;
- // The event timeout used in perf tests because encoding 2160p
- // |kNumFramesToEncodeForPerformance| frames take much time.
- constexpr base::TimeDelta kPerfEventTimeout = base::Seconds(180);
- // Default output folder used to store performance metrics.
- constexpr const base::FilePath::CharType* kDefaultOutputFolder =
- FILE_PATH_LITERAL("perf_metrics");
- // Struct storing various time-related statistics.
- struct PerformanceTimeStats {
- PerformanceTimeStats() {}
- explicit PerformanceTimeStats(const std::vector<double>& times);
- double avg_ms_ = 0.0;
- double percentile_25_ms_ = 0.0;
- double percentile_50_ms_ = 0.0;
- double percentile_75_ms_ = 0.0;
- };
- PerformanceTimeStats::PerformanceTimeStats(const std::vector<double>& times) {
- if (times.empty())
- return;
- avg_ms_ = std::accumulate(times.begin(), times.end(), 0.0) / times.size();
- std::vector<double> sorted_times = times;
- std::sort(sorted_times.begin(), sorted_times.end());
- percentile_25_ms_ = sorted_times[sorted_times.size() / 4];
- percentile_50_ms_ = sorted_times[sorted_times.size() / 2];
- percentile_75_ms_ = sorted_times[(sorted_times.size() * 3) / 4];
- }
- // TODO(dstaessens): Investigate using more appropriate metrics for encoding.
- struct PerformanceMetrics {
- // Write the collected performance metrics to the console.
- void WriteToConsole() const;
- // Write the collected performance metrics to file.
- void WriteToFile() const;
- // Total measurement duration.
- base::TimeDelta total_duration_;
- // The number of bitstreams encoded.
- size_t bitstreams_encoded_ = 0;
- // The overall number of bitstreams encoded per second.
- double bitstreams_per_second_ = 0.0;
- // List of times between subsequent bitstream buffer deliveries. This is
- // important in real-time encoding scenarios, where the delivery time should
- // be less than the frame rate used.
- std::vector<double> bitstream_delivery_times_;
- // Statistics related to the time between bitstream buffer deliveries.
- PerformanceTimeStats bitstream_delivery_stats_;
- // List of times between queuing an encode operation and getting back the
- // encoded bitstream buffer.
- std::vector<double> bitstream_encode_times_;
- // Statistics related to the encode times.
- PerformanceTimeStats bitstream_encode_stats_;
- };
- // The performance evaluator can be plugged into the video encoder to collect
- // various performance metrics.
- class PerformanceEvaluator : public BitstreamProcessor {
- public:
- // Create a new performance evaluator.
- PerformanceEvaluator() {}
- void ProcessBitstream(scoped_refptr<BitstreamRef> bitstream,
- size_t frame_index) override;
- bool WaitUntilDone() override { return true; }
- // Start/Stop collecting performance metrics.
- void StartMeasuring();
- void StopMeasuring();
- // Get the collected performance metrics.
- const PerformanceMetrics& Metrics() const { return perf_metrics_; }
- private:
- // Start/end time of the measurement period.
- base::TimeTicks start_time_;
- base::TimeTicks end_time_;
- // Time at which the previous bitstream was delivered.
- base::TimeTicks prev_bitstream_delivery_time_;
- // Collection of various performance metrics.
- PerformanceMetrics perf_metrics_;
- };
- void PerformanceEvaluator::ProcessBitstream(
- scoped_refptr<BitstreamRef> bitstream,
- size_t frame_index) {
- base::TimeTicks now = base::TimeTicks::Now();
- base::TimeDelta delivery_time = (now - prev_bitstream_delivery_time_);
- perf_metrics_.bitstream_delivery_times_.push_back(
- delivery_time.InMillisecondsF());
- prev_bitstream_delivery_time_ = now;
- // TODO(hiroh): |encode_time| on upper spatial layer in SVC encoding becomes
- // larger because the bitstram is produced after lower spatial layers are
- // produced. |encode_time| should be aggregated per spatial layer.
- base::TimeDelta encode_time =
- base::TimeTicks::Now() - bitstream->source_timestamp;
- perf_metrics_.bitstream_encode_times_.push_back(
- encode_time.InMillisecondsF());
- }
- void PerformanceEvaluator::StartMeasuring() {
- start_time_ = base::TimeTicks::Now();
- prev_bitstream_delivery_time_ = start_time_;
- }
- void PerformanceEvaluator::StopMeasuring() {
- DCHECK_EQ(perf_metrics_.bitstream_delivery_times_.size(),
- perf_metrics_.bitstream_encode_times_.size());
- end_time_ = base::TimeTicks::Now();
- perf_metrics_.total_duration_ = end_time_ - start_time_;
- perf_metrics_.bitstreams_encoded_ =
- perf_metrics_.bitstream_encode_times_.size();
- perf_metrics_.bitstreams_per_second_ =
- perf_metrics_.bitstreams_encoded_ /
- perf_metrics_.total_duration_.InSecondsF();
- // Calculate delivery and encode time metrics.
- perf_metrics_.bitstream_delivery_stats_ =
- PerformanceTimeStats(perf_metrics_.bitstream_delivery_times_);
- perf_metrics_.bitstream_encode_stats_ =
- PerformanceTimeStats(perf_metrics_.bitstream_encode_times_);
- }
- void PerformanceMetrics::WriteToConsole() const {
- std::cout << "Bitstreams encoded: " << bitstreams_encoded_ << std::endl;
- std::cout << "Total duration: " << total_duration_.InMillisecondsF()
- << "ms" << std::endl;
- std::cout << "FPS: " << bitstreams_per_second_
- << std::endl;
- std::cout << "Bitstream delivery time - average: "
- << bitstream_delivery_stats_.avg_ms_ << "ms" << std::endl;
- std::cout << "Bitstream delivery time - percentile 25: "
- << bitstream_delivery_stats_.percentile_25_ms_ << "ms" << std::endl;
- std::cout << "Bitstream delivery time - percentile 50: "
- << bitstream_delivery_stats_.percentile_50_ms_ << "ms" << std::endl;
- std::cout << "Bitstream delivery time - percentile 75: "
- << bitstream_delivery_stats_.percentile_75_ms_ << "ms" << std::endl;
- std::cout << "Bitstream encode time - average: "
- << bitstream_encode_stats_.avg_ms_ << "ms" << std::endl;
- std::cout << "Bitstream encode time - percentile 25: "
- << bitstream_encode_stats_.percentile_25_ms_ << "ms" << std::endl;
- std::cout << "Bitstream encode time - percentile 50: "
- << bitstream_encode_stats_.percentile_50_ms_ << "ms" << std::endl;
- std::cout << "Bitstream encode time - percentile 75: "
- << bitstream_encode_stats_.percentile_75_ms_ << "ms" << std::endl;
- }
- void PerformanceMetrics::WriteToFile() const {
- base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
- if (!DirectoryExists(output_folder_path))
- base::CreateDirectory(output_folder_path);
- output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);
- // Write performance metrics to json.
- base::Value metrics(base::Value::Type::DICTIONARY);
- metrics.SetKey("BitstreamsEncoded",
- base::Value(base::checked_cast<int>(bitstreams_encoded_)));
- metrics.SetKey("TotalDurationMs",
- base::Value(total_duration_.InMillisecondsF()));
- metrics.SetKey("FPS", base::Value(bitstreams_per_second_));
- metrics.SetKey("BitstreamDeliveryTimeAverage",
- base::Value(bitstream_delivery_stats_.avg_ms_));
- metrics.SetKey("BitstreamDeliveryTimePercentile25",
- base::Value(bitstream_delivery_stats_.percentile_25_ms_));
- metrics.SetKey("BitstreamDeliveryTimePercentile50",
- base::Value(bitstream_delivery_stats_.percentile_50_ms_));
- metrics.SetKey("BitstreamDeliveryTimePercentile75",
- base::Value(bitstream_delivery_stats_.percentile_75_ms_));
- metrics.SetKey("BitstreamEncodeTimeAverage",
- base::Value(bitstream_encode_stats_.avg_ms_));
- metrics.SetKey("BitstreamEncodeTimePercentile25",
- base::Value(bitstream_encode_stats_.percentile_25_ms_));
- metrics.SetKey("BitstreamEncodeTimePercentile50",
- base::Value(bitstream_encode_stats_.percentile_50_ms_));
- metrics.SetKey("BitstreamEncodeTimePercentile75",
- base::Value(bitstream_encode_stats_.percentile_75_ms_));
- // Write bitstream delivery times to json.
- base::Value delivery_times(base::Value::Type::LIST);
- for (double bitstream_delivery_time : bitstream_delivery_times_) {
- delivery_times.Append(bitstream_delivery_time);
- }
- metrics.SetKey("BitstreamDeliveryTimes", std::move(delivery_times));
- // Write bitstream encodes times to json.
- base::Value encode_times(base::Value::Type::LIST);
- for (double bitstream_encode_time : bitstream_encode_times_) {
- encode_times.Append(bitstream_encode_time);
- }
- metrics.SetKey("BitstreamEncodeTimes", std::move(encode_times));
- // Write json to file.
- std::string metrics_str;
- ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
- metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
- base::FilePath metrics_file_path = output_folder_path.Append(
- g_env->GetTestOutputFilePath().AddExtension(FILE_PATH_LITERAL(".json")));
- // Make sure that the directory into which json is saved is created.
- LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
- base::File metrics_output_file(
- base::FilePath(metrics_file_path),
- base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
- int bytes_written = metrics_output_file.WriteAtCurrentPos(
- metrics_str.data(), metrics_str.length());
- ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
- VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
- }
- struct BitstreamQualityMetrics {
- BitstreamQualityMetrics(const PSNRVideoFrameValidator* const psnr_validator,
- const SSIMVideoFrameValidator* const ssim_validator,
- const absl::optional<size_t>& spatial_idx,
- const absl::optional<size_t>& temporal_idx);
- void Output(uint32_t target_bitrate, uint32_t actual_bitrate);
- absl::optional<size_t> spatial_idx;
- absl::optional<size_t> temporal_idx;
- private:
- struct QualityStats {
- QualityStats() = default;
- QualityStats(const QualityStats&) = default;
- QualityStats& operator=(const QualityStats&) = default;
- double avg = 0;
- double percentile_25 = 0;
- double percentile_50 = 0;
- double percentile_75 = 0;
- std::vector<double> values_in_order;
- };
- static QualityStats ComputeQualityStats(
- const std::map<size_t, double>& values);
- void WriteToConsole(const std::string& svc_text,
- const BitstreamQualityMetrics::QualityStats& psnr_stats,
- const BitstreamQualityMetrics::QualityStats& ssim_stats,
- uint32_t target_bitrate,
- uint32_t actual_bitrate) const;
- void WriteToFile(const std::string& svc_text,
- const BitstreamQualityMetrics::QualityStats& psnr_stats,
- const BitstreamQualityMetrics::QualityStats& ssim_stats,
- uint32_t target_bitrate,
- uint32_t actual_bitrate) const;
- const PSNRVideoFrameValidator* const psnr_validator;
- const SSIMVideoFrameValidator* const ssim_validator;
- };
- BitstreamQualityMetrics::BitstreamQualityMetrics(
- const PSNRVideoFrameValidator* const psnr_validator,
- const SSIMVideoFrameValidator* const ssim_validator,
- const absl::optional<size_t>& spatial_idx,
- const absl::optional<size_t>& temporal_idx)
- : spatial_idx(spatial_idx),
- temporal_idx(temporal_idx),
- psnr_validator(psnr_validator),
- ssim_validator(ssim_validator) {}
- // static
- BitstreamQualityMetrics::QualityStats
- BitstreamQualityMetrics::ComputeQualityStats(
- const std::map<size_t, double>& values) {
- if (values.empty())
- return QualityStats();
- std::vector<double> sorted_values;
- std::vector<std::pair<size_t, double>> index_and_value;
- sorted_values.reserve(values.size());
- index_and_value.reserve(values.size());
- for (const auto& v : values) {
- sorted_values.push_back(v.second);
- index_and_value.emplace_back(v.first, v.second);
- }
- std::sort(sorted_values.begin(), sorted_values.end());
- std::sort(index_and_value.begin(), index_and_value.end());
- QualityStats stats;
- stats.avg = std::accumulate(sorted_values.begin(), sorted_values.end(), 0.0) /
- sorted_values.size();
- stats.percentile_25 = sorted_values[sorted_values.size() / 4];
- stats.percentile_50 = sorted_values[sorted_values.size() / 2];
- stats.percentile_75 = sorted_values[(sorted_values.size() * 3) / 4];
- stats.values_in_order.resize(index_and_value.size());
- for (size_t i = 0; i < index_and_value.size(); ++i)
- stats.values_in_order[i] = index_and_value[i].second;
- return stats;
- }
- void BitstreamQualityMetrics::Output(uint32_t target_bitrate,
- uint32_t actual_bitrate) {
- std::string svc_text;
- if (spatial_idx)
- svc_text += "L" + base::NumberToString(*spatial_idx + 1);
- if (temporal_idx)
- svc_text += "T" + base::NumberToString(*temporal_idx + 1);
- auto psnr_stats = ComputeQualityStats(psnr_validator->GetPSNRValues());
- auto ssim_stats = ComputeQualityStats(ssim_validator->GetSSIMValues());
- WriteToConsole(svc_text, psnr_stats, ssim_stats, target_bitrate,
- actual_bitrate);
- WriteToFile(svc_text, psnr_stats, ssim_stats, target_bitrate, actual_bitrate);
- }
- void BitstreamQualityMetrics::WriteToConsole(
- const std::string& svc_text,
- const BitstreamQualityMetrics::QualityStats& psnr_stats,
- const BitstreamQualityMetrics::QualityStats& ssim_stats,
- uint32_t target_bitrate,
- uint32_t actual_bitrate) const {
- const auto default_ssize = std::cout.precision();
- std::cout << "[ Result " << svc_text << "]" << std::endl;
- std::cout << "Bitrate: " << actual_bitrate << " (target: " << target_bitrate
- << ")" << std::endl;
- std::cout << "Bitrate deviation: " << std::fixed << std::setprecision(2)
- << (actual_bitrate * 100.0 / target_bitrate) - 100.0 << " %"
- << std::endl;
- std::cout << std::fixed << std::setprecision(4);
- std::cout << "SSIM - average: " << ssim_stats.avg << std::endl;
- std::cout << "SSIM - percentile 25: " << ssim_stats.percentile_25
- << std::endl;
- std::cout << "SSIM - percentile 50: " << ssim_stats.percentile_50
- << std::endl;
- std::cout << "SSIM - percentile 75: " << ssim_stats.percentile_75
- << std::endl;
- std::cout << "PSNR - average: " << psnr_stats.avg << std::endl;
- std::cout << "PSNR - percentile 25: " << psnr_stats.percentile_25
- << std::endl;
- std::cout << "PSNR - percentile 50: " << psnr_stats.percentile_50
- << std::endl;
- std::cout << "PSNR - percentile 75: " << psnr_stats.percentile_75
- << std::endl;
- std::cout.precision(default_ssize);
- }
- void BitstreamQualityMetrics::WriteToFile(
- const std::string& svc_text,
- const BitstreamQualityMetrics::QualityStats& psnr_stats,
- const BitstreamQualityMetrics::QualityStats& ssim_stats,
- uint32_t target_bitrate,
- uint32_t actual_bitrate) const {
- base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
- if (!DirectoryExists(output_folder_path))
- base::CreateDirectory(output_folder_path);
- output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);
- // Write quality metrics to json.
- base::Value metrics(base::Value::Type::DICTIONARY);
- if (!svc_text.empty())
- metrics.SetKey("SVC", base::Value(svc_text));
- metrics.SetKey("Bitrate",
- base::Value(base::checked_cast<int>(actual_bitrate)));
- metrics.SetKey(
- "BitrateDeviation",
- base::Value((actual_bitrate * 100.0 / target_bitrate) - 100.0));
- metrics.SetKey("SSIMAverage", base::Value(ssim_stats.avg));
- metrics.SetKey("PSNRAverage", base::Value(psnr_stats.avg));
- // Write ssim values bitstream delivery times to json.
- base::Value ssim_values(base::Value::Type::LIST);
- for (double value : ssim_stats.values_in_order)
- ssim_values.Append(value);
- metrics.SetKey("SSIMValues", std::move(ssim_values));
- // Write psnr values to json.
- base::Value psnr_values(base::Value::Type::LIST);
- for (double value : psnr_stats.values_in_order)
- psnr_values.Append(value);
- metrics.SetKey("PSNRValues", std::move(psnr_values));
- // Write json to file.
- std::string metrics_str;
- ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
- metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
- base::FilePath metrics_file_path = output_folder_path.Append(
- g_env->GetTestOutputFilePath()
- .AddExtension(svc_text.empty() ? "" : "." + svc_text)
- .AddExtension(FILE_PATH_LITERAL(".json")));
- // Make sure that the directory into which json is saved is created.
- LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
- base::File metrics_output_file(
- base::FilePath(metrics_file_path),
- base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
- int bytes_written = metrics_output_file.WriteAtCurrentPos(
- metrics_str.data(), metrics_str.length());
- ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
- VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
- }
- // Video encode test class. Performs setup and teardown for each single test.
- // It measures the performance in encoding NV12 GpuMemoryBuffer based
- // VideoFrame.
- class VideoEncoderTest : public ::testing::Test {
- public:
- // Creates VideoEncoder for encoding NV12 GpuMemoryBuffer based VideoFrames.
- // The input VideoFrames are provided every 1 / |encoder_rate| seconds if it
- // is specified. Or they are provided as soon as the previous input VideoFrame
- // is consumed by VideoEncoder. |measure_quality| measures SSIM and PSNR
- // values of encoded bitstream comparing the original input VideoFrames.
- std::unique_ptr<VideoEncoder> CreateVideoEncoder(
- absl::optional<uint32_t> encode_rate,
- bool measure_quality) {
- Video* video = g_env->GenerateNV12Video();
- VideoCodecProfile profile = g_env->Profile();
- const media::VideoBitrateAllocation& bitrate = g_env->BitrateAllocation();
- const std::vector<VideoEncodeAccelerator::Config::SpatialLayer>&
- spatial_layers = g_env->SpatialLayers();
- std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
- if (measure_quality) {
- bitstream_processors = CreateBitstreamProcessorsForQualityPerformance(
- video, profile, spatial_layers);
- } else {
- auto performance_evaluator = std::make_unique<PerformanceEvaluator>();
- performance_evaluator_ = performance_evaluator.get();
- bitstream_processors.push_back(std::move(performance_evaluator));
- }
- LOG_ASSERT(!bitstream_processors.empty())
- << "Failed to create bitstream processors";
- VideoEncoderClientConfig config(video, profile, spatial_layers, bitrate,
- g_env->Reverse());
- config.input_storage_type =
- VideoEncodeAccelerator::Config::StorageType::kGpuMemoryBuffer;
- config.num_frames_to_encode = kNumFramesToEncodeForPerformance;
- if (encode_rate) {
- config.encode_interval = base::Seconds(1u) / encode_rate.value();
- }
- auto video_encoder =
- VideoEncoder::Create(config, std::move(bitstream_processors));
- LOG_ASSERT(video_encoder);
- LOG_ASSERT(video_encoder->Initialize(video));
- return video_encoder;
- }
- protected:
- raw_ptr<PerformanceEvaluator> performance_evaluator_;
- std::vector<BitstreamQualityMetrics> quality_metrics_;
- private:
- std::unique_ptr<BitstreamValidator> CreateBitstreamValidator(
- const VideoCodecProfile profile,
- const gfx::Rect& visible_rect,
- const absl::optional<size_t>& spatial_layer_index_to_decode,
- const absl::optional<size_t>& temporal_layer_index_to_decode,
- const std::vector<gfx::Size>& spatial_layer_resolutions) {
- std::vector<std::unique_ptr<VideoFrameProcessor>> video_frame_processors;
- VideoFrameValidator::GetModelFrameCB get_model_frame_cb =
- base::BindRepeating(&VideoEncoderTest::GetModelFrame,
- base::Unretained(this), visible_rect);
- auto ssim_validator = SSIMVideoFrameValidator::Create(
- get_model_frame_cb, /*corrupt_frame_processor=*/nullptr,
- VideoFrameValidator::ValidationMode::kAverage,
- /*tolerance=*/0.0);
- LOG_ASSERT(ssim_validator);
- auto psnr_validator = PSNRVideoFrameValidator::Create(
- get_model_frame_cb, /*corrupt_frame_processor=*/nullptr,
- VideoFrameValidator::ValidationMode::kAverage,
- /*tolerance=*/0.0);
- LOG_ASSERT(psnr_validator);
- quality_metrics_.push_back(BitstreamQualityMetrics(
- psnr_validator.get(), ssim_validator.get(),
- spatial_layer_index_to_decode, temporal_layer_index_to_decode));
- video_frame_processors.push_back(std::move(ssim_validator));
- video_frame_processors.push_back(std::move(psnr_validator));
- VideoDecoderConfig decoder_config(
- VideoCodecProfileToVideoCodec(profile), profile,
- VideoDecoderConfig::AlphaMode::kIsOpaque, VideoColorSpace(),
- kNoTransformation, visible_rect.size(), visible_rect,
- visible_rect.size(), EmptyExtraData(), EncryptionScheme::kUnencrypted);
- return BitstreamValidator::Create(
- decoder_config, kNumFramesToEncodeForPerformance - 1,
- std::move(video_frame_processors), spatial_layer_index_to_decode,
- temporal_layer_index_to_decode, spatial_layer_resolutions);
- }
- // Create bitstream processors for quality performance tests.
- std::vector<std::unique_ptr<BitstreamProcessor>>
- CreateBitstreamProcessorsForQualityPerformance(
- Video* video,
- VideoCodecProfile profile,
- const std::vector<VideoEncodeAccelerator::Config::SpatialLayer>&
- spatial_layers) {
- std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
- raw_data_helper_ = RawDataHelper::Create(video, g_env->Reverse());
- if (!raw_data_helper_) {
- LOG(ERROR) << "Failed to create raw data helper";
- return bitstream_processors;
- }
- if (spatial_layers.empty()) {
- // Simple stream encoding.
- bitstream_processors.push_back(CreateBitstreamValidator(
- profile, gfx::Rect(video->Resolution()),
- /*spatial_layer_index_to_decode=*/absl::nullopt,
- /*temporal_layer_index_to_decode=*/absl::nullopt,
- /*spatial_layer_resolutions=*/{}));
- LOG_ASSERT(!!bitstream_processors.back());
- } else {
- // Temporal/Spatial layer encoding.
- std::vector<gfx::Size> spatial_layer_resolutions;
- for (const auto& sl : spatial_layers)
- spatial_layer_resolutions.emplace_back(sl.width, sl.height);
- for (size_t sid = 0; sid < spatial_layers.size(); ++sid) {
- for (size_t tid = 0; tid < spatial_layers[sid].num_of_temporal_layers;
- ++tid) {
- bitstream_processors.push_back(CreateBitstreamValidator(
- profile, gfx::Rect(spatial_layer_resolutions[sid]), sid, tid,
- spatial_layer_resolutions));
- LOG_ASSERT(!!bitstream_processors.back());
- }
- }
- }
- return bitstream_processors;
- }
- scoped_refptr<const VideoFrame> GetModelFrame(const gfx::Rect& visible_rect,
- size_t frame_index) {
- LOG_ASSERT(raw_data_helper_);
- auto frame = raw_data_helper_->GetFrame(frame_index);
- if (!frame)
- return nullptr;
- if (visible_rect.size() == frame->visible_rect().size())
- return frame;
- return ScaleVideoFrame(frame.get(), visible_rect.size());
- }
- std::unique_ptr<RawDataHelper> raw_data_helper_;
- };
- } // namespace
- // Encode |kNumFramesToEncodeForPerformance| frames while measuring uncapped
- // performance. This test will encode a video as fast as possible, and gives an
- // idea about the maximum output of the encoder.
- TEST_F(VideoEncoderTest, MeasureUncappedPerformance) {
- auto encoder = CreateVideoEncoder(/*encode_rate=*/absl::nullopt,
- /*measure_quality=*/false);
- encoder->SetEventWaitTimeout(kPerfEventTimeout);
- performance_evaluator_->StartMeasuring();
- encoder->Encode();
- EXPECT_TRUE(encoder->WaitForFlushDone());
- performance_evaluator_->StopMeasuring();
- auto metrics = performance_evaluator_->Metrics();
- metrics.WriteToConsole();
- metrics.WriteToFile();
- EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
- EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
- }
- // Encode |kNumFramesToEncodeForPerformance| frames while measuring capped
- // performance. This test will encode a video at a fixed ratio, 30fps.
- // This test can be used to measure the cpu metrics during encoding.
- TEST_F(VideoEncoderTest, MeasureCappedPerformance) {
- const uint32_t kEncodeRate = 30;
- auto encoder = CreateVideoEncoder(/*encode_rate=*/kEncodeRate,
- /*measure_quality=*/false);
- encoder->SetEventWaitTimeout(kPerfEventTimeout);
- performance_evaluator_->StartMeasuring();
- encoder->Encode();
- EXPECT_TRUE(encoder->WaitForFlushDone());
- performance_evaluator_->StopMeasuring();
- auto metrics = performance_evaluator_->Metrics();
- metrics.WriteToConsole();
- metrics.WriteToFile();
- EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
- EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
- }
- TEST_F(VideoEncoderTest, MeasureProducedBitstreamQuality) {
- auto encoder = CreateVideoEncoder(/*encode_rate=*/absl::nullopt,
- /*measure_quality=*/true);
- encoder->SetEventWaitTimeout(kPerfEventTimeout);
- encoder->Encode();
- EXPECT_TRUE(encoder->WaitForFlushDone());
- EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
- EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
- EXPECT_TRUE(encoder->WaitForBitstreamProcessors());
- const VideoEncoderStats stats = encoder->GetStats();
- for (auto& metrics : quality_metrics_) {
- absl::optional<size_t> spatial_idx = metrics.spatial_idx;
- absl::optional<size_t> temporal_idx = metrics.temporal_idx;
- uint32_t target_bitrate = 0;
- uint32_t actual_bitrate = 0;
- if (!spatial_idx && !temporal_idx) {
- target_bitrate = g_env->BitrateAllocation().GetSumBps();
- actual_bitrate = stats.Bitrate();
- } else {
- CHECK(spatial_idx && temporal_idx);
- // Target and actual bitrates in temporal layer encoding are the sum of
- // bitrates of the temporal layers in the spatial layer.
- for (size_t tid = 0; tid <= *temporal_idx; ++tid) {
- target_bitrate +=
- g_env->BitrateAllocation().GetBitrateBps(*spatial_idx, tid);
- actual_bitrate += stats.LayerBitrate(*spatial_idx, tid);
- }
- }
- metrics.Output(target_bitrate, actual_bitrate);
- }
- }
- // TODO(b/211783279) The |performance_evaluator_| only keeps track of the last
- // created encoder. We should instead keep track of multiple evaluators, and
- // then decide how to aggregate/report those metrics.
- TEST_F(VideoEncoderTest,
- MeasureUncappedPerformance_MultipleConcurrentEncoders) {
- // Run two encoders for larger resolutions to avoid creating shared memory
- // buffers during the test on lower end devices.
- constexpr gfx::Size k1080p(1920, 1080);
- const size_t kMinSupportedConcurrentEncoders =
- g_env->Video()->Resolution().GetArea() >= k1080p.GetArea() ? 2 : 3;
- std::vector<std::unique_ptr<VideoEncoder>> encoders(
- kMinSupportedConcurrentEncoders);
- for (size_t i = 0; i < kMinSupportedConcurrentEncoders; ++i) {
- encoders[i] = CreateVideoEncoder(/*encode_rate=*/absl::nullopt,
- /*measure_quality=*/false);
- encoders[i]->SetEventWaitTimeout(kPerfEventTimeout);
- }
- performance_evaluator_->StartMeasuring();
- for (auto&& encoder : encoders)
- encoder->Encode();
- for (auto&& encoder : encoders) {
- EXPECT_TRUE(encoder->WaitForFlushDone());
- EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
- EXPECT_EQ(encoder->GetFrameReleasedCount(),
- kNumFramesToEncodeForPerformance);
- }
- performance_evaluator_->StopMeasuring();
- auto metrics = performance_evaluator_->Metrics();
- metrics.WriteToConsole();
- metrics.WriteToFile();
- }
- } // namespace test
- } // namespace media
- int main(int argc, char** argv) {
- // Set the default test data path.
- media::test::Video::SetTestDataPath(media::GetTestDataPath());
- // Print the help message if requested. This needs to be done before
- // initializing gtest, to overwrite the default gtest help message.
- base::CommandLine::Init(argc, argv);
- const base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
- LOG_ASSERT(cmd_line);
- if (cmd_line->HasSwitch("help")) {
- std::cout << media::test::usage_msg << "\n" << media::test::help_msg;
- return 0;
- }
- // Check if a video was specified on the command line.
- base::CommandLine::StringVector args = cmd_line->GetArgs();
- base::FilePath video_path =
- (args.size() >= 1) ? base::FilePath(args[0])
- : base::FilePath(media::test::kDefaultTestVideoPath);
- base::FilePath video_metadata_path =
- (args.size() >= 2) ? base::FilePath(args[1]) : base::FilePath();
- std::string codec = "h264";
- size_t num_spatial_layers = 1u;
- size_t num_temporal_layers = 1u;
- media::Bitrate::Mode bitrate_mode = media::Bitrate::Mode::kConstant;
- bool reverse = false;
- absl::optional<uint32_t> encode_bitrate;
- std::vector<base::Feature> disabled_features;
- // Parse command line arguments.
- base::FilePath::StringType output_folder = media::test::kDefaultOutputFolder;
- base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
- for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
- it != switches.end(); ++it) {
- if (it->first.find("gtest_") == 0 || // Handled by GoogleTest
- it->first == "v" || it->first == "vmodule") { // Handled by Chrome
- continue;
- }
- if (it->first == "output_folder") {
- output_folder = it->second;
- } else if (it->first == "codec") {
- codec = it->second;
- } else if (it->first == "num_spatial_layers") {
- if (!base::StringToSizeT(it->second, &num_spatial_layers)) {
- std::cout << "invalid number of spatial layers: " << it->second << "\n";
- return EXIT_FAILURE;
- }
- if (num_spatial_layers > media::test::kMaxSpatialLayers) {
- std::cout << "unsupported number of spatial layers: " << it->second
- << "\n";
- return EXIT_FAILURE;
- }
- } else if (it->first == "num_temporal_layers") {
- if (!base::StringToSizeT(it->second, &num_temporal_layers)) {
- std::cout << "invalid number of temporal layers: " << it->second
- << "\n";
- return EXIT_FAILURE;
- }
- if (num_spatial_layers > media::test::kMaxTemporalLayers) {
- std::cout << "unsupported number of temporal layers: " << it->second
- << "\n";
- return EXIT_FAILURE;
- }
- } else if (it->first == "bitrate_mode") {
- if (it->second == "vbr") {
- bitrate_mode = media::Bitrate::Mode::kVariable;
- } else if (it->second != "cbr") {
- std::cout << "unknown bitrate mode \"" << it->second
- << "\", possible values are \"cbr|vbr\"\n";
- return EXIT_FAILURE;
- }
- } else if (it->first == "reverse") {
- reverse = true;
- } else if (it->first == "bitrate") {
- unsigned value;
- if (!base::StringToUint(it->second, &value)) {
- std::cout << "invalid bitrate " << it->second << "\n"
- << media::test::usage_msg;
- return EXIT_FAILURE;
- }
- encode_bitrate = base::checked_cast<uint32_t>(value);
- } else if (it->first == "disable_vaapi_lock") {
- disabled_features.push_back(media::kGlobalVaapiLock);
- } else {
- std::cout << "unknown option: --" << it->first << "\n"
- << media::test::usage_msg;
- return EXIT_FAILURE;
- }
- }
- testing::InitGoogleTest(&argc, argv);
- // Set up our test environment.
- media::test::VideoEncoderTestEnvironment* test_environment =
- media::test::VideoEncoderTestEnvironment::Create(
- video_path, video_metadata_path, false, base::FilePath(output_folder),
- codec, num_temporal_layers, num_spatial_layers,
- false /* output_bitstream */, encode_bitrate, bitrate_mode, reverse,
- media::test::FrameOutputConfig(),
- /*enabled_features=*/{}, disabled_features);
- if (!test_environment)
- return EXIT_FAILURE;
- media::test::g_env = static_cast<media::test::VideoEncoderTestEnvironment*>(
- testing::AddGlobalTestEnvironment(test_environment));
- return RUN_ALL_TESTS();
- }
|