123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548 |
- // Copyright 2019 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include <algorithm>
- #include <numeric>
- #include <vector>
- #include "base/command_line.h"
- #include "base/files/file_util.h"
- #include "base/json/json_writer.h"
- #include "base/memory/raw_ptr.h"
- #include "base/time/time.h"
- #include "build/build_config.h"
- #include "media/base/media_switches.h"
- #include "media/base/test_data_util.h"
- #include "media/gpu/buildflags.h"
- #include "media/gpu/test/video.h"
- #include "media/gpu/test/video_player/decoder_listener.h"
- #include "media/gpu/test/video_player/decoder_wrapper.h"
- #include "media/gpu/test/video_player/frame_renderer_dummy.h"
- #include "media/gpu/test/video_player/video_player_test_environment.h"
- #include "sandbox/linux/services/resource_limits.h"
- #include "testing/gtest/include/gtest/gtest.h"
- namespace media {
- namespace test {
- namespace {
- // Video decoder perf tests usage message. Make sure to also update the
- // documentation under docs/media/gpu/video_decoder_perf_test_usage.md when
- // making changes here.
- constexpr const char* usage_msg =
- R"(usage: video_decode_accelerator_perf_tests
- [-v=<level>] [--vmodule=<config>] [--output_folder]
- ([--use-legacy]|[--use_vd_vda]) [--linear_output]
- [--disable_vaapi_lock]
- [--gtest_help] [--help]
- [<video path>] [<video metadata path>]
- )";
- // Video decoder perf tests help message.
- const std::string help_msg =
- std::string(
- R"""(Run the video decode accelerator performance tests on the video
- specified by <video path>. If no <video path> is given the default
- "test-25fps.h264" video will be used.
- The <video metadata path> should specify the location of a json file
- containing the video's metadata, such as frame checksums. By default
- <video path>.json will be used.
- The following arguments are supported:
- -v enable verbose mode, e.g. -v=2.
- --vmodule enable verbose mode for the specified module,
- e.g. --vmodule=*media/gpu*=2.
- --output_folder overwrite the output folder used to store
- performance metrics, if not specified results
- will be stored in the current working directory.
- --use-legacy use the legacy VDA-based video decoders.
- --use_vd_vda use the new VD-based video decoders with a
- wrapper that translates to the VDA interface,
- used to test interaction with older components
- expecting the VDA interface.
- --linear_output use linear buffers as the final output of the
- decoder which may require the use of an image
- processor internally. This flag only works in
- conjunction with --use_vd_vda.
- Disabled by default.
- --disable_vaapi_lock disable the global VA-API lock if applicable,
- i.e., only on devices that use the VA-API with a libva
- backend that's known to be thread-safe and only in
- portions of the Chrome stack that should be able to
- deal with the absence of the lock
- (not the VaapiVideoDecodeAccelerator).)""") +
- #if defined(ARCH_CPU_ARM_FAMILY)
- R"""(
- --disable-libyuv use hw format conversion instead of libYUV.
- libYUV will be used by default, unless the
- video decoder format is not supported;
- in that case the code will try to use the
- v4l2 image processor.)""" +
- #endif // defined(ARCH_CPU_ARM_FAMILY)
- R"""(
- --gtest_help display the gtest help and exit.
- --help display this help and exit.
- )""";
- media::test::VideoPlayerTestEnvironment* g_env;
- // Default output folder used to store performance metrics.
- constexpr const base::FilePath::CharType* kDefaultOutputFolder =
- FILE_PATH_LITERAL("perf_metrics");
- constexpr base::TimeDelta kMultipleDecodersTimeout = base::Seconds(120);
- // Struct storing various time-related statistics.
- struct PerformanceTimeStats {
- PerformanceTimeStats() {}
- explicit PerformanceTimeStats(const std::vector<double>& times);
- double avg_ms_ = 0.0;
- double percentile_25_ms_ = 0.0;
- double percentile_50_ms_ = 0.0;
- double percentile_75_ms_ = 0.0;
- };
- PerformanceTimeStats::PerformanceTimeStats(const std::vector<double>& times) {
- avg_ms_ = std::accumulate(times.begin(), times.end(), 0.0) / times.size();
- std::vector<double> sorted_times = times;
- std::sort(sorted_times.begin(), sorted_times.end());
- percentile_25_ms_ = sorted_times[sorted_times.size() / 4];
- percentile_50_ms_ = sorted_times[sorted_times.size() / 2];
- percentile_75_ms_ = sorted_times[(sorted_times.size() * 3) / 4];
- }
- struct PerformanceMetrics {
- // Total measurement duration.
- base::TimeDelta total_duration_;
- // The number of frames decoded.
- size_t frames_decoded_ = 0;
- // The overall number of frames decoded per second.
- double frames_per_second_ = 0.0;
- // The number of frames dropped because of the decoder running behind, only
- // relevant for capped performance tests.
- size_t frames_dropped_ = 0;
- // The percentage of frames dropped because of the decoder running behind,
- // only relevant for capped performance tests.
- double dropped_frame_percentage_ = 0.0;
- // Statistics about the time between subsequent frame deliveries.
- PerformanceTimeStats delivery_time_stats_;
- // Statistics about the time between decode start and frame deliveries.
- PerformanceTimeStats decode_time_stats_;
- };
- // The performance evaluator can be plugged into the video player to collect
- // various performance metrics.
- // TODO(dstaessens@) Check and post warning when CPU frequency scaling is
- // enabled as this affects test results.
- class PerformanceEvaluator : public VideoFrameProcessor {
- public:
- // Create a new performance evaluator. The caller should makes sure
- // |frame_renderer| outlives the performance evaluator.
- explicit PerformanceEvaluator(const FrameRendererDummy* const frame_renderer)
- : frame_renderer_(frame_renderer) {}
- // Interface VideoFrameProcessor
- void ProcessVideoFrame(scoped_refptr<const VideoFrame> video_frame,
- size_t frame_index) override;
- bool WaitUntilDone() override { return true; }
- // Start/Stop collecting performance metrics.
- void StartMeasuring();
- void StopMeasuring();
- // Write the collected performance metrics to file.
- void WriteMetricsToFile() const;
- private:
- // Start/end time of the measurement period.
- base::TimeTicks start_time_;
- base::TimeTicks end_time_;
- // Time at which the previous frame was delivered.
- base::TimeTicks prev_frame_delivery_time_;
- // List of times between subsequent frame deliveries.
- std::vector<double> frame_delivery_times_;
- // List of times between decode start and frame delivery.
- std::vector<double> frame_decode_times_;
- // Collection of various performance metrics.
- PerformanceMetrics perf_metrics_;
- // Frame renderer used to get the dropped frame rate, owned by the creator of
- // the performance evaluator.
- const raw_ptr<const FrameRendererDummy> frame_renderer_;
- };
- void PerformanceEvaluator::ProcessVideoFrame(
- scoped_refptr<const VideoFrame> video_frame,
- size_t frame_index) {
- base::TimeTicks now = base::TimeTicks::Now();
- base::TimeDelta delivery_time = (now - prev_frame_delivery_time_);
- frame_delivery_times_.push_back(delivery_time.InMillisecondsF());
- prev_frame_delivery_time_ = now;
- base::TimeDelta decode_time = now.since_origin() - video_frame->timestamp();
- frame_decode_times_.push_back(decode_time.InMillisecondsF());
- perf_metrics_.frames_decoded_++;
- }
- void PerformanceEvaluator::StartMeasuring() {
- start_time_ = base::TimeTicks::Now();
- prev_frame_delivery_time_ = start_time_;
- }
- void PerformanceEvaluator::StopMeasuring() {
- end_time_ = base::TimeTicks::Now();
- perf_metrics_.total_duration_ = end_time_ - start_time_;
- perf_metrics_.frames_per_second_ = perf_metrics_.frames_decoded_ /
- perf_metrics_.total_duration_.InSecondsF();
- perf_metrics_.frames_dropped_ = frame_renderer_->FramesDropped();
- // Calculate the dropped frame percentage.
- perf_metrics_.dropped_frame_percentage_ =
- static_cast<double>(perf_metrics_.frames_dropped_) /
- static_cast<double>(
- std::max<size_t>(perf_metrics_.frames_decoded_, 1ul)) *
- 100.0;
- // Calculate delivery and decode time metrics.
- perf_metrics_.delivery_time_stats_ =
- PerformanceTimeStats(frame_delivery_times_);
- perf_metrics_.decode_time_stats_ = PerformanceTimeStats(frame_decode_times_);
- std::cout << "Frames decoded: " << perf_metrics_.frames_decoded_
- << std::endl;
- std::cout << "Total duration: "
- << perf_metrics_.total_duration_.InMillisecondsF() << "ms"
- << std::endl;
- std::cout << "FPS: " << perf_metrics_.frames_per_second_
- << std::endl;
- std::cout << "Frames Dropped: " << perf_metrics_.frames_dropped_
- << std::endl;
- std::cout << "Dropped frame percentage: "
- << perf_metrics_.dropped_frame_percentage_ << "%" << std::endl;
- std::cout << "Frame delivery time - average: "
- << perf_metrics_.delivery_time_stats_.avg_ms_ << "ms" << std::endl;
- std::cout << "Frame delivery time - percentile 25: "
- << perf_metrics_.delivery_time_stats_.percentile_25_ms_ << "ms"
- << std::endl;
- std::cout << "Frame delivery time - percentile 50: "
- << perf_metrics_.delivery_time_stats_.percentile_50_ms_ << "ms"
- << std::endl;
- std::cout << "Frame delivery time - percentile 75: "
- << perf_metrics_.delivery_time_stats_.percentile_75_ms_ << "ms"
- << std::endl;
- std::cout << "Frame decode time - average: "
- << perf_metrics_.decode_time_stats_.avg_ms_ << "ms" << std::endl;
- std::cout << "Frame decode time - percentile 25: "
- << perf_metrics_.decode_time_stats_.percentile_25_ms_ << "ms"
- << std::endl;
- std::cout << "Frame decode time - percentile 50: "
- << perf_metrics_.decode_time_stats_.percentile_50_ms_ << "ms"
- << std::endl;
- std::cout << "Frame decode time - percentile 75: "
- << perf_metrics_.decode_time_stats_.percentile_75_ms_ << "ms"
- << std::endl;
- }
- void PerformanceEvaluator::WriteMetricsToFile() const {
- base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
- if (!DirectoryExists(output_folder_path))
- base::CreateDirectory(output_folder_path);
- output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);
- // Write performance metrics to json.
- base::Value metrics(base::Value::Type::DICTIONARY);
- metrics.SetKey(
- "FramesDecoded",
- base::Value(base::checked_cast<int>(perf_metrics_.frames_decoded_)));
- metrics.SetKey("TotalDurationMs",
- base::Value(perf_metrics_.total_duration_.InMillisecondsF()));
- metrics.SetKey("FPS", base::Value(perf_metrics_.frames_per_second_));
- metrics.SetKey(
- "FramesDropped",
- base::Value(base::checked_cast<int>(perf_metrics_.frames_dropped_)));
- metrics.SetKey("DroppedFramePercentage",
- base::Value(perf_metrics_.dropped_frame_percentage_));
- metrics.SetKey("FrameDeliveryTimeAverage",
- base::Value(perf_metrics_.delivery_time_stats_.avg_ms_));
- metrics.SetKey(
- "FrameDeliveryTimePercentile25",
- base::Value(perf_metrics_.delivery_time_stats_.percentile_25_ms_));
- metrics.SetKey(
- "FrameDeliveryTimePercentile50",
- base::Value(perf_metrics_.delivery_time_stats_.percentile_50_ms_));
- metrics.SetKey(
- "FrameDeliveryTimePercentile75",
- base::Value(perf_metrics_.delivery_time_stats_.percentile_75_ms_));
- metrics.SetKey("FrameDecodeTimeAverage",
- base::Value(perf_metrics_.decode_time_stats_.avg_ms_));
- metrics.SetKey(
- "FrameDecodeTimePercentile25",
- base::Value(perf_metrics_.decode_time_stats_.percentile_25_ms_));
- metrics.SetKey(
- "FrameDecodeTimePercentile50",
- base::Value(perf_metrics_.decode_time_stats_.percentile_50_ms_));
- metrics.SetKey(
- "FrameDecodeTimePercentile75",
- base::Value(perf_metrics_.decode_time_stats_.percentile_75_ms_));
- // Write frame delivery times to json.
- base::Value delivery_times(base::Value::Type::LIST);
- for (double frame_delivery_time : frame_delivery_times_) {
- delivery_times.Append(frame_delivery_time);
- }
- metrics.SetKey("FrameDeliveryTimes", std::move(delivery_times));
- // Write frame decodes times to json.
- base::Value decode_times(base::Value::Type::LIST);
- for (double frame_decode_time : frame_decode_times_) {
- decode_times.Append(frame_decode_time);
- }
- metrics.SetKey("FrameDecodeTimes", std::move(decode_times));
- // Write json to file.
- std::string metrics_str;
- ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
- metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
- base::FilePath metrics_file_path = output_folder_path.Append(
- g_env->GetTestOutputFilePath().AddExtension(FILE_PATH_LITERAL(".json")));
- // Make sure that the directory into which json is saved is created.
- LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
- base::File metrics_output_file(
- base::FilePath(metrics_file_path),
- base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
- int bytes_written = metrics_output_file.WriteAtCurrentPos(
- metrics_str.data(), metrics_str.length());
- ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
- VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
- }
- // Video decode test class. Performs setup and teardown for each single test.
- class VideoDecoderTest : public ::testing::Test {
- public:
- // Create a new video player instance. |render_frame_rate| is the rate at
- // which the video player will simulate rendering frames, if 0 no rendering is
- // simulated. The |vsync_rate| is used during simulated rendering, if 0 Vsync
- // is disabled.
- std::unique_ptr<DecoderListener> CreateDecoderListener(
- const Video* video,
- uint32_t render_frame_rate = 0,
- uint32_t vsync_rate = 0) {
- LOG_ASSERT(video);
- // Create dummy frame renderer, simulates rendering at specified frame rate.
- base::TimeDelta frame_duration;
- base::TimeDelta vsync_interval_duration;
- if (render_frame_rate > 0) {
- frame_duration = base::Seconds(1) / render_frame_rate;
- vsync_interval_duration = base::Seconds(1) / vsync_rate;
- }
- auto frame_renderer =
- FrameRendererDummy::Create(frame_duration, vsync_interval_duration);
- std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors;
- auto performance_evaluator =
- std::make_unique<PerformanceEvaluator>(frame_renderer.get());
- performance_evaluator_ = performance_evaluator.get();
- frame_processors.push_back(std::move(performance_evaluator));
- // Use the new VD-based video decoders if requested.
- DecoderWrapperConfig config;
- config.implementation = g_env->GetDecoderImplementation();
- config.linear_output = g_env->ShouldOutputLinearBuffers();
- auto video_player = DecoderListener::Create(
- config, std::move(frame_renderer), std::move(frame_processors));
- LOG_ASSERT(video_player);
- LOG_ASSERT(video_player->Initialize(video));
- // Make sure the event timeout is at least as long as the video's duration.
- video_player->SetEventWaitTimeout(
- std::max(kDefaultEventWaitTimeout, g_env->Video()->GetDuration()));
- return video_player;
- }
- raw_ptr<PerformanceEvaluator> performance_evaluator_;
- };
- } // namespace
- // Play video from start to end while measuring uncapped performance. This test
- // will decode a video as fast as possible, and gives an idea about the maximum
- // output of the decoder.
- TEST_F(VideoDecoderTest, MeasureUncappedPerformance) {
- auto tvp = CreateDecoderListener(g_env->Video());
- performance_evaluator_->StartMeasuring();
- tvp->Play();
- EXPECT_TRUE(tvp->WaitForFlushDone());
- performance_evaluator_->StopMeasuring();
- performance_evaluator_->WriteMetricsToFile();
- EXPECT_EQ(tvp->GetFlushDoneCount(), 1u);
- EXPECT_EQ(tvp->GetFrameDecodedCount(), g_env->Video()->NumFrames());
- }
- // Play video from start to end while measuring capped performance. This test
- // will simulate rendering the video at its actual frame rate, and will
- // calculate the number of frames that were dropped. Vsync is enabled at 60 FPS.
- TEST_F(VideoDecoderTest, MeasureCappedPerformance) {
- auto tvp =
- CreateDecoderListener(g_env->Video(), g_env->Video()->FrameRate(), 60);
- performance_evaluator_->StartMeasuring();
- tvp->Play();
- EXPECT_TRUE(tvp->WaitForFlushDone());
- tvp->WaitForRenderer();
- performance_evaluator_->StopMeasuring();
- performance_evaluator_->WriteMetricsToFile();
- EXPECT_EQ(tvp->GetFlushDoneCount(), 1u);
- EXPECT_EQ(tvp->GetFrameDecodedCount(), g_env->Video()->NumFrames());
- }
- // TODO(b/211783279) The |performance_evaluator_| only keeps track of the last
- // created decoder. We should instead keep track of multiple evaluators, and
- // then decide how to aggregate/report those metrics.
- // Play multiple videos simultaneously from start to finish.
- TEST_F(VideoDecoderTest, MeasureUncappedPerformance_TenConcurrentDecoders) {
- // Set RLIMIT_NOFILE soft limit to its hard limit value.
- if (sandbox::ResourceLimits::AdjustCurrent(
- RLIMIT_NOFILE, std::numeric_limits<long long int>::max())) {
- DPLOG(ERROR) << "Unable to increase soft limit of RLIMIT_NOFILE";
- }
- constexpr size_t kNumConcurrentDecoders = 10;
- std::vector<std::unique_ptr<DecoderListener>> players(kNumConcurrentDecoders);
- for (auto&& player : players) {
- player = CreateDecoderListener(g_env->Video());
- // Increase the timeout for older machines that cannot decode as
- // efficiently.
- player->SetEventWaitTimeout(kMultipleDecodersTimeout);
- }
- performance_evaluator_->StartMeasuring();
- for (auto&& player : players)
- player->Play();
- for (auto&& player : players) {
- EXPECT_TRUE(player->WaitForFlushDone());
- EXPECT_EQ(player->GetFlushDoneCount(), 1u);
- EXPECT_EQ(player->GetFrameDecodedCount(), g_env->Video()->NumFrames());
- }
- performance_evaluator_->StopMeasuring();
- performance_evaluator_->WriteMetricsToFile();
- }
- } // namespace test
- } // namespace media
- int main(int argc, char** argv) {
- // Set the default test data path.
- media::test::Video::SetTestDataPath(media::GetTestDataPath());
- // Print the help message if requested. This needs to be done before
- // initializing gtest, to overwrite the default gtest help message.
- base::CommandLine::Init(argc, argv);
- base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
- LOG_ASSERT(cmd_line);
- if (cmd_line->HasSwitch("help")) {
- std::cout << media::test::usage_msg << "\n" << media::test::help_msg;
- return 0;
- }
- // Check if a video was specified on the command line.
- base::CommandLine::StringVector args = cmd_line->GetArgs();
- base::FilePath video_path =
- (args.size() >= 1) ? base::FilePath(args[0]) : base::FilePath();
- base::FilePath video_metadata_path =
- (args.size() >= 2) ? base::FilePath(args[1]) : base::FilePath();
- // Parse command line arguments.
- base::FilePath::StringType output_folder = media::test::kDefaultOutputFolder;
- bool use_legacy = false;
- bool use_vd_vda = false;
- bool linear_output = false;
- std::vector<base::Feature> disabled_features;
- std::vector<base::Feature> enabled_features;
- #if defined(ARCH_CPU_ARM_FAMILY)
- enabled_features.push_back(media::kPreferLibYuvImageProcessor);
- #endif // defined(ARCH_CPU_ARM_FAMILY)
- media::test::DecoderImplementation implementation =
- media::test::DecoderImplementation::kVD;
- base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
- for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
- it != switches.end(); ++it) {
- if (it->first.find("gtest_") == 0 || // Handled by GoogleTest
- it->first == "v" || it->first == "vmodule") { // Handled by Chrome
- continue;
- }
- if (it->first == "output_folder") {
- output_folder = it->second;
- } else if (it->first == "use-legacy") {
- use_legacy = true;
- implementation = media::test::DecoderImplementation::kVDA;
- } else if (it->first == "use_vd_vda") {
- use_vd_vda = true;
- implementation = media::test::DecoderImplementation::kVDVDA;
- } else if (it->first == "linear_output") {
- linear_output = true;
- } else if (it->first == "disable_vaapi_lock") {
- disabled_features.push_back(media::kGlobalVaapiLock);
- #if defined(ARCH_CPU_ARM_FAMILY)
- } else if (it->first == "disable-libyuv") {
- enabled_features.clear();
- #endif // defined(ARCH_CPU_ARM_FAMILY)
- } else {
- std::cout << "unknown option: --" << it->first << "\n"
- << media::test::usage_msg;
- return EXIT_FAILURE;
- }
- }
- if (use_legacy && use_vd_vda) {
- std::cout << "--use-legacy and --use_vd_vda cannot be enabled together.\n"
- << media::test::usage_msg;
- return EXIT_FAILURE;
- }
- if (linear_output && !use_vd_vda) {
- std::cout << "--linear_output must be used with the VDVDA (--use_vd_vda)\n"
- "implementation.\n"
- << media::test::usage_msg;
- return EXIT_FAILURE;
- }
- testing::InitGoogleTest(&argc, argv);
- // Add the command line flag for HEVC testing which will be checked by the
- // video decoder to allow clear HEVC decoding.
- cmd_line->AppendSwitch("enable-clear-hevc-for-testing");
- // Set up our test environment.
- media::test::VideoPlayerTestEnvironment* test_environment =
- media::test::VideoPlayerTestEnvironment::Create(
- video_path, video_metadata_path, /*validator_type=*/
- media::test::VideoPlayerTestEnvironment::ValidatorType::kNone,
- implementation, linear_output, base::FilePath(output_folder),
- media::test::FrameOutputConfig(), enabled_features,
- disabled_features);
- if (!test_environment)
- return EXIT_FAILURE;
- media::test::g_env = static_cast<media::test::VideoPlayerTestEnvironment*>(
- testing::AddGlobalTestEnvironment(test_environment));
- return RUN_ALL_TESTS();
- }
|