123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457 |
- // Copyright (c) 2019 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "gpu/ipc/service/gpu_watchdog_thread.h"
- #include "base/test/task_environment.h"
- #include "base/power_monitor/power_monitor.h"
- #include "base/power_monitor/power_monitor_source.h"
- #include "base/system/sys_info.h"
- #include "base/task/current_thread.h"
- #include "base/test/power_monitor_test.h"
- #include "base/threading/thread_task_runner_handle.h"
- #include "base/time/time.h"
- #include "build/build_config.h"
- #include "testing/gtest/include/gtest/gtest.h"
- #if BUILDFLAG(IS_WIN)
- #include "base/win/windows_version.h"
- #endif
- #if BUILDFLAG(IS_MAC)
- #include "base/mac/mac_util.h"
- #endif
- namespace gpu {
- namespace {
- // |kExtraGPUJobTimeForTesting| is the extra time the gpu main/test thread
- // spends after GpuWatchdogTimeout. Theoretically, any extra time such as 1 ms
- // should be enough to trigger the watchdog kill. However, it can cause test
- // flakiness when the time is too short.
- constexpr auto kGpuWatchdogTimeoutForTesting = base::Milliseconds(120);
- constexpr auto kExtraGPUJobTimeForTesting = base::Milliseconds(500);
- // For slow machines like Win 7, Mac 10.xx and Android L/M/N.
- [[maybe_unused]] constexpr auto kGpuWatchdogTimeoutForTestingSlow =
- base::Milliseconds(240);
- [[maybe_unused]] constexpr auto kExtraGPUJobTimeForTestingSlow =
- base::Milliseconds(1000);
- // For Fuchsia in which GpuWatchdogTest.GpuInitializationAndRunningTasks test
- // is flaky.
- [[maybe_unused]] constexpr auto kGpuWatchdogTimeoutForTestingSlowest =
- base::Milliseconds(1000);
- [[maybe_unused]] constexpr auto kExtraGPUJobTimeForTestingSlowest =
- base::Milliseconds(4000);
- // On Windows, the gpu watchdog check if the main thread has used the full
- // thread time. We want to detect the case in which the main thread is swapped
- // out by the OS scheduler. The task on windows is simiulated by reading
- // TimeTicks instead of Sleep().
- void SimpleTask(base::TimeDelta duration, base::TimeDelta extra_time) {
- #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC)
- auto start_timetick = base::TimeTicks::Now();
- do {
- } while ((base::TimeTicks::Now() - start_timetick) < duration);
- base::PlatformThread::Sleep(extra_time);
- #else
- base::PlatformThread::Sleep(duration + extra_time);
- #endif
- }
- } // namespace
- class GpuWatchdogTest : public testing::Test {
- public:
- GpuWatchdogTest() {}
- void LongTaskWithReportProgress(base::TimeDelta duration,
- base::TimeDelta report_delta);
- #if BUILDFLAG(IS_ANDROID)
- void LongTaskFromBackgroundToForeground(
- base::TimeDelta duration,
- base::TimeDelta extra_time,
- base::TimeDelta time_to_switch_to_foreground);
- #endif
- // Implements testing::Test
- void SetUp() override;
- protected:
- ~GpuWatchdogTest() override = default;
- base::test::SingleThreadTaskEnvironment task_environment_;
- base::RunLoop run_loop;
- std::unique_ptr<gpu::GpuWatchdogThread> watchdog_thread_;
- base::TimeDelta timeout_ = kGpuWatchdogTimeoutForTesting;
- base::TimeDelta extra_gpu_job_time_ = kExtraGPUJobTimeForTesting;
- base::TimeDelta full_thread_time_on_windows_ = base::TimeDelta();
- };
- class GpuWatchdogPowerTest : public GpuWatchdogTest {
- public:
- GpuWatchdogPowerTest() {}
- void LongTaskOnResume(base::TimeDelta duration,
- base::TimeDelta extra_time,
- base::TimeDelta time_to_power_resume);
- // Implements testing::Test
- void SetUp() override;
- void TearDown() override;
- protected:
- ~GpuWatchdogPowerTest() override = default;
- base::test::ScopedPowerMonitorTestSource power_monitor_source_;
- };
- void GpuWatchdogTest::SetUp() {
- ASSERT_TRUE(base::ThreadTaskRunnerHandle::IsSet());
- ASSERT_TRUE(base::CurrentThread::IsSet());
- enum TimeOutType {
- kNormal,
- kSlow,
- kSlowest,
- };
- TimeOutType timeout_type = kNormal;
- #if BUILDFLAG(IS_WIN)
- // Win7
- if (base::win::GetVersion() < base::win::Version::WIN10) {
- timeout_type = kSlow;
- }
- #elif BUILDFLAG(IS_MAC)
- // Use slow timeout for Mac versions < 11.00 and for MacBookPro model <
- // MacBookPro14,1
- int os_version = base::mac::internal::MacOSVersion();
- if (os_version <= 1100) {
- // Check MacOS version.
- timeout_type = kSlow;
- } else {
- // Check Mac machine model version.
- std::string model_str = base::SysInfo::HardwareModelName();
- size_t found_position = model_str.find("MacBookPro");
- constexpr size_t model_version_pos = 10;
- // A MacBookPro is found.
- if (found_position == 0 && model_str.size() > model_version_pos) {
- // model_ver_str = "MacBookProXX,X", model_ver_str = "XX,X"
- std::string model_ver_str = model_str.substr(model_version_pos);
- int major_model_ver = std::atoi(model_ver_str.c_str());
- // For model version < 14,1
- if (major_model_ver < 14) {
- timeout_type = kSlow;
- }
- }
- }
- #elif BUILDFLAG(IS_ANDROID)
- int32_t major_version = 0;
- int32_t minor_version = 0;
- int32_t bugfix_version = 0;
- base::SysInfo::OperatingSystemVersionNumbers(&major_version, &minor_version,
- &bugfix_version);
- // For Android version < Android Pie (Version 9)
- if (major_version < 9) {
- timeout_type = kSlow;
- }
- #elif BUILDFLAG(IS_FUCHSIA)
- timeout_type = kSlowest;
- #endif
- if (timeout_type == kSlow) {
- timeout_ = kGpuWatchdogTimeoutForTestingSlow;
- extra_gpu_job_time_ = kExtraGPUJobTimeForTestingSlow;
- } else if (timeout_type == kSlowest) {
- timeout_ = kGpuWatchdogTimeoutForTestingSlowest;
- extra_gpu_job_time_ = kExtraGPUJobTimeForTestingSlowest;
- }
- #if BUILDFLAG(IS_WIN)
- full_thread_time_on_windows_ = timeout_ * kMaxCountOfMoreGpuThreadTimeAllowed;
- #endif
- watchdog_thread_ = gpu::GpuWatchdogThread::Create(
- /*start_backgrounded=*/false,
- /*timeout=*/timeout_,
- /*init_factor=*/kInitFactor,
- /*restart_factor=*/kRestartFactor,
- /*test_mode=*/true, /*thread_name=*/"GpuWatchdog");
- }
- void GpuWatchdogPowerTest::SetUp() {
- GpuWatchdogTest::SetUp();
- // Report GPU init complete.
- watchdog_thread_->OnInitComplete();
- }
- void GpuWatchdogPowerTest::TearDown() {
- GpuWatchdogTest::TearDown();
- watchdog_thread_.reset();
- }
- // This task will run for duration_ms milliseconds. It will also call watchdog
- // ReportProgress() every report_delta_ms milliseconds.
- void GpuWatchdogTest::LongTaskWithReportProgress(base::TimeDelta duration,
- base::TimeDelta report_delta) {
- base::TimeTicks start = base::TimeTicks::Now();
- base::TimeTicks end;
- do {
- SimpleTask(report_delta, /*extra_time=*/base::TimeDelta());
- watchdog_thread_->ReportProgress();
- end = base::TimeTicks::Now();
- } while (end - start <= duration);
- }
- #if BUILDFLAG(IS_ANDROID)
- void GpuWatchdogTest::LongTaskFromBackgroundToForeground(
- base::TimeDelta duration,
- base::TimeDelta extra_time,
- base::TimeDelta time_to_switch_to_foreground) {
- // Chrome is running in the background first.
- watchdog_thread_->OnBackgrounded();
- SimpleTask(time_to_switch_to_foreground, /*extra_time=*/base::TimeDelta());
- // Now switch Chrome to the foreground after the specified time
- watchdog_thread_->OnForegrounded();
- SimpleTask(duration, extra_time);
- }
- #endif
- void GpuWatchdogPowerTest::LongTaskOnResume(
- base::TimeDelta duration,
- base::TimeDelta extra_time,
- base::TimeDelta time_to_power_resume) {
- // Stay in power suspension mode first.
- power_monitor_source_.GenerateSuspendEvent();
- SimpleTask(time_to_power_resume, /*extra_time=*/base::TimeDelta());
- // Now wake up on power resume.
- power_monitor_source_.GenerateResumeEvent();
- // Continue the GPU task for the remaining time.
- SimpleTask(duration, extra_time);
- }
- // Normal GPU Initialization.
- TEST_F(GpuWatchdogTest, GpuInitializationComplete) {
- // Assume GPU initialization takes a quarter of WatchdogTimeout.
- auto normal_task_time = timeout_ / 4;
- SimpleTask(normal_task_time, /*extra_time=*/base::TimeDelta());
- watchdog_thread_->OnInitComplete();
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_FALSE(result);
- }
- // GPU Hang In Initialization.
- TEST_F(GpuWatchdogTest, GpuInitializationHang) {
- auto allowed_time =
- timeout_ * (kInitFactor + 1) + full_thread_time_on_windows_;
- // GPU init takes longer than timeout.
- SimpleTask(allowed_time, /*extra_time=*/extra_gpu_job_time_);
- // Gpu hangs. OnInitComplete() is not called
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_TRUE(result);
- // retry on failure.
- }
- // Normal GPU Initialization and Running Task.
- TEST_F(GpuWatchdogTest, GpuInitializationAndRunningTasks) {
- // Assume GPU initialization takes quarter of WatchdogTimeout time.
- auto normal_task_time = timeout_ / 4;
- SimpleTask(normal_task_time, /*extra_time=*/base::TimeDelta());
- watchdog_thread_->OnInitComplete();
- // Start running GPU tasks. Watchdog function WillProcessTask(),
- // DidProcessTask() and ReportProgress() are tested.
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE, base::BindOnce(&SimpleTask, normal_task_time,
- /*extra_time=*/base::TimeDelta()));
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE, base::BindOnce(&SimpleTask, normal_task_time,
- /*extra_time=*/base::TimeDelta()));
- // This long task takes 6X timeout to finish, longer than timeout. But it
- // reports progress every quarter of watchdog |timeout_|, so this is an
- // expected normal behavior.
- auto normal_long_task_time = timeout_ * 6;
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE, base::BindOnce(&GpuWatchdogTest::LongTaskWithReportProgress,
- base::Unretained(this), normal_long_task_time,
- /*report_progress_time*/ timeout_ / 4));
- task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
- run_loop.QuitClosure());
- run_loop.Run();
- // Everything should be fine. No GPU hang detected.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_FALSE(result);
- }
- // GPU Hang when running a task.
- TEST_F(GpuWatchdogTest, GpuRunningATaskHang) {
- // Report gpu init complete
- watchdog_thread_->OnInitComplete();
- // Start running a GPU task.
- auto allowed_time = timeout_ * 2 + full_thread_time_on_windows_;
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE,
- base::BindOnce(&SimpleTask, allowed_time, extra_gpu_job_time_));
- task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
- run_loop.QuitClosure());
- run_loop.Run();
- // This GPU task takes too long. A GPU hang should be detected.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_TRUE(result);
- }
- #if BUILDFLAG(IS_ANDROID)
- TEST_F(GpuWatchdogTest, ChromeInBackground) {
- // Chrome starts in the background.
- watchdog_thread_->OnBackgrounded();
- // Gpu init takes longer than 6x watchdog |timeout_|. This is normal since
- // Chrome is running in the background.
- auto normal_long_task_time = timeout_ * 6;
- SimpleTask(normal_long_task_time, /*extra_time=*/base::TimeDelta());
- // Report GPU init complete.
- watchdog_thread_->OnInitComplete();
- // Run a task that takes 6x watchdog |timeout_| longer.This is normal since
- // Chrome is running in the background.
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE, base::BindOnce(&SimpleTask, normal_long_task_time,
- /*extra_time=*/base::TimeDelta()));
- task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
- run_loop.QuitClosure());
- run_loop.Run();
- // The gpu might be slow when running in the background. This is ok.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_FALSE(result);
- }
- TEST_F(GpuWatchdogTest, GpuSwitchingToForegroundHang) {
- // Report GPU init complete.
- watchdog_thread_->OnInitComplete();
- // A task stays in the background for watchdog |timeout_| then switches to the
- // foreground and runs longer than the first-time foreground watchdog timeout
- // allowed.
- auto allowed_time = timeout_ * (kRestartFactor + 1);
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE,
- base::BindOnce(&GpuWatchdogTest::LongTaskFromBackgroundToForeground,
- base::Unretained(this), /*duration*/ allowed_time,
- /*extra_time=*/extra_gpu_job_time_,
- /*time_to_switch_to_foreground*/ timeout_ / 4));
- task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
- run_loop.QuitClosure());
- run_loop.Run();
- // It takes too long to finish a task after switching to the foreground.
- // A GPU hang should be detected.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_TRUE(result);
- }
- #endif
- TEST_F(GpuWatchdogTest, GpuInitializationPause) {
- // Running for watchdog |timeout_|/4 in the beginning of GPU init.
- SimpleTask(timeout_ / 4,
- /*extra_time=*/base::TimeDelta());
- watchdog_thread_->PauseWatchdog();
- // The Gpu init continues for another 6x watchdog |timeout_| after the pause.
- // This is normal since watchdog is paused.
- auto normal_long_task_time = timeout_ * 6;
- SimpleTask(normal_long_task_time, /*extra_time=*/base::TimeDelta());
- // No GPU hang is detected when the watchdog is paused.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_FALSE(result);
- // Continue the watchdog now.
- watchdog_thread_->ResumeWatchdog();
- // The Gpu init continues for longer than allowed init time.
- auto allowed_time =
- timeout_ * (kInitFactor + 1) + full_thread_time_on_windows_;
- SimpleTask(allowed_time, /*extra_time=*/extra_gpu_job_time_);
- // A GPU hang should be detected.
- result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_TRUE(result);
- }
- TEST_F(GpuWatchdogPowerTest, GpuOnSuspend) {
- // watchdog_thread_->OnInitComplete() is called in SetUp
- // Enter power suspension mode.
- power_monitor_source_.GenerateSuspendEvent();
- // Run a task that takes 6x watchdog |timeout_|.
- auto normal_long_task_time = timeout_ * 6;
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE, base::BindOnce(&SimpleTask, normal_long_task_time,
- /*extra_time=*/base::TimeDelta()));
- task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
- run_loop.QuitClosure());
- run_loop.Run();
- // A task might take long time to finish after entering suspension mode.
- // This one is not a GPU hang.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_FALSE(result);
- }
- TEST_F(GpuWatchdogPowerTest, GpuOnResumeHang) {
- // watchdog_thread_->OnInitComplete() is called in SetUp
- // This task stays in the suspension mode for watchdog |timeout_|/4, and it
- // wakes up on power resume and then runs a job that is longer than the
- // watchdog resume restart timeout.
- auto allowed_time =
- timeout_ * (kRestartFactor + 1) + full_thread_time_on_windows_;
- task_environment_.GetMainThreadTaskRunner()->PostTask(
- FROM_HERE, base::BindOnce(&GpuWatchdogPowerTest::LongTaskOnResume,
- base::Unretained(this),
- /*duration*/ allowed_time,
- /*extra_time=*/extra_gpu_job_time_,
- /*time_to_power_resume*/ timeout_ / 4));
- task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
- run_loop.QuitClosure());
- run_loop.Run();
- // It takes too long to finish this task after power resume. A GPU hang should
- // be detected.
- bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
- EXPECT_TRUE(result);
- }
- } // namespace gpu
|