gpu_watchdog_thread_unittest.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457
  1. // Copyright (c) 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "gpu/ipc/service/gpu_watchdog_thread.h"
  5. #include "base/test/task_environment.h"
  6. #include "base/power_monitor/power_monitor.h"
  7. #include "base/power_monitor/power_monitor_source.h"
  8. #include "base/system/sys_info.h"
  9. #include "base/task/current_thread.h"
  10. #include "base/test/power_monitor_test.h"
  11. #include "base/threading/thread_task_runner_handle.h"
  12. #include "base/time/time.h"
  13. #include "build/build_config.h"
  14. #include "testing/gtest/include/gtest/gtest.h"
  15. #if BUILDFLAG(IS_WIN)
  16. #include "base/win/windows_version.h"
  17. #endif
  18. #if BUILDFLAG(IS_MAC)
  19. #include "base/mac/mac_util.h"
  20. #endif
  21. namespace gpu {
  22. namespace {
  23. // |kExtraGPUJobTimeForTesting| is the extra time the gpu main/test thread
  24. // spends after GpuWatchdogTimeout. Theoretically, any extra time such as 1 ms
  25. // should be enough to trigger the watchdog kill. However, it can cause test
  26. // flakiness when the time is too short.
  27. constexpr auto kGpuWatchdogTimeoutForTesting = base::Milliseconds(120);
  28. constexpr auto kExtraGPUJobTimeForTesting = base::Milliseconds(500);
  29. // For slow machines like Win 7, Mac 10.xx and Android L/M/N.
  30. [[maybe_unused]] constexpr auto kGpuWatchdogTimeoutForTestingSlow =
  31. base::Milliseconds(240);
  32. [[maybe_unused]] constexpr auto kExtraGPUJobTimeForTestingSlow =
  33. base::Milliseconds(1000);
  34. // For Fuchsia in which GpuWatchdogTest.GpuInitializationAndRunningTasks test
  35. // is flaky.
  36. [[maybe_unused]] constexpr auto kGpuWatchdogTimeoutForTestingSlowest =
  37. base::Milliseconds(1000);
  38. [[maybe_unused]] constexpr auto kExtraGPUJobTimeForTestingSlowest =
  39. base::Milliseconds(4000);
  40. // On Windows, the gpu watchdog check if the main thread has used the full
  41. // thread time. We want to detect the case in which the main thread is swapped
  42. // out by the OS scheduler. The task on windows is simiulated by reading
  43. // TimeTicks instead of Sleep().
  44. void SimpleTask(base::TimeDelta duration, base::TimeDelta extra_time) {
  45. #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC)
  46. auto start_timetick = base::TimeTicks::Now();
  47. do {
  48. } while ((base::TimeTicks::Now() - start_timetick) < duration);
  49. base::PlatformThread::Sleep(extra_time);
  50. #else
  51. base::PlatformThread::Sleep(duration + extra_time);
  52. #endif
  53. }
  54. } // namespace
  55. class GpuWatchdogTest : public testing::Test {
  56. public:
  57. GpuWatchdogTest() {}
  58. void LongTaskWithReportProgress(base::TimeDelta duration,
  59. base::TimeDelta report_delta);
  60. #if BUILDFLAG(IS_ANDROID)
  61. void LongTaskFromBackgroundToForeground(
  62. base::TimeDelta duration,
  63. base::TimeDelta extra_time,
  64. base::TimeDelta time_to_switch_to_foreground);
  65. #endif
  66. // Implements testing::Test
  67. void SetUp() override;
  68. protected:
  69. ~GpuWatchdogTest() override = default;
  70. base::test::SingleThreadTaskEnvironment task_environment_;
  71. base::RunLoop run_loop;
  72. std::unique_ptr<gpu::GpuWatchdogThread> watchdog_thread_;
  73. base::TimeDelta timeout_ = kGpuWatchdogTimeoutForTesting;
  74. base::TimeDelta extra_gpu_job_time_ = kExtraGPUJobTimeForTesting;
  75. base::TimeDelta full_thread_time_on_windows_ = base::TimeDelta();
  76. };
  77. class GpuWatchdogPowerTest : public GpuWatchdogTest {
  78. public:
  79. GpuWatchdogPowerTest() {}
  80. void LongTaskOnResume(base::TimeDelta duration,
  81. base::TimeDelta extra_time,
  82. base::TimeDelta time_to_power_resume);
  83. // Implements testing::Test
  84. void SetUp() override;
  85. void TearDown() override;
  86. protected:
  87. ~GpuWatchdogPowerTest() override = default;
  88. base::test::ScopedPowerMonitorTestSource power_monitor_source_;
  89. };
  90. void GpuWatchdogTest::SetUp() {
  91. ASSERT_TRUE(base::ThreadTaskRunnerHandle::IsSet());
  92. ASSERT_TRUE(base::CurrentThread::IsSet());
  93. enum TimeOutType {
  94. kNormal,
  95. kSlow,
  96. kSlowest,
  97. };
  98. TimeOutType timeout_type = kNormal;
  99. #if BUILDFLAG(IS_WIN)
  100. // Win7
  101. if (base::win::GetVersion() < base::win::Version::WIN10) {
  102. timeout_type = kSlow;
  103. }
  104. #elif BUILDFLAG(IS_MAC)
  105. // Use slow timeout for Mac versions < 11.00 and for MacBookPro model <
  106. // MacBookPro14,1
  107. int os_version = base::mac::internal::MacOSVersion();
  108. if (os_version <= 1100) {
  109. // Check MacOS version.
  110. timeout_type = kSlow;
  111. } else {
  112. // Check Mac machine model version.
  113. std::string model_str = base::SysInfo::HardwareModelName();
  114. size_t found_position = model_str.find("MacBookPro");
  115. constexpr size_t model_version_pos = 10;
  116. // A MacBookPro is found.
  117. if (found_position == 0 && model_str.size() > model_version_pos) {
  118. // model_ver_str = "MacBookProXX,X", model_ver_str = "XX,X"
  119. std::string model_ver_str = model_str.substr(model_version_pos);
  120. int major_model_ver = std::atoi(model_ver_str.c_str());
  121. // For model version < 14,1
  122. if (major_model_ver < 14) {
  123. timeout_type = kSlow;
  124. }
  125. }
  126. }
  127. #elif BUILDFLAG(IS_ANDROID)
  128. int32_t major_version = 0;
  129. int32_t minor_version = 0;
  130. int32_t bugfix_version = 0;
  131. base::SysInfo::OperatingSystemVersionNumbers(&major_version, &minor_version,
  132. &bugfix_version);
  133. // For Android version < Android Pie (Version 9)
  134. if (major_version < 9) {
  135. timeout_type = kSlow;
  136. }
  137. #elif BUILDFLAG(IS_FUCHSIA)
  138. timeout_type = kSlowest;
  139. #endif
  140. if (timeout_type == kSlow) {
  141. timeout_ = kGpuWatchdogTimeoutForTestingSlow;
  142. extra_gpu_job_time_ = kExtraGPUJobTimeForTestingSlow;
  143. } else if (timeout_type == kSlowest) {
  144. timeout_ = kGpuWatchdogTimeoutForTestingSlowest;
  145. extra_gpu_job_time_ = kExtraGPUJobTimeForTestingSlowest;
  146. }
  147. #if BUILDFLAG(IS_WIN)
  148. full_thread_time_on_windows_ = timeout_ * kMaxCountOfMoreGpuThreadTimeAllowed;
  149. #endif
  150. watchdog_thread_ = gpu::GpuWatchdogThread::Create(
  151. /*start_backgrounded=*/false,
  152. /*timeout=*/timeout_,
  153. /*init_factor=*/kInitFactor,
  154. /*restart_factor=*/kRestartFactor,
  155. /*test_mode=*/true, /*thread_name=*/"GpuWatchdog");
  156. }
  157. void GpuWatchdogPowerTest::SetUp() {
  158. GpuWatchdogTest::SetUp();
  159. // Report GPU init complete.
  160. watchdog_thread_->OnInitComplete();
  161. }
  162. void GpuWatchdogPowerTest::TearDown() {
  163. GpuWatchdogTest::TearDown();
  164. watchdog_thread_.reset();
  165. }
  166. // This task will run for duration_ms milliseconds. It will also call watchdog
  167. // ReportProgress() every report_delta_ms milliseconds.
  168. void GpuWatchdogTest::LongTaskWithReportProgress(base::TimeDelta duration,
  169. base::TimeDelta report_delta) {
  170. base::TimeTicks start = base::TimeTicks::Now();
  171. base::TimeTicks end;
  172. do {
  173. SimpleTask(report_delta, /*extra_time=*/base::TimeDelta());
  174. watchdog_thread_->ReportProgress();
  175. end = base::TimeTicks::Now();
  176. } while (end - start <= duration);
  177. }
  178. #if BUILDFLAG(IS_ANDROID)
  179. void GpuWatchdogTest::LongTaskFromBackgroundToForeground(
  180. base::TimeDelta duration,
  181. base::TimeDelta extra_time,
  182. base::TimeDelta time_to_switch_to_foreground) {
  183. // Chrome is running in the background first.
  184. watchdog_thread_->OnBackgrounded();
  185. SimpleTask(time_to_switch_to_foreground, /*extra_time=*/base::TimeDelta());
  186. // Now switch Chrome to the foreground after the specified time
  187. watchdog_thread_->OnForegrounded();
  188. SimpleTask(duration, extra_time);
  189. }
  190. #endif
  191. void GpuWatchdogPowerTest::LongTaskOnResume(
  192. base::TimeDelta duration,
  193. base::TimeDelta extra_time,
  194. base::TimeDelta time_to_power_resume) {
  195. // Stay in power suspension mode first.
  196. power_monitor_source_.GenerateSuspendEvent();
  197. SimpleTask(time_to_power_resume, /*extra_time=*/base::TimeDelta());
  198. // Now wake up on power resume.
  199. power_monitor_source_.GenerateResumeEvent();
  200. // Continue the GPU task for the remaining time.
  201. SimpleTask(duration, extra_time);
  202. }
  203. // Normal GPU Initialization.
  204. TEST_F(GpuWatchdogTest, GpuInitializationComplete) {
  205. // Assume GPU initialization takes a quarter of WatchdogTimeout.
  206. auto normal_task_time = timeout_ / 4;
  207. SimpleTask(normal_task_time, /*extra_time=*/base::TimeDelta());
  208. watchdog_thread_->OnInitComplete();
  209. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  210. EXPECT_FALSE(result);
  211. }
  212. // GPU Hang In Initialization.
  213. TEST_F(GpuWatchdogTest, GpuInitializationHang) {
  214. auto allowed_time =
  215. timeout_ * (kInitFactor + 1) + full_thread_time_on_windows_;
  216. // GPU init takes longer than timeout.
  217. SimpleTask(allowed_time, /*extra_time=*/extra_gpu_job_time_);
  218. // Gpu hangs. OnInitComplete() is not called
  219. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  220. EXPECT_TRUE(result);
  221. // retry on failure.
  222. }
  223. // Normal GPU Initialization and Running Task.
  224. TEST_F(GpuWatchdogTest, GpuInitializationAndRunningTasks) {
  225. // Assume GPU initialization takes quarter of WatchdogTimeout time.
  226. auto normal_task_time = timeout_ / 4;
  227. SimpleTask(normal_task_time, /*extra_time=*/base::TimeDelta());
  228. watchdog_thread_->OnInitComplete();
  229. // Start running GPU tasks. Watchdog function WillProcessTask(),
  230. // DidProcessTask() and ReportProgress() are tested.
  231. task_environment_.GetMainThreadTaskRunner()->PostTask(
  232. FROM_HERE, base::BindOnce(&SimpleTask, normal_task_time,
  233. /*extra_time=*/base::TimeDelta()));
  234. task_environment_.GetMainThreadTaskRunner()->PostTask(
  235. FROM_HERE, base::BindOnce(&SimpleTask, normal_task_time,
  236. /*extra_time=*/base::TimeDelta()));
  237. // This long task takes 6X timeout to finish, longer than timeout. But it
  238. // reports progress every quarter of watchdog |timeout_|, so this is an
  239. // expected normal behavior.
  240. auto normal_long_task_time = timeout_ * 6;
  241. task_environment_.GetMainThreadTaskRunner()->PostTask(
  242. FROM_HERE, base::BindOnce(&GpuWatchdogTest::LongTaskWithReportProgress,
  243. base::Unretained(this), normal_long_task_time,
  244. /*report_progress_time*/ timeout_ / 4));
  245. task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
  246. run_loop.QuitClosure());
  247. run_loop.Run();
  248. // Everything should be fine. No GPU hang detected.
  249. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  250. EXPECT_FALSE(result);
  251. }
  252. // GPU Hang when running a task.
  253. TEST_F(GpuWatchdogTest, GpuRunningATaskHang) {
  254. // Report gpu init complete
  255. watchdog_thread_->OnInitComplete();
  256. // Start running a GPU task.
  257. auto allowed_time = timeout_ * 2 + full_thread_time_on_windows_;
  258. task_environment_.GetMainThreadTaskRunner()->PostTask(
  259. FROM_HERE,
  260. base::BindOnce(&SimpleTask, allowed_time, extra_gpu_job_time_));
  261. task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
  262. run_loop.QuitClosure());
  263. run_loop.Run();
  264. // This GPU task takes too long. A GPU hang should be detected.
  265. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  266. EXPECT_TRUE(result);
  267. }
  268. #if BUILDFLAG(IS_ANDROID)
  269. TEST_F(GpuWatchdogTest, ChromeInBackground) {
  270. // Chrome starts in the background.
  271. watchdog_thread_->OnBackgrounded();
  272. // Gpu init takes longer than 6x watchdog |timeout_|. This is normal since
  273. // Chrome is running in the background.
  274. auto normal_long_task_time = timeout_ * 6;
  275. SimpleTask(normal_long_task_time, /*extra_time=*/base::TimeDelta());
  276. // Report GPU init complete.
  277. watchdog_thread_->OnInitComplete();
  278. // Run a task that takes 6x watchdog |timeout_| longer.This is normal since
  279. // Chrome is running in the background.
  280. task_environment_.GetMainThreadTaskRunner()->PostTask(
  281. FROM_HERE, base::BindOnce(&SimpleTask, normal_long_task_time,
  282. /*extra_time=*/base::TimeDelta()));
  283. task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
  284. run_loop.QuitClosure());
  285. run_loop.Run();
  286. // The gpu might be slow when running in the background. This is ok.
  287. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  288. EXPECT_FALSE(result);
  289. }
  290. TEST_F(GpuWatchdogTest, GpuSwitchingToForegroundHang) {
  291. // Report GPU init complete.
  292. watchdog_thread_->OnInitComplete();
  293. // A task stays in the background for watchdog |timeout_| then switches to the
  294. // foreground and runs longer than the first-time foreground watchdog timeout
  295. // allowed.
  296. auto allowed_time = timeout_ * (kRestartFactor + 1);
  297. task_environment_.GetMainThreadTaskRunner()->PostTask(
  298. FROM_HERE,
  299. base::BindOnce(&GpuWatchdogTest::LongTaskFromBackgroundToForeground,
  300. base::Unretained(this), /*duration*/ allowed_time,
  301. /*extra_time=*/extra_gpu_job_time_,
  302. /*time_to_switch_to_foreground*/ timeout_ / 4));
  303. task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
  304. run_loop.QuitClosure());
  305. run_loop.Run();
  306. // It takes too long to finish a task after switching to the foreground.
  307. // A GPU hang should be detected.
  308. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  309. EXPECT_TRUE(result);
  310. }
  311. #endif
  312. TEST_F(GpuWatchdogTest, GpuInitializationPause) {
  313. // Running for watchdog |timeout_|/4 in the beginning of GPU init.
  314. SimpleTask(timeout_ / 4,
  315. /*extra_time=*/base::TimeDelta());
  316. watchdog_thread_->PauseWatchdog();
  317. // The Gpu init continues for another 6x watchdog |timeout_| after the pause.
  318. // This is normal since watchdog is paused.
  319. auto normal_long_task_time = timeout_ * 6;
  320. SimpleTask(normal_long_task_time, /*extra_time=*/base::TimeDelta());
  321. // No GPU hang is detected when the watchdog is paused.
  322. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  323. EXPECT_FALSE(result);
  324. // Continue the watchdog now.
  325. watchdog_thread_->ResumeWatchdog();
  326. // The Gpu init continues for longer than allowed init time.
  327. auto allowed_time =
  328. timeout_ * (kInitFactor + 1) + full_thread_time_on_windows_;
  329. SimpleTask(allowed_time, /*extra_time=*/extra_gpu_job_time_);
  330. // A GPU hang should be detected.
  331. result = watchdog_thread_->IsGpuHangDetectedForTesting();
  332. EXPECT_TRUE(result);
  333. }
  334. TEST_F(GpuWatchdogPowerTest, GpuOnSuspend) {
  335. // watchdog_thread_->OnInitComplete() is called in SetUp
  336. // Enter power suspension mode.
  337. power_monitor_source_.GenerateSuspendEvent();
  338. // Run a task that takes 6x watchdog |timeout_|.
  339. auto normal_long_task_time = timeout_ * 6;
  340. task_environment_.GetMainThreadTaskRunner()->PostTask(
  341. FROM_HERE, base::BindOnce(&SimpleTask, normal_long_task_time,
  342. /*extra_time=*/base::TimeDelta()));
  343. task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
  344. run_loop.QuitClosure());
  345. run_loop.Run();
  346. // A task might take long time to finish after entering suspension mode.
  347. // This one is not a GPU hang.
  348. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  349. EXPECT_FALSE(result);
  350. }
  351. TEST_F(GpuWatchdogPowerTest, GpuOnResumeHang) {
  352. // watchdog_thread_->OnInitComplete() is called in SetUp
  353. // This task stays in the suspension mode for watchdog |timeout_|/4, and it
  354. // wakes up on power resume and then runs a job that is longer than the
  355. // watchdog resume restart timeout.
  356. auto allowed_time =
  357. timeout_ * (kRestartFactor + 1) + full_thread_time_on_windows_;
  358. task_environment_.GetMainThreadTaskRunner()->PostTask(
  359. FROM_HERE, base::BindOnce(&GpuWatchdogPowerTest::LongTaskOnResume,
  360. base::Unretained(this),
  361. /*duration*/ allowed_time,
  362. /*extra_time=*/extra_gpu_job_time_,
  363. /*time_to_power_resume*/ timeout_ / 4));
  364. task_environment_.GetMainThreadTaskRunner()->PostTask(FROM_HERE,
  365. run_loop.QuitClosure());
  366. run_loop.Run();
  367. // It takes too long to finish this task after power resume. A GPU hang should
  368. // be detected.
  369. bool result = watchdog_thread_->IsGpuHangDetectedForTesting();
  370. EXPECT_TRUE(result);
  371. }
  372. } // namespace gpu