gpu_watchdog_thread.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef GPU_IPC_SERVICE_GPU_WATCHDOG_THREAD_H_
  5. #define GPU_IPC_SERVICE_GPU_WATCHDOG_THREAD_H_
  6. #include "base/atomicops.h"
  7. #include "base/memory/raw_ptr.h"
  8. #include "base/memory/ref_counted.h"
  9. #include "base/memory/weak_ptr.h"
  10. #include "base/metrics/histogram_macros.h"
  11. #include "base/power_monitor/power_observer.h"
  12. #include "base/task/task_observer.h"
  13. #include "base/threading/thread.h"
  14. #include "base/time/time.h"
  15. #include "build/build_config.h"
  16. #include "build/chromecast_buildflags.h"
  17. #include "gpu/ipc/common/gpu_watchdog_timeout.h"
  18. #include "gpu/ipc/service/gpu_ipc_service_export.h"
  19. #include "ui/gfx/native_widget_types.h"
  20. #include "ui/gl/progress_reporter.h"
  21. namespace gpu {
  22. // These values are persisted to logs. Entries should not be renumbered and
  23. // numeric values should never be reused.
  24. enum class GpuWatchdogThreadEvent {
  25. kGpuWatchdogStart,
  26. kGpuWatchdogKill,
  27. kGpuWatchdogEnd,
  28. kMaxValue = kGpuWatchdogEnd,
  29. };
  30. // These values are persisted to logs. Entries should not be renumbered and
  31. // numeric values should never be reused.
  32. enum class GpuWatchdogTimeoutEvent {
  33. // Recorded each time OnWatchdogTimeout() is called.
  34. kTimeout = 0,
  35. // Recorded when a GPU main thread is killed for a detected hang.
  36. kKill = 1,
  37. // Window only: Recorded when a hang is detected but we allow the GPU main
  38. // thread to continue until it spent the full
  39. // thread time doing the work.
  40. kMoreThreadTime = 2,
  41. // Windows only: The GPU makes progress after givenmore thread time. The GPU
  42. // main thread is not killed.
  43. kProgressAfterMoreThreadTime = 3,
  44. // Deprecated. A gpu hang is detected but watchdog waits for 60 seconds before
  45. // taking action.
  46. // kTimeoutWait = 4,
  47. // Deprecated. The GPU makes progress within 60 sec in OnWatchdogTimeout().
  48. // The GPU main thread is not killed.
  49. // kProgressAfterWait = 5,
  50. // Just continue if it's not on the TTY of our host X11 server.
  51. kContinueOnNonHostServerTty = 6,
  52. // Windows only: After detecting GPU hang and continuing running through
  53. // OnGpuWatchdogTimeout for the max cycles, the GPU main thread still cannot
  54. // get the full thread time.
  55. kLessThanFullThreadTimeAfterCapped = 7,
  56. // Windows only: The GPU main thread went through the
  57. // kLessThanFullThreadTimeAfterCapped stage before the process is killed.
  58. kKillOnLessThreadTime = 8,
  59. // OnWatchdogTimeout() is called long after the expected time. The GPU is not
  60. // killed this time because of the slow system.
  61. kSlowWatchdogThread = 9,
  62. kNoKillForGpuProgressDuringCrashDumping = 10,
  63. kMaxValue = kNoKillForGpuProgressDuringCrashDumping,
  64. };
  65. #if BUILDFLAG(IS_WIN)
  66. // If the actual time the watched GPU thread spent doing actual work is less
  67. // than the watchdog timeout, the GPU thread can continue running through
  68. // OnGPUWatchdogTimeout for at most 4 times before the gpu thread is killed.
  69. constexpr int kMaxCountOfMoreGpuThreadTimeAllowed = 3;
  70. #endif
  71. constexpr int kMaxExtraCyclesBeforeKill = 0;
  72. // If the scheduled timeout function is delayed by more than
  73. // kUnreasonableTimeoutDelay, we assume the system is in a unexpected state and
  74. // the GPU watchdog will NOT terminate the GPU process if no progress is made in
  75. // the GPU main thread or in the GPU display compositor thread. This is used in
  76. // determining SlowWatchdogThread.
  77. constexpr base::TimeDelta kUnreasonableTimeoutDelay = base::Seconds(5);
  78. // A thread that intermitently sends tasks to a group of watched message loops
  79. // and deliberately crashes if one of them does not respond after a timeout.
  80. class GPU_IPC_SERVICE_EXPORT GpuWatchdogThread
  81. : public base::Thread,
  82. public base::PowerSuspendObserver,
  83. public base::TaskObserver,
  84. public gl::ProgressReporter {
  85. public:
  86. static std::unique_ptr<GpuWatchdogThread> Create(
  87. bool start_backgrounded,
  88. const std::string& thread_name);
  89. static std::unique_ptr<GpuWatchdogThread> Create(
  90. bool start_backgrounded,
  91. base::TimeDelta timeout,
  92. int init_factor,
  93. int restart_factor,
  94. bool test_mode,
  95. const std::string& thread_name);
  96. GpuWatchdogThread(const GpuWatchdogThread&) = delete;
  97. GpuWatchdogThread& operator=(const GpuWatchdogThread&) = delete;
  98. ~GpuWatchdogThread() override;
  99. // Notifies the watchdog when Chrome is backgrounded / foregrounded. Should
  100. // only be used if Chrome is completely backgrounded and not expected to
  101. // render (all windows backgrounded and not producing frames).
  102. void OnBackgrounded();
  103. void OnForegrounded();
  104. // The watchdog starts armed to catch startup hangs, and needs to be disarmed
  105. // once init is complete, before executing tasks.
  106. void OnInitComplete();
  107. // Notifies the watchdog when the GPU child process is being destroyed.
  108. // This function is called directly from
  109. // viz::GpuServiceImpl::~GpuServiceImpl()
  110. void OnGpuProcessTearDown();
  111. // Pause the GPU watchdog to stop the timeout task. If the current heavy task
  112. // is not running on the GPU driver, the watchdog can be paused to avoid
  113. // unneeded crash.
  114. void PauseWatchdog();
  115. // Continue the watchdog after a pause.
  116. void ResumeWatchdog();
  117. // For gpu testing only. Return status for the watchdog tests
  118. bool IsGpuHangDetectedForTesting();
  119. // Implements base::Thread.
  120. void Init() override;
  121. void CleanUp() override;
  122. // Implements gl::ProgressReporter.
  123. void ReportProgress() override;
  124. // Implements TaskObserver.
  125. void WillProcessTask(const base::PendingTask& pending_task,
  126. bool was_blocked_or_low_priority) override;
  127. void DidProcessTask(const base::PendingTask& pending_task) override;
  128. // Implements base::PowerSuspendObserver.
  129. void OnSuspend() override;
  130. void OnResume() override;
  131. protected:
  132. GpuWatchdogThread();
  133. private:
  134. enum PauseResumeSource {
  135. kAndroidBackgroundForeground = 0,
  136. kPowerSuspendResume = 1,
  137. kGeneralGpuFlow = 2,
  138. };
  139. GpuWatchdogThread(base::TimeDelta timeout,
  140. int init_factor,
  141. int restart_factor,
  142. bool test_mode,
  143. const std::string& thread_name);
  144. void AddPowerObserver();
  145. void RestartWatchdogTimeoutTask(PauseResumeSource source_of_request);
  146. void StopWatchdogTimeoutTask(PauseResumeSource source_of_request);
  147. void UpdateInitializationFlag();
  148. void Arm();
  149. void Disarm();
  150. void InProgress();
  151. bool IsArmed();
  152. base::subtle::Atomic32 ReadArmDisarmCounter();
  153. void OnWatchdogTimeout();
  154. bool SlowWatchdogThread();
  155. bool WatchedThreadNeedsMoreThreadTime(bool no_gpu_hang_detected);
  156. #if BUILDFLAG(IS_WIN)
  157. base::ThreadTicks GetWatchedThreadTime();
  158. #endif
  159. // Do not change the function name. It is used for [GPU HANG] crash reports.
  160. void DeliberatelyTerminateToRecoverFromHang();
  161. void ContinueWithNextWatchdogTimeoutTask();
  162. // Records "GPU.WatchdogThread.Event".
  163. void GpuWatchdogThreadEventHistogram(GpuWatchdogThreadEvent thread_event);
  164. // Histogram recorded in OnWatchdogTimeout()
  165. // Records "GPU.WatchdogThread.Timeout"
  166. void GpuWatchdogTimeoutHistogram(GpuWatchdogTimeoutEvent timeout_event);
  167. #if BUILDFLAG(IS_WIN)
  168. // Histograms recorded for WatchedThreadNeedsMoreThreadTime() function.
  169. void WatchedThreadNeedsMoreThreadTimeHistogram(
  170. bool no_gpu_hang_detected,
  171. bool start_of_more_thread_time);
  172. #endif
  173. // Used for metrics. It's 1 minute after the event.
  174. bool WithinOneMinFromPowerResumed();
  175. bool WithinOneMinFromForegrounded();
  176. #if BUILDFLAG(IS_LINUX) && !BUILDFLAG(IS_CASTOS)
  177. void UpdateActiveTTY();
  178. #endif
  179. // The watchdog continues when it's not on the TTY of our host X11 server.
  180. bool ContinueOnNonHostX11ServerTty();
  181. // This counter is only written on the gpu thread, and read on both threads.
  182. volatile base::subtle::Atomic32 arm_disarm_counter_ = 0;
  183. // The counter number read in the last OnWatchdogTimeout() on the watchdog
  184. // thread.
  185. int32_t last_arm_disarm_counter_ = 0;
  186. // Timeout on the watchdog thread to check if gpu hangs.
  187. base::TimeDelta watchdog_timeout_;
  188. // The one-time watchdog timeout multiplier in the gpu initialization.
  189. const int watchdog_init_factor_;
  190. // The one-time watchdog timeout multiplier after the watchdog pauses and
  191. // restarts.
  192. const int watchdog_restart_factor_;
  193. // The time the gpu watchdog was created.
  194. base::TimeTicks watchdog_start_timeticks_;
  195. // The time the last OnSuspend and OnResume was called.
  196. base::TimeTicks power_suspend_timeticks_;
  197. base::TimeTicks power_resume_timeticks_;
  198. // The time the last OnBackgrounded and OnForegrounded was called.
  199. base::TimeTicks backgrounded_timeticks_;
  200. base::TimeTicks foregrounded_timeticks_;
  201. // The time PauseWatchdog and ResumeWatchdog was called.
  202. base::TimeTicks watchdog_pause_timeticks_;
  203. base::TimeTicks watchdog_resume_timeticks_;
  204. // TimeTicks: Tracking the amount of time a task runs. Executing delayed
  205. // tasks at the right time.
  206. // ThreadTicks: Use this timer to (approximately) measure how much time the
  207. // calling thread spent doing actual work vs. being de-scheduled.
  208. // The time the last OnWatchdogTimeout() was called.
  209. base::TimeTicks last_on_watchdog_timeout_timeticks_;
  210. // The wall-clock time the next OnWatchdogTimeout() will be called.
  211. base::Time next_on_watchdog_timeout_time_;
  212. #if BUILDFLAG(IS_WIN)
  213. base::ThreadTicks last_on_watchdog_timeout_thread_ticks_;
  214. // The difference between the timeout and the actual time the watched thread
  215. // spent doing actual work.
  216. base::TimeDelta remaining_watched_thread_ticks_;
  217. // The Windows thread hanndle of the watched GPU main thread.
  218. void* watched_thread_handle_ = nullptr;
  219. // After GPU hang detected, how many times has the GPU thread been allowed to
  220. // continue due to not enough thread time.
  221. int count_of_more_gpu_thread_time_allowed_ = 0;
  222. // After detecting GPU hang and continuing running through
  223. // OnGpuWatchdogTimeout for the max cycles, the GPU main thread still cannot
  224. // get the full thread time.
  225. bool less_than_full_thread_time_after_capped_ = false;
  226. #endif
  227. #if BUILDFLAG(IS_LINUX) && !BUILDFLAG(IS_CASTOS)
  228. raw_ptr<FILE, DanglingUntriaged> tty_file_ = nullptr;
  229. int host_tty_ = -1;
  230. int active_tty_ = -1;
  231. int last_active_tty_ = -1;
  232. #endif
  233. // The system has entered the power suspension mode.
  234. bool in_power_suspension_ = false;
  235. // The GPU process has started tearing down. Accessed only in the gpu process.
  236. bool in_gpu_process_teardown_ = false;
  237. // Chrome is running on the background on Android. Gpu is probably very slow
  238. // or stalled.
  239. bool is_backgrounded_ = false;
  240. // The GPU watchdog is paused. The timeout task is temporarily stopped.
  241. bool is_paused_ = false;
  242. // whether GpuWatchdogThreadEvent::kGpuWatchdogStart has been recorded.
  243. bool is_watchdog_start_histogram_recorded_ = false;
  244. // Read/Write by the watchdog thread only after initialized in the
  245. // constructor.
  246. bool in_gpu_initialization_ = false;
  247. // For the experiment and the debugging purpose
  248. size_t num_of_timeout_after_power_resume_ = 0;
  249. size_t num_of_timeout_after_foregrounded_ = 0;
  250. bool foregrounded_event_ = false;
  251. bool power_resumed_event_ = false;
  252. // The watched thread name string used for UMA and crash key.
  253. std::string watched_thread_name_str_uma_;
  254. // The thread id string of the watched thread.
  255. std::string watched_thread_id_str_;
  256. // For gpu testing only.
  257. const bool is_test_mode_;
  258. // Set by the watchdog thread and Read by the test thread.
  259. base::AtomicFlag test_result_timeout_and_gpu_hang_;
  260. SEQUENCE_CHECKER(watchdog_thread_sequence_checker_);
  261. SEQUENCE_CHECKER(watched_thread_sequence_checker_);
  262. base::WeakPtr<GpuWatchdogThread> weak_ptr_;
  263. base::WeakPtrFactory<GpuWatchdogThread> weak_factory_{this};
  264. };
  265. } // namespace gpu
  266. #endif // GPU_IPC_SERVICE_GPU_WATCHDOG_THREAD_H_