stats.h 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifdef CONFIG_SCHEDSTATS
  3. /*
  4. * Expects runqueue lock to be held for atomicity of update
  5. */
  6. static inline void
  7. rq_sched_info_arrive(struct rq *rq, unsigned long long delta)
  8. {
  9. if (rq) {
  10. rq->rq_sched_info.run_delay += delta;
  11. rq->rq_sched_info.pcount++;
  12. }
  13. }
  14. /*
  15. * Expects runqueue lock to be held for atomicity of update
  16. */
  17. static inline void
  18. rq_sched_info_depart(struct rq *rq, unsigned long long delta)
  19. {
  20. if (rq)
  21. rq->rq_cpu_time += delta;
  22. }
  23. static inline void
  24. rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
  25. {
  26. if (rq)
  27. rq->rq_sched_info.run_delay += delta;
  28. }
  29. #define schedstat_enabled() static_branch_unlikely(&sched_schedstats)
  30. #define __schedstat_inc(var) do { var++; } while (0)
  31. #define schedstat_inc(var) do { if (schedstat_enabled()) { var++; } } while (0)
  32. #define __schedstat_add(var, amt) do { var += (amt); } while (0)
  33. #define schedstat_add(var, amt) do { if (schedstat_enabled()) { var += (amt); } } while (0)
  34. #define __schedstat_set(var, val) do { var = (val); } while (0)
  35. #define schedstat_set(var, val) do { if (schedstat_enabled()) { var = (val); } } while (0)
  36. #define schedstat_val(var) (var)
  37. #define schedstat_val_or_zero(var) ((schedstat_enabled()) ? (var) : 0)
  38. #else /* !CONFIG_SCHEDSTATS: */
  39. static inline void rq_sched_info_arrive (struct rq *rq, unsigned long long delta) { }
  40. static inline void rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) { }
  41. static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delta) { }
  42. # define schedstat_enabled() 0
  43. # define __schedstat_inc(var) do { } while (0)
  44. # define schedstat_inc(var) do { } while (0)
  45. # define __schedstat_add(var, amt) do { } while (0)
  46. # define schedstat_add(var, amt) do { } while (0)
  47. # define __schedstat_set(var, val) do { } while (0)
  48. # define schedstat_set(var, val) do { } while (0)
  49. # define schedstat_val(var) 0
  50. # define schedstat_val_or_zero(var) 0
  51. #endif /* CONFIG_SCHEDSTATS */
  52. #ifdef CONFIG_PSI
  53. /*
  54. * PSI tracks state that persists across sleeps, such as iowaits and
  55. * memory stalls. As a result, it has to distinguish between sleeps,
  56. * where a task's runnable state changes, and requeues, where a task
  57. * and its state are being moved between CPUs and runqueues.
  58. */
  59. static inline void psi_enqueue(struct task_struct *p, bool wakeup)
  60. {
  61. int clear = 0, set = TSK_RUNNING;
  62. if (static_branch_likely(&psi_disabled))
  63. return;
  64. if (!wakeup || p->sched_psi_wake_requeue) {
  65. if (p->in_memstall)
  66. set |= TSK_MEMSTALL;
  67. if (p->sched_psi_wake_requeue)
  68. p->sched_psi_wake_requeue = 0;
  69. } else {
  70. if (p->in_iowait)
  71. clear |= TSK_IOWAIT;
  72. }
  73. psi_task_change(p, clear, set);
  74. }
  75. static inline void psi_dequeue(struct task_struct *p, bool sleep)
  76. {
  77. int clear = TSK_RUNNING, set = 0;
  78. if (static_branch_likely(&psi_disabled))
  79. return;
  80. if (!sleep) {
  81. if (p->in_memstall)
  82. clear |= TSK_MEMSTALL;
  83. } else {
  84. /*
  85. * When a task sleeps, schedule() dequeues it before
  86. * switching to the next one. Merge the clearing of
  87. * TSK_RUNNING and TSK_ONCPU to save an unnecessary
  88. * psi_task_change() call in psi_sched_switch().
  89. */
  90. clear |= TSK_ONCPU;
  91. if (p->in_iowait)
  92. set |= TSK_IOWAIT;
  93. }
  94. psi_task_change(p, clear, set);
  95. }
  96. static inline void psi_ttwu_dequeue(struct task_struct *p)
  97. {
  98. if (static_branch_likely(&psi_disabled))
  99. return;
  100. /*
  101. * Is the task being migrated during a wakeup? Make sure to
  102. * deregister its sleep-persistent psi states from the old
  103. * queue, and let psi_enqueue() know it has to requeue.
  104. */
  105. if (unlikely(p->in_iowait || p->in_memstall)) {
  106. struct rq_flags rf;
  107. struct rq *rq;
  108. int clear = 0;
  109. if (p->in_iowait)
  110. clear |= TSK_IOWAIT;
  111. if (p->in_memstall)
  112. clear |= TSK_MEMSTALL;
  113. rq = __task_rq_lock(p, &rf);
  114. psi_task_change(p, clear, 0);
  115. p->sched_psi_wake_requeue = 1;
  116. __task_rq_unlock(rq, &rf);
  117. }
  118. }
  119. static inline void psi_sched_switch(struct task_struct *prev,
  120. struct task_struct *next,
  121. bool sleep)
  122. {
  123. if (static_branch_likely(&psi_disabled))
  124. return;
  125. psi_task_switch(prev, next, sleep);
  126. }
  127. static inline void psi_task_tick(struct rq *rq)
  128. {
  129. if (static_branch_likely(&psi_disabled))
  130. return;
  131. if (unlikely(rq->curr->in_memstall))
  132. psi_memstall_tick(rq->curr, cpu_of(rq));
  133. }
  134. #else /* CONFIG_PSI */
  135. static inline void psi_enqueue(struct task_struct *p, bool wakeup) {}
  136. static inline void psi_dequeue(struct task_struct *p, bool sleep) {}
  137. static inline void psi_ttwu_dequeue(struct task_struct *p) {}
  138. static inline void psi_sched_switch(struct task_struct *prev,
  139. struct task_struct *next,
  140. bool sleep) {}
  141. static inline void psi_task_tick(struct rq *rq) {}
  142. #endif /* CONFIG_PSI */
  143. #ifdef CONFIG_SCHED_INFO
  144. static inline void sched_info_reset_dequeued(struct task_struct *t)
  145. {
  146. t->sched_info.last_queued = 0;
  147. }
  148. /*
  149. * We are interested in knowing how long it was from the *first* time a
  150. * task was queued to the time that it finally hit a CPU, we call this routine
  151. * from dequeue_task() to account for possible rq->clock skew across CPUs. The
  152. * delta taken on each CPU would annul the skew.
  153. */
  154. static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t)
  155. {
  156. unsigned long long now = rq_clock(rq), delta = 0;
  157. if (sched_info_on()) {
  158. if (t->sched_info.last_queued)
  159. delta = now - t->sched_info.last_queued;
  160. }
  161. sched_info_reset_dequeued(t);
  162. t->sched_info.run_delay += delta;
  163. rq_sched_info_dequeued(rq, delta);
  164. }
  165. /*
  166. * Called when a task finally hits the CPU. We can now calculate how
  167. * long it was waiting to run. We also note when it began so that we
  168. * can keep stats on how long its timeslice is.
  169. */
  170. static void sched_info_arrive(struct rq *rq, struct task_struct *t)
  171. {
  172. unsigned long long now = rq_clock(rq), delta = 0;
  173. if (t->sched_info.last_queued)
  174. delta = now - t->sched_info.last_queued;
  175. sched_info_reset_dequeued(t);
  176. t->sched_info.run_delay += delta;
  177. t->sched_info.last_arrival = now;
  178. t->sched_info.pcount++;
  179. rq_sched_info_arrive(rq, delta);
  180. }
  181. /*
  182. * This function is only called from enqueue_task(), but also only updates
  183. * the timestamp if it is already not set. It's assumed that
  184. * sched_info_dequeued() will clear that stamp when appropriate.
  185. */
  186. static inline void sched_info_queued(struct rq *rq, struct task_struct *t)
  187. {
  188. if (sched_info_on()) {
  189. if (!t->sched_info.last_queued)
  190. t->sched_info.last_queued = rq_clock(rq);
  191. }
  192. }
  193. /*
  194. * Called when a process ceases being the active-running process involuntarily
  195. * due, typically, to expiring its time slice (this may also be called when
  196. * switching to the idle task). Now we can calculate how long we ran.
  197. * Also, if the process is still in the TASK_RUNNING state, call
  198. * sched_info_queued() to mark that it has now again started waiting on
  199. * the runqueue.
  200. */
  201. static inline void sched_info_depart(struct rq *rq, struct task_struct *t)
  202. {
  203. unsigned long long delta = rq_clock(rq) - t->sched_info.last_arrival;
  204. rq_sched_info_depart(rq, delta);
  205. if (t->state == TASK_RUNNING)
  206. sched_info_queued(rq, t);
  207. }
  208. /*
  209. * Called when tasks are switched involuntarily due, typically, to expiring
  210. * their time slice. (This may also be called when switching to or from
  211. * the idle task.) We are only called when prev != next.
  212. */
  213. static inline void
  214. __sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next)
  215. {
  216. /*
  217. * prev now departs the CPU. It's not interesting to record
  218. * stats about how efficient we were at scheduling the idle
  219. * process, however.
  220. */
  221. if (prev != rq->idle)
  222. sched_info_depart(rq, prev);
  223. if (next != rq->idle)
  224. sched_info_arrive(rq, next);
  225. }
  226. static inline void
  227. sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next)
  228. {
  229. if (sched_info_on())
  230. __sched_info_switch(rq, prev, next);
  231. }
  232. #else /* !CONFIG_SCHED_INFO: */
  233. # define sched_info_queued(rq, t) do { } while (0)
  234. # define sched_info_reset_dequeued(t) do { } while (0)
  235. # define sched_info_dequeued(rq, t) do { } while (0)
  236. # define sched_info_depart(rq, t) do { } while (0)
  237. # define sched_info_arrive(rq, next) do { } while (0)
  238. # define sched_info_switch(rq, t, next) do { } while (0)
  239. #endif /* CONFIG_SCHED_INFO */