debug.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kernel/sched/debug.c
  4. *
  5. * Print the CFS rbtree and other debugging details
  6. *
  7. * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
  8. */
  9. #include "sched.h"
  10. /*
  11. * This allows printing both to /proc/sched_debug and
  12. * to the console
  13. */
  14. #define SEQ_printf(m, x...) \
  15. do { \
  16. if (m) \
  17. seq_printf(m, x); \
  18. else \
  19. pr_cont(x); \
  20. } while (0)
  21. /*
  22. * Ease the printing of nsec fields:
  23. */
  24. static long long nsec_high(unsigned long long nsec)
  25. {
  26. if ((long long)nsec < 0) {
  27. nsec = -nsec;
  28. do_div(nsec, 1000000);
  29. return -nsec;
  30. }
  31. do_div(nsec, 1000000);
  32. return nsec;
  33. }
  34. static unsigned long nsec_low(unsigned long long nsec)
  35. {
  36. if ((long long)nsec < 0)
  37. nsec = -nsec;
  38. return do_div(nsec, 1000000);
  39. }
  40. #define SPLIT_NS(x) nsec_high(x), nsec_low(x)
  41. #define SCHED_FEAT(name, enabled) \
  42. #name ,
  43. const char * const sched_feat_names[] = {
  44. #include "features.h"
  45. };
  46. EXPORT_SYMBOL_GPL(sched_feat_names);
  47. #undef SCHED_FEAT
  48. static int sched_feat_show(struct seq_file *m, void *v)
  49. {
  50. int i;
  51. for (i = 0; i < __SCHED_FEAT_NR; i++) {
  52. if (!(sysctl_sched_features & (1UL << i)))
  53. seq_puts(m, "NO_");
  54. seq_printf(m, "%s ", sched_feat_names[i]);
  55. }
  56. seq_puts(m, "\n");
  57. return 0;
  58. }
  59. #ifdef CONFIG_JUMP_LABEL
  60. #define jump_label_key__true STATIC_KEY_INIT_TRUE
  61. #define jump_label_key__false STATIC_KEY_INIT_FALSE
  62. #define SCHED_FEAT(name, enabled) \
  63. jump_label_key__##enabled ,
  64. struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
  65. #include "features.h"
  66. };
  67. EXPORT_SYMBOL_GPL(sched_feat_keys);
  68. #undef SCHED_FEAT
  69. static void sched_feat_disable(int i)
  70. {
  71. static_key_disable_cpuslocked(&sched_feat_keys[i]);
  72. }
  73. static void sched_feat_enable(int i)
  74. {
  75. static_key_enable_cpuslocked(&sched_feat_keys[i]);
  76. }
  77. #else
  78. static void sched_feat_disable(int i) { };
  79. static void sched_feat_enable(int i) { };
  80. #endif /* CONFIG_JUMP_LABEL */
  81. static int sched_feat_set(char *cmp)
  82. {
  83. int i;
  84. int neg = 0;
  85. if (strncmp(cmp, "NO_", 3) == 0) {
  86. neg = 1;
  87. cmp += 3;
  88. }
  89. i = match_string(sched_feat_names, __SCHED_FEAT_NR, cmp);
  90. if (i < 0)
  91. return i;
  92. if (neg) {
  93. sysctl_sched_features &= ~(1UL << i);
  94. sched_feat_disable(i);
  95. } else {
  96. sysctl_sched_features |= (1UL << i);
  97. sched_feat_enable(i);
  98. }
  99. return 0;
  100. }
  101. static ssize_t
  102. sched_feat_write(struct file *filp, const char __user *ubuf,
  103. size_t cnt, loff_t *ppos)
  104. {
  105. char buf[64];
  106. char *cmp;
  107. int ret;
  108. struct inode *inode;
  109. if (cnt > 63)
  110. cnt = 63;
  111. if (copy_from_user(&buf, ubuf, cnt))
  112. return -EFAULT;
  113. buf[cnt] = 0;
  114. cmp = strstrip(buf);
  115. /* Ensure the static_key remains in a consistent state */
  116. inode = file_inode(filp);
  117. cpus_read_lock();
  118. inode_lock(inode);
  119. ret = sched_feat_set(cmp);
  120. inode_unlock(inode);
  121. cpus_read_unlock();
  122. if (ret < 0)
  123. return ret;
  124. *ppos += cnt;
  125. return cnt;
  126. }
  127. static int sched_feat_open(struct inode *inode, struct file *filp)
  128. {
  129. return single_open(filp, sched_feat_show, NULL);
  130. }
  131. static const struct file_operations sched_feat_fops = {
  132. .open = sched_feat_open,
  133. .write = sched_feat_write,
  134. .read = seq_read,
  135. .llseek = seq_lseek,
  136. .release = single_release,
  137. };
  138. __read_mostly bool sched_debug_enabled;
  139. static __init int sched_init_debug(void)
  140. {
  141. debugfs_create_file("sched_features", 0644, NULL, NULL,
  142. &sched_feat_fops);
  143. debugfs_create_bool("sched_debug", 0644, NULL,
  144. &sched_debug_enabled);
  145. return 0;
  146. }
  147. late_initcall(sched_init_debug);
  148. #ifdef CONFIG_SMP
  149. #ifdef CONFIG_SYSCTL
  150. static struct ctl_table sd_ctl_dir[] = {
  151. {
  152. .procname = "sched_domain",
  153. .mode = 0555,
  154. },
  155. {}
  156. };
  157. static struct ctl_table sd_ctl_root[] = {
  158. {
  159. .procname = "kernel",
  160. .mode = 0555,
  161. .child = sd_ctl_dir,
  162. },
  163. {}
  164. };
  165. static struct ctl_table *sd_alloc_ctl_entry(int n)
  166. {
  167. struct ctl_table *entry =
  168. kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
  169. return entry;
  170. }
  171. static void sd_free_ctl_entry(struct ctl_table **tablep)
  172. {
  173. struct ctl_table *entry;
  174. /*
  175. * In the intermediate directories, both the child directory and
  176. * procname are dynamically allocated and could fail but the mode
  177. * will always be set. In the lowest directory the names are
  178. * static strings and all have proc handlers.
  179. */
  180. for (entry = *tablep; entry->mode; entry++) {
  181. if (entry->child)
  182. sd_free_ctl_entry(&entry->child);
  183. if (entry->proc_handler == NULL)
  184. kfree(entry->procname);
  185. }
  186. kfree(*tablep);
  187. *tablep = NULL;
  188. }
  189. static void
  190. set_table_entry(struct ctl_table *entry,
  191. const char *procname, void *data, int maxlen,
  192. umode_t mode, proc_handler *proc_handler)
  193. {
  194. entry->procname = procname;
  195. entry->data = data;
  196. entry->maxlen = maxlen;
  197. entry->mode = mode;
  198. entry->proc_handler = proc_handler;
  199. }
  200. static int sd_ctl_doflags(struct ctl_table *table, int write,
  201. void *buffer, size_t *lenp, loff_t *ppos)
  202. {
  203. unsigned long flags = *(unsigned long *)table->data;
  204. size_t data_size = 0;
  205. size_t len = 0;
  206. char *tmp, *buf;
  207. int idx;
  208. if (write)
  209. return 0;
  210. for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
  211. char *name = sd_flag_debug[idx].name;
  212. /* Name plus whitespace */
  213. data_size += strlen(name) + 1;
  214. }
  215. if (*ppos > data_size) {
  216. *lenp = 0;
  217. return 0;
  218. }
  219. buf = kcalloc(data_size + 1, sizeof(*buf), GFP_KERNEL);
  220. if (!buf)
  221. return -ENOMEM;
  222. for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
  223. char *name = sd_flag_debug[idx].name;
  224. len += snprintf(buf + len, strlen(name) + 2, "%s ", name);
  225. }
  226. tmp = buf + *ppos;
  227. len -= *ppos;
  228. if (len > *lenp)
  229. len = *lenp;
  230. if (len)
  231. memcpy(buffer, tmp, len);
  232. if (len < *lenp) {
  233. ((char *)buffer)[len] = '\n';
  234. len++;
  235. }
  236. *lenp = len;
  237. *ppos += len;
  238. kfree(buf);
  239. return 0;
  240. }
  241. static struct ctl_table *
  242. sd_alloc_ctl_domain_table(struct sched_domain *sd)
  243. {
  244. struct ctl_table *table = sd_alloc_ctl_entry(9);
  245. if (table == NULL)
  246. return NULL;
  247. set_table_entry(&table[0], "min_interval", &sd->min_interval, sizeof(long), 0644, proc_doulongvec_minmax);
  248. set_table_entry(&table[1], "max_interval", &sd->max_interval, sizeof(long), 0644, proc_doulongvec_minmax);
  249. set_table_entry(&table[2], "busy_factor", &sd->busy_factor, sizeof(int), 0644, proc_dointvec_minmax);
  250. set_table_entry(&table[3], "imbalance_pct", &sd->imbalance_pct, sizeof(int), 0644, proc_dointvec_minmax);
  251. set_table_entry(&table[4], "cache_nice_tries", &sd->cache_nice_tries, sizeof(int), 0644, proc_dointvec_minmax);
  252. set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0444, sd_ctl_doflags);
  253. set_table_entry(&table[6], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax);
  254. set_table_entry(&table[7], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring);
  255. /* &table[8] is terminator */
  256. return table;
  257. }
  258. static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
  259. {
  260. struct ctl_table *entry, *table;
  261. struct sched_domain *sd;
  262. int domain_num = 0, i;
  263. char buf[32];
  264. for_each_domain(cpu, sd)
  265. domain_num++;
  266. entry = table = sd_alloc_ctl_entry(domain_num + 1);
  267. if (table == NULL)
  268. return NULL;
  269. i = 0;
  270. for_each_domain(cpu, sd) {
  271. snprintf(buf, 32, "domain%d", i);
  272. entry->procname = kstrdup(buf, GFP_KERNEL);
  273. entry->mode = 0555;
  274. entry->child = sd_alloc_ctl_domain_table(sd);
  275. entry++;
  276. i++;
  277. }
  278. return table;
  279. }
  280. static cpumask_var_t sd_sysctl_cpus;
  281. static struct ctl_table_header *sd_sysctl_header;
  282. void register_sched_domain_sysctl(void)
  283. {
  284. static struct ctl_table *cpu_entries;
  285. static struct ctl_table **cpu_idx;
  286. static bool init_done = false;
  287. char buf[32];
  288. int i;
  289. if (!cpu_entries) {
  290. cpu_entries = sd_alloc_ctl_entry(num_possible_cpus() + 1);
  291. if (!cpu_entries)
  292. return;
  293. WARN_ON(sd_ctl_dir[0].child);
  294. sd_ctl_dir[0].child = cpu_entries;
  295. }
  296. if (!cpu_idx) {
  297. struct ctl_table *e = cpu_entries;
  298. cpu_idx = kcalloc(nr_cpu_ids, sizeof(struct ctl_table*), GFP_KERNEL);
  299. if (!cpu_idx)
  300. return;
  301. /* deal with sparse possible map */
  302. for_each_possible_cpu(i) {
  303. cpu_idx[i] = e;
  304. e++;
  305. }
  306. }
  307. if (!cpumask_available(sd_sysctl_cpus)) {
  308. if (!alloc_cpumask_var(&sd_sysctl_cpus, GFP_KERNEL))
  309. return;
  310. }
  311. if (!init_done) {
  312. init_done = true;
  313. /* init to possible to not have holes in @cpu_entries */
  314. cpumask_copy(sd_sysctl_cpus, cpu_possible_mask);
  315. }
  316. for_each_cpu(i, sd_sysctl_cpus) {
  317. struct ctl_table *e = cpu_idx[i];
  318. if (e->child)
  319. sd_free_ctl_entry(&e->child);
  320. if (!e->procname) {
  321. snprintf(buf, 32, "cpu%d", i);
  322. e->procname = kstrdup(buf, GFP_KERNEL);
  323. }
  324. e->mode = 0555;
  325. e->child = sd_alloc_ctl_cpu_table(i);
  326. __cpumask_clear_cpu(i, sd_sysctl_cpus);
  327. }
  328. WARN_ON(sd_sysctl_header);
  329. sd_sysctl_header = register_sysctl_table(sd_ctl_root);
  330. }
  331. void dirty_sched_domain_sysctl(int cpu)
  332. {
  333. if (cpumask_available(sd_sysctl_cpus))
  334. __cpumask_set_cpu(cpu, sd_sysctl_cpus);
  335. }
  336. /* may be called multiple times per register */
  337. void unregister_sched_domain_sysctl(void)
  338. {
  339. unregister_sysctl_table(sd_sysctl_header);
  340. sd_sysctl_header = NULL;
  341. }
  342. #endif /* CONFIG_SYSCTL */
  343. #endif /* CONFIG_SMP */
  344. #ifdef CONFIG_FAIR_GROUP_SCHED
  345. static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
  346. {
  347. struct sched_entity *se = tg->se[cpu];
  348. #define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
  349. #define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)schedstat_val(F))
  350. #define PN(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
  351. #define PN_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(F)))
  352. if (!se)
  353. return;
  354. PN(se->exec_start);
  355. PN(se->vruntime);
  356. PN(se->sum_exec_runtime);
  357. if (schedstat_enabled()) {
  358. PN_SCHEDSTAT(se->statistics.wait_start);
  359. PN_SCHEDSTAT(se->statistics.sleep_start);
  360. PN_SCHEDSTAT(se->statistics.block_start);
  361. PN_SCHEDSTAT(se->statistics.sleep_max);
  362. PN_SCHEDSTAT(se->statistics.block_max);
  363. PN_SCHEDSTAT(se->statistics.exec_max);
  364. PN_SCHEDSTAT(se->statistics.slice_max);
  365. PN_SCHEDSTAT(se->statistics.wait_max);
  366. PN_SCHEDSTAT(se->statistics.wait_sum);
  367. P_SCHEDSTAT(se->statistics.wait_count);
  368. }
  369. P(se->load.weight);
  370. #ifdef CONFIG_SMP
  371. P(se->avg.load_avg);
  372. P(se->avg.util_avg);
  373. P(se->avg.runnable_avg);
  374. #endif
  375. #undef PN_SCHEDSTAT
  376. #undef PN
  377. #undef P_SCHEDSTAT
  378. #undef P
  379. }
  380. #endif
  381. #ifdef CONFIG_CGROUP_SCHED
  382. static DEFINE_SPINLOCK(sched_debug_lock);
  383. static char group_path[PATH_MAX];
  384. static void task_group_path(struct task_group *tg, char *path, int plen)
  385. {
  386. if (autogroup_path(tg, path, plen))
  387. return;
  388. cgroup_path(tg->css.cgroup, path, plen);
  389. }
  390. /*
  391. * Only 1 SEQ_printf_task_group_path() caller can use the full length
  392. * group_path[] for cgroup path. Other simultaneous callers will have
  393. * to use a shorter stack buffer. A "..." suffix is appended at the end
  394. * of the stack buffer so that it will show up in case the output length
  395. * matches the given buffer size to indicate possible path name truncation.
  396. */
  397. #define SEQ_printf_task_group_path(m, tg, fmt...) \
  398. { \
  399. if (spin_trylock(&sched_debug_lock)) { \
  400. task_group_path(tg, group_path, sizeof(group_path)); \
  401. SEQ_printf(m, fmt, group_path); \
  402. spin_unlock(&sched_debug_lock); \
  403. } else { \
  404. char buf[128]; \
  405. char *bufend = buf + sizeof(buf) - 3; \
  406. task_group_path(tg, buf, bufend - buf); \
  407. strcpy(bufend - 1, "..."); \
  408. SEQ_printf(m, fmt, buf); \
  409. } \
  410. }
  411. #endif
  412. static void
  413. print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
  414. {
  415. if (rq->curr == p)
  416. SEQ_printf(m, ">R");
  417. else
  418. SEQ_printf(m, " %c", task_state_to_char(p));
  419. SEQ_printf(m, " %15s %5d %9Ld.%06ld %9Ld %5d ",
  420. p->comm, task_pid_nr(p),
  421. SPLIT_NS(p->se.vruntime),
  422. (long long)(p->nvcsw + p->nivcsw),
  423. p->prio);
  424. SEQ_printf(m, "%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld",
  425. SPLIT_NS(schedstat_val_or_zero(p->se.statistics.wait_sum)),
  426. SPLIT_NS(p->se.sum_exec_runtime),
  427. SPLIT_NS(schedstat_val_or_zero(p->se.statistics.sum_sleep_runtime)));
  428. #ifdef CONFIG_NUMA_BALANCING
  429. SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
  430. #endif
  431. #ifdef CONFIG_CGROUP_SCHED
  432. SEQ_printf_task_group_path(m, task_group(p), " %s")
  433. #endif
  434. SEQ_printf(m, "\n");
  435. }
  436. static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
  437. {
  438. struct task_struct *g, *p;
  439. SEQ_printf(m, "\n");
  440. SEQ_printf(m, "runnable tasks:\n");
  441. SEQ_printf(m, " S task PID tree-key switches prio"
  442. " wait-time sum-exec sum-sleep\n");
  443. SEQ_printf(m, "-------------------------------------------------------"
  444. "------------------------------------------------------\n");
  445. rcu_read_lock();
  446. for_each_process_thread(g, p) {
  447. if (task_cpu(p) != rq_cpu)
  448. continue;
  449. print_task(m, rq, p);
  450. }
  451. rcu_read_unlock();
  452. }
  453. void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
  454. {
  455. s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
  456. spread, rq0_min_vruntime, spread0;
  457. struct rq *rq = cpu_rq(cpu);
  458. struct sched_entity *last;
  459. unsigned long flags;
  460. #ifdef CONFIG_FAIR_GROUP_SCHED
  461. SEQ_printf(m, "\n");
  462. SEQ_printf_task_group_path(m, cfs_rq->tg, "cfs_rq[%d]:%s\n", cpu);
  463. #else
  464. SEQ_printf(m, "\n");
  465. SEQ_printf(m, "cfs_rq[%d]:\n", cpu);
  466. #endif
  467. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
  468. SPLIT_NS(cfs_rq->exec_clock));
  469. raw_spin_lock_irqsave(&rq->lock, flags);
  470. if (rb_first_cached(&cfs_rq->tasks_timeline))
  471. MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime;
  472. last = __pick_last_entity(cfs_rq);
  473. if (last)
  474. max_vruntime = last->vruntime;
  475. min_vruntime = cfs_rq->min_vruntime;
  476. rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;
  477. raw_spin_unlock_irqrestore(&rq->lock, flags);
  478. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime",
  479. SPLIT_NS(MIN_vruntime));
  480. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime",
  481. SPLIT_NS(min_vruntime));
  482. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime",
  483. SPLIT_NS(max_vruntime));
  484. spread = max_vruntime - MIN_vruntime;
  485. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread",
  486. SPLIT_NS(spread));
  487. spread0 = min_vruntime - rq0_min_vruntime;
  488. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0",
  489. SPLIT_NS(spread0));
  490. SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
  491. cfs_rq->nr_spread_over);
  492. SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
  493. SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
  494. #ifdef CONFIG_SMP
  495. SEQ_printf(m, " .%-30s: %lu\n", "load_avg",
  496. cfs_rq->avg.load_avg);
  497. SEQ_printf(m, " .%-30s: %lu\n", "runnable_avg",
  498. cfs_rq->avg.runnable_avg);
  499. SEQ_printf(m, " .%-30s: %lu\n", "util_avg",
  500. cfs_rq->avg.util_avg);
  501. SEQ_printf(m, " .%-30s: %u\n", "util_est_enqueued",
  502. cfs_rq->avg.util_est.enqueued);
  503. SEQ_printf(m, " .%-30s: %ld\n", "removed.load_avg",
  504. cfs_rq->removed.load_avg);
  505. SEQ_printf(m, " .%-30s: %ld\n", "removed.util_avg",
  506. cfs_rq->removed.util_avg);
  507. SEQ_printf(m, " .%-30s: %ld\n", "removed.runnable_avg",
  508. cfs_rq->removed.runnable_avg);
  509. #ifdef CONFIG_FAIR_GROUP_SCHED
  510. SEQ_printf(m, " .%-30s: %lu\n", "tg_load_avg_contrib",
  511. cfs_rq->tg_load_avg_contrib);
  512. SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg",
  513. atomic_long_read(&cfs_rq->tg->load_avg));
  514. #endif
  515. #endif
  516. #ifdef CONFIG_CFS_BANDWIDTH
  517. SEQ_printf(m, " .%-30s: %d\n", "throttled",
  518. cfs_rq->throttled);
  519. SEQ_printf(m, " .%-30s: %d\n", "throttle_count",
  520. cfs_rq->throttle_count);
  521. #endif
  522. #ifdef CONFIG_FAIR_GROUP_SCHED
  523. print_cfs_group_stats(m, cpu, cfs_rq->tg);
  524. #endif
  525. }
  526. void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
  527. {
  528. #ifdef CONFIG_RT_GROUP_SCHED
  529. SEQ_printf(m, "\n");
  530. SEQ_printf_task_group_path(m, rt_rq->tg, "rt_rq[%d]:%s\n", cpu);
  531. #else
  532. SEQ_printf(m, "\n");
  533. SEQ_printf(m, "rt_rq[%d]:\n", cpu);
  534. #endif
  535. #define P(x) \
  536. SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
  537. #define PU(x) \
  538. SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(rt_rq->x))
  539. #define PN(x) \
  540. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
  541. PU(rt_nr_running);
  542. #ifdef CONFIG_SMP
  543. PU(rt_nr_migratory);
  544. #endif
  545. P(rt_throttled);
  546. PN(rt_time);
  547. PN(rt_runtime);
  548. #undef PN
  549. #undef PU
  550. #undef P
  551. }
  552. void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq)
  553. {
  554. struct dl_bw *dl_bw;
  555. SEQ_printf(m, "\n");
  556. SEQ_printf(m, "dl_rq[%d]:\n", cpu);
  557. #define PU(x) \
  558. SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(dl_rq->x))
  559. PU(dl_nr_running);
  560. #ifdef CONFIG_SMP
  561. PU(dl_nr_migratory);
  562. dl_bw = &cpu_rq(cpu)->rd->dl_bw;
  563. #else
  564. dl_bw = &dl_rq->dl_bw;
  565. #endif
  566. SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->bw", dl_bw->bw);
  567. SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->total_bw", dl_bw->total_bw);
  568. #undef PU
  569. }
  570. static void print_cpu(struct seq_file *m, int cpu)
  571. {
  572. struct rq *rq = cpu_rq(cpu);
  573. #ifdef CONFIG_X86
  574. {
  575. unsigned int freq = cpu_khz ? : 1;
  576. SEQ_printf(m, "cpu#%d, %u.%03u MHz\n",
  577. cpu, freq / 1000, (freq % 1000));
  578. }
  579. #else
  580. SEQ_printf(m, "cpu#%d\n", cpu);
  581. #endif
  582. #define P(x) \
  583. do { \
  584. if (sizeof(rq->x) == 4) \
  585. SEQ_printf(m, " .%-30s: %ld\n", #x, (long)(rq->x)); \
  586. else \
  587. SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rq->x));\
  588. } while (0)
  589. #define PN(x) \
  590. SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x))
  591. P(nr_running);
  592. P(nr_switches);
  593. P(nr_uninterruptible);
  594. PN(next_balance);
  595. SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));
  596. PN(clock);
  597. PN(clock_task);
  598. #undef P
  599. #undef PN
  600. #ifdef CONFIG_SMP
  601. #define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n);
  602. P64(avg_idle);
  603. P64(max_idle_balance_cost);
  604. #undef P64
  605. #endif
  606. #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, schedstat_val(rq->n));
  607. if (schedstat_enabled()) {
  608. P(yld_count);
  609. P(sched_count);
  610. P(sched_goidle);
  611. P(ttwu_count);
  612. P(ttwu_local);
  613. }
  614. #undef P
  615. print_cfs_stats(m, cpu);
  616. print_rt_stats(m, cpu);
  617. print_dl_stats(m, cpu);
  618. print_rq(m, rq, cpu);
  619. SEQ_printf(m, "\n");
  620. }
  621. static const char *sched_tunable_scaling_names[] = {
  622. "none",
  623. "logarithmic",
  624. "linear"
  625. };
  626. static void sched_debug_header(struct seq_file *m)
  627. {
  628. u64 ktime, sched_clk, cpu_clk;
  629. unsigned long flags;
  630. local_irq_save(flags);
  631. ktime = ktime_to_ns(ktime_get());
  632. sched_clk = sched_clock();
  633. cpu_clk = local_clock();
  634. local_irq_restore(flags);
  635. SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n",
  636. init_utsname()->release,
  637. (int)strcspn(init_utsname()->version, " "),
  638. init_utsname()->version);
  639. #define P(x) \
  640. SEQ_printf(m, "%-40s: %Ld\n", #x, (long long)(x))
  641. #define PN(x) \
  642. SEQ_printf(m, "%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
  643. PN(ktime);
  644. PN(sched_clk);
  645. PN(cpu_clk);
  646. P(jiffies);
  647. #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
  648. P(sched_clock_stable());
  649. #endif
  650. #undef PN
  651. #undef P
  652. SEQ_printf(m, "\n");
  653. SEQ_printf(m, "sysctl_sched\n");
  654. #define P(x) \
  655. SEQ_printf(m, " .%-40s: %Ld\n", #x, (long long)(x))
  656. #define PN(x) \
  657. SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
  658. PN(sysctl_sched_latency);
  659. PN(sysctl_sched_min_granularity);
  660. PN(sysctl_sched_wakeup_granularity);
  661. P(sysctl_sched_child_runs_first);
  662. P(sysctl_sched_features);
  663. #undef PN
  664. #undef P
  665. SEQ_printf(m, " .%-40s: %d (%s)\n",
  666. "sysctl_sched_tunable_scaling",
  667. sysctl_sched_tunable_scaling,
  668. sched_tunable_scaling_names[sysctl_sched_tunable_scaling]);
  669. SEQ_printf(m, "\n");
  670. }
  671. static int sched_debug_show(struct seq_file *m, void *v)
  672. {
  673. int cpu = (unsigned long)(v - 2);
  674. if (cpu != -1)
  675. print_cpu(m, cpu);
  676. else
  677. sched_debug_header(m);
  678. return 0;
  679. }
  680. void sysrq_sched_debug_show(void)
  681. {
  682. int cpu;
  683. sched_debug_header(NULL);
  684. for_each_online_cpu(cpu) {
  685. /*
  686. * Need to reset softlockup watchdogs on all CPUs, because
  687. * another CPU might be blocked waiting for us to process
  688. * an IPI or stop_machine.
  689. */
  690. touch_nmi_watchdog();
  691. touch_all_softlockup_watchdogs();
  692. print_cpu(NULL, cpu);
  693. }
  694. }
  695. /*
  696. * This itererator needs some explanation.
  697. * It returns 1 for the header position.
  698. * This means 2 is CPU 0.
  699. * In a hotplugged system some CPUs, including CPU 0, may be missing so we have
  700. * to use cpumask_* to iterate over the CPUs.
  701. */
  702. static void *sched_debug_start(struct seq_file *file, loff_t *offset)
  703. {
  704. unsigned long n = *offset;
  705. if (n == 0)
  706. return (void *) 1;
  707. n--;
  708. if (n > 0)
  709. n = cpumask_next(n - 1, cpu_online_mask);
  710. else
  711. n = cpumask_first(cpu_online_mask);
  712. *offset = n + 1;
  713. if (n < nr_cpu_ids)
  714. return (void *)(unsigned long)(n + 2);
  715. return NULL;
  716. }
  717. static void *sched_debug_next(struct seq_file *file, void *data, loff_t *offset)
  718. {
  719. (*offset)++;
  720. return sched_debug_start(file, offset);
  721. }
  722. static void sched_debug_stop(struct seq_file *file, void *data)
  723. {
  724. }
  725. static const struct seq_operations sched_debug_sops = {
  726. .start = sched_debug_start,
  727. .next = sched_debug_next,
  728. .stop = sched_debug_stop,
  729. .show = sched_debug_show,
  730. };
  731. static int __init init_sched_debug_procfs(void)
  732. {
  733. if (!proc_create_seq("sched_debug", 0444, NULL, &sched_debug_sops))
  734. return -ENOMEM;
  735. return 0;
  736. }
  737. __initcall(init_sched_debug_procfs);
  738. #define __PS(S, F) SEQ_printf(m, "%-45s:%21Ld\n", S, (long long)(F))
  739. #define __P(F) __PS(#F, F)
  740. #define P(F) __PS(#F, p->F)
  741. #define PM(F, M) __PS(#F, p->F & (M))
  742. #define __PSN(S, F) SEQ_printf(m, "%-45s:%14Ld.%06ld\n", S, SPLIT_NS((long long)(F)))
  743. #define __PN(F) __PSN(#F, F)
  744. #define PN(F) __PSN(#F, p->F)
  745. #ifdef CONFIG_NUMA_BALANCING
  746. void print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
  747. unsigned long tpf, unsigned long gsf, unsigned long gpf)
  748. {
  749. SEQ_printf(m, "numa_faults node=%d ", node);
  750. SEQ_printf(m, "task_private=%lu task_shared=%lu ", tpf, tsf);
  751. SEQ_printf(m, "group_private=%lu group_shared=%lu\n", gpf, gsf);
  752. }
  753. #endif
  754. static void sched_show_numa(struct task_struct *p, struct seq_file *m)
  755. {
  756. #ifdef CONFIG_NUMA_BALANCING
  757. if (p->mm)
  758. P(mm->numa_scan_seq);
  759. P(numa_pages_migrated);
  760. P(numa_preferred_nid);
  761. P(total_numa_faults);
  762. SEQ_printf(m, "current_node=%d, numa_group_id=%d\n",
  763. task_node(p), task_numa_group_id(p));
  764. show_numa_stats(p, m);
  765. #endif
  766. }
  767. void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
  768. struct seq_file *m)
  769. {
  770. unsigned long nr_switches;
  771. SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns),
  772. get_nr_threads(p));
  773. SEQ_printf(m,
  774. "---------------------------------------------------------"
  775. "----------\n");
  776. #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->F))
  777. #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->F))
  778. PN(se.exec_start);
  779. PN(se.vruntime);
  780. PN(se.sum_exec_runtime);
  781. nr_switches = p->nvcsw + p->nivcsw;
  782. P(se.nr_migrations);
  783. if (schedstat_enabled()) {
  784. u64 avg_atom, avg_per_cpu;
  785. PN_SCHEDSTAT(se.statistics.sum_sleep_runtime);
  786. PN_SCHEDSTAT(se.statistics.wait_start);
  787. PN_SCHEDSTAT(se.statistics.sleep_start);
  788. PN_SCHEDSTAT(se.statistics.block_start);
  789. PN_SCHEDSTAT(se.statistics.sleep_max);
  790. PN_SCHEDSTAT(se.statistics.block_max);
  791. PN_SCHEDSTAT(se.statistics.exec_max);
  792. PN_SCHEDSTAT(se.statistics.slice_max);
  793. PN_SCHEDSTAT(se.statistics.wait_max);
  794. PN_SCHEDSTAT(se.statistics.wait_sum);
  795. P_SCHEDSTAT(se.statistics.wait_count);
  796. PN_SCHEDSTAT(se.statistics.iowait_sum);
  797. P_SCHEDSTAT(se.statistics.iowait_count);
  798. P_SCHEDSTAT(se.statistics.nr_migrations_cold);
  799. P_SCHEDSTAT(se.statistics.nr_failed_migrations_affine);
  800. P_SCHEDSTAT(se.statistics.nr_failed_migrations_running);
  801. P_SCHEDSTAT(se.statistics.nr_failed_migrations_hot);
  802. P_SCHEDSTAT(se.statistics.nr_forced_migrations);
  803. P_SCHEDSTAT(se.statistics.nr_wakeups);
  804. P_SCHEDSTAT(se.statistics.nr_wakeups_sync);
  805. P_SCHEDSTAT(se.statistics.nr_wakeups_migrate);
  806. P_SCHEDSTAT(se.statistics.nr_wakeups_local);
  807. P_SCHEDSTAT(se.statistics.nr_wakeups_remote);
  808. P_SCHEDSTAT(se.statistics.nr_wakeups_affine);
  809. P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts);
  810. P_SCHEDSTAT(se.statistics.nr_wakeups_passive);
  811. P_SCHEDSTAT(se.statistics.nr_wakeups_idle);
  812. avg_atom = p->se.sum_exec_runtime;
  813. if (nr_switches)
  814. avg_atom = div64_ul(avg_atom, nr_switches);
  815. else
  816. avg_atom = -1LL;
  817. avg_per_cpu = p->se.sum_exec_runtime;
  818. if (p->se.nr_migrations) {
  819. avg_per_cpu = div64_u64(avg_per_cpu,
  820. p->se.nr_migrations);
  821. } else {
  822. avg_per_cpu = -1LL;
  823. }
  824. __PN(avg_atom);
  825. __PN(avg_per_cpu);
  826. }
  827. __P(nr_switches);
  828. __PS("nr_voluntary_switches", p->nvcsw);
  829. __PS("nr_involuntary_switches", p->nivcsw);
  830. P(se.load.weight);
  831. #ifdef CONFIG_SMP
  832. P(se.avg.load_sum);
  833. P(se.avg.runnable_sum);
  834. P(se.avg.util_sum);
  835. P(se.avg.load_avg);
  836. P(se.avg.runnable_avg);
  837. P(se.avg.util_avg);
  838. P(se.avg.last_update_time);
  839. P(se.avg.util_est.ewma);
  840. PM(se.avg.util_est.enqueued, ~UTIL_AVG_UNCHANGED);
  841. #endif
  842. #ifdef CONFIG_UCLAMP_TASK
  843. __PS("uclamp.min", p->uclamp_req[UCLAMP_MIN].value);
  844. __PS("uclamp.max", p->uclamp_req[UCLAMP_MAX].value);
  845. __PS("effective uclamp.min", uclamp_eff_value(p, UCLAMP_MIN));
  846. __PS("effective uclamp.max", uclamp_eff_value(p, UCLAMP_MAX));
  847. #endif
  848. P(policy);
  849. P(prio);
  850. if (task_has_dl_policy(p)) {
  851. P(dl.runtime);
  852. P(dl.deadline);
  853. }
  854. #undef PN_SCHEDSTAT
  855. #undef P_SCHEDSTAT
  856. {
  857. unsigned int this_cpu = raw_smp_processor_id();
  858. u64 t0, t1;
  859. t0 = cpu_clock(this_cpu);
  860. t1 = cpu_clock(this_cpu);
  861. __PS("clock-delta", t1-t0);
  862. }
  863. sched_show_numa(p, m);
  864. }
  865. void proc_sched_set_task(struct task_struct *p)
  866. {
  867. #ifdef CONFIG_SCHEDSTATS
  868. memset(&p->se.statistics, 0, sizeof(p->se.statistics));
  869. #endif
  870. }