profile.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/kernel/profile.c
  4. * Simple profiling. Manages a direct-mapped profile hit count buffer,
  5. * with configurable resolution, support for restricting the cpus on
  6. * which profiling is done, and switching between cpu time and
  7. * schedule() calls via kernel command line parameters passed at boot.
  8. *
  9. * Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
  10. * Red Hat, July 2004
  11. * Consolidation of architecture support code for profiling,
  12. * Nadia Yvette Chambers, Oracle, July 2004
  13. * Amortized hit count accounting via per-cpu open-addressed hashtables
  14. * to resolve timer interrupt livelocks, Nadia Yvette Chambers,
  15. * Oracle, 2004
  16. */
  17. #include <linux/export.h>
  18. #include <linux/profile.h>
  19. #include <linux/memblock.h>
  20. #include <linux/notifier.h>
  21. #include <linux/mm.h>
  22. #include <linux/cpumask.h>
  23. #include <linux/cpu.h>
  24. #include <linux/highmem.h>
  25. #include <linux/mutex.h>
  26. #include <linux/slab.h>
  27. #include <linux/vmalloc.h>
  28. #include <linux/sched/stat.h>
  29. #include <asm/sections.h>
  30. #include <asm/irq_regs.h>
  31. #include <asm/ptrace.h>
  32. struct profile_hit {
  33. u32 pc, hits;
  34. };
  35. #define PROFILE_GRPSHIFT 3
  36. #define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
  37. #define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
  38. #define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
  39. static atomic_t *prof_buffer;
  40. static unsigned long prof_len;
  41. static unsigned short int prof_shift;
  42. int prof_on __read_mostly;
  43. EXPORT_SYMBOL_GPL(prof_on);
  44. static cpumask_var_t prof_cpu_mask;
  45. #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
  46. static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
  47. static DEFINE_PER_CPU(int, cpu_profile_flip);
  48. static DEFINE_MUTEX(profile_flip_mutex);
  49. #endif /* CONFIG_SMP */
  50. int profile_setup(char *str)
  51. {
  52. static const char schedstr[] = "schedule";
  53. static const char sleepstr[] = "sleep";
  54. static const char kvmstr[] = "kvm";
  55. int par;
  56. if (!strncmp(str, sleepstr, strlen(sleepstr))) {
  57. #ifdef CONFIG_SCHEDSTATS
  58. force_schedstat_enabled();
  59. prof_on = SLEEP_PROFILING;
  60. if (str[strlen(sleepstr)] == ',')
  61. str += strlen(sleepstr) + 1;
  62. if (get_option(&str, &par))
  63. prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
  64. pr_info("kernel sleep profiling enabled (shift: %u)\n",
  65. prof_shift);
  66. #else
  67. pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
  68. #endif /* CONFIG_SCHEDSTATS */
  69. } else if (!strncmp(str, schedstr, strlen(schedstr))) {
  70. prof_on = SCHED_PROFILING;
  71. if (str[strlen(schedstr)] == ',')
  72. str += strlen(schedstr) + 1;
  73. if (get_option(&str, &par))
  74. prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
  75. pr_info("kernel schedule profiling enabled (shift: %u)\n",
  76. prof_shift);
  77. } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
  78. prof_on = KVM_PROFILING;
  79. if (str[strlen(kvmstr)] == ',')
  80. str += strlen(kvmstr) + 1;
  81. if (get_option(&str, &par))
  82. prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
  83. pr_info("kernel KVM profiling enabled (shift: %u)\n",
  84. prof_shift);
  85. } else if (get_option(&str, &par)) {
  86. prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
  87. prof_on = CPU_PROFILING;
  88. pr_info("kernel profiling enabled (shift: %u)\n",
  89. prof_shift);
  90. }
  91. return 1;
  92. }
  93. __setup("profile=", profile_setup);
  94. int __ref profile_init(void)
  95. {
  96. int buffer_bytes;
  97. if (!prof_on)
  98. return 0;
  99. /* only text is profiled */
  100. prof_len = (_etext - _stext) >> prof_shift;
  101. buffer_bytes = prof_len*sizeof(atomic_t);
  102. if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
  103. return -ENOMEM;
  104. cpumask_copy(prof_cpu_mask, cpu_possible_mask);
  105. prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
  106. if (prof_buffer)
  107. return 0;
  108. prof_buffer = alloc_pages_exact(buffer_bytes,
  109. GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
  110. if (prof_buffer)
  111. return 0;
  112. prof_buffer = vzalloc(buffer_bytes);
  113. if (prof_buffer)
  114. return 0;
  115. free_cpumask_var(prof_cpu_mask);
  116. return -ENOMEM;
  117. }
  118. /* Profile event notifications */
  119. static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
  120. static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
  121. static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
  122. void profile_task_exit(struct task_struct *task)
  123. {
  124. blocking_notifier_call_chain(&task_exit_notifier, 0, task);
  125. }
  126. int profile_handoff_task(struct task_struct *task)
  127. {
  128. int ret;
  129. ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
  130. return (ret == NOTIFY_OK) ? 1 : 0;
  131. }
  132. void profile_munmap(unsigned long addr)
  133. {
  134. blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
  135. }
  136. int task_handoff_register(struct notifier_block *n)
  137. {
  138. return atomic_notifier_chain_register(&task_free_notifier, n);
  139. }
  140. EXPORT_SYMBOL_GPL(task_handoff_register);
  141. int task_handoff_unregister(struct notifier_block *n)
  142. {
  143. return atomic_notifier_chain_unregister(&task_free_notifier, n);
  144. }
  145. EXPORT_SYMBOL_GPL(task_handoff_unregister);
  146. int profile_event_register(enum profile_type type, struct notifier_block *n)
  147. {
  148. int err = -EINVAL;
  149. switch (type) {
  150. case PROFILE_TASK_EXIT:
  151. err = blocking_notifier_chain_register(
  152. &task_exit_notifier, n);
  153. break;
  154. case PROFILE_MUNMAP:
  155. err = blocking_notifier_chain_register(
  156. &munmap_notifier, n);
  157. break;
  158. }
  159. return err;
  160. }
  161. EXPORT_SYMBOL_GPL(profile_event_register);
  162. int profile_event_unregister(enum profile_type type, struct notifier_block *n)
  163. {
  164. int err = -EINVAL;
  165. switch (type) {
  166. case PROFILE_TASK_EXIT:
  167. err = blocking_notifier_chain_unregister(
  168. &task_exit_notifier, n);
  169. break;
  170. case PROFILE_MUNMAP:
  171. err = blocking_notifier_chain_unregister(
  172. &munmap_notifier, n);
  173. break;
  174. }
  175. return err;
  176. }
  177. EXPORT_SYMBOL_GPL(profile_event_unregister);
  178. #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
  179. /*
  180. * Each cpu has a pair of open-addressed hashtables for pending
  181. * profile hits. read_profile() IPI's all cpus to request them
  182. * to flip buffers and flushes their contents to prof_buffer itself.
  183. * Flip requests are serialized by the profile_flip_mutex. The sole
  184. * use of having a second hashtable is for avoiding cacheline
  185. * contention that would otherwise happen during flushes of pending
  186. * profile hits required for the accuracy of reported profile hits
  187. * and so resurrect the interrupt livelock issue.
  188. *
  189. * The open-addressed hashtables are indexed by profile buffer slot
  190. * and hold the number of pending hits to that profile buffer slot on
  191. * a cpu in an entry. When the hashtable overflows, all pending hits
  192. * are accounted to their corresponding profile buffer slots with
  193. * atomic_add() and the hashtable emptied. As numerous pending hits
  194. * may be accounted to a profile buffer slot in a hashtable entry,
  195. * this amortizes a number of atomic profile buffer increments likely
  196. * to be far larger than the number of entries in the hashtable,
  197. * particularly given that the number of distinct profile buffer
  198. * positions to which hits are accounted during short intervals (e.g.
  199. * several seconds) is usually very small. Exclusion from buffer
  200. * flipping is provided by interrupt disablement (note that for
  201. * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
  202. * process context).
  203. * The hash function is meant to be lightweight as opposed to strong,
  204. * and was vaguely inspired by ppc64 firmware-supported inverted
  205. * pagetable hash functions, but uses a full hashtable full of finite
  206. * collision chains, not just pairs of them.
  207. *
  208. * -- nyc
  209. */
  210. static void __profile_flip_buffers(void *unused)
  211. {
  212. int cpu = smp_processor_id();
  213. per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
  214. }
  215. static void profile_flip_buffers(void)
  216. {
  217. int i, j, cpu;
  218. mutex_lock(&profile_flip_mutex);
  219. j = per_cpu(cpu_profile_flip, get_cpu());
  220. put_cpu();
  221. on_each_cpu(__profile_flip_buffers, NULL, 1);
  222. for_each_online_cpu(cpu) {
  223. struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
  224. for (i = 0; i < NR_PROFILE_HIT; ++i) {
  225. if (!hits[i].hits) {
  226. if (hits[i].pc)
  227. hits[i].pc = 0;
  228. continue;
  229. }
  230. atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
  231. hits[i].hits = hits[i].pc = 0;
  232. }
  233. }
  234. mutex_unlock(&profile_flip_mutex);
  235. }
  236. static void profile_discard_flip_buffers(void)
  237. {
  238. int i, cpu;
  239. mutex_lock(&profile_flip_mutex);
  240. i = per_cpu(cpu_profile_flip, get_cpu());
  241. put_cpu();
  242. on_each_cpu(__profile_flip_buffers, NULL, 1);
  243. for_each_online_cpu(cpu) {
  244. struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
  245. memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
  246. }
  247. mutex_unlock(&profile_flip_mutex);
  248. }
  249. static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
  250. {
  251. unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
  252. int i, j, cpu;
  253. struct profile_hit *hits;
  254. pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
  255. i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
  256. secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
  257. cpu = get_cpu();
  258. hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
  259. if (!hits) {
  260. put_cpu();
  261. return;
  262. }
  263. /*
  264. * We buffer the global profiler buffer into a per-CPU
  265. * queue and thus reduce the number of global (and possibly
  266. * NUMA-alien) accesses. The write-queue is self-coalescing:
  267. */
  268. local_irq_save(flags);
  269. do {
  270. for (j = 0; j < PROFILE_GRPSZ; ++j) {
  271. if (hits[i + j].pc == pc) {
  272. hits[i + j].hits += nr_hits;
  273. goto out;
  274. } else if (!hits[i + j].hits) {
  275. hits[i + j].pc = pc;
  276. hits[i + j].hits = nr_hits;
  277. goto out;
  278. }
  279. }
  280. i = (i + secondary) & (NR_PROFILE_HIT - 1);
  281. } while (i != primary);
  282. /*
  283. * Add the current hit(s) and flush the write-queue out
  284. * to the global buffer:
  285. */
  286. atomic_add(nr_hits, &prof_buffer[pc]);
  287. for (i = 0; i < NR_PROFILE_HIT; ++i) {
  288. atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
  289. hits[i].pc = hits[i].hits = 0;
  290. }
  291. out:
  292. local_irq_restore(flags);
  293. put_cpu();
  294. }
  295. static int profile_dead_cpu(unsigned int cpu)
  296. {
  297. struct page *page;
  298. int i;
  299. if (cpumask_available(prof_cpu_mask))
  300. cpumask_clear_cpu(cpu, prof_cpu_mask);
  301. for (i = 0; i < 2; i++) {
  302. if (per_cpu(cpu_profile_hits, cpu)[i]) {
  303. page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
  304. per_cpu(cpu_profile_hits, cpu)[i] = NULL;
  305. __free_page(page);
  306. }
  307. }
  308. return 0;
  309. }
  310. static int profile_prepare_cpu(unsigned int cpu)
  311. {
  312. int i, node = cpu_to_mem(cpu);
  313. struct page *page;
  314. per_cpu(cpu_profile_flip, cpu) = 0;
  315. for (i = 0; i < 2; i++) {
  316. if (per_cpu(cpu_profile_hits, cpu)[i])
  317. continue;
  318. page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
  319. if (!page) {
  320. profile_dead_cpu(cpu);
  321. return -ENOMEM;
  322. }
  323. per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);
  324. }
  325. return 0;
  326. }
  327. static int profile_online_cpu(unsigned int cpu)
  328. {
  329. if (cpumask_available(prof_cpu_mask))
  330. cpumask_set_cpu(cpu, prof_cpu_mask);
  331. return 0;
  332. }
  333. #else /* !CONFIG_SMP */
  334. #define profile_flip_buffers() do { } while (0)
  335. #define profile_discard_flip_buffers() do { } while (0)
  336. static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
  337. {
  338. unsigned long pc;
  339. pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
  340. atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
  341. }
  342. #endif /* !CONFIG_SMP */
  343. void profile_hits(int type, void *__pc, unsigned int nr_hits)
  344. {
  345. if (prof_on != type || !prof_buffer)
  346. return;
  347. do_profile_hits(type, __pc, nr_hits);
  348. }
  349. EXPORT_SYMBOL_GPL(profile_hits);
  350. void profile_tick(int type)
  351. {
  352. struct pt_regs *regs = get_irq_regs();
  353. if (!user_mode(regs) && cpumask_available(prof_cpu_mask) &&
  354. cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
  355. profile_hit(type, (void *)profile_pc(regs));
  356. }
  357. #ifdef CONFIG_PROC_FS
  358. #include <linux/proc_fs.h>
  359. #include <linux/seq_file.h>
  360. #include <linux/uaccess.h>
  361. static int prof_cpu_mask_proc_show(struct seq_file *m, void *v)
  362. {
  363. seq_printf(m, "%*pb\n", cpumask_pr_args(prof_cpu_mask));
  364. return 0;
  365. }
  366. static int prof_cpu_mask_proc_open(struct inode *inode, struct file *file)
  367. {
  368. return single_open(file, prof_cpu_mask_proc_show, NULL);
  369. }
  370. static ssize_t prof_cpu_mask_proc_write(struct file *file,
  371. const char __user *buffer, size_t count, loff_t *pos)
  372. {
  373. cpumask_var_t new_value;
  374. int err;
  375. if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
  376. return -ENOMEM;
  377. err = cpumask_parse_user(buffer, count, new_value);
  378. if (!err) {
  379. cpumask_copy(prof_cpu_mask, new_value);
  380. err = count;
  381. }
  382. free_cpumask_var(new_value);
  383. return err;
  384. }
  385. static const struct proc_ops prof_cpu_mask_proc_ops = {
  386. .proc_open = prof_cpu_mask_proc_open,
  387. .proc_read = seq_read,
  388. .proc_lseek = seq_lseek,
  389. .proc_release = single_release,
  390. .proc_write = prof_cpu_mask_proc_write,
  391. };
  392. void create_prof_cpu_mask(void)
  393. {
  394. /* create /proc/irq/prof_cpu_mask */
  395. proc_create("irq/prof_cpu_mask", 0600, NULL, &prof_cpu_mask_proc_ops);
  396. }
  397. /*
  398. * This function accesses profiling information. The returned data is
  399. * binary: the sampling step and the actual contents of the profile
  400. * buffer. Use of the program readprofile is recommended in order to
  401. * get meaningful info out of these data.
  402. */
  403. static ssize_t
  404. read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  405. {
  406. unsigned long p = *ppos;
  407. ssize_t read;
  408. char *pnt;
  409. unsigned long sample_step = 1UL << prof_shift;
  410. profile_flip_buffers();
  411. if (p >= (prof_len+1)*sizeof(unsigned int))
  412. return 0;
  413. if (count > (prof_len+1)*sizeof(unsigned int) - p)
  414. count = (prof_len+1)*sizeof(unsigned int) - p;
  415. read = 0;
  416. while (p < sizeof(unsigned int) && count > 0) {
  417. if (put_user(*((char *)(&sample_step)+p), buf))
  418. return -EFAULT;
  419. buf++; p++; count--; read++;
  420. }
  421. pnt = (char *)prof_buffer + p - sizeof(atomic_t);
  422. if (copy_to_user(buf, (void *)pnt, count))
  423. return -EFAULT;
  424. read += count;
  425. *ppos += read;
  426. return read;
  427. }
  428. /*
  429. * Writing to /proc/profile resets the counters
  430. *
  431. * Writing a 'profiling multiplier' value into it also re-sets the profiling
  432. * interrupt frequency, on architectures that support this.
  433. */
  434. static ssize_t write_profile(struct file *file, const char __user *buf,
  435. size_t count, loff_t *ppos)
  436. {
  437. #ifdef CONFIG_SMP
  438. extern int setup_profiling_timer(unsigned int multiplier);
  439. if (count == sizeof(int)) {
  440. unsigned int multiplier;
  441. if (copy_from_user(&multiplier, buf, sizeof(int)))
  442. return -EFAULT;
  443. if (setup_profiling_timer(multiplier))
  444. return -EINVAL;
  445. }
  446. #endif
  447. profile_discard_flip_buffers();
  448. memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
  449. return count;
  450. }
  451. static const struct proc_ops profile_proc_ops = {
  452. .proc_read = read_profile,
  453. .proc_write = write_profile,
  454. .proc_lseek = default_llseek,
  455. };
  456. int __ref create_proc_profile(void)
  457. {
  458. struct proc_dir_entry *entry;
  459. #ifdef CONFIG_SMP
  460. enum cpuhp_state online_state;
  461. #endif
  462. int err = 0;
  463. if (!prof_on)
  464. return 0;
  465. #ifdef CONFIG_SMP
  466. err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
  467. profile_prepare_cpu, profile_dead_cpu);
  468. if (err)
  469. return err;
  470. err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
  471. profile_online_cpu, NULL);
  472. if (err < 0)
  473. goto err_state_prep;
  474. online_state = err;
  475. err = 0;
  476. #endif
  477. entry = proc_create("profile", S_IWUSR | S_IRUGO,
  478. NULL, &profile_proc_ops);
  479. if (!entry)
  480. goto err_state_onl;
  481. proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
  482. return err;
  483. err_state_onl:
  484. #ifdef CONFIG_SMP
  485. cpuhp_remove_state(online_state);
  486. err_state_prep:
  487. cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
  488. #endif
  489. return err;
  490. }
  491. subsys_initcall(create_proc_profile);
  492. #endif /* CONFIG_PROC_FS */