callchain.c 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Performance events callchain code, extracted from core.c:
  4. *
  5. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  6. * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  7. * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  8. * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  9. */
  10. #include <linux/perf_event.h>
  11. #include <linux/slab.h>
  12. #include <linux/sched/task_stack.h>
  13. #include "internal.h"
  14. struct callchain_cpus_entries {
  15. struct rcu_head rcu_head;
  16. struct perf_callchain_entry *cpu_entries[];
  17. };
  18. int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
  19. int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
  20. static inline size_t perf_callchain_entry__sizeof(void)
  21. {
  22. return (sizeof(struct perf_callchain_entry) +
  23. sizeof(__u64) * (sysctl_perf_event_max_stack +
  24. sysctl_perf_event_max_contexts_per_stack));
  25. }
  26. static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
  27. static atomic_t nr_callchain_events;
  28. static DEFINE_MUTEX(callchain_mutex);
  29. static struct callchain_cpus_entries *callchain_cpus_entries;
  30. __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
  31. struct pt_regs *regs)
  32. {
  33. }
  34. __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
  35. struct pt_regs *regs)
  36. {
  37. }
  38. static void release_callchain_buffers_rcu(struct rcu_head *head)
  39. {
  40. struct callchain_cpus_entries *entries;
  41. int cpu;
  42. entries = container_of(head, struct callchain_cpus_entries, rcu_head);
  43. for_each_possible_cpu(cpu)
  44. kfree(entries->cpu_entries[cpu]);
  45. kfree(entries);
  46. }
  47. static void release_callchain_buffers(void)
  48. {
  49. struct callchain_cpus_entries *entries;
  50. entries = callchain_cpus_entries;
  51. RCU_INIT_POINTER(callchain_cpus_entries, NULL);
  52. call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
  53. }
  54. static int alloc_callchain_buffers(void)
  55. {
  56. int cpu;
  57. int size;
  58. struct callchain_cpus_entries *entries;
  59. /*
  60. * We can't use the percpu allocation API for data that can be
  61. * accessed from NMI. Use a temporary manual per cpu allocation
  62. * until that gets sorted out.
  63. */
  64. size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
  65. entries = kzalloc(size, GFP_KERNEL);
  66. if (!entries)
  67. return -ENOMEM;
  68. size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
  69. for_each_possible_cpu(cpu) {
  70. entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
  71. cpu_to_node(cpu));
  72. if (!entries->cpu_entries[cpu])
  73. goto fail;
  74. }
  75. rcu_assign_pointer(callchain_cpus_entries, entries);
  76. return 0;
  77. fail:
  78. for_each_possible_cpu(cpu)
  79. kfree(entries->cpu_entries[cpu]);
  80. kfree(entries);
  81. return -ENOMEM;
  82. }
  83. int get_callchain_buffers(int event_max_stack)
  84. {
  85. int err = 0;
  86. int count;
  87. mutex_lock(&callchain_mutex);
  88. count = atomic_inc_return(&nr_callchain_events);
  89. if (WARN_ON_ONCE(count < 1)) {
  90. err = -EINVAL;
  91. goto exit;
  92. }
  93. /*
  94. * If requesting per event more than the global cap,
  95. * return a different error to help userspace figure
  96. * this out.
  97. *
  98. * And also do it here so that we have &callchain_mutex held.
  99. */
  100. if (event_max_stack > sysctl_perf_event_max_stack) {
  101. err = -EOVERFLOW;
  102. goto exit;
  103. }
  104. if (count == 1)
  105. err = alloc_callchain_buffers();
  106. exit:
  107. if (err)
  108. atomic_dec(&nr_callchain_events);
  109. mutex_unlock(&callchain_mutex);
  110. return err;
  111. }
  112. void put_callchain_buffers(void)
  113. {
  114. if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
  115. release_callchain_buffers();
  116. mutex_unlock(&callchain_mutex);
  117. }
  118. }
  119. struct perf_callchain_entry *get_callchain_entry(int *rctx)
  120. {
  121. int cpu;
  122. struct callchain_cpus_entries *entries;
  123. *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
  124. if (*rctx == -1)
  125. return NULL;
  126. entries = rcu_dereference(callchain_cpus_entries);
  127. if (!entries) {
  128. put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx);
  129. return NULL;
  130. }
  131. cpu = smp_processor_id();
  132. return (((void *)entries->cpu_entries[cpu]) +
  133. (*rctx * perf_callchain_entry__sizeof()));
  134. }
  135. void
  136. put_callchain_entry(int rctx)
  137. {
  138. put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
  139. }
  140. struct perf_callchain_entry *
  141. get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
  142. u32 max_stack, bool crosstask, bool add_mark)
  143. {
  144. struct perf_callchain_entry *entry;
  145. struct perf_callchain_entry_ctx ctx;
  146. int rctx;
  147. entry = get_callchain_entry(&rctx);
  148. if (!entry)
  149. return NULL;
  150. ctx.entry = entry;
  151. ctx.max_stack = max_stack;
  152. ctx.nr = entry->nr = init_nr;
  153. ctx.contexts = 0;
  154. ctx.contexts_maxed = false;
  155. if (kernel && !user_mode(regs)) {
  156. if (add_mark)
  157. perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
  158. perf_callchain_kernel(&ctx, regs);
  159. }
  160. if (user) {
  161. if (!user_mode(regs)) {
  162. if (current->mm)
  163. regs = task_pt_regs(current);
  164. else
  165. regs = NULL;
  166. }
  167. if (regs) {
  168. mm_segment_t fs;
  169. if (crosstask)
  170. goto exit_put;
  171. if (add_mark)
  172. perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
  173. fs = force_uaccess_begin();
  174. perf_callchain_user(&ctx, regs);
  175. force_uaccess_end(fs);
  176. }
  177. }
  178. exit_put:
  179. put_callchain_entry(rctx);
  180. return entry;
  181. }
  182. /*
  183. * Used for sysctl_perf_event_max_stack and
  184. * sysctl_perf_event_max_contexts_per_stack.
  185. */
  186. int perf_event_max_stack_handler(struct ctl_table *table, int write,
  187. void *buffer, size_t *lenp, loff_t *ppos)
  188. {
  189. int *value = table->data;
  190. int new_value = *value, ret;
  191. struct ctl_table new_table = *table;
  192. new_table.data = &new_value;
  193. ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
  194. if (ret || !write)
  195. return ret;
  196. mutex_lock(&callchain_mutex);
  197. if (atomic_read(&nr_callchain_events))
  198. ret = -EBUSY;
  199. else
  200. *value = new_value;
  201. mutex_unlock(&callchain_mutex);
  202. return ret;
  203. }