ftrace.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2008 Matt Fleming <matt@console-pimps.org>
  4. * Copyright (C) 2008 Paul Mundt <lethal@linux-sh.org>
  5. *
  6. * Code for replacing ftrace calls with jumps.
  7. *
  8. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  9. *
  10. * Thanks goes to Ingo Molnar, for suggesting the idea.
  11. * Mathieu Desnoyers, for suggesting postponing the modifications.
  12. * Arjan van de Ven, for keeping me straight, and explaining to me
  13. * the dangers of modifying code on the run.
  14. */
  15. #include <linux/uaccess.h>
  16. #include <linux/ftrace.h>
  17. #include <linux/string.h>
  18. #include <linux/init.h>
  19. #include <linux/io.h>
  20. #include <linux/kernel.h>
  21. #include <asm/ftrace.h>
  22. #include <asm/cacheflush.h>
  23. #include <asm/unistd.h>
  24. #include <trace/syscall.h>
  25. #ifdef CONFIG_DYNAMIC_FTRACE
  26. static unsigned char ftrace_replaced_code[MCOUNT_INSN_SIZE];
  27. static unsigned char ftrace_nop[4];
  28. /*
  29. * If we're trying to nop out a call to a function, we instead
  30. * place a call to the address after the memory table.
  31. *
  32. * 8c011060 <a>:
  33. * 8c011060: 02 d1 mov.l 8c01106c <a+0xc>,r1
  34. * 8c011062: 22 4f sts.l pr,@-r15
  35. * 8c011064: 02 c7 mova 8c011070 <a+0x10>,r0
  36. * 8c011066: 2b 41 jmp @r1
  37. * 8c011068: 2a 40 lds r0,pr
  38. * 8c01106a: 09 00 nop
  39. * 8c01106c: 68 24 .word 0x2468 <--- ip
  40. * 8c01106e: 1d 8c .word 0x8c1d
  41. * 8c011070: 26 4f lds.l @r15+,pr <--- ip + MCOUNT_INSN_SIZE
  42. *
  43. * We write 0x8c011070 to 0x8c01106c so that on entry to a() we branch
  44. * past the _mcount call and continue executing code like normal.
  45. */
  46. static unsigned char *ftrace_nop_replace(unsigned long ip)
  47. {
  48. __raw_writel(ip + MCOUNT_INSN_SIZE, ftrace_nop);
  49. return ftrace_nop;
  50. }
  51. static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
  52. {
  53. /* Place the address in the memory table. */
  54. __raw_writel(addr, ftrace_replaced_code);
  55. /*
  56. * No locking needed, this must be called via kstop_machine
  57. * which in essence is like running on a uniprocessor machine.
  58. */
  59. return ftrace_replaced_code;
  60. }
  61. /*
  62. * Modifying code must take extra care. On an SMP machine, if
  63. * the code being modified is also being executed on another CPU
  64. * that CPU will have undefined results and possibly take a GPF.
  65. * We use kstop_machine to stop other CPUS from exectuing code.
  66. * But this does not stop NMIs from happening. We still need
  67. * to protect against that. We separate out the modification of
  68. * the code to take care of this.
  69. *
  70. * Two buffers are added: An IP buffer and a "code" buffer.
  71. *
  72. * 1) Put the instruction pointer into the IP buffer
  73. * and the new code into the "code" buffer.
  74. * 2) Wait for any running NMIs to finish and set a flag that says
  75. * we are modifying code, it is done in an atomic operation.
  76. * 3) Write the code
  77. * 4) clear the flag.
  78. * 5) Wait for any running NMIs to finish.
  79. *
  80. * If an NMI is executed, the first thing it does is to call
  81. * "ftrace_nmi_enter". This will check if the flag is set to write
  82. * and if it is, it will write what is in the IP and "code" buffers.
  83. *
  84. * The trick is, it does not matter if everyone is writing the same
  85. * content to the code location. Also, if a CPU is executing code
  86. * it is OK to write to that code location if the contents being written
  87. * are the same as what exists.
  88. */
  89. #define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */
  90. static atomic_t nmi_running = ATOMIC_INIT(0);
  91. static int mod_code_status; /* holds return value of text write */
  92. static void *mod_code_ip; /* holds the IP to write to */
  93. static void *mod_code_newcode; /* holds the text to write to the IP */
  94. static void clear_mod_flag(void)
  95. {
  96. int old = atomic_read(&nmi_running);
  97. for (;;) {
  98. int new = old & ~MOD_CODE_WRITE_FLAG;
  99. if (old == new)
  100. break;
  101. old = atomic_cmpxchg(&nmi_running, old, new);
  102. }
  103. }
  104. static void ftrace_mod_code(void)
  105. {
  106. /*
  107. * Yes, more than one CPU process can be writing to mod_code_status.
  108. * (and the code itself)
  109. * But if one were to fail, then they all should, and if one were
  110. * to succeed, then they all should.
  111. */
  112. mod_code_status = copy_to_kernel_nofault(mod_code_ip, mod_code_newcode,
  113. MCOUNT_INSN_SIZE);
  114. /* if we fail, then kill any new writers */
  115. if (mod_code_status)
  116. clear_mod_flag();
  117. }
  118. void arch_ftrace_nmi_enter(void)
  119. {
  120. if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
  121. smp_rmb();
  122. ftrace_mod_code();
  123. }
  124. /* Must have previous changes seen before executions */
  125. smp_mb();
  126. }
  127. void arch_ftrace_nmi_exit(void)
  128. {
  129. /* Finish all executions before clearing nmi_running */
  130. smp_mb();
  131. atomic_dec(&nmi_running);
  132. }
  133. static void wait_for_nmi_and_set_mod_flag(void)
  134. {
  135. if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
  136. return;
  137. do {
  138. cpu_relax();
  139. } while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
  140. }
  141. static void wait_for_nmi(void)
  142. {
  143. if (!atomic_read(&nmi_running))
  144. return;
  145. do {
  146. cpu_relax();
  147. } while (atomic_read(&nmi_running));
  148. }
  149. static int
  150. do_ftrace_mod_code(unsigned long ip, void *new_code)
  151. {
  152. mod_code_ip = (void *)ip;
  153. mod_code_newcode = new_code;
  154. /* The buffers need to be visible before we let NMIs write them */
  155. smp_mb();
  156. wait_for_nmi_and_set_mod_flag();
  157. /* Make sure all running NMIs have finished before we write the code */
  158. smp_mb();
  159. ftrace_mod_code();
  160. /* Make sure the write happens before clearing the bit */
  161. smp_mb();
  162. clear_mod_flag();
  163. wait_for_nmi();
  164. return mod_code_status;
  165. }
  166. static int ftrace_modify_code(unsigned long ip, unsigned char *old_code,
  167. unsigned char *new_code)
  168. {
  169. unsigned char replaced[MCOUNT_INSN_SIZE];
  170. /*
  171. * Note:
  172. * We are paranoid about modifying text, as if a bug was to happen, it
  173. * could cause us to read or write to someplace that could cause harm.
  174. * Carefully read and modify the code with probe_kernel_*(), and make
  175. * sure what we read is what we expected it to be before modifying it.
  176. */
  177. /* read the text we want to modify */
  178. if (copy_from_kernel_nofault(replaced, (void *)ip, MCOUNT_INSN_SIZE))
  179. return -EFAULT;
  180. /* Make sure it is what we expect it to be */
  181. if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
  182. return -EINVAL;
  183. /* replace the text with the new text */
  184. if (do_ftrace_mod_code(ip, new_code))
  185. return -EPERM;
  186. flush_icache_range(ip, ip + MCOUNT_INSN_SIZE);
  187. return 0;
  188. }
  189. int ftrace_update_ftrace_func(ftrace_func_t func)
  190. {
  191. unsigned long ip = (unsigned long)(&ftrace_call) + MCOUNT_INSN_OFFSET;
  192. unsigned char old[MCOUNT_INSN_SIZE], *new;
  193. memcpy(old, (unsigned char *)ip, MCOUNT_INSN_SIZE);
  194. new = ftrace_call_replace(ip, (unsigned long)func);
  195. return ftrace_modify_code(ip, old, new);
  196. }
  197. int ftrace_make_nop(struct module *mod,
  198. struct dyn_ftrace *rec, unsigned long addr)
  199. {
  200. unsigned char *new, *old;
  201. unsigned long ip = rec->ip;
  202. old = ftrace_call_replace(ip, addr);
  203. new = ftrace_nop_replace(ip);
  204. return ftrace_modify_code(rec->ip, old, new);
  205. }
  206. int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
  207. {
  208. unsigned char *new, *old;
  209. unsigned long ip = rec->ip;
  210. old = ftrace_nop_replace(ip);
  211. new = ftrace_call_replace(ip, addr);
  212. return ftrace_modify_code(rec->ip, old, new);
  213. }
  214. int __init ftrace_dyn_arch_init(void)
  215. {
  216. return 0;
  217. }
  218. #endif /* CONFIG_DYNAMIC_FTRACE */
  219. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  220. #ifdef CONFIG_DYNAMIC_FTRACE
  221. extern void ftrace_graph_call(void);
  222. static int ftrace_mod(unsigned long ip, unsigned long old_addr,
  223. unsigned long new_addr)
  224. {
  225. unsigned char code[MCOUNT_INSN_SIZE];
  226. if (copy_from_kernel_nofault(code, (void *)ip, MCOUNT_INSN_SIZE))
  227. return -EFAULT;
  228. if (old_addr != __raw_readl((unsigned long *)code))
  229. return -EINVAL;
  230. __raw_writel(new_addr, ip);
  231. return 0;
  232. }
  233. int ftrace_enable_ftrace_graph_caller(void)
  234. {
  235. unsigned long ip, old_addr, new_addr;
  236. ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
  237. old_addr = (unsigned long)(&skip_trace);
  238. new_addr = (unsigned long)(&ftrace_graph_caller);
  239. return ftrace_mod(ip, old_addr, new_addr);
  240. }
  241. int ftrace_disable_ftrace_graph_caller(void)
  242. {
  243. unsigned long ip, old_addr, new_addr;
  244. ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
  245. old_addr = (unsigned long)(&ftrace_graph_caller);
  246. new_addr = (unsigned long)(&skip_trace);
  247. return ftrace_mod(ip, old_addr, new_addr);
  248. }
  249. #endif /* CONFIG_DYNAMIC_FTRACE */
  250. /*
  251. * Hook the return address and push it in the stack of return addrs
  252. * in the current thread info.
  253. *
  254. * This is the main routine for the function graph tracer. The function
  255. * graph tracer essentially works like this:
  256. *
  257. * parent is the stack address containing self_addr's return address.
  258. * We pull the real return address out of parent and store it in
  259. * current's ret_stack. Then, we replace the return address on the stack
  260. * with the address of return_to_handler. self_addr is the function that
  261. * called mcount.
  262. *
  263. * When self_addr returns, it will jump to return_to_handler which calls
  264. * ftrace_return_to_handler. ftrace_return_to_handler will pull the real
  265. * return address off of current's ret_stack and jump to it.
  266. */
  267. void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
  268. {
  269. unsigned long old;
  270. int faulted;
  271. unsigned long return_hooker = (unsigned long)&return_to_handler;
  272. if (unlikely(ftrace_graph_is_dead()))
  273. return;
  274. if (unlikely(atomic_read(&current->tracing_graph_pause)))
  275. return;
  276. /*
  277. * Protect against fault, even if it shouldn't
  278. * happen. This tool is too much intrusive to
  279. * ignore such a protection.
  280. */
  281. __asm__ __volatile__(
  282. "1: \n\t"
  283. "mov.l @%2, %0 \n\t"
  284. "2: \n\t"
  285. "mov.l %3, @%2 \n\t"
  286. "mov #0, %1 \n\t"
  287. "3: \n\t"
  288. ".section .fixup, \"ax\" \n\t"
  289. "4: \n\t"
  290. "mov.l 5f, %0 \n\t"
  291. "jmp @%0 \n\t"
  292. " mov #1, %1 \n\t"
  293. ".balign 4 \n\t"
  294. "5: .long 3b \n\t"
  295. ".previous \n\t"
  296. ".section __ex_table,\"a\" \n\t"
  297. ".long 1b, 4b \n\t"
  298. ".long 2b, 4b \n\t"
  299. ".previous \n\t"
  300. : "=&r" (old), "=r" (faulted)
  301. : "r" (parent), "r" (return_hooker)
  302. );
  303. if (unlikely(faulted)) {
  304. ftrace_graph_stop();
  305. WARN_ON(1);
  306. return;
  307. }
  308. if (function_graph_enter(old, self_addr, 0, NULL))
  309. __raw_writel(old, parent);
  310. }
  311. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */