entry-ftrace.S 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. #include <asm/assembler.h>
  3. #include <asm/ftrace.h>
  4. #include <asm/unwind.h>
  5. #include "entry-header.S"
  6. /*
  7. * When compiling with -pg, gcc inserts a call to the mcount routine at the
  8. * start of every function. In mcount, apart from the function's address (in
  9. * lr), we need to get hold of the function's caller's address.
  10. *
  11. * Newer GCCs (4.4+) solve this problem by using a version of mcount with call
  12. * sites like:
  13. *
  14. * push {lr}
  15. * bl __gnu_mcount_nc
  16. *
  17. * With these compilers, frame pointers are not necessary.
  18. *
  19. * mcount can be thought of as a function called in the middle of a subroutine
  20. * call. As such, it needs to be transparent for both the caller and the
  21. * callee: the original lr needs to be restored when leaving mcount, and no
  22. * registers should be clobbered.
  23. *
  24. * When using dynamic ftrace, we patch out the mcount call by a "pop {lr}"
  25. * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c).
  26. */
  27. .macro mcount_adjust_addr rd, rn
  28. bic \rd, \rn, #1 @ clear the Thumb bit if present
  29. sub \rd, \rd, #MCOUNT_INSN_SIZE
  30. .endm
  31. .macro __mcount suffix
  32. mcount_enter
  33. ldr r0, =ftrace_trace_function
  34. ldr r2, [r0]
  35. adr r0, .Lftrace_stub
  36. cmp r0, r2
  37. bne 1f
  38. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  39. ldr r1, =ftrace_graph_return
  40. ldr r2, [r1]
  41. cmp r0, r2
  42. bne ftrace_graph_caller\suffix
  43. ldr r1, =ftrace_graph_entry
  44. ldr r2, [r1]
  45. ldr r0, =ftrace_graph_entry_stub
  46. cmp r0, r2
  47. bne ftrace_graph_caller\suffix
  48. #endif
  49. mcount_exit
  50. 1: mcount_get_lr r1 @ lr of instrumented func
  51. mcount_adjust_addr r0, lr @ instrumented function
  52. badr lr, 2f
  53. mov pc, r2
  54. 2: mcount_exit
  55. .endm
  56. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  57. .macro __ftrace_regs_caller
  58. str lr, [sp, #-8]! @ store LR as PC and make space for CPSR/OLD_R0,
  59. @ OLD_R0 will overwrite previous LR
  60. ldr lr, [sp, #8] @ get previous LR
  61. str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR
  62. str lr, [sp, #-4]! @ store previous LR as LR
  63. add lr, sp, #16 @ move in LR the value of SP as it was
  64. @ before the push {lr} of the mcount mechanism
  65. push {r0-r11, ip, lr}
  66. @ stack content at this point:
  67. @ 0 4 48 52 56 60 64 68 72
  68. @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 |
  69. mov r3, sp @ struct pt_regs*
  70. ldr r2, =function_trace_op
  71. ldr r2, [r2] @ pointer to the current
  72. @ function tracing op
  73. ldr r1, [sp, #S_LR] @ lr of instrumented func
  74. ldr lr, [sp, #S_PC] @ get LR
  75. mcount_adjust_addr r0, lr @ instrumented function
  76. .globl ftrace_regs_call
  77. ftrace_regs_call:
  78. bl ftrace_stub
  79. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  80. .globl ftrace_graph_regs_call
  81. ftrace_graph_regs_call:
  82. mov r0, r0
  83. #endif
  84. @ pop saved regs
  85. pop {r0-r11, ip, lr} @ restore r0 through r12
  86. ldr lr, [sp], #4 @ restore LR
  87. ldr pc, [sp], #12
  88. .endm
  89. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  90. .macro __ftrace_graph_regs_caller
  91. sub r0, fp, #4 @ lr of instrumented routine (parent)
  92. @ called from __ftrace_regs_caller
  93. ldr r1, [sp, #S_PC] @ instrumented routine (func)
  94. mcount_adjust_addr r1, r1
  95. mov r2, fp @ frame pointer
  96. bl prepare_ftrace_return
  97. @ pop registers saved in ftrace_regs_caller
  98. pop {r0-r11, ip, lr} @ restore r0 through r12
  99. ldr lr, [sp], #4 @ restore LR
  100. ldr pc, [sp], #12
  101. .endm
  102. #endif
  103. #endif
  104. .macro __ftrace_caller suffix
  105. mcount_enter
  106. mcount_get_lr r1 @ lr of instrumented func
  107. mcount_adjust_addr r0, lr @ instrumented function
  108. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  109. ldr r2, =function_trace_op
  110. ldr r2, [r2] @ pointer to the current
  111. @ function tracing op
  112. mov r3, #0 @ regs is NULL
  113. #endif
  114. .globl ftrace_call\suffix
  115. ftrace_call\suffix:
  116. bl ftrace_stub
  117. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  118. .globl ftrace_graph_call\suffix
  119. ftrace_graph_call\suffix:
  120. mov r0, r0
  121. #endif
  122. mcount_exit
  123. .endm
  124. .macro __ftrace_graph_caller
  125. sub r0, fp, #4 @ &lr of instrumented routine (&parent)
  126. #ifdef CONFIG_DYNAMIC_FTRACE
  127. @ called from __ftrace_caller, saved in mcount_enter
  128. ldr r1, [sp, #16] @ instrumented routine (func)
  129. mcount_adjust_addr r1, r1
  130. #else
  131. @ called from __mcount, untouched in lr
  132. mcount_adjust_addr r1, lr @ instrumented routine (func)
  133. #endif
  134. mov r2, fp @ frame pointer
  135. bl prepare_ftrace_return
  136. mcount_exit
  137. .endm
  138. /*
  139. * __gnu_mcount_nc
  140. */
  141. .macro mcount_enter
  142. /*
  143. * This pad compensates for the push {lr} at the call site. Note that we are
  144. * unable to unwind through a function which does not otherwise save its lr.
  145. */
  146. UNWIND(.pad #4)
  147. stmdb sp!, {r0-r3, lr}
  148. UNWIND(.save {r0-r3, lr})
  149. .endm
  150. .macro mcount_get_lr reg
  151. ldr \reg, [sp, #20]
  152. .endm
  153. .macro mcount_exit
  154. ldmia sp!, {r0-r3}
  155. ldr lr, [sp, #4]
  156. ldr pc, [sp], #8
  157. .endm
  158. ENTRY(__gnu_mcount_nc)
  159. UNWIND(.fnstart)
  160. #ifdef CONFIG_DYNAMIC_FTRACE
  161. push {lr}
  162. ldr lr, [sp, #4]
  163. ldr pc, [sp], #8
  164. #else
  165. __mcount
  166. #endif
  167. UNWIND(.fnend)
  168. ENDPROC(__gnu_mcount_nc)
  169. #ifdef CONFIG_DYNAMIC_FTRACE
  170. ENTRY(ftrace_caller)
  171. UNWIND(.fnstart)
  172. __ftrace_caller
  173. UNWIND(.fnend)
  174. ENDPROC(ftrace_caller)
  175. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  176. ENTRY(ftrace_regs_caller)
  177. UNWIND(.fnstart)
  178. __ftrace_regs_caller
  179. UNWIND(.fnend)
  180. ENDPROC(ftrace_regs_caller)
  181. #endif
  182. #endif
  183. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  184. ENTRY(ftrace_graph_caller)
  185. UNWIND(.fnstart)
  186. __ftrace_graph_caller
  187. UNWIND(.fnend)
  188. ENDPROC(ftrace_graph_caller)
  189. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  190. ENTRY(ftrace_graph_regs_caller)
  191. UNWIND(.fnstart)
  192. __ftrace_graph_regs_caller
  193. UNWIND(.fnend)
  194. ENDPROC(ftrace_graph_regs_caller)
  195. #endif
  196. #endif
  197. .purgem mcount_enter
  198. .purgem mcount_get_lr
  199. .purgem mcount_exit
  200. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  201. .globl return_to_handler
  202. return_to_handler:
  203. stmdb sp!, {r0-r3}
  204. mov r0, fp @ frame pointer
  205. bl ftrace_return_to_handler
  206. mov lr, r0 @ r0 has real ret addr
  207. ldmia sp!, {r0-r3}
  208. ret lr
  209. #endif
  210. ENTRY(ftrace_stub)
  211. .Lftrace_stub:
  212. ret lr
  213. ENDPROC(ftrace_stub)