entry-header.S 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #include <linux/init.h>
  3. #include <linux/linkage.h>
  4. #include <asm/assembler.h>
  5. #include <asm/asm-offsets.h>
  6. #include <asm/errno.h>
  7. #include <asm/thread_info.h>
  8. #include <asm/uaccess-asm.h>
  9. #include <asm/v7m.h>
  10. @ Bad Abort numbers
  11. @ -----------------
  12. @
  13. #define BAD_PREFETCH 0
  14. #define BAD_DATA 1
  15. #define BAD_ADDREXCPTN 2
  16. #define BAD_IRQ 3
  17. #define BAD_UNDEFINSTR 4
  18. @
  19. @ Most of the stack format comes from struct pt_regs, but with
  20. @ the addition of 8 bytes for storing syscall args 5 and 6.
  21. @ This _must_ remain a multiple of 8 for EABI.
  22. @
  23. #define S_OFF 8
  24. /*
  25. * The SWI code relies on the fact that R0 is at the bottom of the stack
  26. * (due to slow/fast restore user regs).
  27. */
  28. #if S_R0 != 0
  29. #error "Please fix"
  30. #endif
  31. .macro zero_fp
  32. #ifdef CONFIG_FRAME_POINTER
  33. mov fp, #0
  34. #endif
  35. .endm
  36. #ifdef CONFIG_ALIGNMENT_TRAP
  37. #define ATRAP(x...) x
  38. #else
  39. #define ATRAP(x...)
  40. #endif
  41. .macro alignment_trap, rtmp1, rtmp2, label
  42. #ifdef CONFIG_ALIGNMENT_TRAP
  43. mrc p15, 0, \rtmp2, c1, c0, 0
  44. ldr \rtmp1, \label
  45. ldr \rtmp1, [\rtmp1]
  46. teq \rtmp1, \rtmp2
  47. mcrne p15, 0, \rtmp1, c1, c0, 0
  48. #endif
  49. .endm
  50. #ifdef CONFIG_CPU_V7M
  51. /*
  52. * ARMv7-M exception entry/exit macros.
  53. *
  54. * xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are
  55. * automatically saved on the current stack (32 words) before
  56. * switching to the exception stack (SP_main).
  57. *
  58. * If exception is taken while in user mode, SP_main is
  59. * empty. Otherwise, SP_main is aligned to 64 bit automatically
  60. * (CCR.STKALIGN set).
  61. *
  62. * Linux assumes that the interrupts are disabled when entering an
  63. * exception handler and it may BUG if this is not the case. Interrupts
  64. * are disabled during entry and reenabled in the exit macro.
  65. *
  66. * v7m_exception_slow_exit is used when returning from SVC or PendSV.
  67. * When returning to kernel mode, we don't return from exception.
  68. */
  69. .macro v7m_exception_entry
  70. @ determine the location of the registers saved by the core during
  71. @ exception entry. Depending on the mode the cpu was in when the
  72. @ exception happend that is either on the main or the process stack.
  73. @ Bit 2 of EXC_RETURN stored in the lr register specifies which stack
  74. @ was used.
  75. tst lr, #EXC_RET_STACK_MASK
  76. mrsne r12, psp
  77. moveq r12, sp
  78. @ we cannot rely on r0-r3 and r12 matching the value saved in the
  79. @ exception frame because of tail-chaining. So these have to be
  80. @ reloaded.
  81. ldmia r12!, {r0-r3}
  82. @ Linux expects to have irqs off. Do it here before taking stack space
  83. cpsid i
  84. sub sp, #PT_REGS_SIZE-S_IP
  85. stmdb sp!, {r0-r11}
  86. @ load saved r12, lr, return address and xPSR.
  87. @ r0-r7 are used for signals and never touched from now on. Clobbering
  88. @ r8-r12 is OK.
  89. mov r9, r12
  90. ldmia r9!, {r8, r10-r12}
  91. @ calculate the original stack pointer value.
  92. @ r9 currently points to the memory location just above the auto saved
  93. @ xPSR.
  94. @ The cpu might automatically 8-byte align the stack. Bit 9
  95. @ of the saved xPSR specifies if stack aligning took place. In this case
  96. @ another 32-bit value is included in the stack.
  97. tst r12, V7M_xPSR_FRAMEPTRALIGN
  98. addne r9, r9, #4
  99. @ store saved r12 using str to have a register to hold the base for stm
  100. str r8, [sp, #S_IP]
  101. add r8, sp, #S_SP
  102. @ store r13-r15, xPSR
  103. stmia r8!, {r9-r12}
  104. @ store old_r0
  105. str r0, [r8]
  106. .endm
  107. /*
  108. * PENDSV and SVCALL are configured to have the same exception
  109. * priorities. As a kernel thread runs at SVCALL execution priority it
  110. * can never be preempted and so we will never have to return to a
  111. * kernel thread here.
  112. */
  113. .macro v7m_exception_slow_exit ret_r0
  114. cpsid i
  115. ldr lr, =exc_ret
  116. ldr lr, [lr]
  117. @ read original r12, sp, lr, pc and xPSR
  118. add r12, sp, #S_IP
  119. ldmia r12, {r1-r5}
  120. @ an exception frame is always 8-byte aligned. To tell the hardware if
  121. @ the sp to be restored is aligned or not set bit 9 of the saved xPSR
  122. @ accordingly.
  123. tst r2, #4
  124. subne r2, r2, #4
  125. orrne r5, V7M_xPSR_FRAMEPTRALIGN
  126. biceq r5, V7M_xPSR_FRAMEPTRALIGN
  127. @ ensure bit 0 is cleared in the PC, otherwise behaviour is
  128. @ unpredictable
  129. bic r4, #1
  130. @ write basic exception frame
  131. stmdb r2!, {r1, r3-r5}
  132. ldmia sp, {r1, r3-r5}
  133. .if \ret_r0
  134. stmdb r2!, {r0, r3-r5}
  135. .else
  136. stmdb r2!, {r1, r3-r5}
  137. .endif
  138. @ restore process sp
  139. msr psp, r2
  140. @ restore original r4-r11
  141. ldmia sp!, {r0-r11}
  142. @ restore main sp
  143. add sp, sp, #PT_REGS_SIZE-S_IP
  144. cpsie i
  145. bx lr
  146. .endm
  147. #endif /* CONFIG_CPU_V7M */
  148. @
  149. @ Store/load the USER SP and LR registers by switching to the SYS
  150. @ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not
  151. @ available. Should only be called from SVC mode
  152. @
  153. .macro store_user_sp_lr, rd, rtemp, offset = 0
  154. mrs \rtemp, cpsr
  155. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  156. msr cpsr_c, \rtemp @ switch to the SYS mode
  157. str sp, [\rd, #\offset] @ save sp_usr
  158. str lr, [\rd, #\offset + 4] @ save lr_usr
  159. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  160. msr cpsr_c, \rtemp @ switch back to the SVC mode
  161. .endm
  162. .macro load_user_sp_lr, rd, rtemp, offset = 0
  163. mrs \rtemp, cpsr
  164. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  165. msr cpsr_c, \rtemp @ switch to the SYS mode
  166. ldr sp, [\rd, #\offset] @ load sp_usr
  167. ldr lr, [\rd, #\offset + 4] @ load lr_usr
  168. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  169. msr cpsr_c, \rtemp @ switch back to the SVC mode
  170. .endm
  171. .macro svc_exit, rpsr, irq = 0
  172. .if \irq != 0
  173. @ IRQs already off
  174. #ifdef CONFIG_TRACE_IRQFLAGS
  175. @ The parent context IRQs must have been enabled to get here in
  176. @ the first place, so there's no point checking the PSR I bit.
  177. bl trace_hardirqs_on
  178. #endif
  179. .else
  180. @ IRQs off again before pulling preserved data off the stack
  181. disable_irq_notrace
  182. #ifdef CONFIG_TRACE_IRQFLAGS
  183. tst \rpsr, #PSR_I_BIT
  184. bleq trace_hardirqs_on
  185. tst \rpsr, #PSR_I_BIT
  186. blne trace_hardirqs_off
  187. #endif
  188. .endif
  189. uaccess_exit tsk, r0, r1
  190. #ifndef CONFIG_THUMB2_KERNEL
  191. @ ARM mode SVC restore
  192. msr spsr_cxsf, \rpsr
  193. #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
  194. @ We must avoid clrex due to Cortex-A15 erratum #830321
  195. sub r0, sp, #4 @ uninhabited address
  196. strex r1, r2, [r0] @ clear the exclusive monitor
  197. #endif
  198. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  199. #else
  200. @ Thumb mode SVC restore
  201. ldr lr, [sp, #S_SP] @ top of the stack
  202. ldrd r0, r1, [sp, #S_LR] @ calling lr and pc
  203. @ We must avoid clrex due to Cortex-A15 erratum #830321
  204. strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor
  205. stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context
  206. ldmia sp, {r0 - r12}
  207. mov sp, lr
  208. ldr lr, [sp], #4
  209. rfeia sp!
  210. #endif
  211. .endm
  212. @
  213. @ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit
  214. @
  215. @ This macro acts in a similar manner to svc_exit but switches to FIQ
  216. @ mode to restore the final part of the register state.
  217. @
  218. @ We cannot use the normal svc_exit procedure because that would
  219. @ clobber spsr_svc (FIQ could be delivered during the first few
  220. @ instructions of vector_swi meaning its contents have not been
  221. @ saved anywhere).
  222. @
  223. @ Note that, unlike svc_exit, this macro also does not allow a caller
  224. @ supplied rpsr. This is because the FIQ exceptions are not re-entrant
  225. @ and the handlers cannot call into the scheduler (meaning the value
  226. @ on the stack remains correct).
  227. @
  228. .macro svc_exit_via_fiq
  229. uaccess_exit tsk, r0, r1
  230. #ifndef CONFIG_THUMB2_KERNEL
  231. @ ARM mode restore
  232. mov r0, sp
  233. ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will
  234. @ clobber state restored below)
  235. msr cpsr_c, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT
  236. add r8, r0, #S_PC
  237. ldr r9, [r0, #S_PSR]
  238. msr spsr_cxsf, r9
  239. ldr r0, [r0, #S_R0]
  240. ldmia r8, {pc}^
  241. #else
  242. @ Thumb mode restore
  243. add r0, sp, #S_R2
  244. ldr lr, [sp, #S_LR]
  245. ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will
  246. @ clobber state restored below)
  247. ldmia r0, {r2 - r12}
  248. mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT
  249. msr cpsr_c, r1
  250. sub r0, #S_R2
  251. add r8, r0, #S_PC
  252. ldmia r0, {r0 - r1}
  253. rfeia r8
  254. #endif
  255. .endm
  256. .macro restore_user_regs, fast = 0, offset = 0
  257. uaccess_enable r1, isb=0
  258. #ifndef CONFIG_THUMB2_KERNEL
  259. @ ARM mode restore
  260. mov r2, sp
  261. ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr
  262. ldr lr, [r2, #\offset + S_PC]! @ get pc
  263. tst r1, #PSR_I_BIT | 0x0f
  264. bne 1f
  265. msr spsr_cxsf, r1 @ save in spsr_svc
  266. #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
  267. @ We must avoid clrex due to Cortex-A15 erratum #830321
  268. strex r1, r2, [r2] @ clear the exclusive monitor
  269. #endif
  270. .if \fast
  271. ldmdb r2, {r1 - lr}^ @ get calling r1 - lr
  272. .else
  273. ldmdb r2, {r0 - lr}^ @ get calling r0 - lr
  274. .endif
  275. mov r0, r0 @ ARMv5T and earlier require a nop
  276. @ after ldm {}^
  277. add sp, sp, #\offset + PT_REGS_SIZE
  278. movs pc, lr @ return & move spsr_svc into cpsr
  279. 1: bug "Returning to usermode but unexpected PSR bits set?", \@
  280. #elif defined(CONFIG_CPU_V7M)
  281. @ V7M restore.
  282. @ Note that we don't need to do clrex here as clearing the local
  283. @ monitor is part of the exception entry and exit sequence.
  284. .if \offset
  285. add sp, #\offset
  286. .endif
  287. v7m_exception_slow_exit ret_r0 = \fast
  288. #else
  289. @ Thumb mode restore
  290. mov r2, sp
  291. load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr
  292. ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr
  293. ldr lr, [sp, #\offset + S_PC] @ get pc
  294. add sp, sp, #\offset + S_SP
  295. tst r1, #PSR_I_BIT | 0x0f
  296. bne 1f
  297. msr spsr_cxsf, r1 @ save in spsr_svc
  298. @ We must avoid clrex due to Cortex-A15 erratum #830321
  299. strex r1, r2, [sp] @ clear the exclusive monitor
  300. .if \fast
  301. ldmdb sp, {r1 - r12} @ get calling r1 - r12
  302. .else
  303. ldmdb sp, {r0 - r12} @ get calling r0 - r12
  304. .endif
  305. add sp, sp, #PT_REGS_SIZE - S_SP
  306. movs pc, lr @ return & move spsr_svc into cpsr
  307. 1: bug "Returning to usermode but unexpected PSR bits set?", \@
  308. #endif /* !CONFIG_THUMB2_KERNEL */
  309. .endm
  310. /*
  311. * Context tracking subsystem. Used to instrument transitions
  312. * between user and kernel mode.
  313. */
  314. .macro ct_user_exit, save = 1
  315. #ifdef CONFIG_CONTEXT_TRACKING
  316. .if \save
  317. stmdb sp!, {r0-r3, ip, lr}
  318. bl context_tracking_user_exit
  319. ldmia sp!, {r0-r3, ip, lr}
  320. .else
  321. bl context_tracking_user_exit
  322. .endif
  323. #endif
  324. .endm
  325. .macro ct_user_enter, save = 1
  326. #ifdef CONFIG_CONTEXT_TRACKING
  327. .if \save
  328. stmdb sp!, {r0-r3, ip, lr}
  329. bl context_tracking_user_enter
  330. ldmia sp!, {r0-r3, ip, lr}
  331. .else
  332. bl context_tracking_user_enter
  333. .endif
  334. #endif
  335. .endm
  336. .macro invoke_syscall, table, nr, tmp, ret, reload=0
  337. #ifdef CONFIG_CPU_SPECTRE
  338. mov \tmp, \nr
  339. cmp \tmp, #NR_syscalls @ check upper syscall limit
  340. movcs \tmp, #0
  341. csdb
  342. badr lr, \ret @ return address
  343. .if \reload
  344. add r1, sp, #S_R0 + S_OFF @ pointer to regs
  345. ldmiacc r1, {r0 - r6} @ reload r0-r6
  346. stmiacc sp, {r4, r5} @ update stack arguments
  347. .endif
  348. ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine
  349. #else
  350. cmp \nr, #NR_syscalls @ check upper syscall limit
  351. badr lr, \ret @ return address
  352. .if \reload
  353. add r1, sp, #S_R0 + S_OFF @ pointer to regs
  354. ldmiacc r1, {r0 - r6} @ reload r0-r6
  355. stmiacc sp, {r4, r5} @ update stack arguments
  356. .endif
  357. ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine
  358. #endif
  359. .endm
  360. /*
  361. * These are the registers used in the syscall handler, and allow us to
  362. * have in theory up to 7 arguments to a function - r0 to r6.
  363. *
  364. * r7 is reserved for the system call number for thumb mode.
  365. *
  366. * Note that tbl == why is intentional.
  367. *
  368. * We must set at least "tsk" and "why" when calling ret_with_reschedule.
  369. */
  370. scno .req r7 @ syscall number
  371. tbl .req r8 @ syscall table pointer
  372. why .req r8 @ Linux syscall (!= 0)
  373. tsk .req r9 @ current thread_info