entry-armv.S 25 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082
  1. /*
  2. * linux/arch/arm/kernel/entry-armv.S
  3. *
  4. * Copyright (C) 1996,1997,1998 Russell King.
  5. * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
  6. * nommu support by Hyok S. Choi (hyok.choi@samsung.com)
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License version 2 as
  10. * published by the Free Software Foundation.
  11. *
  12. * Low-level vector interface routines
  13. *
  14. * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction that causes
  15. * it to save wrong values... Be aware!
  16. */
  17. #include <asm/memory.h>
  18. #include <asm/glue.h>
  19. #include <asm/vfpmacros.h>
  20. #include <asm/arch/entry-macro.S>
  21. #include <asm/thread_notify.h>
  22. #include "entry-header.S"
  23. /*
  24. * Interrupt handling. Preserves r7, r8, r9
  25. */
  26. .macro irq_handler
  27. get_irqnr_preamble r5, lr
  28. 1: get_irqnr_and_base r0, r6, r5, lr
  29. movne r1, sp
  30. @
  31. @ routine called with r0 = irq number, r1 = struct pt_regs *
  32. @
  33. adrne lr, 1b
  34. bne asm_do_IRQ
  35. #ifdef CONFIG_SMP
  36. /*
  37. * XXX
  38. *
  39. * this macro assumes that irqstat (r6) and base (r5) are
  40. * preserved from get_irqnr_and_base above
  41. */
  42. test_for_ipi r0, r6, r5, lr
  43. movne r0, sp
  44. adrne lr, 1b
  45. bne do_IPI
  46. #ifdef CONFIG_LOCAL_TIMERS
  47. test_for_ltirq r0, r6, r5, lr
  48. movne r0, sp
  49. adrne lr, 1b
  50. bne do_local_timer
  51. #endif
  52. #endif
  53. .endm
  54. /*
  55. * Invalid mode handlers
  56. */
  57. .macro inv_entry, reason
  58. sub sp, sp, #S_FRAME_SIZE
  59. stmib sp, {r1 - lr}
  60. mov r1, #\reason
  61. .endm
  62. __pabt_invalid:
  63. inv_entry BAD_PREFETCH
  64. b common_invalid
  65. __dabt_invalid:
  66. inv_entry BAD_DATA
  67. b common_invalid
  68. __irq_invalid:
  69. inv_entry BAD_IRQ
  70. b common_invalid
  71. __und_invalid:
  72. inv_entry BAD_UNDEFINSTR
  73. @
  74. @ XXX fall through to common_invalid
  75. @
  76. @
  77. @ common_invalid - generic code for failed exception (re-entrant version of handlers)
  78. @
  79. common_invalid:
  80. zero_fp
  81. ldmia r0, {r4 - r6}
  82. add r0, sp, #S_PC @ here for interlock avoidance
  83. mov r7, #-1 @ "" "" "" ""
  84. str r4, [sp] @ save preserved r0
  85. stmia r0, {r5 - r7} @ lr_<exception>,
  86. @ cpsr_<exception>, "old_r0"
  87. mov r0, sp
  88. b bad_mode
  89. /*
  90. * SVC mode handlers
  91. */
  92. #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
  93. #define SPFIX(code...) code
  94. #else
  95. #define SPFIX(code...)
  96. #endif
  97. .macro svc_entry
  98. sub sp, sp, #S_FRAME_SIZE
  99. SPFIX( tst sp, #4 )
  100. SPFIX( bicne sp, sp, #4 )
  101. stmib sp, {r1 - r12}
  102. ldmia r0, {r1 - r3}
  103. add r5, sp, #S_SP @ here for interlock avoidance
  104. mov r4, #-1 @ "" "" "" ""
  105. add r0, sp, #S_FRAME_SIZE @ "" "" "" ""
  106. SPFIX( addne r0, r0, #4 )
  107. str r1, [sp] @ save the "real" r0 copied
  108. @ from the exception stack
  109. mov r1, lr
  110. @
  111. @ We are now ready to fill in the remaining blanks on the stack:
  112. @
  113. @ r0 - sp_svc
  114. @ r1 - lr_svc
  115. @ r2 - lr_<exception>, already fixed up for correct return/restart
  116. @ r3 - spsr_<exception>
  117. @ r4 - orig_r0 (see pt_regs definition in ptrace.h)
  118. @
  119. stmia r5, {r0 - r4}
  120. .endm
  121. .align 5
  122. __dabt_svc:
  123. svc_entry
  124. @
  125. @ get ready to re-enable interrupts if appropriate
  126. @
  127. mrs r9, cpsr
  128. tst r3, #PSR_I_BIT
  129. biceq r9, r9, #PSR_I_BIT
  130. @
  131. @ Call the processor-specific abort handler:
  132. @
  133. @ r2 - aborted context pc
  134. @ r3 - aborted context cpsr
  135. @
  136. @ The abort handler must return the aborted address in r0, and
  137. @ the fault status register in r1. r9 must be preserved.
  138. @
  139. #ifdef MULTI_ABORT
  140. ldr r4, .LCprocfns
  141. mov lr, pc
  142. ldr pc, [r4]
  143. #else
  144. bl CPU_ABORT_HANDLER
  145. #endif
  146. @
  147. @ set desired IRQ state, then call main handler
  148. @
  149. msr cpsr_c, r9
  150. mov r2, sp
  151. bl do_DataAbort
  152. @
  153. @ IRQs off again before pulling preserved data off the stack
  154. @
  155. disable_irq
  156. @
  157. @ restore SPSR and restart the instruction
  158. @
  159. ldr r0, [sp, #S_PSR]
  160. msr spsr_cxsf, r0
  161. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  162. .align 5
  163. __irq_svc:
  164. svc_entry
  165. #ifdef CONFIG_TRACE_IRQFLAGS
  166. bl trace_hardirqs_off
  167. #endif
  168. #ifdef CONFIG_PREEMPT
  169. get_thread_info tsk
  170. ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
  171. add r7, r8, #1 @ increment it
  172. str r7, [tsk, #TI_PREEMPT]
  173. #endif
  174. irq_handler
  175. #ifdef CONFIG_PREEMPT
  176. ldr r0, [tsk, #TI_FLAGS] @ get flags
  177. tst r0, #_TIF_NEED_RESCHED
  178. blne svc_preempt
  179. preempt_return:
  180. ldr r0, [tsk, #TI_PREEMPT] @ read preempt value
  181. str r8, [tsk, #TI_PREEMPT] @ restore preempt count
  182. teq r0, r7
  183. strne r0, [r0, -r0] @ bug()
  184. #endif
  185. ldr r0, [sp, #S_PSR] @ irqs are already disabled
  186. msr spsr_cxsf, r0
  187. #ifdef CONFIG_TRACE_IRQFLAGS
  188. tst r0, #PSR_I_BIT
  189. bleq trace_hardirqs_on
  190. #endif
  191. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  192. .ltorg
  193. #ifdef CONFIG_PREEMPT
  194. svc_preempt:
  195. teq r8, #0 @ was preempt count = 0
  196. ldreq r6, .LCirq_stat
  197. movne pc, lr @ no
  198. ldr r0, [r6, #4] @ local_irq_count
  199. ldr r1, [r6, #8] @ local_bh_count
  200. adds r0, r0, r1
  201. movne pc, lr
  202. mov r7, #0 @ preempt_schedule_irq
  203. str r7, [tsk, #TI_PREEMPT] @ expects preempt_count == 0
  204. 1: bl preempt_schedule_irq @ irq en/disable is done inside
  205. ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
  206. tst r0, #_TIF_NEED_RESCHED
  207. beq preempt_return @ go again
  208. b 1b
  209. #endif
  210. .align 5
  211. __und_svc:
  212. svc_entry
  213. @
  214. @ call emulation code, which returns using r9 if it has emulated
  215. @ the instruction, or the more conventional lr if we are to treat
  216. @ this as a real undefined instruction
  217. @
  218. @ r0 - instruction
  219. @
  220. ldr r0, [r2, #-4]
  221. adr r9, 1f
  222. bl call_fpe
  223. mov r0, sp @ struct pt_regs *regs
  224. bl do_undefinstr
  225. @
  226. @ IRQs off again before pulling preserved data off the stack
  227. @
  228. 1: disable_irq
  229. @
  230. @ restore SPSR and restart the instruction
  231. @
  232. ldr lr, [sp, #S_PSR] @ Get SVC cpsr
  233. msr spsr_cxsf, lr
  234. ldmia sp, {r0 - pc}^ @ Restore SVC registers
  235. .align 5
  236. __pabt_svc:
  237. svc_entry
  238. @
  239. @ re-enable interrupts if appropriate
  240. @
  241. mrs r9, cpsr
  242. tst r3, #PSR_I_BIT
  243. biceq r9, r9, #PSR_I_BIT
  244. msr cpsr_c, r9
  245. @
  246. @ set args, then call main handler
  247. @
  248. @ r0 - address of faulting instruction
  249. @ r1 - pointer to registers on stack
  250. @
  251. mov r0, r2 @ address (pc)
  252. mov r1, sp @ regs
  253. bl do_PrefetchAbort @ call abort handler
  254. @
  255. @ IRQs off again before pulling preserved data off the stack
  256. @
  257. disable_irq
  258. @
  259. @ restore SPSR and restart the instruction
  260. @
  261. ldr r0, [sp, #S_PSR]
  262. msr spsr_cxsf, r0
  263. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  264. .align 5
  265. .LCcralign:
  266. .word cr_alignment
  267. #ifdef MULTI_ABORT
  268. .LCprocfns:
  269. .word processor
  270. #endif
  271. .LCfp:
  272. .word fp_enter
  273. #ifdef CONFIG_PREEMPT
  274. .LCirq_stat:
  275. .word irq_stat
  276. #endif
  277. /*
  278. * User mode handlers
  279. *
  280. * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE
  281. */
  282. #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7)
  283. #error "sizeof(struct pt_regs) must be a multiple of 8"
  284. #endif
  285. .macro usr_entry
  286. sub sp, sp, #S_FRAME_SIZE
  287. stmib sp, {r1 - r12}
  288. ldmia r0, {r1 - r3}
  289. add r0, sp, #S_PC @ here for interlock avoidance
  290. mov r4, #-1 @ "" "" "" ""
  291. str r1, [sp] @ save the "real" r0 copied
  292. @ from the exception stack
  293. #if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
  294. #ifndef CONFIG_MMU
  295. #warning "NPTL on non MMU needs fixing"
  296. #else
  297. @ make sure our user space atomic helper is aborted
  298. cmp r2, #TASK_SIZE
  299. bichs r3, r3, #PSR_Z_BIT
  300. #endif
  301. #endif
  302. @
  303. @ We are now ready to fill in the remaining blanks on the stack:
  304. @
  305. @ r2 - lr_<exception>, already fixed up for correct return/restart
  306. @ r3 - spsr_<exception>
  307. @ r4 - orig_r0 (see pt_regs definition in ptrace.h)
  308. @
  309. @ Also, separately save sp_usr and lr_usr
  310. @
  311. stmia r0, {r2 - r4}
  312. stmdb r0, {sp, lr}^
  313. @
  314. @ Enable the alignment trap while in kernel mode
  315. @
  316. alignment_trap r0
  317. @
  318. @ Clear FP to mark the first stack frame
  319. @
  320. zero_fp
  321. .endm
  322. .align 5
  323. __dabt_usr:
  324. usr_entry
  325. @
  326. @ Call the processor-specific abort handler:
  327. @
  328. @ r2 - aborted context pc
  329. @ r3 - aborted context cpsr
  330. @
  331. @ The abort handler must return the aborted address in r0, and
  332. @ the fault status register in r1.
  333. @
  334. #ifdef MULTI_ABORT
  335. ldr r4, .LCprocfns
  336. mov lr, pc
  337. ldr pc, [r4]
  338. #else
  339. bl CPU_ABORT_HANDLER
  340. #endif
  341. @
  342. @ IRQs on, then call the main handler
  343. @
  344. enable_irq
  345. mov r2, sp
  346. adr lr, ret_from_exception
  347. b do_DataAbort
  348. .align 5
  349. __irq_usr:
  350. usr_entry
  351. #ifdef CONFIG_TRACE_IRQFLAGS
  352. bl trace_hardirqs_off
  353. #endif
  354. get_thread_info tsk
  355. #ifdef CONFIG_PREEMPT
  356. ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
  357. add r7, r8, #1 @ increment it
  358. str r7, [tsk, #TI_PREEMPT]
  359. #endif
  360. irq_handler
  361. #ifdef CONFIG_PREEMPT
  362. ldr r0, [tsk, #TI_PREEMPT]
  363. str r8, [tsk, #TI_PREEMPT]
  364. teq r0, r7
  365. strne r0, [r0, -r0]
  366. #endif
  367. #ifdef CONFIG_TRACE_IRQFLAGS
  368. bl trace_hardirqs_on
  369. #endif
  370. mov why, #0
  371. b ret_to_user
  372. .ltorg
  373. .align 5
  374. __und_usr:
  375. usr_entry
  376. tst r3, #PSR_T_BIT @ Thumb mode?
  377. bne __und_usr_unknown @ ignore FP
  378. sub r4, r2, #4
  379. @
  380. @ fall through to the emulation code, which returns using r9 if
  381. @ it has emulated the instruction, or the more conventional lr
  382. @ if we are to treat this as a real undefined instruction
  383. @
  384. @ r0 - instruction
  385. @
  386. 1: ldrt r0, [r4]
  387. adr r9, ret_from_exception
  388. adr lr, __und_usr_unknown
  389. @
  390. @ fallthrough to call_fpe
  391. @
  392. /*
  393. * The out of line fixup for the ldrt above.
  394. */
  395. .section .fixup, "ax"
  396. 2: mov pc, r9
  397. .previous
  398. .section __ex_table,"a"
  399. .long 1b, 2b
  400. .previous
  401. /*
  402. * Check whether the instruction is a co-processor instruction.
  403. * If yes, we need to call the relevant co-processor handler.
  404. *
  405. * Note that we don't do a full check here for the co-processor
  406. * instructions; all instructions with bit 27 set are well
  407. * defined. The only instructions that should fault are the
  408. * co-processor instructions. However, we have to watch out
  409. * for the ARM6/ARM7 SWI bug.
  410. *
  411. * Emulators may wish to make use of the following registers:
  412. * r0 = instruction opcode.
  413. * r2 = PC+4
  414. * r9 = normal "successful" return address
  415. * r10 = this threads thread_info structure.
  416. * lr = unrecognised instruction return address
  417. */
  418. call_fpe:
  419. tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27
  420. #if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
  421. and r8, r0, #0x0f000000 @ mask out op-code bits
  422. teqne r8, #0x0f000000 @ SWI (ARM6/7 bug)?
  423. #endif
  424. moveq pc, lr
  425. get_thread_info r10 @ get current thread
  426. and r8, r0, #0x00000f00 @ mask out CP number
  427. mov r7, #1
  428. add r6, r10, #TI_USED_CP
  429. strb r7, [r6, r8, lsr #8] @ set appropriate used_cp[]
  430. #ifdef CONFIG_IWMMXT
  431. @ Test if we need to give access to iWMMXt coprocessors
  432. ldr r5, [r10, #TI_FLAGS]
  433. rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only
  434. movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1)
  435. bcs iwmmxt_task_enable
  436. #endif
  437. add pc, pc, r8, lsr #6
  438. mov r0, r0
  439. mov pc, lr @ CP#0
  440. b do_fpe @ CP#1 (FPE)
  441. b do_fpe @ CP#2 (FPE)
  442. mov pc, lr @ CP#3
  443. #ifdef CONFIG_CRUNCH
  444. b crunch_task_enable @ CP#4 (MaverickCrunch)
  445. b crunch_task_enable @ CP#5 (MaverickCrunch)
  446. b crunch_task_enable @ CP#6 (MaverickCrunch)
  447. #else
  448. mov pc, lr @ CP#4
  449. mov pc, lr @ CP#5
  450. mov pc, lr @ CP#6
  451. #endif
  452. mov pc, lr @ CP#7
  453. mov pc, lr @ CP#8
  454. mov pc, lr @ CP#9
  455. #ifdef CONFIG_VFP
  456. b do_vfp @ CP#10 (VFP)
  457. b do_vfp @ CP#11 (VFP)
  458. #else
  459. mov pc, lr @ CP#10 (VFP)
  460. mov pc, lr @ CP#11 (VFP)
  461. #endif
  462. mov pc, lr @ CP#12
  463. mov pc, lr @ CP#13
  464. mov pc, lr @ CP#14 (Debug)
  465. mov pc, lr @ CP#15 (Control)
  466. do_fpe:
  467. enable_irq
  468. ldr r4, .LCfp
  469. add r10, r10, #TI_FPSTATE @ r10 = workspace
  470. ldr pc, [r4] @ Call FP module USR entry point
  471. /*
  472. * The FP module is called with these registers set:
  473. * r0 = instruction
  474. * r2 = PC+4
  475. * r9 = normal "successful" return address
  476. * r10 = FP workspace
  477. * lr = unrecognised FP instruction return address
  478. */
  479. .data
  480. ENTRY(fp_enter)
  481. .word no_fp
  482. .text
  483. no_fp: mov pc, lr
  484. __und_usr_unknown:
  485. mov r0, sp
  486. adr lr, ret_from_exception
  487. b do_undefinstr
  488. .align 5
  489. __pabt_usr:
  490. usr_entry
  491. enable_irq @ Enable interrupts
  492. mov r0, r2 @ address (pc)
  493. mov r1, sp @ regs
  494. bl do_PrefetchAbort @ call abort handler
  495. /* fall through */
  496. /*
  497. * This is the return code to user mode for abort handlers
  498. */
  499. ENTRY(ret_from_exception)
  500. get_thread_info tsk
  501. mov why, #0
  502. b ret_to_user
  503. /*
  504. * Register switch for ARMv3 and ARMv4 processors
  505. * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info
  506. * previous and next are guaranteed not to be the same.
  507. */
  508. ENTRY(__switch_to)
  509. add ip, r1, #TI_CPU_SAVE
  510. ldr r3, [r2, #TI_TP_VALUE]
  511. stmia ip!, {r4 - sl, fp, sp, lr} @ Store most regs on stack
  512. #ifdef CONFIG_MMU
  513. ldr r6, [r2, #TI_CPU_DOMAIN]
  514. #endif
  515. #if __LINUX_ARM_ARCH__ >= 6
  516. #ifdef CONFIG_CPU_32v6K
  517. clrex
  518. #else
  519. strex r5, r4, [ip] @ Clear exclusive monitor
  520. #endif
  521. #endif
  522. #if defined(CONFIG_HAS_TLS_REG)
  523. mcr p15, 0, r3, c13, c0, 3 @ set TLS register
  524. #elif !defined(CONFIG_TLS_REG_EMUL)
  525. mov r4, #0xffff0fff
  526. str r3, [r4, #-15] @ TLS val at 0xffff0ff0
  527. #endif
  528. #ifdef CONFIG_MMU
  529. mcr p15, 0, r6, c3, c0, 0 @ Set domain register
  530. #endif
  531. mov r5, r0
  532. add r4, r2, #TI_CPU_SAVE
  533. ldr r0, =thread_notify_head
  534. mov r1, #THREAD_NOTIFY_SWITCH
  535. bl atomic_notifier_call_chain
  536. mov r0, r5
  537. ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously
  538. __INIT
  539. /*
  540. * User helpers.
  541. *
  542. * These are segment of kernel provided user code reachable from user space
  543. * at a fixed address in kernel memory. This is used to provide user space
  544. * with some operations which require kernel help because of unimplemented
  545. * native feature and/or instructions in many ARM CPUs. The idea is for
  546. * this code to be executed directly in user mode for best efficiency but
  547. * which is too intimate with the kernel counter part to be left to user
  548. * libraries. In fact this code might even differ from one CPU to another
  549. * depending on the available instruction set and restrictions like on
  550. * SMP systems. In other words, the kernel reserves the right to change
  551. * this code as needed without warning. Only the entry points and their
  552. * results are guaranteed to be stable.
  553. *
  554. * Each segment is 32-byte aligned and will be moved to the top of the high
  555. * vector page. New segments (if ever needed) must be added in front of
  556. * existing ones. This mechanism should be used only for things that are
  557. * really small and justified, and not be abused freely.
  558. *
  559. * User space is expected to implement those things inline when optimizing
  560. * for a processor that has the necessary native support, but only if such
  561. * resulting binaries are already to be incompatible with earlier ARM
  562. * processors due to the use of unsupported instructions other than what
  563. * is provided here. In other words don't make binaries unable to run on
  564. * earlier processors just for the sake of not using these kernel helpers
  565. * if your compiled code is not going to use the new instructions for other
  566. * purpose.
  567. */
  568. .macro usr_ret, reg
  569. #ifdef CONFIG_ARM_THUMB
  570. bx \reg
  571. #else
  572. mov pc, \reg
  573. #endif
  574. .endm
  575. .align 5
  576. .globl __kuser_helper_start
  577. __kuser_helper_start:
  578. /*
  579. * Reference prototype:
  580. *
  581. * void __kernel_memory_barrier(void)
  582. *
  583. * Input:
  584. *
  585. * lr = return address
  586. *
  587. * Output:
  588. *
  589. * none
  590. *
  591. * Clobbered:
  592. *
  593. * the Z flag might be lost
  594. *
  595. * Definition and user space usage example:
  596. *
  597. * typedef void (__kernel_dmb_t)(void);
  598. * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0)
  599. *
  600. * Apply any needed memory barrier to preserve consistency with data modified
  601. * manually and __kuser_cmpxchg usage.
  602. *
  603. * This could be used as follows:
  604. *
  605. * #define __kernel_dmb() \
  606. * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \
  607. * : : : "r0", "lr","cc" )
  608. */
  609. __kuser_memory_barrier: @ 0xffff0fa0
  610. #if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_SMP)
  611. mcr p15, 0, r0, c7, c10, 5 @ dmb
  612. #endif
  613. usr_ret lr
  614. .align 5
  615. /*
  616. * Reference prototype:
  617. *
  618. * int __kernel_cmpxchg(int oldval, int newval, int *ptr)
  619. *
  620. * Input:
  621. *
  622. * r0 = oldval
  623. * r1 = newval
  624. * r2 = ptr
  625. * lr = return address
  626. *
  627. * Output:
  628. *
  629. * r0 = returned value (zero or non-zero)
  630. * C flag = set if r0 == 0, clear if r0 != 0
  631. *
  632. * Clobbered:
  633. *
  634. * r3, ip, flags
  635. *
  636. * Definition and user space usage example:
  637. *
  638. * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr);
  639. * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0)
  640. *
  641. * Atomically store newval in *ptr if *ptr is equal to oldval for user space.
  642. * Return zero if *ptr was changed or non-zero if no exchange happened.
  643. * The C flag is also set if *ptr was changed to allow for assembly
  644. * optimization in the calling code.
  645. *
  646. * Notes:
  647. *
  648. * - This routine already includes memory barriers as needed.
  649. *
  650. * - A failure might be transient, i.e. it is possible, although unlikely,
  651. * that "failure" be returned even if *ptr == oldval.
  652. *
  653. * For example, a user space atomic_add implementation could look like this:
  654. *
  655. * #define atomic_add(ptr, val) \
  656. * ({ register unsigned int *__ptr asm("r2") = (ptr); \
  657. * register unsigned int __result asm("r1"); \
  658. * asm volatile ( \
  659. * "1: @ atomic_add\n\t" \
  660. * "ldr r0, [r2]\n\t" \
  661. * "mov r3, #0xffff0fff\n\t" \
  662. * "add lr, pc, #4\n\t" \
  663. * "add r1, r0, %2\n\t" \
  664. * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \
  665. * "bcc 1b" \
  666. * : "=&r" (__result) \
  667. * : "r" (__ptr), "rIL" (val) \
  668. * : "r0","r3","ip","lr","cc","memory" ); \
  669. * __result; })
  670. */
  671. __kuser_cmpxchg: @ 0xffff0fc0
  672. #if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
  673. /*
  674. * Poor you. No fast solution possible...
  675. * The kernel itself must perform the operation.
  676. * A special ghost syscall is used for that (see traps.c).
  677. */
  678. stmfd sp!, {r7, lr}
  679. mov r7, #0xff00 @ 0xfff0 into r7 for EABI
  680. orr r7, r7, #0xf0
  681. swi #0x9ffff0
  682. ldmfd sp!, {r7, pc}
  683. #elif __LINUX_ARM_ARCH__ < 6
  684. /*
  685. * Theory of operation:
  686. *
  687. * We set the Z flag before loading oldval. If ever an exception
  688. * occurs we can not be sure the loaded value will still be the same
  689. * when the exception returns, therefore the user exception handler
  690. * will clear the Z flag whenever the interrupted user code was
  691. * actually from the kernel address space (see the usr_entry macro).
  692. *
  693. * The post-increment on the str is used to prevent a race with an
  694. * exception happening just after the str instruction which would
  695. * clear the Z flag although the exchange was done.
  696. */
  697. #ifdef CONFIG_MMU
  698. teq ip, ip @ set Z flag
  699. ldr ip, [r2] @ load current val
  700. add r3, r2, #1 @ prepare store ptr
  701. teqeq ip, r0 @ compare with oldval if still allowed
  702. streq r1, [r3, #-1]! @ store newval if still allowed
  703. subs r0, r2, r3 @ if r2 == r3 the str occured
  704. #else
  705. #warning "NPTL on non MMU needs fixing"
  706. mov r0, #-1
  707. adds r0, r0, #0
  708. #endif
  709. usr_ret lr
  710. #else
  711. #ifdef CONFIG_SMP
  712. mcr p15, 0, r0, c7, c10, 5 @ dmb
  713. #endif
  714. ldrex r3, [r2]
  715. subs r3, r3, r0
  716. strexeq r3, r1, [r2]
  717. rsbs r0, r3, #0
  718. #ifdef CONFIG_SMP
  719. mcr p15, 0, r0, c7, c10, 5 @ dmb
  720. #endif
  721. usr_ret lr
  722. #endif
  723. .align 5
  724. /*
  725. * Reference prototype:
  726. *
  727. * int __kernel_get_tls(void)
  728. *
  729. * Input:
  730. *
  731. * lr = return address
  732. *
  733. * Output:
  734. *
  735. * r0 = TLS value
  736. *
  737. * Clobbered:
  738. *
  739. * the Z flag might be lost
  740. *
  741. * Definition and user space usage example:
  742. *
  743. * typedef int (__kernel_get_tls_t)(void);
  744. * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0)
  745. *
  746. * Get the TLS value as previously set via the __ARM_NR_set_tls syscall.
  747. *
  748. * This could be used as follows:
  749. *
  750. * #define __kernel_get_tls() \
  751. * ({ register unsigned int __val asm("r0"); \
  752. * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \
  753. * : "=r" (__val) : : "lr","cc" ); \
  754. * __val; })
  755. */
  756. __kuser_get_tls: @ 0xffff0fe0
  757. #if !defined(CONFIG_HAS_TLS_REG) && !defined(CONFIG_TLS_REG_EMUL)
  758. ldr r0, [pc, #(16 - 8)] @ TLS stored at 0xffff0ff0
  759. #else
  760. mrc p15, 0, r0, c13, c0, 3 @ read TLS register
  761. #endif
  762. usr_ret lr
  763. .rep 5
  764. .word 0 @ pad up to __kuser_helper_version
  765. .endr
  766. /*
  767. * Reference declaration:
  768. *
  769. * extern unsigned int __kernel_helper_version;
  770. *
  771. * Definition and user space usage example:
  772. *
  773. * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc)
  774. *
  775. * User space may read this to determine the curent number of helpers
  776. * available.
  777. */
  778. __kuser_helper_version: @ 0xffff0ffc
  779. .word ((__kuser_helper_end - __kuser_helper_start) >> 5)
  780. .globl __kuser_helper_end
  781. __kuser_helper_end:
  782. /*
  783. * Vector stubs.
  784. *
  785. * This code is copied to 0xffff0200 so we can use branches in the
  786. * vectors, rather than ldr's. Note that this code must not
  787. * exceed 0x300 bytes.
  788. *
  789. * Common stub entry macro:
  790. * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
  791. *
  792. * SP points to a minimal amount of processor-private memory, the address
  793. * of which is copied into r0 for the mode specific abort handler.
  794. */
  795. .macro vector_stub, name, mode, correction=0
  796. .align 5
  797. vector_\name:
  798. .if \correction
  799. sub lr, lr, #\correction
  800. .endif
  801. @
  802. @ Save r0, lr_<exception> (parent PC) and spsr_<exception>
  803. @ (parent CPSR)
  804. @
  805. stmia sp, {r0, lr} @ save r0, lr
  806. mrs lr, spsr
  807. str lr, [sp, #8] @ save spsr
  808. @
  809. @ Prepare for SVC32 mode. IRQs remain disabled.
  810. @
  811. mrs r0, cpsr
  812. eor r0, r0, #(\mode ^ SVC_MODE)
  813. msr spsr_cxsf, r0
  814. @
  815. @ the branch table must immediately follow this code
  816. @
  817. and lr, lr, #0x0f
  818. mov r0, sp
  819. ldr lr, [pc, lr, lsl #2]
  820. movs pc, lr @ branch to handler in SVC mode
  821. .endm
  822. .globl __stubs_start
  823. __stubs_start:
  824. /*
  825. * Interrupt dispatcher
  826. */
  827. vector_stub irq, IRQ_MODE, 4
  828. .long __irq_usr @ 0 (USR_26 / USR_32)
  829. .long __irq_invalid @ 1 (FIQ_26 / FIQ_32)
  830. .long __irq_invalid @ 2 (IRQ_26 / IRQ_32)
  831. .long __irq_svc @ 3 (SVC_26 / SVC_32)
  832. .long __irq_invalid @ 4
  833. .long __irq_invalid @ 5
  834. .long __irq_invalid @ 6
  835. .long __irq_invalid @ 7
  836. .long __irq_invalid @ 8
  837. .long __irq_invalid @ 9
  838. .long __irq_invalid @ a
  839. .long __irq_invalid @ b
  840. .long __irq_invalid @ c
  841. .long __irq_invalid @ d
  842. .long __irq_invalid @ e
  843. .long __irq_invalid @ f
  844. /*
  845. * Data abort dispatcher
  846. * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
  847. */
  848. vector_stub dabt, ABT_MODE, 8
  849. .long __dabt_usr @ 0 (USR_26 / USR_32)
  850. .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32)
  851. .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32)
  852. .long __dabt_svc @ 3 (SVC_26 / SVC_32)
  853. .long __dabt_invalid @ 4
  854. .long __dabt_invalid @ 5
  855. .long __dabt_invalid @ 6
  856. .long __dabt_invalid @ 7
  857. .long __dabt_invalid @ 8
  858. .long __dabt_invalid @ 9
  859. .long __dabt_invalid @ a
  860. .long __dabt_invalid @ b
  861. .long __dabt_invalid @ c
  862. .long __dabt_invalid @ d
  863. .long __dabt_invalid @ e
  864. .long __dabt_invalid @ f
  865. /*
  866. * Prefetch abort dispatcher
  867. * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
  868. */
  869. vector_stub pabt, ABT_MODE, 4
  870. .long __pabt_usr @ 0 (USR_26 / USR_32)
  871. .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32)
  872. .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32)
  873. .long __pabt_svc @ 3 (SVC_26 / SVC_32)
  874. .long __pabt_invalid @ 4
  875. .long __pabt_invalid @ 5
  876. .long __pabt_invalid @ 6
  877. .long __pabt_invalid @ 7
  878. .long __pabt_invalid @ 8
  879. .long __pabt_invalid @ 9
  880. .long __pabt_invalid @ a
  881. .long __pabt_invalid @ b
  882. .long __pabt_invalid @ c
  883. .long __pabt_invalid @ d
  884. .long __pabt_invalid @ e
  885. .long __pabt_invalid @ f
  886. /*
  887. * Undef instr entry dispatcher
  888. * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
  889. */
  890. vector_stub und, UND_MODE
  891. .long __und_usr @ 0 (USR_26 / USR_32)
  892. .long __und_invalid @ 1 (FIQ_26 / FIQ_32)
  893. .long __und_invalid @ 2 (IRQ_26 / IRQ_32)
  894. .long __und_svc @ 3 (SVC_26 / SVC_32)
  895. .long __und_invalid @ 4
  896. .long __und_invalid @ 5
  897. .long __und_invalid @ 6
  898. .long __und_invalid @ 7
  899. .long __und_invalid @ 8
  900. .long __und_invalid @ 9
  901. .long __und_invalid @ a
  902. .long __und_invalid @ b
  903. .long __und_invalid @ c
  904. .long __und_invalid @ d
  905. .long __und_invalid @ e
  906. .long __und_invalid @ f
  907. .align 5
  908. /*=============================================================================
  909. * Undefined FIQs
  910. *-----------------------------------------------------------------------------
  911. * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC
  912. * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg.
  913. * Basically to switch modes, we *HAVE* to clobber one register... brain
  914. * damage alert! I don't think that we can execute any code in here in any
  915. * other mode than FIQ... Ok you can switch to another mode, but you can't
  916. * get out of that mode without clobbering one register.
  917. */
  918. vector_fiq:
  919. disable_fiq
  920. subs pc, lr, #4
  921. /*=============================================================================
  922. * Address exception handler
  923. *-----------------------------------------------------------------------------
  924. * These aren't too critical.
  925. * (they're not supposed to happen, and won't happen in 32-bit data mode).
  926. */
  927. vector_addrexcptn:
  928. b vector_addrexcptn
  929. /*
  930. * We group all the following data together to optimise
  931. * for CPUs with separate I & D caches.
  932. */
  933. .align 5
  934. .LCvswi:
  935. .word vector_swi
  936. .globl __stubs_end
  937. __stubs_end:
  938. .equ stubs_offset, __vectors_start + 0x200 - __stubs_start
  939. .globl __vectors_start
  940. __vectors_start:
  941. swi SYS_ERROR0
  942. b vector_und + stubs_offset
  943. ldr pc, .LCvswi + stubs_offset
  944. b vector_pabt + stubs_offset
  945. b vector_dabt + stubs_offset
  946. b vector_addrexcptn + stubs_offset
  947. b vector_irq + stubs_offset
  948. b vector_fiq + stubs_offset
  949. .globl __vectors_end
  950. __vectors_end:
  951. .data
  952. .globl cr_alignment
  953. .globl cr_no_alignment
  954. cr_alignment:
  955. .space 4
  956. cr_no_alignment:
  957. .space 4