align.S 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485
  1. /*
  2. * arch/xtensa/kernel/align.S
  3. *
  4. * Handle unalignment exceptions in kernel space.
  5. *
  6. * This file is subject to the terms and conditions of the GNU General
  7. * Public License. See the file "COPYING" in the main directory of
  8. * this archive for more details.
  9. *
  10. * Copyright (C) 2001 - 2005 Tensilica, Inc.
  11. * Copyright (C) 2014 Cadence Design Systems Inc.
  12. *
  13. * Rewritten by Chris Zankel <chris@zankel.net>
  14. *
  15. * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
  16. * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca>
  17. */
  18. #include <linux/linkage.h>
  19. #include <asm/current.h>
  20. #include <asm/asm-offsets.h>
  21. #include <asm/asmmacro.h>
  22. #include <asm/processor.h>
  23. #if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION
  24. /* First-level exception handler for unaligned exceptions.
  25. *
  26. * Note: This handler works only for kernel exceptions. Unaligned user
  27. * access should get a seg fault.
  28. */
  29. /* Big and little endian 16-bit values are located in
  30. * different halves of a register. HWORD_START helps to
  31. * abstract the notion of extracting a 16-bit value from a
  32. * register.
  33. * We also have to define new shifting instructions because
  34. * lsb and msb are on 'opposite' ends in a register for
  35. * different endian machines.
  36. *
  37. * Assume a memory region in ascending address:
  38. * 0 1 2 3|4 5 6 7
  39. *
  40. * When loading one word into a register, the content of that register is:
  41. * LE 3 2 1 0, 7 6 5 4
  42. * BE 0 1 2 3, 4 5 6 7
  43. *
  44. * Masking the bits of the higher/lower address means:
  45. * LE X X 0 0, 0 0 X X
  46. * BE 0 0 X X, X X 0 0
  47. *
  48. * Shifting to higher/lower addresses, means:
  49. * LE shift left / shift right
  50. * BE shift right / shift left
  51. *
  52. * Extracting 16 bits from a 32 bit reg. value to higher/lower address means:
  53. * LE mask 0 0 X X / shift left
  54. * BE shift left / mask 0 0 X X
  55. */
  56. #define UNALIGNED_USER_EXCEPTION
  57. #if XCHAL_HAVE_BE
  58. #define HWORD_START 16
  59. #define INSN_OP0 28
  60. #define INSN_T 24
  61. #define INSN_OP1 16
  62. .macro __ssa8r r; ssa8l \r; .endm
  63. .macro __sh r, s; srl \r, \s; .endm
  64. .macro __sl r, s; sll \r, \s; .endm
  65. .macro __exth r, s; extui \r, \s, 0, 16; .endm
  66. .macro __extl r, s; slli \r, \s, 16; .endm
  67. #else
  68. #define HWORD_START 0
  69. #define INSN_OP0 0
  70. #define INSN_T 4
  71. #define INSN_OP1 12
  72. .macro __ssa8r r; ssa8b \r; .endm
  73. .macro __sh r, s; sll \r, \s; .endm
  74. .macro __sl r, s; srl \r, \s; .endm
  75. .macro __exth r, s; slli \r, \s, 16; .endm
  76. .macro __extl r, s; extui \r, \s, 0, 16; .endm
  77. #endif
  78. /*
  79. * xxxx xxxx = imm8 field
  80. * yyyy = imm4 field
  81. * ssss = s field
  82. * tttt = t field
  83. *
  84. * 16 0
  85. * -------------------
  86. * L32I.N yyyy ssss tttt 1000
  87. * S32I.N yyyy ssss tttt 1001
  88. *
  89. * 23 0
  90. * -----------------------------
  91. * res 0000 0010
  92. * L16UI xxxx xxxx 0001 ssss tttt 0010
  93. * L32I xxxx xxxx 0010 ssss tttt 0010
  94. * XXX 0011 ssss tttt 0010
  95. * XXX 0100 ssss tttt 0010
  96. * S16I xxxx xxxx 0101 ssss tttt 0010
  97. * S32I xxxx xxxx 0110 ssss tttt 0010
  98. * XXX 0111 ssss tttt 0010
  99. * XXX 1000 ssss tttt 0010
  100. * L16SI xxxx xxxx 1001 ssss tttt 0010
  101. * XXX 1010 0010
  102. * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported
  103. * XXX 1100 0010
  104. * XXX 1101 0010
  105. * XXX 1110 0010
  106. * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported
  107. * -----------------------------
  108. * ^ ^ ^
  109. * sub-opcode (NIBBLE_R) -+ | |
  110. * t field (NIBBLE_T) -----------+ |
  111. * major opcode (NIBBLE_OP0) --------------+
  112. */
  113. #define OP0_L32I_N 0x8 /* load immediate narrow */
  114. #define OP0_S32I_N 0x9 /* store immediate narrow */
  115. #define OP1_SI_MASK 0x4 /* OP1 bit set for stores */
  116. #define OP1_SI_BIT 2 /* OP1 bit number for stores */
  117. #define OP1_L32I 0x2
  118. #define OP1_L16UI 0x1
  119. #define OP1_L16SI 0x9
  120. #define OP1_L32AI 0xb
  121. #define OP1_S32I 0x6
  122. #define OP1_S16I 0x5
  123. #define OP1_S32RI 0xf
  124. /*
  125. * Entry condition:
  126. *
  127. * a0: trashed, original value saved on stack (PT_AREG0)
  128. * a1: a1
  129. * a2: new stack pointer, original in DEPC
  130. * a3: a3
  131. * depc: a2, original value saved on stack (PT_DEPC)
  132. * excsave_1: dispatch table
  133. *
  134. * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  135. * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
  136. */
  137. .literal_position
  138. ENTRY(fast_unaligned)
  139. /* Note: We don't expect the address to be aligned on a word
  140. * boundary. After all, the processor generated that exception
  141. * and it would be a hardware fault.
  142. */
  143. /* Save some working register */
  144. s32i a4, a2, PT_AREG4
  145. s32i a5, a2, PT_AREG5
  146. s32i a6, a2, PT_AREG6
  147. s32i a7, a2, PT_AREG7
  148. s32i a8, a2, PT_AREG8
  149. rsr a0, depc
  150. s32i a0, a2, PT_AREG2
  151. s32i a3, a2, PT_AREG3
  152. rsr a3, excsave1
  153. movi a4, fast_unaligned_fixup
  154. s32i a4, a3, EXC_TABLE_FIXUP
  155. /* Keep value of SAR in a0 */
  156. rsr a0, sar
  157. rsr a8, excvaddr # load unaligned memory address
  158. /* Now, identify one of the following load/store instructions.
  159. *
  160. * The only possible danger of a double exception on the
  161. * following l32i instructions is kernel code in vmalloc
  162. * memory. The processor was just executing at the EPC_1
  163. * address, and indeed, already fetched the instruction. That
  164. * guarantees a TLB mapping, which hasn't been replaced by
  165. * this unaligned exception handler that uses only static TLB
  166. * mappings. However, high-level interrupt handlers might
  167. * modify TLB entries, so for the generic case, we register a
  168. * TABLE_FIXUP handler here, too.
  169. */
  170. /* a3...a6 saved on stack, a2 = SP */
  171. /* Extract the instruction that caused the unaligned access. */
  172. rsr a7, epc1 # load exception address
  173. movi a3, ~3
  174. and a3, a3, a7 # mask lower bits
  175. l32i a4, a3, 0 # load 2 words
  176. l32i a5, a3, 4
  177. __ssa8 a7
  178. __src_b a4, a4, a5 # a4 has the instruction
  179. /* Analyze the instruction (load or store?). */
  180. extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble
  181. #if XCHAL_HAVE_DENSITY
  182. _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump
  183. addi a6, a5, -OP0_S32I_N
  184. _beqz a6, .Lstore # S32I.N, do a store
  185. #endif
  186. /* 'store indicator bit' not set, jump */
  187. _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload
  188. /* Store: Jump to table entry to get the value in the source register.*/
  189. .Lstore:movi a5, .Lstore_table # table
  190. extui a6, a4, INSN_T, 4 # get source register
  191. addx8 a5, a6, a5
  192. jx a5 # jump into table
  193. /* Load: Load memory address. */
  194. .Lload: movi a3, ~3
  195. and a3, a3, a8 # align memory address
  196. __ssa8 a8
  197. #ifdef UNALIGNED_USER_EXCEPTION
  198. addi a3, a3, 8
  199. l32e a5, a3, -8
  200. l32e a6, a3, -4
  201. #else
  202. l32i a5, a3, 0
  203. l32i a6, a3, 4
  204. #endif
  205. __src_b a3, a5, a6 # a3 has the data word
  206. #if XCHAL_HAVE_DENSITY
  207. addi a7, a7, 2 # increment PC (assume 16-bit insn)
  208. extui a5, a4, INSN_OP0, 4
  209. _beqi a5, OP0_L32I_N, 1f # l32i.n: jump
  210. addi a7, a7, 1
  211. #else
  212. addi a7, a7, 3
  213. #endif
  214. extui a5, a4, INSN_OP1, 4
  215. _beqi a5, OP1_L32I, 1f # l32i: jump
  216. extui a3, a3, 0, 16 # extract lower 16 bits
  217. _beqi a5, OP1_L16UI, 1f
  218. addi a5, a5, -OP1_L16SI
  219. _bnez a5, .Linvalid_instruction_load
  220. /* sign extend value */
  221. slli a3, a3, 16
  222. srai a3, a3, 16
  223. /* Set target register. */
  224. 1:
  225. extui a4, a4, INSN_T, 4 # extract target register
  226. movi a5, .Lload_table
  227. addx8 a4, a4, a5
  228. jx a4 # jump to entry for target register
  229. .align 8
  230. .Lload_table:
  231. s32i a3, a2, PT_AREG0; _j .Lexit; .align 8
  232. mov a1, a3; _j .Lexit; .align 8 # fishy??
  233. s32i a3, a2, PT_AREG2; _j .Lexit; .align 8
  234. s32i a3, a2, PT_AREG3; _j .Lexit; .align 8
  235. s32i a3, a2, PT_AREG4; _j .Lexit; .align 8
  236. s32i a3, a2, PT_AREG5; _j .Lexit; .align 8
  237. s32i a3, a2, PT_AREG6; _j .Lexit; .align 8
  238. s32i a3, a2, PT_AREG7; _j .Lexit; .align 8
  239. s32i a3, a2, PT_AREG8; _j .Lexit; .align 8
  240. mov a9, a3 ; _j .Lexit; .align 8
  241. mov a10, a3 ; _j .Lexit; .align 8
  242. mov a11, a3 ; _j .Lexit; .align 8
  243. mov a12, a3 ; _j .Lexit; .align 8
  244. mov a13, a3 ; _j .Lexit; .align 8
  245. mov a14, a3 ; _j .Lexit; .align 8
  246. mov a15, a3 ; _j .Lexit; .align 8
  247. .Lstore_table:
  248. l32i a3, a2, PT_AREG0; _j 1f; .align 8
  249. mov a3, a1; _j 1f; .align 8 # fishy??
  250. l32i a3, a2, PT_AREG2; _j 1f; .align 8
  251. l32i a3, a2, PT_AREG3; _j 1f; .align 8
  252. l32i a3, a2, PT_AREG4; _j 1f; .align 8
  253. l32i a3, a2, PT_AREG5; _j 1f; .align 8
  254. l32i a3, a2, PT_AREG6; _j 1f; .align 8
  255. l32i a3, a2, PT_AREG7; _j 1f; .align 8
  256. l32i a3, a2, PT_AREG8; _j 1f; .align 8
  257. mov a3, a9 ; _j 1f; .align 8
  258. mov a3, a10 ; _j 1f; .align 8
  259. mov a3, a11 ; _j 1f; .align 8
  260. mov a3, a12 ; _j 1f; .align 8
  261. mov a3, a13 ; _j 1f; .align 8
  262. mov a3, a14 ; _j 1f; .align 8
  263. mov a3, a15 ; _j 1f; .align 8
  264. /* We cannot handle this exception. */
  265. .extern _kernel_exception
  266. .Linvalid_instruction_load:
  267. .Linvalid_instruction_store:
  268. movi a4, 0
  269. rsr a3, excsave1
  270. s32i a4, a3, EXC_TABLE_FIXUP
  271. /* Restore a4...a8 and SAR, set SP, and jump to default exception. */
  272. l32i a8, a2, PT_AREG8
  273. l32i a7, a2, PT_AREG7
  274. l32i a6, a2, PT_AREG6
  275. l32i a5, a2, PT_AREG5
  276. l32i a4, a2, PT_AREG4
  277. wsr a0, sar
  278. mov a1, a2
  279. rsr a0, ps
  280. bbsi.l a0, PS_UM_BIT, 2f # jump if user mode
  281. movi a0, _kernel_exception
  282. jx a0
  283. 2: movi a0, _user_exception
  284. jx a0
  285. 1: # a7: instruction pointer, a4: instruction, a3: value
  286. movi a6, 0 # mask: ffffffff:00000000
  287. #if XCHAL_HAVE_DENSITY
  288. addi a7, a7, 2 # incr. PC,assume 16-bit instruction
  289. extui a5, a4, INSN_OP0, 4 # extract OP0
  290. addi a5, a5, -OP0_S32I_N
  291. _beqz a5, 1f # s32i.n: jump
  292. addi a7, a7, 1 # increment PC, 32-bit instruction
  293. #else
  294. addi a7, a7, 3 # increment PC, 32-bit instruction
  295. #endif
  296. extui a5, a4, INSN_OP1, 4 # extract OP1
  297. _beqi a5, OP1_S32I, 1f # jump if 32 bit store
  298. _bnei a5, OP1_S16I, .Linvalid_instruction_store
  299. movi a5, -1
  300. __extl a3, a3 # get 16-bit value
  301. __exth a6, a5 # get 16-bit mask ffffffff:ffff0000
  302. /* Get memory address */
  303. 1:
  304. movi a4, ~3
  305. and a4, a4, a8 # align memory address
  306. /* Insert value into memory */
  307. movi a5, -1 # mask: ffffffff:XXXX0000
  308. #ifdef UNALIGNED_USER_EXCEPTION
  309. addi a4, a4, 8
  310. #endif
  311. __ssa8r a8
  312. __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE)
  313. __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE)
  314. #ifdef UNALIGNED_USER_EXCEPTION
  315. l32e a5, a4, -8
  316. #else
  317. l32i a5, a4, 0 # load lower address word
  318. #endif
  319. and a5, a5, a8 # mask
  320. __sh a8, a3 # shift value
  321. or a5, a5, a8 # or with original value
  322. #ifdef UNALIGNED_USER_EXCEPTION
  323. s32e a5, a4, -8
  324. l32e a8, a4, -4
  325. #else
  326. s32i a5, a4, 0 # store
  327. l32i a8, a4, 4 # same for upper address word
  328. #endif
  329. __sl a5, a3
  330. and a6, a8, a6
  331. or a6, a6, a5
  332. #ifdef UNALIGNED_USER_EXCEPTION
  333. s32e a6, a4, -4
  334. #else
  335. s32i a6, a4, 4
  336. #endif
  337. .Lexit:
  338. #if XCHAL_HAVE_LOOPS
  339. rsr a4, lend # check if we reached LEND
  340. bne a7, a4, 1f
  341. rsr a4, lcount # and LCOUNT != 0
  342. beqz a4, 1f
  343. addi a4, a4, -1 # decrement LCOUNT and set
  344. rsr a7, lbeg # set PC to LBEGIN
  345. wsr a4, lcount
  346. #endif
  347. 1: wsr a7, epc1 # skip emulated instruction
  348. /* Update icount if we're single-stepping in userspace. */
  349. rsr a4, icountlevel
  350. beqz a4, 1f
  351. bgeui a4, LOCKLEVEL + 1, 1f
  352. rsr a4, icount
  353. addi a4, a4, 1
  354. wsr a4, icount
  355. 1:
  356. movi a4, 0
  357. rsr a3, excsave1
  358. s32i a4, a3, EXC_TABLE_FIXUP
  359. /* Restore working register */
  360. l32i a8, a2, PT_AREG8
  361. l32i a7, a2, PT_AREG7
  362. l32i a6, a2, PT_AREG6
  363. l32i a5, a2, PT_AREG5
  364. l32i a4, a2, PT_AREG4
  365. l32i a3, a2, PT_AREG3
  366. /* restore SAR and return */
  367. wsr a0, sar
  368. l32i a0, a2, PT_AREG0
  369. l32i a2, a2, PT_AREG2
  370. rfe
  371. ENDPROC(fast_unaligned)
  372. ENTRY(fast_unaligned_fixup)
  373. l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
  374. wsr a3, excsave1
  375. l32i a8, a2, PT_AREG8
  376. l32i a7, a2, PT_AREG7
  377. l32i a6, a2, PT_AREG6
  378. l32i a5, a2, PT_AREG5
  379. l32i a4, a2, PT_AREG4
  380. l32i a0, a2, PT_AREG2
  381. xsr a0, depc # restore depc and a0
  382. wsr a0, sar
  383. rsr a0, exccause
  384. s32i a0, a2, PT_DEPC # mark as a regular exception
  385. rsr a0, ps
  386. bbsi.l a0, PS_UM_BIT, 1f # jump if user mode
  387. rsr a0, exccause
  388. addx4 a0, a0, a3 # find entry in table
  389. l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler
  390. l32i a3, a2, PT_AREG3
  391. jx a0
  392. 1:
  393. rsr a0, exccause
  394. addx4 a0, a0, a3 # find entry in table
  395. l32i a0, a0, EXC_TABLE_FAST_USER # load handler
  396. l32i a3, a2, PT_AREG3
  397. jx a0
  398. ENDPROC(fast_unaligned_fixup)
  399. #endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION */