sun4v_tlb_miss.S 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
  3. *
  4. * Copyright (C) 2006 <davem@davemloft.net>
  5. */
  6. .text
  7. .align 32
  8. /* Load ITLB fault information into VADDR and CTX, using BASE. */
  9. #define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
  10. ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
  11. ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX;
  12. /* Load DTLB fault information into VADDR and CTX, using BASE. */
  13. #define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
  14. ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
  15. ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX;
  16. /* DEST = (VADDR >> 22)
  17. *
  18. * Branch to ZERO_CTX_LABEL if context is zero.
  19. */
  20. #define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
  21. srlx VADDR, 22, DEST; \
  22. brz,pn CTX, ZERO_CTX_LABEL; \
  23. nop;
  24. /* Create TSB pointer. This is something like:
  25. *
  26. * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
  27. * tsb_base = tsb_reg & ~0x7UL;
  28. * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
  29. * tsb_ptr = tsb_base + (tsb_index * 16);
  30. */
  31. #define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
  32. and TSB_PTR, 0x7, TMP1; \
  33. mov 512, TMP2; \
  34. andn TSB_PTR, 0x7, TSB_PTR; \
  35. sllx TMP2, TMP1, TMP2; \
  36. srlx VADDR, HASH_SHIFT, TMP1; \
  37. sub TMP2, 1, TMP2; \
  38. and TMP1, TMP2, TMP1; \
  39. sllx TMP1, 4, TMP1; \
  40. add TSB_PTR, TMP1, TSB_PTR;
  41. sun4v_itlb_miss:
  42. /* Load MMU Miss base into %g2. */
  43. ldxa [%g0] ASI_SCRATCHPAD, %g2
  44. /* Load UTSB reg into %g1. */
  45. mov SCRATCHPAD_UTSBREG1, %g1
  46. ldxa [%g1] ASI_SCRATCHPAD, %g1
  47. LOAD_ITLB_INFO(%g2, %g4, %g5)
  48. COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
  49. COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
  50. /* Load TSB tag/pte into %g2/%g3 and compare the tag. */
  51. ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
  52. cmp %g2, %g6
  53. bne,a,pn %xcc, tsb_miss_page_table_walk
  54. mov FAULT_CODE_ITLB, %g3
  55. andcc %g3, _PAGE_EXEC_4V, %g0
  56. be,a,pn %xcc, tsb_do_fault
  57. mov FAULT_CODE_ITLB, %g3
  58. /* We have a valid entry, make hypervisor call to load
  59. * I-TLB and return from trap.
  60. *
  61. * %g3: PTE
  62. * %g4: vaddr
  63. */
  64. sun4v_itlb_load:
  65. ldxa [%g0] ASI_SCRATCHPAD, %g6
  66. mov %o0, %g1 ! save %o0
  67. mov %o1, %g2 ! save %o1
  68. mov %o2, %g5 ! save %o2
  69. mov %o3, %g7 ! save %o3
  70. mov %g4, %o0 ! vaddr
  71. ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx
  72. mov %g3, %o2 ! PTE
  73. mov HV_MMU_IMMU, %o3 ! flags
  74. ta HV_MMU_MAP_ADDR_TRAP
  75. brnz,pn %o0, sun4v_itlb_error
  76. mov %g2, %o1 ! restore %o1
  77. mov %g1, %o0 ! restore %o0
  78. mov %g5, %o2 ! restore %o2
  79. mov %g7, %o3 ! restore %o3
  80. retry
  81. sun4v_dtlb_miss:
  82. /* Load MMU Miss base into %g2. */
  83. ldxa [%g0] ASI_SCRATCHPAD, %g2
  84. /* Load UTSB reg into %g1. */
  85. mov SCRATCHPAD_UTSBREG1, %g1
  86. ldxa [%g1] ASI_SCRATCHPAD, %g1
  87. LOAD_DTLB_INFO(%g2, %g4, %g5)
  88. COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
  89. COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
  90. /* Load TSB tag/pte into %g2/%g3 and compare the tag. */
  91. ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
  92. cmp %g2, %g6
  93. bne,a,pn %xcc, tsb_miss_page_table_walk
  94. mov FAULT_CODE_DTLB, %g3
  95. /* We have a valid entry, make hypervisor call to load
  96. * D-TLB and return from trap.
  97. *
  98. * %g3: PTE
  99. * %g4: vaddr
  100. */
  101. sun4v_dtlb_load:
  102. ldxa [%g0] ASI_SCRATCHPAD, %g6
  103. mov %o0, %g1 ! save %o0
  104. mov %o1, %g2 ! save %o1
  105. mov %o2, %g5 ! save %o2
  106. mov %o3, %g7 ! save %o3
  107. mov %g4, %o0 ! vaddr
  108. ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx
  109. mov %g3, %o2 ! PTE
  110. mov HV_MMU_DMMU, %o3 ! flags
  111. ta HV_MMU_MAP_ADDR_TRAP
  112. brnz,pn %o0, sun4v_dtlb_error
  113. mov %g2, %o1 ! restore %o1
  114. mov %g1, %o0 ! restore %o0
  115. mov %g5, %o2 ! restore %o2
  116. mov %g7, %o3 ! restore %o3
  117. retry
  118. sun4v_dtlb_prot:
  119. SET_GL(1)
  120. /* Load MMU Miss base into %g5. */
  121. ldxa [%g0] ASI_SCRATCHPAD, %g5
  122. ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
  123. rdpr %tl, %g1
  124. cmp %g1, 1
  125. bgu,pn %xcc, winfix_trampoline
  126. mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
  127. ba,pt %xcc, sparc64_realfault_common
  128. nop
  129. /* Called from trap table:
  130. * %g4: vaddr
  131. * %g5: context
  132. * %g6: TAG TARGET
  133. */
  134. sun4v_itsb_miss:
  135. mov SCRATCHPAD_UTSBREG1, %g1
  136. ldxa [%g1] ASI_SCRATCHPAD, %g1
  137. brz,pn %g5, kvmap_itlb_4v
  138. mov FAULT_CODE_ITLB, %g3
  139. ba,a,pt %xcc, sun4v_tsb_miss_common
  140. /* Called from trap table:
  141. * %g4: vaddr
  142. * %g5: context
  143. * %g6: TAG TARGET
  144. */
  145. sun4v_dtsb_miss:
  146. mov SCRATCHPAD_UTSBREG1, %g1
  147. ldxa [%g1] ASI_SCRATCHPAD, %g1
  148. brz,pn %g5, kvmap_dtlb_4v
  149. mov FAULT_CODE_DTLB, %g3
  150. /* fallthrough */
  151. sun4v_tsb_miss_common:
  152. COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
  153. sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2
  154. #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
  155. mov SCRATCHPAD_UTSBREG2, %g5
  156. ldxa [%g5] ASI_SCRATCHPAD, %g5
  157. cmp %g5, -1
  158. be,pt %xcc, 80f
  159. nop
  160. COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7)
  161. /* That clobbered %g2, reload it. */
  162. ldxa [%g0] ASI_SCRATCHPAD, %g2
  163. sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2
  164. 80: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
  165. #endif
  166. ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath
  167. ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
  168. sun4v_itlb_error:
  169. rdpr %tl, %g1
  170. cmp %g1, 1
  171. ble,pt %icc, sun4v_bad_ra
  172. or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1
  173. sethi %hi(sun4v_err_itlb_vaddr), %g1
  174. stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
  175. sethi %hi(sun4v_err_itlb_ctx), %g1
  176. ldxa [%g0] ASI_SCRATCHPAD, %g6
  177. ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1
  178. stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
  179. sethi %hi(sun4v_err_itlb_pte), %g1
  180. stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)]
  181. sethi %hi(sun4v_err_itlb_error), %g1
  182. stx %o0, [%g1 + %lo(sun4v_err_itlb_error)]
  183. sethi %hi(1f), %g7
  184. rdpr %tl, %g4
  185. ba,pt %xcc, etraptl1
  186. 1: or %g7, %lo(1f), %g7
  187. mov %l4, %o1
  188. call sun4v_itlb_error_report
  189. add %sp, PTREGS_OFF, %o0
  190. /* NOTREACHED */
  191. sun4v_dtlb_error:
  192. rdpr %tl, %g1
  193. cmp %g1, 1
  194. ble,pt %icc, sun4v_bad_ra
  195. or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1
  196. sethi %hi(sun4v_err_dtlb_vaddr), %g1
  197. stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
  198. sethi %hi(sun4v_err_dtlb_ctx), %g1
  199. ldxa [%g0] ASI_SCRATCHPAD, %g6
  200. ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1
  201. stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
  202. sethi %hi(sun4v_err_dtlb_pte), %g1
  203. stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
  204. sethi %hi(sun4v_err_dtlb_error), %g1
  205. stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)]
  206. sethi %hi(1f), %g7
  207. rdpr %tl, %g4
  208. ba,pt %xcc, etraptl1
  209. 1: or %g7, %lo(1f), %g7
  210. mov %l4, %o1
  211. call sun4v_dtlb_error_report
  212. add %sp, PTREGS_OFF, %o0
  213. /* NOTREACHED */
  214. sun4v_bad_ra:
  215. or %g0, %g4, %g5
  216. ba,pt %xcc, sparc64_realfault_common
  217. or %g1, %g0, %g4
  218. /* NOTREACHED */
  219. /* Instruction Access Exception, tl0. */
  220. sun4v_iacc:
  221. ldxa [%g0] ASI_SCRATCHPAD, %g2
  222. ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
  223. ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
  224. ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
  225. sllx %g3, 16, %g3
  226. or %g5, %g3, %g5
  227. ba,pt %xcc, etrap
  228. rd %pc, %g7
  229. mov %l4, %o1
  230. mov %l5, %o2
  231. call sun4v_insn_access_exception
  232. add %sp, PTREGS_OFF, %o0
  233. ba,a,pt %xcc, rtrap
  234. /* Instruction Access Exception, tl1. */
  235. sun4v_iacc_tl1:
  236. ldxa [%g0] ASI_SCRATCHPAD, %g2
  237. ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
  238. ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
  239. ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
  240. sllx %g3, 16, %g3
  241. or %g5, %g3, %g5
  242. ba,pt %xcc, etraptl1
  243. rd %pc, %g7
  244. mov %l4, %o1
  245. mov %l5, %o2
  246. call sun4v_insn_access_exception_tl1
  247. add %sp, PTREGS_OFF, %o0
  248. ba,a,pt %xcc, rtrap
  249. /* Data Access Exception, tl0. */
  250. sun4v_dacc:
  251. ldxa [%g0] ASI_SCRATCHPAD, %g2
  252. ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
  253. ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
  254. ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
  255. sllx %g3, 16, %g3
  256. or %g5, %g3, %g5
  257. ba,pt %xcc, etrap
  258. rd %pc, %g7
  259. mov %l4, %o1
  260. mov %l5, %o2
  261. call sun4v_data_access_exception
  262. add %sp, PTREGS_OFF, %o0
  263. ba,a,pt %xcc, rtrap
  264. /* Data Access Exception, tl1. */
  265. sun4v_dacc_tl1:
  266. ldxa [%g0] ASI_SCRATCHPAD, %g2
  267. ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
  268. ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
  269. ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
  270. sllx %g3, 16, %g3
  271. or %g5, %g3, %g5
  272. ba,pt %xcc, etraptl1
  273. rd %pc, %g7
  274. mov %l4, %o1
  275. mov %l5, %o2
  276. call sun4v_data_access_exception_tl1
  277. add %sp, PTREGS_OFF, %o0
  278. ba,a,pt %xcc, rtrap
  279. /* Memory Address Unaligned. */
  280. sun4v_mna:
  281. /* Window fixup? */
  282. rdpr %tl, %g2
  283. cmp %g2, 1
  284. ble,pt %icc, 1f
  285. nop
  286. SET_GL(1)
  287. ldxa [%g0] ASI_SCRATCHPAD, %g2
  288. ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
  289. mov HV_FAULT_TYPE_UNALIGNED, %g3
  290. ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4
  291. sllx %g3, 16, %g3
  292. or %g4, %g3, %g4
  293. ba,pt %xcc, winfix_mna
  294. rdpr %tpc, %g3
  295. /* not reached */
  296. 1: ldxa [%g0] ASI_SCRATCHPAD, %g2
  297. mov HV_FAULT_TYPE_UNALIGNED, %g3
  298. ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
  299. ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
  300. sllx %g3, 16, %g3
  301. or %g5, %g3, %g5
  302. ba,pt %xcc, etrap
  303. rd %pc, %g7
  304. mov %l4, %o1
  305. mov %l5, %o2
  306. call sun4v_do_mna
  307. add %sp, PTREGS_OFF, %o0
  308. ba,a,pt %xcc, rtrap
  309. nop
  310. /* Privileged Action. */
  311. sun4v_privact:
  312. ba,pt %xcc, etrap
  313. rd %pc, %g7
  314. call do_privact
  315. add %sp, PTREGS_OFF, %o0
  316. ba,a,pt %xcc, rtrap
  317. /* Unaligned ldd float, tl0. */
  318. sun4v_lddfmna:
  319. ldxa [%g0] ASI_SCRATCHPAD, %g2
  320. ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
  321. ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
  322. ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
  323. sllx %g3, 16, %g3
  324. or %g5, %g3, %g5
  325. ba,pt %xcc, etrap
  326. rd %pc, %g7
  327. mov %l4, %o1
  328. mov %l5, %o2
  329. call handle_lddfmna
  330. add %sp, PTREGS_OFF, %o0
  331. ba,a,pt %xcc, rtrap
  332. /* Unaligned std float, tl0. */
  333. sun4v_stdfmna:
  334. ldxa [%g0] ASI_SCRATCHPAD, %g2
  335. ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
  336. ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
  337. ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
  338. sllx %g3, 16, %g3
  339. or %g5, %g3, %g5
  340. ba,pt %xcc, etrap
  341. rd %pc, %g7
  342. mov %l4, %o1
  343. mov %l5, %o2
  344. call handle_stdfmna
  345. add %sp, PTREGS_OFF, %o0
  346. ba,a,pt %xcc, rtrap
  347. #define BRANCH_ALWAYS 0x10680000
  348. #define NOP 0x01000000
  349. #define SUN4V_DO_PATCH(OLD, NEW) \
  350. sethi %hi(NEW), %g1; \
  351. or %g1, %lo(NEW), %g1; \
  352. sethi %hi(OLD), %g2; \
  353. or %g2, %lo(OLD), %g2; \
  354. sub %g1, %g2, %g1; \
  355. sethi %hi(BRANCH_ALWAYS), %g3; \
  356. sll %g1, 11, %g1; \
  357. srl %g1, 11 + 2, %g1; \
  358. or %g3, %lo(BRANCH_ALWAYS), %g3; \
  359. or %g3, %g1, %g3; \
  360. stw %g3, [%g2]; \
  361. sethi %hi(NOP), %g3; \
  362. or %g3, %lo(NOP), %g3; \
  363. stw %g3, [%g2 + 0x4]; \
  364. flush %g2;
  365. .globl sun4v_patch_tlb_handlers
  366. .type sun4v_patch_tlb_handlers,#function
  367. sun4v_patch_tlb_handlers:
  368. SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
  369. SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
  370. SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
  371. SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
  372. SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
  373. SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
  374. SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
  375. SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
  376. SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
  377. SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
  378. SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
  379. SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
  380. SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
  381. SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
  382. SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
  383. retl
  384. nop
  385. .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers