ktlb.S 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
  3. *
  4. * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
  5. * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
  6. * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
  7. * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  8. */
  9. #include <linux/pgtable.h>
  10. #include <asm/head.h>
  11. #include <asm/asi.h>
  12. #include <asm/page.h>
  13. #include <asm/tsb.h>
  14. .text
  15. .align 32
  16. kvmap_itlb:
  17. /* g6: TAG TARGET */
  18. mov TLB_TAG_ACCESS, %g4
  19. ldxa [%g4] ASI_IMMU, %g4
  20. /* The kernel executes in context zero, therefore we do not
  21. * need to clear the context ID bits out of %g4 here.
  22. */
  23. /* sun4v_itlb_miss branches here with the missing virtual
  24. * address already loaded into %g4
  25. */
  26. kvmap_itlb_4v:
  27. /* Catch kernel NULL pointer calls. */
  28. sethi %hi(PAGE_SIZE), %g5
  29. cmp %g4, %g5
  30. blu,pn %xcc, kvmap_itlb_longpath
  31. nop
  32. KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
  33. kvmap_itlb_tsb_miss:
  34. sethi %hi(LOW_OBP_ADDRESS), %g5
  35. cmp %g4, %g5
  36. blu,pn %xcc, kvmap_itlb_vmalloc_addr
  37. mov 0x1, %g5
  38. sllx %g5, 32, %g5
  39. cmp %g4, %g5
  40. blu,pn %xcc, kvmap_itlb_obp
  41. nop
  42. kvmap_itlb_vmalloc_addr:
  43. KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
  44. TSB_LOCK_TAG(%g1, %g2, %g7)
  45. TSB_WRITE(%g1, %g5, %g6)
  46. /* fallthrough to TLB load */
  47. kvmap_itlb_load:
  48. 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
  49. retry
  50. .section .sun4v_2insn_patch, "ax"
  51. .word 661b
  52. nop
  53. nop
  54. .previous
  55. /* For sun4v the ASI_ITLB_DATA_IN store and the retry
  56. * instruction get nop'd out and we get here to branch
  57. * to the sun4v tlb load code. The registers are setup
  58. * as follows:
  59. *
  60. * %g4: vaddr
  61. * %g5: PTE
  62. * %g6: TAG
  63. *
  64. * The sun4v TLB load wants the PTE in %g3 so we fix that
  65. * up here.
  66. */
  67. ba,pt %xcc, sun4v_itlb_load
  68. mov %g5, %g3
  69. kvmap_itlb_longpath:
  70. 661: rdpr %pstate, %g5
  71. wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
  72. .section .sun4v_2insn_patch, "ax"
  73. .word 661b
  74. SET_GL(1)
  75. nop
  76. .previous
  77. rdpr %tpc, %g5
  78. ba,pt %xcc, sparc64_realfault_common
  79. mov FAULT_CODE_ITLB, %g4
  80. kvmap_itlb_obp:
  81. OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
  82. TSB_LOCK_TAG(%g1, %g2, %g7)
  83. TSB_WRITE(%g1, %g5, %g6)
  84. ba,pt %xcc, kvmap_itlb_load
  85. nop
  86. kvmap_dtlb_obp:
  87. OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
  88. TSB_LOCK_TAG(%g1, %g2, %g7)
  89. TSB_WRITE(%g1, %g5, %g6)
  90. ba,pt %xcc, kvmap_dtlb_load
  91. nop
  92. kvmap_linear_early:
  93. sethi %hi(kern_linear_pte_xor), %g7
  94. ldx [%g7 + %lo(kern_linear_pte_xor)], %g2
  95. ba,pt %xcc, kvmap_dtlb_tsb4m_load
  96. xor %g2, %g4, %g5
  97. .align 32
  98. kvmap_dtlb_tsb4m_load:
  99. TSB_LOCK_TAG(%g1, %g2, %g7)
  100. TSB_WRITE(%g1, %g5, %g6)
  101. ba,pt %xcc, kvmap_dtlb_load
  102. nop
  103. kvmap_dtlb:
  104. /* %g6: TAG TARGET */
  105. mov TLB_TAG_ACCESS, %g4
  106. ldxa [%g4] ASI_DMMU, %g4
  107. /* The kernel executes in context zero, therefore we do not
  108. * need to clear the context ID bits out of %g4 here.
  109. */
  110. /* sun4v_dtlb_miss branches here with the missing virtual
  111. * address already loaded into %g4
  112. */
  113. kvmap_dtlb_4v:
  114. brgez,pn %g4, kvmap_dtlb_nonlinear
  115. nop
  116. #ifdef CONFIG_DEBUG_PAGEALLOC
  117. /* Index through the base page size TSB even for linear
  118. * mappings when using page allocation debugging.
  119. */
  120. KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
  121. #else
  122. /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
  123. KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
  124. #endif
  125. /* Linear mapping TSB lookup failed. Fallthrough to kernel
  126. * page table based lookup.
  127. */
  128. .globl kvmap_linear_patch
  129. kvmap_linear_patch:
  130. ba,a,pt %xcc, kvmap_linear_early
  131. kvmap_dtlb_vmalloc_addr:
  132. KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
  133. TSB_LOCK_TAG(%g1, %g2, %g7)
  134. TSB_WRITE(%g1, %g5, %g6)
  135. /* fallthrough to TLB load */
  136. kvmap_dtlb_load:
  137. 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
  138. retry
  139. .section .sun4v_2insn_patch, "ax"
  140. .word 661b
  141. nop
  142. nop
  143. .previous
  144. /* For sun4v the ASI_DTLB_DATA_IN store and the retry
  145. * instruction get nop'd out and we get here to branch
  146. * to the sun4v tlb load code. The registers are setup
  147. * as follows:
  148. *
  149. * %g4: vaddr
  150. * %g5: PTE
  151. * %g6: TAG
  152. *
  153. * The sun4v TLB load wants the PTE in %g3 so we fix that
  154. * up here.
  155. */
  156. ba,pt %xcc, sun4v_dtlb_load
  157. mov %g5, %g3
  158. #ifdef CONFIG_SPARSEMEM_VMEMMAP
  159. kvmap_vmemmap:
  160. KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
  161. ba,a,pt %xcc, kvmap_dtlb_load
  162. #endif
  163. kvmap_dtlb_nonlinear:
  164. /* Catch kernel NULL pointer derefs. */
  165. sethi %hi(PAGE_SIZE), %g5
  166. cmp %g4, %g5
  167. bleu,pn %xcc, kvmap_dtlb_longpath
  168. nop
  169. #ifdef CONFIG_SPARSEMEM_VMEMMAP
  170. /* Do not use the TSB for vmemmap. */
  171. sethi %hi(VMEMMAP_BASE), %g5
  172. ldx [%g5 + %lo(VMEMMAP_BASE)], %g5
  173. cmp %g4,%g5
  174. bgeu,pn %xcc, kvmap_vmemmap
  175. nop
  176. #endif
  177. KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
  178. kvmap_dtlb_tsbmiss:
  179. sethi %hi(MODULES_VADDR), %g5
  180. cmp %g4, %g5
  181. blu,pn %xcc, kvmap_dtlb_longpath
  182. sethi %hi(VMALLOC_END), %g5
  183. ldx [%g5 + %lo(VMALLOC_END)], %g5
  184. cmp %g4, %g5
  185. bgeu,pn %xcc, kvmap_dtlb_longpath
  186. nop
  187. kvmap_check_obp:
  188. sethi %hi(LOW_OBP_ADDRESS), %g5
  189. cmp %g4, %g5
  190. blu,pn %xcc, kvmap_dtlb_vmalloc_addr
  191. mov 0x1, %g5
  192. sllx %g5, 32, %g5
  193. cmp %g4, %g5
  194. blu,pn %xcc, kvmap_dtlb_obp
  195. nop
  196. ba,pt %xcc, kvmap_dtlb_vmalloc_addr
  197. nop
  198. kvmap_dtlb_longpath:
  199. 661: rdpr %pstate, %g5
  200. wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
  201. .section .sun4v_2insn_patch, "ax"
  202. .word 661b
  203. SET_GL(1)
  204. ldxa [%g0] ASI_SCRATCHPAD, %g5
  205. .previous
  206. rdpr %tl, %g3
  207. cmp %g3, 1
  208. 661: mov TLB_TAG_ACCESS, %g4
  209. ldxa [%g4] ASI_DMMU, %g5
  210. .section .sun4v_2insn_patch, "ax"
  211. .word 661b
  212. ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
  213. nop
  214. .previous
  215. /* The kernel executes in context zero, therefore we do not
  216. * need to clear the context ID bits out of %g5 here.
  217. */
  218. be,pt %xcc, sparc64_realfault_common
  219. mov FAULT_CODE_DTLB, %g4
  220. ba,pt %xcc, winfix_trampoline
  221. nop