copy_page.S 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* clear_page.S: UltraSparc optimized copy page.
  3. *
  4. * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
  5. * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
  6. */
  7. #include <asm/visasm.h>
  8. #include <asm/thread_info.h>
  9. #include <asm/page.h>
  10. #include <linux/pgtable.h>
  11. #include <asm/spitfire.h>
  12. #include <asm/head.h>
  13. #include <asm/export.h>
  14. /* What we used to do was lock a TLB entry into a specific
  15. * TLB slot, clear the page with interrupts disabled, then
  16. * restore the original TLB entry. This was great for
  17. * disturbing the TLB as little as possible, but it meant
  18. * we had to keep interrupts disabled for a long time.
  19. *
  20. * Now, we simply use the normal TLB loading mechanism,
  21. * and this makes the cpu choose a slot all by itself.
  22. * Then we do a normal TLB flush on exit. We need only
  23. * disable preemption during the clear.
  24. */
  25. #define DCACHE_SIZE (PAGE_SIZE * 2)
  26. #if (PAGE_SHIFT == 13)
  27. #define PAGE_SIZE_REM 0x80
  28. #elif (PAGE_SHIFT == 16)
  29. #define PAGE_SIZE_REM 0x100
  30. #else
  31. #error Wrong PAGE_SHIFT specified
  32. #endif
  33. #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
  34. fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \
  35. fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \
  36. fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \
  37. fsrc2 %reg6, %f60; fsrc2 %reg7, %f62;
  38. .text
  39. .align 32
  40. .globl copy_user_page
  41. .type copy_user_page,#function
  42. EXPORT_SYMBOL(copy_user_page)
  43. copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
  44. lduw [%g6 + TI_PRE_COUNT], %o4
  45. sethi %hi(PAGE_OFFSET), %g2
  46. sethi %hi(PAGE_SIZE), %o3
  47. ldx [%g2 + %lo(PAGE_OFFSET)], %g2
  48. sethi %hi(PAGE_KERNEL_LOCKED), %g3
  49. ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
  50. sub %o0, %g2, %g1 ! dest paddr
  51. sub %o1, %g2, %g2 ! src paddr
  52. and %o2, %o3, %o0 ! vaddr D-cache alias bit
  53. or %g1, %g3, %g1 ! dest TTE data
  54. or %g2, %g3, %g2 ! src TTE data
  55. sethi %hi(TLBTEMP_BASE), %o3
  56. sethi %hi(DCACHE_SIZE), %o1
  57. add %o0, %o3, %o0 ! dest TTE vaddr
  58. add %o4, 1, %o2
  59. add %o0, %o1, %o1 ! src TTE vaddr
  60. /* Disable preemption. */
  61. mov TLB_TAG_ACCESS, %g3
  62. stw %o2, [%g6 + TI_PRE_COUNT]
  63. /* Load TLB entries. */
  64. rdpr %pstate, %o2
  65. wrpr %o2, PSTATE_IE, %pstate
  66. stxa %o0, [%g3] ASI_DMMU
  67. stxa %g1, [%g0] ASI_DTLB_DATA_IN
  68. membar #Sync
  69. stxa %o1, [%g3] ASI_DMMU
  70. stxa %g2, [%g0] ASI_DTLB_DATA_IN
  71. membar #Sync
  72. wrpr %o2, 0x0, %pstate
  73. cheetah_copy_page_insn:
  74. ba,pt %xcc, 9f
  75. nop
  76. 1:
  77. VISEntryHalf
  78. membar #StoreLoad | #StoreStore | #LoadStore
  79. sethi %hi((PAGE_SIZE/64)-2), %o2
  80. mov %o0, %g1
  81. prefetch [%o1 + 0x000], #one_read
  82. or %o2, %lo((PAGE_SIZE/64)-2), %o2
  83. prefetch [%o1 + 0x040], #one_read
  84. prefetch [%o1 + 0x080], #one_read
  85. prefetch [%o1 + 0x0c0], #one_read
  86. ldd [%o1 + 0x000], %f0
  87. prefetch [%o1 + 0x100], #one_read
  88. ldd [%o1 + 0x008], %f2
  89. prefetch [%o1 + 0x140], #one_read
  90. ldd [%o1 + 0x010], %f4
  91. prefetch [%o1 + 0x180], #one_read
  92. fsrc2 %f0, %f16
  93. ldd [%o1 + 0x018], %f6
  94. fsrc2 %f2, %f18
  95. ldd [%o1 + 0x020], %f8
  96. fsrc2 %f4, %f20
  97. ldd [%o1 + 0x028], %f10
  98. fsrc2 %f6, %f22
  99. ldd [%o1 + 0x030], %f12
  100. fsrc2 %f8, %f24
  101. ldd [%o1 + 0x038], %f14
  102. fsrc2 %f10, %f26
  103. ldd [%o1 + 0x040], %f0
  104. 1: ldd [%o1 + 0x048], %f2
  105. fsrc2 %f12, %f28
  106. ldd [%o1 + 0x050], %f4
  107. fsrc2 %f14, %f30
  108. stda %f16, [%o0] ASI_BLK_P
  109. ldd [%o1 + 0x058], %f6
  110. fsrc2 %f0, %f16
  111. ldd [%o1 + 0x060], %f8
  112. fsrc2 %f2, %f18
  113. ldd [%o1 + 0x068], %f10
  114. fsrc2 %f4, %f20
  115. ldd [%o1 + 0x070], %f12
  116. fsrc2 %f6, %f22
  117. ldd [%o1 + 0x078], %f14
  118. fsrc2 %f8, %f24
  119. ldd [%o1 + 0x080], %f0
  120. prefetch [%o1 + 0x180], #one_read
  121. fsrc2 %f10, %f26
  122. subcc %o2, 1, %o2
  123. add %o0, 0x40, %o0
  124. bne,pt %xcc, 1b
  125. add %o1, 0x40, %o1
  126. ldd [%o1 + 0x048], %f2
  127. fsrc2 %f12, %f28
  128. ldd [%o1 + 0x050], %f4
  129. fsrc2 %f14, %f30
  130. stda %f16, [%o0] ASI_BLK_P
  131. ldd [%o1 + 0x058], %f6
  132. fsrc2 %f0, %f16
  133. ldd [%o1 + 0x060], %f8
  134. fsrc2 %f2, %f18
  135. ldd [%o1 + 0x068], %f10
  136. fsrc2 %f4, %f20
  137. ldd [%o1 + 0x070], %f12
  138. fsrc2 %f6, %f22
  139. add %o0, 0x40, %o0
  140. ldd [%o1 + 0x078], %f14
  141. fsrc2 %f8, %f24
  142. fsrc2 %f10, %f26
  143. fsrc2 %f12, %f28
  144. fsrc2 %f14, %f30
  145. stda %f16, [%o0] ASI_BLK_P
  146. membar #Sync
  147. VISExitHalf
  148. ba,pt %xcc, 5f
  149. nop
  150. 9:
  151. VISEntry
  152. ldub [%g6 + TI_FAULT_CODE], %g3
  153. mov %o0, %g1
  154. cmp %g3, 0
  155. rd %asi, %g3
  156. be,a,pt %icc, 1f
  157. wr %g0, ASI_BLK_P, %asi
  158. wr %g0, ASI_BLK_COMMIT_P, %asi
  159. 1: ldda [%o1] ASI_BLK_P, %f0
  160. add %o1, 0x40, %o1
  161. ldda [%o1] ASI_BLK_P, %f16
  162. add %o1, 0x40, %o1
  163. sethi %hi(PAGE_SIZE), %o2
  164. 1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
  165. ldda [%o1] ASI_BLK_P, %f32
  166. stda %f48, [%o0] %asi
  167. add %o1, 0x40, %o1
  168. sub %o2, 0x40, %o2
  169. add %o0, 0x40, %o0
  170. TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
  171. ldda [%o1] ASI_BLK_P, %f0
  172. stda %f48, [%o0] %asi
  173. add %o1, 0x40, %o1
  174. sub %o2, 0x40, %o2
  175. add %o0, 0x40, %o0
  176. TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
  177. ldda [%o1] ASI_BLK_P, %f16
  178. stda %f48, [%o0] %asi
  179. sub %o2, 0x40, %o2
  180. add %o1, 0x40, %o1
  181. cmp %o2, PAGE_SIZE_REM
  182. bne,pt %xcc, 1b
  183. add %o0, 0x40, %o0
  184. #if (PAGE_SHIFT == 16)
  185. TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
  186. ldda [%o1] ASI_BLK_P, %f32
  187. stda %f48, [%o0] %asi
  188. add %o1, 0x40, %o1
  189. sub %o2, 0x40, %o2
  190. add %o0, 0x40, %o0
  191. TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
  192. ldda [%o1] ASI_BLK_P, %f0
  193. stda %f48, [%o0] %asi
  194. add %o1, 0x40, %o1
  195. sub %o2, 0x40, %o2
  196. add %o0, 0x40, %o0
  197. membar #Sync
  198. stda %f32, [%o0] %asi
  199. add %o0, 0x40, %o0
  200. stda %f0, [%o0] %asi
  201. #else
  202. membar #Sync
  203. stda %f0, [%o0] %asi
  204. add %o0, 0x40, %o0
  205. stda %f16, [%o0] %asi
  206. #endif
  207. membar #Sync
  208. wr %g3, 0x0, %asi
  209. VISExit
  210. 5:
  211. stxa %g0, [%g1] ASI_DMMU_DEMAP
  212. membar #Sync
  213. sethi %hi(DCACHE_SIZE), %g2
  214. stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP
  215. membar #Sync
  216. retl
  217. stw %o4, [%g6 + TI_PRE_COUNT]
  218. .size copy_user_page, .-copy_user_page
  219. .globl cheetah_patch_copy_page
  220. cheetah_patch_copy_page:
  221. sethi %hi(0x01000000), %o1 ! NOP
  222. sethi %hi(cheetah_copy_page_insn), %o0
  223. or %o0, %lo(cheetah_copy_page_insn), %o0
  224. stw %o1, [%o0]
  225. membar #StoreStore
  226. flush %o0
  227. retl
  228. nop