copy_template.S 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * linux/arch/arm/lib/copy_template.s
  4. *
  5. * Code template for optimized memory copy functions
  6. *
  7. * Author: Nicolas Pitre
  8. * Created: Sep 28, 2005
  9. * Copyright: MontaVista Software, Inc.
  10. */
  11. /*
  12. * Theory of operation
  13. * -------------------
  14. *
  15. * This file provides the core code for a forward memory copy used in
  16. * the implementation of memcopy(), copy_to_user() and copy_from_user().
  17. *
  18. * The including file must define the following accessor macros
  19. * according to the need of the given function:
  20. *
  21. * ldr1w ptr reg abort
  22. *
  23. * This loads one word from 'ptr', stores it in 'reg' and increments
  24. * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
  25. *
  26. * ldr4w ptr reg1 reg2 reg3 reg4 abort
  27. * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  28. *
  29. * This loads four or eight words starting from 'ptr', stores them
  30. * in provided registers and increments 'ptr' past those words.
  31. * The'abort' argument is used for fixup tables.
  32. *
  33. * ldr1b ptr reg cond abort
  34. *
  35. * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
  36. * It also must apply the condition code if provided, otherwise the
  37. * "al" condition is assumed by default.
  38. *
  39. * str1w ptr reg abort
  40. * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  41. * str1b ptr reg cond abort
  42. *
  43. * Same as their ldr* counterparts, but data is stored to 'ptr' location
  44. * rather than being loaded.
  45. *
  46. * enter reg1 reg2
  47. *
  48. * Preserve the provided registers on the stack plus any additional
  49. * data as needed by the implementation including this code. Called
  50. * upon code entry.
  51. *
  52. * usave reg1 reg2
  53. *
  54. * Unwind annotation macro is corresponding for 'enter' macro.
  55. * It tell unwinder that preserved some provided registers on the stack
  56. * and additional data by a prior 'enter' macro.
  57. *
  58. * exit reg1 reg2
  59. *
  60. * Restore registers with the values previously saved with the
  61. * 'preserv' macro. Called upon code termination.
  62. *
  63. * LDR1W_SHIFT
  64. * STR1W_SHIFT
  65. *
  66. * Correction to be applied to the "ip" register when branching into
  67. * the ldr1w or str1w instructions (some of these macros may expand to
  68. * than one 32bit instruction in Thumb-2)
  69. */
  70. UNWIND( .fnstart )
  71. enter r4, lr
  72. UNWIND( .fnend )
  73. UNWIND( .fnstart )
  74. usave r4, lr @ in first stmdb block
  75. subs r2, r2, #4
  76. blt 8f
  77. ands ip, r0, #3
  78. PLD( pld [r1, #0] )
  79. bne 9f
  80. ands ip, r1, #3
  81. bne 10f
  82. 1: subs r2, r2, #(28)
  83. stmfd sp!, {r5 - r8}
  84. UNWIND( .fnend )
  85. UNWIND( .fnstart )
  86. usave r4, lr
  87. UNWIND( .save {r5 - r8} ) @ in second stmfd block
  88. blt 5f
  89. CALGN( ands ip, r0, #31 )
  90. CALGN( rsb r3, ip, #32 )
  91. CALGN( sbcsne r4, r3, r2 ) @ C is always set here
  92. CALGN( bcs 2f )
  93. CALGN( adr r4, 6f )
  94. CALGN( subs r2, r2, r3 ) @ C gets set
  95. CALGN( add pc, r4, ip )
  96. PLD( pld [r1, #0] )
  97. 2: PLD( subs r2, r2, #96 )
  98. PLD( pld [r1, #28] )
  99. PLD( blt 4f )
  100. PLD( pld [r1, #60] )
  101. PLD( pld [r1, #92] )
  102. 3: PLD( pld [r1, #124] )
  103. 4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  104. subs r2, r2, #32
  105. str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  106. bge 3b
  107. PLD( cmn r2, #96 )
  108. PLD( bge 4b )
  109. 5: ands ip, r2, #28
  110. rsb ip, ip, #32
  111. #if LDR1W_SHIFT > 0
  112. lsl ip, ip, #LDR1W_SHIFT
  113. #endif
  114. addne pc, pc, ip @ C is always clear here
  115. b 7f
  116. 6:
  117. .rept (1 << LDR1W_SHIFT)
  118. W(nop)
  119. .endr
  120. ldr1w r1, r3, abort=20f
  121. ldr1w r1, r4, abort=20f
  122. ldr1w r1, r5, abort=20f
  123. ldr1w r1, r6, abort=20f
  124. ldr1w r1, r7, abort=20f
  125. ldr1w r1, r8, abort=20f
  126. ldr1w r1, lr, abort=20f
  127. #if LDR1W_SHIFT < STR1W_SHIFT
  128. lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
  129. #elif LDR1W_SHIFT > STR1W_SHIFT
  130. lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
  131. #endif
  132. add pc, pc, ip
  133. nop
  134. .rept (1 << STR1W_SHIFT)
  135. W(nop)
  136. .endr
  137. str1w r0, r3, abort=20f
  138. str1w r0, r4, abort=20f
  139. str1w r0, r5, abort=20f
  140. str1w r0, r6, abort=20f
  141. str1w r0, r7, abort=20f
  142. str1w r0, r8, abort=20f
  143. str1w r0, lr, abort=20f
  144. CALGN( bcs 2b )
  145. 7: ldmfd sp!, {r5 - r8}
  146. UNWIND( .fnend ) @ end of second stmfd block
  147. UNWIND( .fnstart )
  148. usave r4, lr @ still in first stmdb block
  149. 8: movs r2, r2, lsl #31
  150. ldr1b r1, r3, ne, abort=21f
  151. ldr1b r1, r4, cs, abort=21f
  152. ldr1b r1, ip, cs, abort=21f
  153. str1b r0, r3, ne, abort=21f
  154. str1b r0, r4, cs, abort=21f
  155. str1b r0, ip, cs, abort=21f
  156. exit r4, pc
  157. 9: rsb ip, ip, #4
  158. cmp ip, #2
  159. ldr1b r1, r3, gt, abort=21f
  160. ldr1b r1, r4, ge, abort=21f
  161. ldr1b r1, lr, abort=21f
  162. str1b r0, r3, gt, abort=21f
  163. str1b r0, r4, ge, abort=21f
  164. subs r2, r2, ip
  165. str1b r0, lr, abort=21f
  166. blt 8b
  167. ands ip, r1, #3
  168. beq 1b
  169. 10: bic r1, r1, #3
  170. cmp ip, #2
  171. ldr1w r1, lr, abort=21f
  172. beq 17f
  173. bgt 18f
  174. UNWIND( .fnend )
  175. .macro forward_copy_shift pull push
  176. UNWIND( .fnstart )
  177. usave r4, lr @ still in first stmdb block
  178. subs r2, r2, #28
  179. blt 14f
  180. CALGN( ands ip, r0, #31 )
  181. CALGN( rsb ip, ip, #32 )
  182. CALGN( sbcsne r4, ip, r2 ) @ C is always set here
  183. CALGN( subcc r2, r2, ip )
  184. CALGN( bcc 15f )
  185. 11: stmfd sp!, {r5 - r9}
  186. UNWIND( .fnend )
  187. UNWIND( .fnstart )
  188. usave r4, lr
  189. UNWIND( .save {r5 - r9} ) @ in new second stmfd block
  190. PLD( pld [r1, #0] )
  191. PLD( subs r2, r2, #96 )
  192. PLD( pld [r1, #28] )
  193. PLD( blt 13f )
  194. PLD( pld [r1, #60] )
  195. PLD( pld [r1, #92] )
  196. 12: PLD( pld [r1, #124] )
  197. 13: ldr4w r1, r4, r5, r6, r7, abort=19f
  198. mov r3, lr, lspull #\pull
  199. subs r2, r2, #32
  200. ldr4w r1, r8, r9, ip, lr, abort=19f
  201. orr r3, r3, r4, lspush #\push
  202. mov r4, r4, lspull #\pull
  203. orr r4, r4, r5, lspush #\push
  204. mov r5, r5, lspull #\pull
  205. orr r5, r5, r6, lspush #\push
  206. mov r6, r6, lspull #\pull
  207. orr r6, r6, r7, lspush #\push
  208. mov r7, r7, lspull #\pull
  209. orr r7, r7, r8, lspush #\push
  210. mov r8, r8, lspull #\pull
  211. orr r8, r8, r9, lspush #\push
  212. mov r9, r9, lspull #\pull
  213. orr r9, r9, ip, lspush #\push
  214. mov ip, ip, lspull #\pull
  215. orr ip, ip, lr, lspush #\push
  216. str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f
  217. bge 12b
  218. PLD( cmn r2, #96 )
  219. PLD( bge 13b )
  220. ldmfd sp!, {r5 - r9}
  221. UNWIND( .fnend ) @ end of the second stmfd block
  222. UNWIND( .fnstart )
  223. usave r4, lr @ still in first stmdb block
  224. 14: ands ip, r2, #28
  225. beq 16f
  226. 15: mov r3, lr, lspull #\pull
  227. ldr1w r1, lr, abort=21f
  228. subs ip, ip, #4
  229. orr r3, r3, lr, lspush #\push
  230. str1w r0, r3, abort=21f
  231. bgt 15b
  232. CALGN( cmp r2, #0 )
  233. CALGN( bge 11b )
  234. 16: sub r1, r1, #(\push / 8)
  235. b 8b
  236. UNWIND( .fnend )
  237. .endm
  238. forward_copy_shift pull=8 push=24
  239. 17: forward_copy_shift pull=16 push=16
  240. 18: forward_copy_shift pull=24 push=8
  241. /*
  242. * Abort preamble and completion macros.
  243. * If a fixup handler is required then those macros must surround it.
  244. * It is assumed that the fixup code will handle the private part of
  245. * the exit macro.
  246. */
  247. .macro copy_abort_preamble
  248. 19: ldmfd sp!, {r5 - r9}
  249. b 21f
  250. 20: ldmfd sp!, {r5 - r8}
  251. 21:
  252. .endm
  253. .macro copy_abort_end
  254. ldmfd sp!, {r4, pc}
  255. .endm