memcpy.S 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * linux/arch/arm/lib/memcpy.S
  4. *
  5. * Author: Nicolas Pitre
  6. * Created: Sep 28, 2005
  7. * Copyright: MontaVista Software, Inc.
  8. */
  9. #include <linux/linkage.h>
  10. #include <asm/assembler.h>
  11. #define LDR1W_SHIFT 0
  12. #define STR1W_SHIFT 0
  13. .macro ldr1w ptr reg abort
  14. W(ldr) \reg, [\ptr], #4
  15. .endm
  16. .macro ldr4w ptr reg1 reg2 reg3 reg4 abort
  17. ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
  18. .endm
  19. .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  20. ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  21. .endm
  22. .macro ldr1b ptr reg cond=al abort
  23. ldrb\cond\() \reg, [\ptr], #1
  24. .endm
  25. .macro str1w ptr reg abort
  26. W(str) \reg, [\ptr], #4
  27. .endm
  28. .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  29. stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  30. .endm
  31. .macro str1b ptr reg cond=al abort
  32. strb\cond\() \reg, [\ptr], #1
  33. .endm
  34. .macro enter reg1 reg2
  35. stmdb sp!, {r0, \reg1, \reg2}
  36. .endm
  37. .macro exit reg1 reg2
  38. ldmfd sp!, {r0, \reg1, \reg2}
  39. .endm
  40. .text
  41. /* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
  42. .syntax unified
  43. #if CONFIG_IS_ENABLED(SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD)
  44. .thumb
  45. .thumb_func
  46. #endif
  47. ENTRY(memcpy)
  48. cmp r0, r1
  49. bxeq lr
  50. enter r4, lr
  51. subs r2, r2, #4
  52. blt 8f
  53. ands ip, r0, #3
  54. PLD( pld [r1, #0] )
  55. bne 9f
  56. ands ip, r1, #3
  57. bne 10f
  58. 1: subs r2, r2, #(28)
  59. stmfd sp!, {r5 - r8}
  60. blt 5f
  61. CALGN( ands ip, r0, #31 )
  62. CALGN( rsb r3, ip, #32 )
  63. CALGN( sbcsne r4, r3, r2 ) @ C is always set here
  64. CALGN( bcs 2f )
  65. CALGN( adr r4, 6f )
  66. CALGN( subs r2, r2, r3 ) @ C gets set
  67. CALGN( add pc, r4, ip )
  68. PLD( pld [r1, #0] )
  69. 2: PLD( subs r2, r2, #96 )
  70. PLD( pld [r1, #28] )
  71. PLD( blt 4f )
  72. PLD( pld [r1, #60] )
  73. PLD( pld [r1, #92] )
  74. 3: PLD( pld [r1, #124] )
  75. 4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  76. subs r2, r2, #32
  77. str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  78. bge 3b
  79. PLD( cmn r2, #96 )
  80. PLD( bge 4b )
  81. 5: ands ip, r2, #28
  82. rsb ip, ip, #32
  83. #if LDR1W_SHIFT > 0
  84. lsl ip, ip, #LDR1W_SHIFT
  85. #endif
  86. addne pc, pc, ip @ C is always clear here
  87. b 7f
  88. 6:
  89. .rept (1 << LDR1W_SHIFT)
  90. W(nop)
  91. .endr
  92. ldr1w r1, r3, abort=20f
  93. ldr1w r1, r4, abort=20f
  94. ldr1w r1, r5, abort=20f
  95. ldr1w r1, r6, abort=20f
  96. ldr1w r1, r7, abort=20f
  97. ldr1w r1, r8, abort=20f
  98. ldr1w r1, lr, abort=20f
  99. #if LDR1W_SHIFT < STR1W_SHIFT
  100. lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
  101. #elif LDR1W_SHIFT > STR1W_SHIFT
  102. lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
  103. #endif
  104. add pc, pc, ip
  105. nop
  106. .rept (1 << STR1W_SHIFT)
  107. W(nop)
  108. .endr
  109. str1w r0, r3, abort=20f
  110. str1w r0, r4, abort=20f
  111. str1w r0, r5, abort=20f
  112. str1w r0, r6, abort=20f
  113. str1w r0, r7, abort=20f
  114. str1w r0, r8, abort=20f
  115. str1w r0, lr, abort=20f
  116. CALGN( bcs 2b )
  117. 7: ldmfd sp!, {r5 - r8}
  118. 8: movs r2, r2, lsl #31
  119. ldr1b r1, r3, ne, abort=21f
  120. ldr1b r1, r4, cs, abort=21f
  121. ldr1b r1, ip, cs, abort=21f
  122. str1b r0, r3, ne, abort=21f
  123. str1b r0, r4, cs, abort=21f
  124. str1b r0, ip, cs, abort=21f
  125. exit r4, lr
  126. bx lr
  127. 9: rsb ip, ip, #4
  128. cmp ip, #2
  129. ldr1b r1, r3, gt, abort=21f
  130. ldr1b r1, r4, ge, abort=21f
  131. ldr1b r1, lr, abort=21f
  132. str1b r0, r3, gt, abort=21f
  133. str1b r0, r4, ge, abort=21f
  134. subs r2, r2, ip
  135. str1b r0, lr, abort=21f
  136. blt 8b
  137. ands ip, r1, #3
  138. beq 1b
  139. 10: bic r1, r1, #3
  140. cmp ip, #2
  141. ldr1w r1, lr, abort=21f
  142. beq 17f
  143. bgt 18f
  144. .macro forward_copy_shift pull push
  145. subs r2, r2, #28
  146. blt 14f
  147. CALGN( ands ip, r0, #31 )
  148. CALGN( rsb ip, ip, #32 )
  149. CALGN( sbcsne r4, ip, r2 ) @ C is always set here
  150. CALGN( subcc r2, r2, ip )
  151. CALGN( bcc 15f )
  152. 11: stmfd sp!, {r5 - r9}
  153. PLD( pld [r1, #0] )
  154. PLD( subs r2, r2, #96 )
  155. PLD( pld [r1, #28] )
  156. PLD( blt 13f )
  157. PLD( pld [r1, #60] )
  158. PLD( pld [r1, #92] )
  159. 12: PLD( pld [r1, #124] )
  160. 13: ldr4w r1, r4, r5, r6, r7, abort=19f
  161. mov r3, lr, lspull #\pull
  162. subs r2, r2, #32
  163. ldr4w r1, r8, r9, ip, lr, abort=19f
  164. orr r3, r3, r4, lspush #\push
  165. mov r4, r4, lspull #\pull
  166. orr r4, r4, r5, lspush #\push
  167. mov r5, r5, lspull #\pull
  168. orr r5, r5, r6, lspush #\push
  169. mov r6, r6, lspull #\pull
  170. orr r6, r6, r7, lspush #\push
  171. mov r7, r7, lspull #\pull
  172. orr r7, r7, r8, lspush #\push
  173. mov r8, r8, lspull #\pull
  174. orr r8, r8, r9, lspush #\push
  175. mov r9, r9, lspull #\pull
  176. orr r9, r9, ip, lspush #\push
  177. mov ip, ip, lspull #\pull
  178. orr ip, ip, lr, lspush #\push
  179. str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
  180. bge 12b
  181. PLD( cmn r2, #96 )
  182. PLD( bge 13b )
  183. ldmfd sp!, {r5 - r9}
  184. 14: ands ip, r2, #28
  185. beq 16f
  186. 15: mov r3, lr, lspull #\pull
  187. ldr1w r1, lr, abort=21f
  188. subs ip, ip, #4
  189. orr r3, r3, lr, lspush #\push
  190. str1w r0, r3, abort=21f
  191. bgt 15b
  192. CALGN( cmp r2, #0 )
  193. CALGN( bge 11b )
  194. 16: sub r1, r1, #(\push / 8)
  195. b 8b
  196. .endm
  197. forward_copy_shift pull=8 push=24
  198. 17: forward_copy_shift pull=16 push=16
  199. 18: forward_copy_shift pull=24 push=8
  200. /*
  201. * Abort preamble and completion macros.
  202. * If a fixup handler is required then those macros must surround it.
  203. * It is assumed that the fixup code will handle the private part of
  204. * the exit macro.
  205. */
  206. .macro copy_abort_preamble
  207. 19: ldmfd sp!, {r5 - r9}
  208. b 21f
  209. 20: ldmfd sp!, {r5 - r8}
  210. 21:
  211. .endm
  212. .macro copy_abort_end
  213. ldmfd sp!, {r4, lr}
  214. bx lr
  215. .endm
  216. ENDPROC(memcpy)