memmove.S 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * linux/arch/arm/lib/memmove.S
  4. *
  5. * Author: Nicolas Pitre
  6. * Created: Sep 28, 2005
  7. * Copyright: (C) MontaVista Software Inc.
  8. */
  9. #include <linux/linkage.h>
  10. #include <asm/assembler.h>
  11. #include <asm/unwind.h>
  12. .text
  13. /*
  14. * Prototype: void *memmove(void *dest, const void *src, size_t n);
  15. *
  16. * Note:
  17. *
  18. * If the memory regions don't overlap, we simply branch to memcpy which is
  19. * normally a bit faster. Otherwise the copy is done going downwards. This
  20. * is a transposition of the code from copy_template.S but with the copy
  21. * occurring in the opposite direction.
  22. */
  23. ENTRY(memmove)
  24. UNWIND( .fnstart )
  25. subs ip, r0, r1
  26. cmphi r2, ip
  27. bls memcpy
  28. stmfd sp!, {r0, r4, lr}
  29. UNWIND( .fnend )
  30. UNWIND( .fnstart )
  31. UNWIND( .save {r0, r4, lr} ) @ in first stmfd block
  32. add r1, r1, r2
  33. add r0, r0, r2
  34. subs r2, r2, #4
  35. blt 8f
  36. ands ip, r0, #3
  37. PLD( pld [r1, #-4] )
  38. bne 9f
  39. ands ip, r1, #3
  40. bne 10f
  41. 1: subs r2, r2, #(28)
  42. stmfd sp!, {r5 - r8}
  43. UNWIND( .fnend )
  44. UNWIND( .fnstart )
  45. UNWIND( .save {r0, r4, lr} )
  46. UNWIND( .save {r5 - r8} ) @ in second stmfd block
  47. blt 5f
  48. CALGN( ands ip, r0, #31 )
  49. CALGN( sbcsne r4, ip, r2 ) @ C is always set here
  50. CALGN( bcs 2f )
  51. CALGN( adr r4, 6f )
  52. CALGN( subs r2, r2, ip ) @ C is set here
  53. CALGN( rsb ip, ip, #32 )
  54. CALGN( add pc, r4, ip )
  55. PLD( pld [r1, #-4] )
  56. 2: PLD( subs r2, r2, #96 )
  57. PLD( pld [r1, #-32] )
  58. PLD( blt 4f )
  59. PLD( pld [r1, #-64] )
  60. PLD( pld [r1, #-96] )
  61. 3: PLD( pld [r1, #-128] )
  62. 4: ldmdb r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
  63. subs r2, r2, #32
  64. stmdb r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
  65. bge 3b
  66. PLD( cmn r2, #96 )
  67. PLD( bge 4b )
  68. 5: ands ip, r2, #28
  69. rsb ip, ip, #32
  70. addne pc, pc, ip @ C is always clear here
  71. b 7f
  72. 6: W(nop)
  73. W(ldr) r3, [r1, #-4]!
  74. W(ldr) r4, [r1, #-4]!
  75. W(ldr) r5, [r1, #-4]!
  76. W(ldr) r6, [r1, #-4]!
  77. W(ldr) r7, [r1, #-4]!
  78. W(ldr) r8, [r1, #-4]!
  79. W(ldr) lr, [r1, #-4]!
  80. add pc, pc, ip
  81. nop
  82. W(nop)
  83. W(str) r3, [r0, #-4]!
  84. W(str) r4, [r0, #-4]!
  85. W(str) r5, [r0, #-4]!
  86. W(str) r6, [r0, #-4]!
  87. W(str) r7, [r0, #-4]!
  88. W(str) r8, [r0, #-4]!
  89. W(str) lr, [r0, #-4]!
  90. CALGN( bcs 2b )
  91. 7: ldmfd sp!, {r5 - r8}
  92. UNWIND( .fnend ) @ end of second stmfd block
  93. UNWIND( .fnstart )
  94. UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
  95. 8: movs r2, r2, lsl #31
  96. ldrbne r3, [r1, #-1]!
  97. ldrbcs r4, [r1, #-1]!
  98. ldrbcs ip, [r1, #-1]
  99. strbne r3, [r0, #-1]!
  100. strbcs r4, [r0, #-1]!
  101. strbcs ip, [r0, #-1]
  102. ldmfd sp!, {r0, r4, pc}
  103. 9: cmp ip, #2
  104. ldrbgt r3, [r1, #-1]!
  105. ldrbge r4, [r1, #-1]!
  106. ldrb lr, [r1, #-1]!
  107. strbgt r3, [r0, #-1]!
  108. strbge r4, [r0, #-1]!
  109. subs r2, r2, ip
  110. strb lr, [r0, #-1]!
  111. blt 8b
  112. ands ip, r1, #3
  113. beq 1b
  114. 10: bic r1, r1, #3
  115. cmp ip, #2
  116. ldr r3, [r1, #0]
  117. beq 17f
  118. blt 18f
  119. UNWIND( .fnend )
  120. .macro backward_copy_shift push pull
  121. UNWIND( .fnstart )
  122. UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
  123. subs r2, r2, #28
  124. blt 14f
  125. CALGN( ands ip, r0, #31 )
  126. CALGN( sbcsne r4, ip, r2 ) @ C is always set here
  127. CALGN( subcc r2, r2, ip )
  128. CALGN( bcc 15f )
  129. 11: stmfd sp!, {r5 - r9}
  130. UNWIND( .fnend )
  131. UNWIND( .fnstart )
  132. UNWIND( .save {r0, r4, lr} )
  133. UNWIND( .save {r5 - r9} ) @ in new second stmfd block
  134. PLD( pld [r1, #-4] )
  135. PLD( subs r2, r2, #96 )
  136. PLD( pld [r1, #-32] )
  137. PLD( blt 13f )
  138. PLD( pld [r1, #-64] )
  139. PLD( pld [r1, #-96] )
  140. 12: PLD( pld [r1, #-128] )
  141. 13: ldmdb r1!, {r7, r8, r9, ip}
  142. mov lr, r3, lspush #\push
  143. subs r2, r2, #32
  144. ldmdb r1!, {r3, r4, r5, r6}
  145. orr lr, lr, ip, lspull #\pull
  146. mov ip, ip, lspush #\push
  147. orr ip, ip, r9, lspull #\pull
  148. mov r9, r9, lspush #\push
  149. orr r9, r9, r8, lspull #\pull
  150. mov r8, r8, lspush #\push
  151. orr r8, r8, r7, lspull #\pull
  152. mov r7, r7, lspush #\push
  153. orr r7, r7, r6, lspull #\pull
  154. mov r6, r6, lspush #\push
  155. orr r6, r6, r5, lspull #\pull
  156. mov r5, r5, lspush #\push
  157. orr r5, r5, r4, lspull #\pull
  158. mov r4, r4, lspush #\push
  159. orr r4, r4, r3, lspull #\pull
  160. stmdb r0!, {r4 - r9, ip, lr}
  161. bge 12b
  162. PLD( cmn r2, #96 )
  163. PLD( bge 13b )
  164. ldmfd sp!, {r5 - r9}
  165. UNWIND( .fnend ) @ end of the second stmfd block
  166. UNWIND( .fnstart )
  167. UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
  168. 14: ands ip, r2, #28
  169. beq 16f
  170. 15: mov lr, r3, lspush #\push
  171. ldr r3, [r1, #-4]!
  172. subs ip, ip, #4
  173. orr lr, lr, r3, lspull #\pull
  174. str lr, [r0, #-4]!
  175. bgt 15b
  176. CALGN( cmp r2, #0 )
  177. CALGN( bge 11b )
  178. 16: add r1, r1, #(\pull / 8)
  179. b 8b
  180. UNWIND( .fnend )
  181. .endm
  182. backward_copy_shift push=8 pull=24
  183. 17: backward_copy_shift push=16 pull=16
  184. 18: backward_copy_shift push=24 pull=8
  185. ENDPROC(memmove)