copy_template.S 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2013 ARM Ltd.
  4. * Copyright (C) 2013 Linaro.
  5. *
  6. * This code is based on glibc cortex strings work originally authored by Linaro
  7. * be found @
  8. *
  9. * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
  10. * files/head:/src/aarch64/
  11. */
  12. /*
  13. * Copy a buffer from src to dest (alignment handled by the hardware)
  14. *
  15. * Parameters:
  16. * x0 - dest
  17. * x1 - src
  18. * x2 - n
  19. * Returns:
  20. * x0 - dest
  21. */
  22. dstin .req x0
  23. src .req x1
  24. count .req x2
  25. tmp1 .req x3
  26. tmp1w .req w3
  27. tmp2 .req x4
  28. tmp2w .req w4
  29. dst .req x6
  30. A_l .req x7
  31. A_h .req x8
  32. B_l .req x9
  33. B_h .req x10
  34. C_l .req x11
  35. C_h .req x12
  36. D_l .req x13
  37. D_h .req x14
  38. mov dst, dstin
  39. cmp count, #16
  40. /*When memory length is less than 16, the accessed are not aligned.*/
  41. b.lo .Ltiny15
  42. neg tmp2, src
  43. ands tmp2, tmp2, #15/* Bytes to reach alignment. */
  44. b.eq .LSrcAligned
  45. sub count, count, tmp2
  46. /*
  47. * Copy the leading memory data from src to dst in an increasing
  48. * address order.By this way,the risk of overwriting the source
  49. * memory data is eliminated when the distance between src and
  50. * dst is less than 16. The memory accesses here are alignment.
  51. */
  52. tbz tmp2, #0, 1f
  53. ldrb1 tmp1w, src, #1
  54. strb1 tmp1w, dst, #1
  55. 1:
  56. tbz tmp2, #1, 2f
  57. ldrh1 tmp1w, src, #2
  58. strh1 tmp1w, dst, #2
  59. 2:
  60. tbz tmp2, #2, 3f
  61. ldr1 tmp1w, src, #4
  62. str1 tmp1w, dst, #4
  63. 3:
  64. tbz tmp2, #3, .LSrcAligned
  65. ldr1 tmp1, src, #8
  66. str1 tmp1, dst, #8
  67. .LSrcAligned:
  68. cmp count, #64
  69. b.ge .Lcpy_over64
  70. /*
  71. * Deal with small copies quickly by dropping straight into the
  72. * exit block.
  73. */
  74. .Ltail63:
  75. /*
  76. * Copy up to 48 bytes of data. At this point we only need the
  77. * bottom 6 bits of count to be accurate.
  78. */
  79. ands tmp1, count, #0x30
  80. b.eq .Ltiny15
  81. cmp tmp1w, #0x20
  82. b.eq 1f
  83. b.lt 2f
  84. ldp1 A_l, A_h, src, #16
  85. stp1 A_l, A_h, dst, #16
  86. 1:
  87. ldp1 A_l, A_h, src, #16
  88. stp1 A_l, A_h, dst, #16
  89. 2:
  90. ldp1 A_l, A_h, src, #16
  91. stp1 A_l, A_h, dst, #16
  92. .Ltiny15:
  93. /*
  94. * Prefer to break one ldp/stp into several load/store to access
  95. * memory in an increasing address order,rather than to load/store 16
  96. * bytes from (src-16) to (dst-16) and to backward the src to aligned
  97. * address,which way is used in original cortex memcpy. If keeping
  98. * the original memcpy process here, memmove need to satisfy the
  99. * precondition that src address is at least 16 bytes bigger than dst
  100. * address,otherwise some source data will be overwritten when memove
  101. * call memcpy directly. To make memmove simpler and decouple the
  102. * memcpy's dependency on memmove, withdrew the original process.
  103. */
  104. tbz count, #3, 1f
  105. ldr1 tmp1, src, #8
  106. str1 tmp1, dst, #8
  107. 1:
  108. tbz count, #2, 2f
  109. ldr1 tmp1w, src, #4
  110. str1 tmp1w, dst, #4
  111. 2:
  112. tbz count, #1, 3f
  113. ldrh1 tmp1w, src, #2
  114. strh1 tmp1w, dst, #2
  115. 3:
  116. tbz count, #0, .Lexitfunc
  117. ldrb1 tmp1w, src, #1
  118. strb1 tmp1w, dst, #1
  119. b .Lexitfunc
  120. .Lcpy_over64:
  121. subs count, count, #128
  122. b.ge .Lcpy_body_large
  123. /*
  124. * Less than 128 bytes to copy, so handle 64 here and then jump
  125. * to the tail.
  126. */
  127. ldp1 A_l, A_h, src, #16
  128. stp1 A_l, A_h, dst, #16
  129. ldp1 B_l, B_h, src, #16
  130. ldp1 C_l, C_h, src, #16
  131. stp1 B_l, B_h, dst, #16
  132. stp1 C_l, C_h, dst, #16
  133. ldp1 D_l, D_h, src, #16
  134. stp1 D_l, D_h, dst, #16
  135. tst count, #0x3f
  136. b.ne .Ltail63
  137. b .Lexitfunc
  138. /*
  139. * Critical loop. Start at a new cache line boundary. Assuming
  140. * 64 bytes per line this ensures the entire loop is in one line.
  141. */
  142. .p2align L1_CACHE_SHIFT
  143. .Lcpy_body_large:
  144. /* pre-get 64 bytes data. */
  145. ldp1 A_l, A_h, src, #16
  146. ldp1 B_l, B_h, src, #16
  147. ldp1 C_l, C_h, src, #16
  148. ldp1 D_l, D_h, src, #16
  149. 1:
  150. /*
  151. * interlace the load of next 64 bytes data block with store of the last
  152. * loaded 64 bytes data.
  153. */
  154. stp1 A_l, A_h, dst, #16
  155. ldp1 A_l, A_h, src, #16
  156. stp1 B_l, B_h, dst, #16
  157. ldp1 B_l, B_h, src, #16
  158. stp1 C_l, C_h, dst, #16
  159. ldp1 C_l, C_h, src, #16
  160. stp1 D_l, D_h, dst, #16
  161. ldp1 D_l, D_h, src, #16
  162. subs count, count, #64
  163. b.ge 1b
  164. stp1 A_l, A_h, dst, #16
  165. stp1 B_l, B_h, dst, #16
  166. stp1 C_l, C_h, dst, #16
  167. stp1 D_l, D_h, dst, #16
  168. tst count, #0x3f
  169. b.ne .Ltail63
  170. .Lexitfunc: