memcpy-archs.S 4.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com)
  4. */
  5. #include <linux/linkage.h>
  6. #ifdef __LITTLE_ENDIAN__
  7. # define SHIFT_1(RX,RY,IMM) asl RX, RY, IMM ; <<
  8. # define SHIFT_2(RX,RY,IMM) lsr RX, RY, IMM ; >>
  9. # define MERGE_1(RX,RY,IMM) asl RX, RY, IMM
  10. # define MERGE_2(RX,RY,IMM)
  11. # define EXTRACT_1(RX,RY,IMM) and RX, RY, 0xFFFF
  12. # define EXTRACT_2(RX,RY,IMM) lsr RX, RY, IMM
  13. #else
  14. # define SHIFT_1(RX,RY,IMM) lsr RX, RY, IMM ; >>
  15. # define SHIFT_2(RX,RY,IMM) asl RX, RY, IMM ; <<
  16. # define MERGE_1(RX,RY,IMM) asl RX, RY, IMM ; <<
  17. # define MERGE_2(RX,RY,IMM) asl RX, RY, IMM ; <<
  18. # define EXTRACT_1(RX,RY,IMM) lsr RX, RY, IMM
  19. # define EXTRACT_2(RX,RY,IMM) lsr RX, RY, 0x08
  20. #endif
  21. #ifdef CONFIG_ARC_HAS_LL64
  22. # define LOADX(DST,RX) ldd.ab DST, [RX, 8]
  23. # define STOREX(SRC,RX) std.ab SRC, [RX, 8]
  24. # define ZOLSHFT 5
  25. # define ZOLAND 0x1F
  26. #else
  27. # define LOADX(DST,RX) ld.ab DST, [RX, 4]
  28. # define STOREX(SRC,RX) st.ab SRC, [RX, 4]
  29. # define ZOLSHFT 4
  30. # define ZOLAND 0xF
  31. #endif
  32. ENTRY_CFI(memcpy)
  33. mov.f 0, r2
  34. ;;; if size is zero
  35. jz.d [blink]
  36. mov r3, r0 ; don;t clobber ret val
  37. ;;; if size <= 8
  38. cmp r2, 8
  39. bls.d @.Lsmallchunk
  40. mov.f lp_count, r2
  41. and.f r4, r0, 0x03
  42. rsub lp_count, r4, 4
  43. lpnz @.Laligndestination
  44. ;; LOOP BEGIN
  45. ldb.ab r5, [r1,1]
  46. sub r2, r2, 1
  47. stb.ab r5, [r3,1]
  48. .Laligndestination:
  49. ;;; Check the alignment of the source
  50. and.f r4, r1, 0x03
  51. bnz.d @.Lsourceunaligned
  52. ;;; CASE 0: Both source and destination are 32bit aligned
  53. ;;; Convert len to Dwords, unfold x4
  54. lsr.f lp_count, r2, ZOLSHFT
  55. lpnz @.Lcopy32_64bytes
  56. ;; LOOP START
  57. LOADX (r6, r1)
  58. LOADX (r8, r1)
  59. LOADX (r10, r1)
  60. LOADX (r4, r1)
  61. STOREX (r6, r3)
  62. STOREX (r8, r3)
  63. STOREX (r10, r3)
  64. STOREX (r4, r3)
  65. .Lcopy32_64bytes:
  66. and.f lp_count, r2, ZOLAND ;Last remaining 31 bytes
  67. .Lsmallchunk:
  68. lpnz @.Lcopyremainingbytes
  69. ;; LOOP START
  70. ldb.ab r5, [r1,1]
  71. stb.ab r5, [r3,1]
  72. .Lcopyremainingbytes:
  73. j [blink]
  74. ;;; END CASE 0
  75. .Lsourceunaligned:
  76. cmp r4, 2
  77. beq.d @.LunalignedOffby2
  78. sub r2, r2, 1
  79. bhi.d @.LunalignedOffby3
  80. ldb.ab r5, [r1, 1]
  81. ;;; CASE 1: The source is unaligned, off by 1
  82. ;; Hence I need to read 1 byte for a 16bit alignment
  83. ;; and 2bytes to reach 32bit alignment
  84. ldh.ab r6, [r1, 2]
  85. sub r2, r2, 2
  86. ;; Convert to words, unfold x2
  87. lsr.f lp_count, r2, 3
  88. MERGE_1 (r6, r6, 8)
  89. MERGE_2 (r5, r5, 24)
  90. or r5, r5, r6
  91. ;; Both src and dst are aligned
  92. lpnz @.Lcopy8bytes_1
  93. ;; LOOP START
  94. ld.ab r6, [r1, 4]
  95. ld.ab r8, [r1,4]
  96. SHIFT_1 (r7, r6, 24)
  97. or r7, r7, r5
  98. SHIFT_2 (r5, r6, 8)
  99. SHIFT_1 (r9, r8, 24)
  100. or r9, r9, r5
  101. SHIFT_2 (r5, r8, 8)
  102. st.ab r7, [r3, 4]
  103. st.ab r9, [r3, 4]
  104. .Lcopy8bytes_1:
  105. ;; Write back the remaining 16bits
  106. EXTRACT_1 (r6, r5, 16)
  107. sth.ab r6, [r3, 2]
  108. ;; Write back the remaining 8bits
  109. EXTRACT_2 (r5, r5, 16)
  110. stb.ab r5, [r3, 1]
  111. and.f lp_count, r2, 0x07 ;Last 8bytes
  112. lpnz @.Lcopybytewise_1
  113. ;; LOOP START
  114. ldb.ab r6, [r1,1]
  115. stb.ab r6, [r3,1]
  116. .Lcopybytewise_1:
  117. j [blink]
  118. .LunalignedOffby2:
  119. ;;; CASE 2: The source is unaligned, off by 2
  120. ldh.ab r5, [r1, 2]
  121. sub r2, r2, 1
  122. ;; Both src and dst are aligned
  123. ;; Convert to words, unfold x2
  124. lsr.f lp_count, r2, 3
  125. #ifdef __BIG_ENDIAN__
  126. asl.nz r5, r5, 16
  127. #endif
  128. lpnz @.Lcopy8bytes_2
  129. ;; LOOP START
  130. ld.ab r6, [r1, 4]
  131. ld.ab r8, [r1,4]
  132. SHIFT_1 (r7, r6, 16)
  133. or r7, r7, r5
  134. SHIFT_2 (r5, r6, 16)
  135. SHIFT_1 (r9, r8, 16)
  136. or r9, r9, r5
  137. SHIFT_2 (r5, r8, 16)
  138. st.ab r7, [r3, 4]
  139. st.ab r9, [r3, 4]
  140. .Lcopy8bytes_2:
  141. #ifdef __BIG_ENDIAN__
  142. lsr.nz r5, r5, 16
  143. #endif
  144. sth.ab r5, [r3, 2]
  145. and.f lp_count, r2, 0x07 ;Last 8bytes
  146. lpnz @.Lcopybytewise_2
  147. ;; LOOP START
  148. ldb.ab r6, [r1,1]
  149. stb.ab r6, [r3,1]
  150. .Lcopybytewise_2:
  151. j [blink]
  152. .LunalignedOffby3:
  153. ;;; CASE 3: The source is unaligned, off by 3
  154. ;;; Hence, I need to read 1byte for achieve the 32bit alignment
  155. ;; Both src and dst are aligned
  156. ;; Convert to words, unfold x2
  157. lsr.f lp_count, r2, 3
  158. #ifdef __BIG_ENDIAN__
  159. asl.ne r5, r5, 24
  160. #endif
  161. lpnz @.Lcopy8bytes_3
  162. ;; LOOP START
  163. ld.ab r6, [r1, 4]
  164. ld.ab r8, [r1,4]
  165. SHIFT_1 (r7, r6, 8)
  166. or r7, r7, r5
  167. SHIFT_2 (r5, r6, 24)
  168. SHIFT_1 (r9, r8, 8)
  169. or r9, r9, r5
  170. SHIFT_2 (r5, r8, 24)
  171. st.ab r7, [r3, 4]
  172. st.ab r9, [r3, 4]
  173. .Lcopy8bytes_3:
  174. #ifdef __BIG_ENDIAN__
  175. lsr.nz r5, r5, 24
  176. #endif
  177. stb.ab r5, [r3, 1]
  178. and.f lp_count, r2, 0x07 ;Last 8bytes
  179. lpnz @.Lcopybytewise_3
  180. ;; LOOP START
  181. ldb.ab r6, [r1,1]
  182. stb.ab r6, [r3,1]
  183. .Lcopybytewise_3:
  184. j [blink]
  185. END_CFI(memcpy)