memcpy.S 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* memcpy.S: Sparc optimized memcpy and memmove code
  3. * Hand optimized from GNU libc's memcpy and memmove
  4. * Copyright (C) 1991,1996 Free Software Foundation
  5. * Copyright (C) 1995 Linus Torvalds (Linus.Torvalds@helsinki.fi)
  6. * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
  7. * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
  8. * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  9. */
  10. #include <asm/export.h>
  11. #define FUNC(x) \
  12. .globl x; \
  13. .type x,@function; \
  14. .align 4; \
  15. x:
  16. /* Both these macros have to start with exactly the same insn */
  17. #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
  18. ldd [%src + (offset) + 0x00], %t0; \
  19. ldd [%src + (offset) + 0x08], %t2; \
  20. ldd [%src + (offset) + 0x10], %t4; \
  21. ldd [%src + (offset) + 0x18], %t6; \
  22. st %t0, [%dst + (offset) + 0x00]; \
  23. st %t1, [%dst + (offset) + 0x04]; \
  24. st %t2, [%dst + (offset) + 0x08]; \
  25. st %t3, [%dst + (offset) + 0x0c]; \
  26. st %t4, [%dst + (offset) + 0x10]; \
  27. st %t5, [%dst + (offset) + 0x14]; \
  28. st %t6, [%dst + (offset) + 0x18]; \
  29. st %t7, [%dst + (offset) + 0x1c];
  30. #define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
  31. ldd [%src + (offset) + 0x00], %t0; \
  32. ldd [%src + (offset) + 0x08], %t2; \
  33. ldd [%src + (offset) + 0x10], %t4; \
  34. ldd [%src + (offset) + 0x18], %t6; \
  35. std %t0, [%dst + (offset) + 0x00]; \
  36. std %t2, [%dst + (offset) + 0x08]; \
  37. std %t4, [%dst + (offset) + 0x10]; \
  38. std %t6, [%dst + (offset) + 0x18];
  39. #define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
  40. ldd [%src - (offset) - 0x10], %t0; \
  41. ldd [%src - (offset) - 0x08], %t2; \
  42. st %t0, [%dst - (offset) - 0x10]; \
  43. st %t1, [%dst - (offset) - 0x0c]; \
  44. st %t2, [%dst - (offset) - 0x08]; \
  45. st %t3, [%dst - (offset) - 0x04];
  46. #define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
  47. ldd [%src - (offset) - 0x10], %t0; \
  48. ldd [%src - (offset) - 0x08], %t2; \
  49. std %t0, [%dst - (offset) - 0x10]; \
  50. std %t2, [%dst - (offset) - 0x08];
  51. #define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
  52. ldub [%src - (offset) - 0x02], %t0; \
  53. ldub [%src - (offset) - 0x01], %t1; \
  54. stb %t0, [%dst - (offset) - 0x02]; \
  55. stb %t1, [%dst - (offset) - 0x01];
  56. .text
  57. .align 4
  58. FUNC(memmove)
  59. EXPORT_SYMBOL(memmove)
  60. cmp %o0, %o1
  61. mov %o0, %g7
  62. bleu 9f
  63. sub %o0, %o1, %o4
  64. add %o1, %o2, %o3
  65. cmp %o3, %o0
  66. bleu 0f
  67. andcc %o4, 3, %o5
  68. add %o1, %o2, %o1
  69. add %o0, %o2, %o0
  70. sub %o1, 1, %o1
  71. sub %o0, 1, %o0
  72. 1: /* reverse_bytes */
  73. ldub [%o1], %o4
  74. subcc %o2, 1, %o2
  75. stb %o4, [%o0]
  76. sub %o1, 1, %o1
  77. bne 1b
  78. sub %o0, 1, %o0
  79. retl
  80. mov %g7, %o0
  81. /* NOTE: This code is executed just for the cases,
  82. where %src (=%o1) & 3 is != 0.
  83. We need to align it to 4. So, for (%src & 3)
  84. 1 we need to do ldub,lduh
  85. 2 lduh
  86. 3 just ldub
  87. so even if it looks weird, the branches
  88. are correct here. -jj
  89. */
  90. 78: /* dword_align */
  91. andcc %o1, 1, %g0
  92. be 4f
  93. andcc %o1, 2, %g0
  94. ldub [%o1], %g2
  95. add %o1, 1, %o1
  96. stb %g2, [%o0]
  97. sub %o2, 1, %o2
  98. bne 3f
  99. add %o0, 1, %o0
  100. 4:
  101. lduh [%o1], %g2
  102. add %o1, 2, %o1
  103. sth %g2, [%o0]
  104. sub %o2, 2, %o2
  105. b 3f
  106. add %o0, 2, %o0
  107. FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
  108. EXPORT_SYMBOL(memcpy)
  109. sub %o0, %o1, %o4
  110. mov %o0, %g7
  111. 9:
  112. andcc %o4, 3, %o5
  113. 0:
  114. bne 86f
  115. cmp %o2, 15
  116. bleu 90f
  117. andcc %o1, 3, %g0
  118. bne 78b
  119. 3:
  120. andcc %o1, 4, %g0
  121. be 2f
  122. mov %o2, %g1
  123. ld [%o1], %o4
  124. sub %g1, 4, %g1
  125. st %o4, [%o0]
  126. add %o1, 4, %o1
  127. add %o0, 4, %o0
  128. 2:
  129. andcc %g1, 0xffffff80, %g0
  130. be 3f
  131. andcc %o0, 4, %g0
  132. be 82f + 4
  133. 5:
  134. MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
  135. MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
  136. MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
  137. MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
  138. sub %g1, 128, %g1
  139. add %o1, 128, %o1
  140. cmp %g1, 128
  141. bge 5b
  142. add %o0, 128, %o0
  143. 3:
  144. andcc %g1, 0x70, %g4
  145. be 80f
  146. andcc %g1, 8, %g0
  147. sethi %hi(80f), %o5
  148. srl %g4, 1, %o4
  149. add %g4, %o4, %o4
  150. add %o1, %g4, %o1
  151. sub %o5, %o4, %o5
  152. jmpl %o5 + %lo(80f), %g0
  153. add %o0, %g4, %o0
  154. 79: /* memcpy_table */
  155. MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
  156. MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
  157. MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
  158. MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
  159. MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
  160. MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
  161. MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
  162. 80: /* memcpy_table_end */
  163. be 81f
  164. andcc %g1, 4, %g0
  165. ldd [%o1], %g2
  166. add %o0, 8, %o0
  167. st %g2, [%o0 - 0x08]
  168. add %o1, 8, %o1
  169. st %g3, [%o0 - 0x04]
  170. 81: /* memcpy_last7 */
  171. be 1f
  172. andcc %g1, 2, %g0
  173. ld [%o1], %g2
  174. add %o1, 4, %o1
  175. st %g2, [%o0]
  176. add %o0, 4, %o0
  177. 1:
  178. be 1f
  179. andcc %g1, 1, %g0
  180. lduh [%o1], %g2
  181. add %o1, 2, %o1
  182. sth %g2, [%o0]
  183. add %o0, 2, %o0
  184. 1:
  185. be 1f
  186. nop
  187. ldub [%o1], %g2
  188. stb %g2, [%o0]
  189. 1:
  190. retl
  191. mov %g7, %o0
  192. 82: /* ldd_std */
  193. MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
  194. MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
  195. MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
  196. MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
  197. subcc %g1, 128, %g1
  198. add %o1, 128, %o1
  199. cmp %g1, 128
  200. bge 82b
  201. add %o0, 128, %o0
  202. andcc %g1, 0x70, %g4
  203. be 84f
  204. andcc %g1, 8, %g0
  205. sethi %hi(84f), %o5
  206. add %o1, %g4, %o1
  207. sub %o5, %g4, %o5
  208. jmpl %o5 + %lo(84f), %g0
  209. add %o0, %g4, %o0
  210. 83: /* amemcpy_table */
  211. MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
  212. MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
  213. MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
  214. MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
  215. MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
  216. MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
  217. MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
  218. 84: /* amemcpy_table_end */
  219. be 85f
  220. andcc %g1, 4, %g0
  221. ldd [%o1], %g2
  222. add %o0, 8, %o0
  223. std %g2, [%o0 - 0x08]
  224. add %o1, 8, %o1
  225. 85: /* amemcpy_last7 */
  226. be 1f
  227. andcc %g1, 2, %g0
  228. ld [%o1], %g2
  229. add %o1, 4, %o1
  230. st %g2, [%o0]
  231. add %o0, 4, %o0
  232. 1:
  233. be 1f
  234. andcc %g1, 1, %g0
  235. lduh [%o1], %g2
  236. add %o1, 2, %o1
  237. sth %g2, [%o0]
  238. add %o0, 2, %o0
  239. 1:
  240. be 1f
  241. nop
  242. ldub [%o1], %g2
  243. stb %g2, [%o0]
  244. 1:
  245. retl
  246. mov %g7, %o0
  247. 86: /* non_aligned */
  248. cmp %o2, 6
  249. bleu 88f
  250. nop
  251. save %sp, -96, %sp
  252. andcc %i0, 3, %g0
  253. be 61f
  254. andcc %i0, 1, %g0
  255. be 60f
  256. andcc %i0, 2, %g0
  257. ldub [%i1], %g5
  258. add %i1, 1, %i1
  259. stb %g5, [%i0]
  260. sub %i2, 1, %i2
  261. bne 61f
  262. add %i0, 1, %i0
  263. 60:
  264. ldub [%i1], %g3
  265. add %i1, 2, %i1
  266. stb %g3, [%i0]
  267. sub %i2, 2, %i2
  268. ldub [%i1 - 1], %g3
  269. add %i0, 2, %i0
  270. stb %g3, [%i0 - 1]
  271. 61:
  272. and %i1, 3, %g2
  273. and %i2, 0xc, %g3
  274. and %i1, -4, %i1
  275. cmp %g3, 4
  276. sll %g2, 3, %g4
  277. mov 32, %g2
  278. be 4f
  279. sub %g2, %g4, %l0
  280. blu 3f
  281. cmp %g3, 0x8
  282. be 2f
  283. srl %i2, 2, %g3
  284. ld [%i1], %i3
  285. add %i0, -8, %i0
  286. ld [%i1 + 4], %i4
  287. b 8f
  288. add %g3, 1, %g3
  289. 2:
  290. ld [%i1], %i4
  291. add %i0, -12, %i0
  292. ld [%i1 + 4], %i5
  293. add %g3, 2, %g3
  294. b 9f
  295. add %i1, -4, %i1
  296. 3:
  297. ld [%i1], %g1
  298. add %i0, -4, %i0
  299. ld [%i1 + 4], %i3
  300. srl %i2, 2, %g3
  301. b 7f
  302. add %i1, 4, %i1
  303. 4:
  304. ld [%i1], %i5
  305. cmp %i2, 7
  306. ld [%i1 + 4], %g1
  307. srl %i2, 2, %g3
  308. bleu 10f
  309. add %i1, 8, %i1
  310. ld [%i1], %i3
  311. add %g3, -1, %g3
  312. 5:
  313. sll %i5, %g4, %g2
  314. srl %g1, %l0, %g5
  315. or %g2, %g5, %g2
  316. st %g2, [%i0]
  317. 7:
  318. ld [%i1 + 4], %i4
  319. sll %g1, %g4, %g2
  320. srl %i3, %l0, %g5
  321. or %g2, %g5, %g2
  322. st %g2, [%i0 + 4]
  323. 8:
  324. ld [%i1 + 8], %i5
  325. sll %i3, %g4, %g2
  326. srl %i4, %l0, %g5
  327. or %g2, %g5, %g2
  328. st %g2, [%i0 + 8]
  329. 9:
  330. ld [%i1 + 12], %g1
  331. sll %i4, %g4, %g2
  332. srl %i5, %l0, %g5
  333. addcc %g3, -4, %g3
  334. or %g2, %g5, %g2
  335. add %i1, 16, %i1
  336. st %g2, [%i0 + 12]
  337. add %i0, 16, %i0
  338. bne,a 5b
  339. ld [%i1], %i3
  340. 10:
  341. sll %i5, %g4, %g2
  342. srl %g1, %l0, %g5
  343. srl %l0, 3, %g3
  344. or %g2, %g5, %g2
  345. sub %i1, %g3, %i1
  346. andcc %i2, 2, %g0
  347. st %g2, [%i0]
  348. be 1f
  349. andcc %i2, 1, %g0
  350. ldub [%i1], %g2
  351. add %i1, 2, %i1
  352. stb %g2, [%i0 + 4]
  353. add %i0, 2, %i0
  354. ldub [%i1 - 1], %g2
  355. stb %g2, [%i0 + 3]
  356. 1:
  357. be 1f
  358. nop
  359. ldub [%i1], %g2
  360. stb %g2, [%i0 + 4]
  361. 1:
  362. ret
  363. restore %g7, %g0, %o0
  364. 88: /* short_end */
  365. and %o2, 0xe, %o3
  366. 20:
  367. sethi %hi(89f), %o5
  368. sll %o3, 3, %o4
  369. add %o0, %o3, %o0
  370. sub %o5, %o4, %o5
  371. add %o1, %o3, %o1
  372. jmpl %o5 + %lo(89f), %g0
  373. andcc %o2, 1, %g0
  374. MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
  375. MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
  376. MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3)
  377. MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3)
  378. MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3)
  379. MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3)
  380. MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3)
  381. 89: /* short_table_end */
  382. be 1f
  383. nop
  384. ldub [%o1], %g2
  385. stb %g2, [%o0]
  386. 1:
  387. retl
  388. mov %g7, %o0
  389. 90: /* short_aligned_end */
  390. bne 88b
  391. andcc %o2, 8, %g0
  392. be 1f
  393. andcc %o2, 4, %g0
  394. ld [%o1 + 0x00], %g2
  395. ld [%o1 + 0x04], %g3
  396. add %o1, 8, %o1
  397. st %g2, [%o0 + 0x00]
  398. st %g3, [%o0 + 0x04]
  399. add %o0, 8, %o0
  400. 1:
  401. b 81b
  402. mov %o2, %g1