NG2memcpy.S 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* NG2memcpy.S: Niagara-2 optimized memcpy.
  3. *
  4. * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
  5. */
  6. #ifdef __KERNEL__
  7. #include <linux/linkage.h>
  8. #include <asm/visasm.h>
  9. #include <asm/asi.h>
  10. #define GLOBAL_SPARE %g7
  11. #else
  12. #define ASI_PNF 0x82
  13. #define ASI_BLK_P 0xf0
  14. #define ASI_BLK_INIT_QUAD_LDD_P 0xe2
  15. #define FPRS_FEF 0x04
  16. #ifdef MEMCPY_DEBUG
  17. #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
  18. clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0;
  19. #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
  20. #else
  21. #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
  22. #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
  23. #endif
  24. #define GLOBAL_SPARE %g5
  25. #endif
  26. #ifndef STORE_ASI
  27. #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
  28. #define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P
  29. #else
  30. #define STORE_ASI 0x80 /* ASI_P */
  31. #endif
  32. #endif
  33. #ifndef EX_LD
  34. #define EX_LD(x,y) x
  35. #endif
  36. #ifndef EX_LD_FP
  37. #define EX_LD_FP(x,y) x
  38. #endif
  39. #ifndef EX_ST
  40. #define EX_ST(x,y) x
  41. #endif
  42. #ifndef EX_ST_FP
  43. #define EX_ST_FP(x,y) x
  44. #endif
  45. #ifndef LOAD
  46. #define LOAD(type,addr,dest) type [addr], dest
  47. #endif
  48. #ifndef LOAD_BLK
  49. #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
  50. #endif
  51. #ifndef STORE
  52. #ifndef MEMCPY_DEBUG
  53. #define STORE(type,src,addr) type src, [addr]
  54. #else
  55. #define STORE(type,src,addr) type##a src, [addr] 0x80
  56. #endif
  57. #endif
  58. #ifndef STORE_BLK
  59. #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
  60. #endif
  61. #ifndef STORE_INIT
  62. #define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI
  63. #endif
  64. #ifndef FUNC_NAME
  65. #define FUNC_NAME NG2memcpy
  66. #endif
  67. #ifndef PREAMBLE
  68. #define PREAMBLE
  69. #endif
  70. #ifndef XCC
  71. #define XCC xcc
  72. #endif
  73. #define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \
  74. faligndata %x0, %x1, %f0; \
  75. faligndata %x1, %x2, %f2; \
  76. faligndata %x2, %x3, %f4; \
  77. faligndata %x3, %x4, %f6; \
  78. faligndata %x4, %x5, %f8; \
  79. faligndata %x5, %x6, %f10; \
  80. faligndata %x6, %x7, %f12; \
  81. faligndata %x7, %x8, %f14;
  82. #define FREG_MOVE_1(x0) \
  83. fsrc2 %x0, %f0;
  84. #define FREG_MOVE_2(x0, x1) \
  85. fsrc2 %x0, %f0; \
  86. fsrc2 %x1, %f2;
  87. #define FREG_MOVE_3(x0, x1, x2) \
  88. fsrc2 %x0, %f0; \
  89. fsrc2 %x1, %f2; \
  90. fsrc2 %x2, %f4;
  91. #define FREG_MOVE_4(x0, x1, x2, x3) \
  92. fsrc2 %x0, %f0; \
  93. fsrc2 %x1, %f2; \
  94. fsrc2 %x2, %f4; \
  95. fsrc2 %x3, %f6;
  96. #define FREG_MOVE_5(x0, x1, x2, x3, x4) \
  97. fsrc2 %x0, %f0; \
  98. fsrc2 %x1, %f2; \
  99. fsrc2 %x2, %f4; \
  100. fsrc2 %x3, %f6; \
  101. fsrc2 %x4, %f8;
  102. #define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
  103. fsrc2 %x0, %f0; \
  104. fsrc2 %x1, %f2; \
  105. fsrc2 %x2, %f4; \
  106. fsrc2 %x3, %f6; \
  107. fsrc2 %x4, %f8; \
  108. fsrc2 %x5, %f10;
  109. #define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
  110. fsrc2 %x0, %f0; \
  111. fsrc2 %x1, %f2; \
  112. fsrc2 %x2, %f4; \
  113. fsrc2 %x3, %f6; \
  114. fsrc2 %x4, %f8; \
  115. fsrc2 %x5, %f10; \
  116. fsrc2 %x6, %f12;
  117. #define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
  118. fsrc2 %x0, %f0; \
  119. fsrc2 %x1, %f2; \
  120. fsrc2 %x2, %f4; \
  121. fsrc2 %x3, %f6; \
  122. fsrc2 %x4, %f8; \
  123. fsrc2 %x5, %f10; \
  124. fsrc2 %x6, %f12; \
  125. fsrc2 %x7, %f14;
  126. #define FREG_LOAD_1(base, x0) \
  127. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
  128. #define FREG_LOAD_2(base, x0, x1) \
  129. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
  130. EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
  131. #define FREG_LOAD_3(base, x0, x1, x2) \
  132. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
  133. EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
  134. EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
  135. #define FREG_LOAD_4(base, x0, x1, x2, x3) \
  136. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
  137. EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
  138. EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
  139. EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
  140. #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
  141. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
  142. EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
  143. EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
  144. EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
  145. EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
  146. #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
  147. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
  148. EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
  149. EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
  150. EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
  151. EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
  152. EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
  153. #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
  154. EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
  155. EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
  156. EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
  157. EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
  158. EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
  159. EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
  160. EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
  161. .register %g2,#scratch
  162. .register %g3,#scratch
  163. .text
  164. #ifndef EX_RETVAL
  165. #define EX_RETVAL(x) x
  166. __restore_fp:
  167. VISExitHalf
  168. __restore_asi:
  169. retl
  170. wr %g0, ASI_AIUS, %asi
  171. ENTRY(NG2_retl_o2)
  172. ba,pt %xcc, __restore_asi
  173. mov %o2, %o0
  174. ENDPROC(NG2_retl_o2)
  175. ENTRY(NG2_retl_o2_plus_1)
  176. ba,pt %xcc, __restore_asi
  177. add %o2, 1, %o0
  178. ENDPROC(NG2_retl_o2_plus_1)
  179. ENTRY(NG2_retl_o2_plus_4)
  180. ba,pt %xcc, __restore_asi
  181. add %o2, 4, %o0
  182. ENDPROC(NG2_retl_o2_plus_4)
  183. ENTRY(NG2_retl_o2_plus_8)
  184. ba,pt %xcc, __restore_asi
  185. add %o2, 8, %o0
  186. ENDPROC(NG2_retl_o2_plus_8)
  187. ENTRY(NG2_retl_o2_plus_o4_plus_1)
  188. add %o4, 1, %o4
  189. ba,pt %xcc, __restore_asi
  190. add %o2, %o4, %o0
  191. ENDPROC(NG2_retl_o2_plus_o4_plus_1)
  192. ENTRY(NG2_retl_o2_plus_o4_plus_8)
  193. add %o4, 8, %o4
  194. ba,pt %xcc, __restore_asi
  195. add %o2, %o4, %o0
  196. ENDPROC(NG2_retl_o2_plus_o4_plus_8)
  197. ENTRY(NG2_retl_o2_plus_o4_plus_16)
  198. add %o4, 16, %o4
  199. ba,pt %xcc, __restore_asi
  200. add %o2, %o4, %o0
  201. ENDPROC(NG2_retl_o2_plus_o4_plus_16)
  202. ENTRY(NG2_retl_o2_plus_g1_fp)
  203. ba,pt %xcc, __restore_fp
  204. add %o2, %g1, %o0
  205. ENDPROC(NG2_retl_o2_plus_g1_fp)
  206. ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
  207. add %g1, 64, %g1
  208. ba,pt %xcc, __restore_fp
  209. add %o2, %g1, %o0
  210. ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
  211. ENTRY(NG2_retl_o2_plus_g1_plus_1)
  212. add %g1, 1, %g1
  213. ba,pt %xcc, __restore_asi
  214. add %o2, %g1, %o0
  215. ENDPROC(NG2_retl_o2_plus_g1_plus_1)
  216. ENTRY(NG2_retl_o2_and_7_plus_o4)
  217. and %o2, 7, %o2
  218. ba,pt %xcc, __restore_asi
  219. add %o2, %o4, %o0
  220. ENDPROC(NG2_retl_o2_and_7_plus_o4)
  221. ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
  222. and %o2, 7, %o2
  223. add %o4, 8, %o4
  224. ba,pt %xcc, __restore_asi
  225. add %o2, %o4, %o0
  226. ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
  227. #endif
  228. .align 64
  229. .globl FUNC_NAME
  230. .type FUNC_NAME,#function
  231. FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
  232. srlx %o2, 31, %g2
  233. cmp %g2, 0
  234. tne %xcc, 5
  235. PREAMBLE
  236. mov %o0, %o3
  237. cmp %o2, 0
  238. be,pn %XCC, 85f
  239. or %o0, %o1, GLOBAL_SPARE
  240. cmp %o2, 16
  241. blu,a,pn %XCC, 80f
  242. or GLOBAL_SPARE, %o2, GLOBAL_SPARE
  243. /* 2 blocks (128 bytes) is the minimum we can do the block
  244. * copy with. We need to ensure that we'll iterate at least
  245. * once in the block copy loop. At worst we'll need to align
  246. * the destination to a 64-byte boundary which can chew up
  247. * to (64 - 1) bytes from the length before we perform the
  248. * block copy loop.
  249. *
  250. * However, the cut-off point, performance wise, is around
  251. * 4 64-byte blocks.
  252. */
  253. cmp %o2, (4 * 64)
  254. blu,pt %XCC, 75f
  255. andcc GLOBAL_SPARE, 0x7, %g0
  256. /* %o0: dst
  257. * %o1: src
  258. * %o2: len (known to be >= 128)
  259. *
  260. * The block copy loops can use %o4, %g2, %g3 as
  261. * temporaries while copying the data. %o5 must
  262. * be preserved between VISEntryHalf and VISExitHalf
  263. */
  264. LOAD(prefetch, %o1 + 0x000, #one_read)
  265. LOAD(prefetch, %o1 + 0x040, #one_read)
  266. LOAD(prefetch, %o1 + 0x080, #one_read)
  267. /* Align destination on 64-byte boundary. */
  268. andcc %o0, (64 - 1), %o4
  269. be,pt %XCC, 2f
  270. sub %o4, 64, %o4
  271. sub %g0, %o4, %o4 ! bytes to align dst
  272. sub %o2, %o4, %o2
  273. 1: subcc %o4, 1, %o4
  274. EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
  275. EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
  276. add %o1, 1, %o1
  277. bne,pt %XCC, 1b
  278. add %o0, 1, %o0
  279. 2:
  280. /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve
  281. * o5 from here until we hit VISExitHalf.
  282. */
  283. VISEntryHalf
  284. membar #Sync
  285. alignaddr %o1, %g0, %g0
  286. add %o1, (64 - 1), %o4
  287. andn %o4, (64 - 1), %o4
  288. andn %o2, (64 - 1), %g1
  289. sub %o2, %g1, %o2
  290. and %o1, (64 - 1), %g2
  291. add %o1, %g1, %o1
  292. sub %o0, %o4, %g3
  293. brz,pt %g2, 190f
  294. cmp %g2, 32
  295. blu,a 5f
  296. cmp %g2, 16
  297. cmp %g2, 48
  298. blu,a 4f
  299. cmp %g2, 40
  300. cmp %g2, 56
  301. blu 170f
  302. nop
  303. ba,a,pt %xcc, 180f
  304. nop
  305. 4: /* 32 <= low bits < 48 */
  306. blu 150f
  307. nop
  308. ba,a,pt %xcc, 160f
  309. nop
  310. 5: /* 0 < low bits < 32 */
  311. blu,a 6f
  312. cmp %g2, 8
  313. cmp %g2, 24
  314. blu 130f
  315. nop
  316. ba,a,pt %xcc, 140f
  317. nop
  318. 6: /* 0 < low bits < 16 */
  319. bgeu 120f
  320. nop
  321. /* fall through for 0 < low bits < 8 */
  322. 110: sub %o4, 64, %g2
  323. EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
  324. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  325. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  326. FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
  327. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  328. FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
  329. subcc %g1, 64, %g1
  330. add %o4, 64, %o4
  331. bne,pt %xcc, 1b
  332. LOAD(prefetch, %o4 + 64, #one_read)
  333. ba,pt %xcc, 195f
  334. nop
  335. 120: sub %o4, 56, %g2
  336. FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
  337. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  338. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  339. FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
  340. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  341. FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
  342. subcc %g1, 64, %g1
  343. add %o4, 64, %o4
  344. bne,pt %xcc, 1b
  345. LOAD(prefetch, %o4 + 64, #one_read)
  346. ba,pt %xcc, 195f
  347. nop
  348. 130: sub %o4, 48, %g2
  349. FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
  350. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  351. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  352. FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
  353. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  354. FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
  355. subcc %g1, 64, %g1
  356. add %o4, 64, %o4
  357. bne,pt %xcc, 1b
  358. LOAD(prefetch, %o4 + 64, #one_read)
  359. ba,pt %xcc, 195f
  360. nop
  361. 140: sub %o4, 40, %g2
  362. FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
  363. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  364. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  365. FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
  366. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  367. FREG_MOVE_5(f22, f24, f26, f28, f30)
  368. subcc %g1, 64, %g1
  369. add %o4, 64, %o4
  370. bne,pt %xcc, 1b
  371. LOAD(prefetch, %o4 + 64, #one_read)
  372. ba,pt %xcc, 195f
  373. nop
  374. 150: sub %o4, 32, %g2
  375. FREG_LOAD_4(%g2, f0, f2, f4, f6)
  376. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  377. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  378. FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
  379. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  380. FREG_MOVE_4(f24, f26, f28, f30)
  381. subcc %g1, 64, %g1
  382. add %o4, 64, %o4
  383. bne,pt %xcc, 1b
  384. LOAD(prefetch, %o4 + 64, #one_read)
  385. ba,pt %xcc, 195f
  386. nop
  387. 160: sub %o4, 24, %g2
  388. FREG_LOAD_3(%g2, f0, f2, f4)
  389. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  390. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  391. FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
  392. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  393. FREG_MOVE_3(f26, f28, f30)
  394. subcc %g1, 64, %g1
  395. add %o4, 64, %o4
  396. bne,pt %xcc, 1b
  397. LOAD(prefetch, %o4 + 64, #one_read)
  398. ba,pt %xcc, 195f
  399. nop
  400. 170: sub %o4, 16, %g2
  401. FREG_LOAD_2(%g2, f0, f2)
  402. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  403. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  404. FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
  405. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  406. FREG_MOVE_2(f28, f30)
  407. subcc %g1, 64, %g1
  408. add %o4, 64, %o4
  409. bne,pt %xcc, 1b
  410. LOAD(prefetch, %o4 + 64, #one_read)
  411. ba,pt %xcc, 195f
  412. nop
  413. 180: sub %o4, 8, %g2
  414. FREG_LOAD_1(%g2, f0)
  415. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  416. EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
  417. FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
  418. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
  419. FREG_MOVE_1(f30)
  420. subcc %g1, 64, %g1
  421. add %o4, 64, %o4
  422. bne,pt %xcc, 1b
  423. LOAD(prefetch, %o4 + 64, #one_read)
  424. ba,pt %xcc, 195f
  425. nop
  426. 190:
  427. 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
  428. subcc %g1, 64, %g1
  429. EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
  430. EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
  431. add %o4, 64, %o4
  432. bne,pt %xcc, 1b
  433. LOAD(prefetch, %o4 + 64, #one_read)
  434. 195:
  435. add %o4, %g3, %o0
  436. membar #Sync
  437. VISExitHalf
  438. /* %o2 contains any final bytes still needed to be copied
  439. * over. If anything is left, we copy it one byte at a time.
  440. */
  441. brz,pt %o2, 85f
  442. sub %o0, %o1, GLOBAL_SPARE
  443. ba,a,pt %XCC, 90f
  444. nop
  445. .align 64
  446. 75: /* 16 < len <= 64 */
  447. bne,pn %XCC, 75f
  448. sub %o0, %o1, GLOBAL_SPARE
  449. 72:
  450. andn %o2, 0xf, %o4
  451. and %o2, 0xf, %o2
  452. 1: subcc %o4, 0x10, %o4
  453. EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
  454. add %o1, 0x08, %o1
  455. EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
  456. sub %o1, 0x08, %o1
  457. EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
  458. add %o1, 0x8, %o1
  459. EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
  460. bgu,pt %XCC, 1b
  461. add %o1, 0x8, %o1
  462. 73: andcc %o2, 0x8, %g0
  463. be,pt %XCC, 1f
  464. nop
  465. sub %o2, 0x8, %o2
  466. EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
  467. EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
  468. add %o1, 0x8, %o1
  469. 1: andcc %o2, 0x4, %g0
  470. be,pt %XCC, 1f
  471. nop
  472. sub %o2, 0x4, %o2
  473. EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
  474. EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
  475. add %o1, 0x4, %o1
  476. 1: cmp %o2, 0
  477. be,pt %XCC, 85f
  478. nop
  479. ba,pt %xcc, 90f
  480. nop
  481. 75:
  482. andcc %o0, 0x7, %g1
  483. sub %g1, 0x8, %g1
  484. be,pn %icc, 2f
  485. sub %g0, %g1, %g1
  486. sub %o2, %g1, %o2
  487. 1: subcc %g1, 1, %g1
  488. EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
  489. EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
  490. bgu,pt %icc, 1b
  491. add %o1, 1, %o1
  492. 2: add %o1, GLOBAL_SPARE, %o0
  493. andcc %o1, 0x7, %g1
  494. bne,pt %icc, 8f
  495. sll %g1, 3, %g1
  496. cmp %o2, 16
  497. bgeu,pt %icc, 72b
  498. nop
  499. ba,a,pt %xcc, 73b
  500. 8: mov 64, GLOBAL_SPARE
  501. andn %o1, 0x7, %o1
  502. EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
  503. sub GLOBAL_SPARE, %g1, GLOBAL_SPARE
  504. andn %o2, 0x7, %o4
  505. sllx %g2, %g1, %g2
  506. 1: add %o1, 0x8, %o1
  507. EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
  508. subcc %o4, 0x8, %o4
  509. srlx %g3, GLOBAL_SPARE, %o5
  510. or %o5, %g2, %o5
  511. EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
  512. add %o0, 0x8, %o0
  513. bgu,pt %icc, 1b
  514. sllx %g3, %g1, %g2
  515. srl %g1, 3, %g1
  516. andcc %o2, 0x7, %o2
  517. be,pn %icc, 85f
  518. add %o1, %g1, %o1
  519. ba,pt %xcc, 90f
  520. sub %o0, %o1, GLOBAL_SPARE
  521. .align 64
  522. 80: /* 0 < len <= 16 */
  523. andcc GLOBAL_SPARE, 0x3, %g0
  524. bne,pn %XCC, 90f
  525. sub %o0, %o1, GLOBAL_SPARE
  526. 1:
  527. subcc %o2, 4, %o2
  528. EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
  529. EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
  530. bgu,pt %XCC, 1b
  531. add %o1, 4, %o1
  532. 85: retl
  533. mov EX_RETVAL(%o3), %o0
  534. .align 32
  535. 90:
  536. subcc %o2, 1, %o2
  537. EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
  538. EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
  539. bgu,pt %XCC, 90b
  540. add %o1, 1, %o1
  541. retl
  542. mov EX_RETVAL(%o3), %o0
  543. .size FUNC_NAME, .-FUNC_NAME