memcpy.s 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485
  1. /* $NetBSD: memcpy.S,v 1.3 1997/11/22 03:27:12 mark Exp $ */
  2. /*-
  3. * Copyright (c) 1997 The NetBSD Foundation, Inc.
  4. * All rights reserved.
  5. *
  6. * This code is derived from software contributed to The NetBSD Foundation
  7. * by Neil A. Carson and Mark Brinicombe
  8. *
  9. * Redistribution and use in source and binary forms, with or without
  10. * modification, are permitted provided that the following conditions
  11. * are met:
  12. * 1. Redistributions of source code must retain the above copyright
  13. * notice, this list of conditions and the following disclaimer.
  14. * 2. Redistributions in binary form must reproduce the above copyright
  15. * notice, this list of conditions and the following disclaimer in the
  16. * documentation and/or other materials provided with the distribution.
  17. * 3. All advertising materials mentioning features or use of this software
  18. * must display the following acknowledgement:
  19. * This product includes software developed by the NetBSD
  20. * Foundation, Inc. and its contributors.
  21. * 4. Neither the name of The NetBSD Foundation nor the names of its
  22. * contributors may be used to endorse or promote products derived
  23. * from this software without specific prior written permission.
  24. *
  25. * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  26. * ``AS IS\'\' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  27. * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  28. * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  29. * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35. * POSSIBILITY OF SUCH DAMAGE.
  36. */
  37. /* #include <machine/asm.h>*/
  38. .globl memcpy
  39. .globl _memcpy
  40. memcpy:
  41. stmfd sp!, {r0, lr}
  42. bl _memcpy
  43. ldmfd sp!, {r0, pc}
  44. .globl memmove
  45. memmove:
  46. stmfd sp!, {r0, lr}
  47. bl _memcpy
  48. ldmfd sp!, {r0, pc}
  49. /*
  50. * This is one fun bit of code ...
  51. * Some easy listening music is suggested while trying to understand this
  52. * code e.g. Iron Maiden
  53. *
  54. * For anyone attempting to understand it :
  55. *
  56. * The core code is implemented here with simple stubs for memcpy()
  57. * memmove() and bcopy().
  58. *
  59. * All local labels are prefixed with Lmemcpy_
  60. * Following the prefix a label starting f is used in the forward copy code
  61. * while a label using b is used in the backwards copy code
  62. * The source and destination addresses determine whether a forward or
  63. * backward copy is performed.
  64. * Separate bits of code are used to deal with the following situations
  65. * for both the forward and backwards copy.
  66. * unaligned source address
  67. * unaligned destination address
  68. * Separate copy routines are used to produce an optimised result for each
  69. * of these cases.
  70. * The copy code will use LDM/STM instructions to copy up to 32 bytes at
  71. * a time where possible.
  72. *
  73. * Note: r12 (aka ip) can be trashed during the function along with
  74. * r0-r3 although r0-r2 have defined uses i.e. src, dest, len through out.
  75. * Additional registers are preserved prior to use i.e. r4, r5 & lr
  76. *
  77. * Apologies for the state of the comments;-)
  78. */
  79. _memcpy:
  80. /* Determine copy direction */
  81. cmp r1, r0
  82. bcc Lmemcpy_backwards
  83. moveq r0, #0 /* Quick abort for len=0 */
  84. moveq pc, lr
  85. stmdb sp!, {r0, lr} /* memcpy() returns dest addr */
  86. subs r2, r2, #4
  87. blt Lmemcpy_fl4 /* less than 4 bytes */
  88. ands r12, r0, #3
  89. bne Lmemcpy_fdestul /* oh unaligned destination addr */
  90. ands r12, r1, #3
  91. bne Lmemcpy_fsrcul /* oh unaligned source addr */
  92. Lmemcpy_ft8:
  93. /* We have aligned source and destination */
  94. subs r2, r2, #8
  95. blt Lmemcpy_fl12 /* less than 12 bytes (4 from above) */
  96. subs r2, r2, #0x14
  97. blt Lmemcpy_fl32 /* less than 32 bytes (12 from above) */
  98. stmdb sp!, {r4, r7, r8, r9, r10} /* borrow r4 */
  99. /* blat 32 bytes at a time */
  100. /* XXX for really big copies perhaps we should use more registers */
  101. Lmemcpy_floop32:
  102. ldmia r1!, {r3, r4, r7, r8, r9, r10, r12, lr}
  103. stmia r0!, {r3, r4, r7, r8, r9, r10, r12, lr}
  104. subs r2, r2, #0x20
  105. bge Lmemcpy_floop32
  106. cmn r2, #0x10
  107. ldmgeia r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */
  108. stmgeia r0!, {r3, r4, r12, lr}
  109. subge r2, r2, #0x10
  110. ldmia sp!, {r4, r7, r8, r9, r10} /* return r4 */
  111. Lmemcpy_fl32:
  112. adds r2, r2, #0x14
  113. /* blat 12 bytes at a time */
  114. Lmemcpy_floop12:
  115. ldmgeia r1!, {r3, r12, lr}
  116. stmgeia r0!, {r3, r12, lr}
  117. subges r2, r2, #0x0c
  118. bge Lmemcpy_floop12
  119. Lmemcpy_fl12:
  120. adds r2, r2, #8
  121. blt Lmemcpy_fl4
  122. subs r2, r2, #4
  123. ldrlt r3, [r1], #4
  124. strlt r3, [r0], #4
  125. ldmgeia r1!, {r3, r12}
  126. stmgeia r0!, {r3, r12}
  127. subge r2, r2, #4
  128. Lmemcpy_fl4:
  129. /* less than 4 bytes to go */
  130. adds r2, r2, #4
  131. ldmeqia sp!, {r0, pc} /* done */
  132. /* copy the crud byte at a time */
  133. cmp r2, #2
  134. ldrb r3, [r1], #1
  135. strb r3, [r0], #1
  136. ldrgeb r3, [r1], #1
  137. strgeb r3, [r0], #1
  138. ldrgtb r3, [r1], #1
  139. strgtb r3, [r0], #1
  140. ldmia sp!, {r0, pc}
  141. /* erg - unaligned destination */
  142. Lmemcpy_fdestul:
  143. rsb r12, r12, #4
  144. cmp r12, #2
  145. /* align destination with byte copies */
  146. ldrb r3, [r1], #1
  147. strb r3, [r0], #1
  148. ldrgeb r3, [r1], #1
  149. strgeb r3, [r0], #1
  150. ldrgtb r3, [r1], #1
  151. strgtb r3, [r0], #1
  152. subs r2, r2, r12
  153. blt Lmemcpy_fl4 /* less the 4 bytes */
  154. ands r12, r1, #3
  155. beq Lmemcpy_ft8 /* we have an aligned source */
  156. /* erg - unaligned source */
  157. /* This is where it gets nasty ... */
  158. Lmemcpy_fsrcul:
  159. bic r1, r1, #3
  160. ldr lr, [r1], #4
  161. cmp r12, #2
  162. bgt Lmemcpy_fsrcul3
  163. beq Lmemcpy_fsrcul2
  164. cmp r2, #0x0c
  165. blt Lmemcpy_fsrcul1loop4
  166. sub r2, r2, #0x0c
  167. stmdb sp!, {r4, r5}
  168. Lmemcpy_fsrcul1loop16:
  169. mov r3, lr, lsr #8
  170. ldmia r1!, {r4, r5, r12, lr}
  171. orr r3, r3, r4, lsl #24
  172. mov r4, r4, lsr #8
  173. orr r4, r4, r5, lsl #24
  174. mov r5, r5, lsr #8
  175. orr r5, r5, r12, lsl #24
  176. mov r12, r12, lsr #8
  177. orr r12, r12, lr, lsl #24
  178. stmia r0!, {r3-r5, r12}
  179. subs r2, r2, #0x10
  180. bge Lmemcpy_fsrcul1loop16
  181. ldmia sp!, {r4, r5}
  182. adds r2, r2, #0x0c
  183. blt Lmemcpy_fsrcul1l4
  184. Lmemcpy_fsrcul1loop4:
  185. mov r12, lr, lsr #8
  186. ldr lr, [r1], #4
  187. orr r12, r12, lr, lsl #24
  188. str r12, [r0], #4
  189. subs r2, r2, #4
  190. bge Lmemcpy_fsrcul1loop4
  191. Lmemcpy_fsrcul1l4:
  192. sub r1, r1, #3
  193. b Lmemcpy_fl4
  194. Lmemcpy_fsrcul2:
  195. cmp r2, #0x0c
  196. blt Lmemcpy_fsrcul2loop4
  197. sub r2, r2, #0x0c
  198. stmdb sp!, {r4, r5}
  199. Lmemcpy_fsrcul2loop16:
  200. mov r3, lr, lsr #16
  201. ldmia r1!, {r4, r5, r12, lr}
  202. orr r3, r3, r4, lsl #16
  203. mov r4, r4, lsr #16
  204. orr r4, r4, r5, lsl #16
  205. mov r5, r5, lsr #16
  206. orr r5, r5, r12, lsl #16
  207. mov r12, r12, lsr #16
  208. orr r12, r12, lr, lsl #16
  209. stmia r0!, {r3-r5, r12}
  210. subs r2, r2, #0x10
  211. bge Lmemcpy_fsrcul2loop16
  212. ldmia sp!, {r4, r5}
  213. adds r2, r2, #0x0c
  214. blt Lmemcpy_fsrcul2l4
  215. Lmemcpy_fsrcul2loop4:
  216. mov r12, lr, lsr #16
  217. ldr lr, [r1], #4
  218. orr r12, r12, lr, lsl #16
  219. str r12, [r0], #4
  220. subs r2, r2, #4
  221. bge Lmemcpy_fsrcul2loop4
  222. Lmemcpy_fsrcul2l4:
  223. sub r1, r1, #2
  224. b Lmemcpy_fl4
  225. Lmemcpy_fsrcul3:
  226. cmp r2, #0x0c
  227. blt Lmemcpy_fsrcul3loop4
  228. sub r2, r2, #0x0c
  229. stmdb sp!, {r4, r5}
  230. Lmemcpy_fsrcul3loop16:
  231. mov r3, lr, lsr #24
  232. ldmia r1!, {r4, r5, r12, lr}
  233. orr r3, r3, r4, lsl #8
  234. mov r4, r4, lsr #24
  235. orr r4, r4, r5, lsl #8
  236. mov r5, r5, lsr #24
  237. orr r5, r5, r12, lsl #8
  238. mov r12, r12, lsr #24
  239. orr r12, r12, lr, lsl #8
  240. stmia r0!, {r3-r5, r12}
  241. subs r2, r2, #0x10
  242. bge Lmemcpy_fsrcul3loop16
  243. ldmia sp!, {r4, r5}
  244. adds r2, r2, #0x0c
  245. blt Lmemcpy_fsrcul3l4
  246. Lmemcpy_fsrcul3loop4:
  247. mov r12, lr, lsr #24
  248. ldr lr, [r1], #4
  249. orr r12, r12, lr, lsl #8
  250. str r12, [r0], #4
  251. subs r2, r2, #4
  252. bge Lmemcpy_fsrcul3loop4
  253. Lmemcpy_fsrcul3l4:
  254. sub r1, r1, #1
  255. b Lmemcpy_fl4
  256. Lmemcpy_backwards:
  257. add r1, r1, r2
  258. add r0, r0, r2
  259. subs r2, r2, #4
  260. blt Lmemcpy_bl4 /* less than 4 bytes */
  261. ands r12, r0, #3
  262. bne Lmemcpy_bdestul /* oh unaligned destination addr */
  263. ands r12, r1, #3
  264. bne Lmemcpy_bsrcul /* oh unaligned source addr */
  265. Lmemcpy_bt8:
  266. /* We have aligned source and destination */
  267. subs r2, r2, #8
  268. blt Lmemcpy_bl12 /* less than 12 bytes (4 from above) */
  269. stmdb sp!, {r4, r7, r8, r9, r10, lr}
  270. subs r2, r2, #0x14 /* less than 32 bytes (12 from above) */
  271. blt Lmemcpy_bl32
  272. /* blat 32 bytes at a time */
  273. /* XXX for really big copies perhaps we should use more registers */
  274. Lmemcpy_bloop32:
  275. ldmdb r1!, {r3, r4, r7, r8, r9, r10, r12, lr}
  276. stmdb r0!, {r3, r4, r7, r8, r9, r10, r12, lr}
  277. subs r2, r2, #0x20
  278. bge Lmemcpy_bloop32
  279. Lmemcpy_bl32:
  280. cmn r2, #0x10
  281. ldmgedb r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */
  282. stmgedb r0!, {r3, r4, r12, lr}
  283. subge r2, r2, #0x10
  284. adds r2, r2, #0x14
  285. ldmgedb r1!, {r3, r12, lr} /* blat a remaining 12 bytes */
  286. stmgedb r0!, {r3, r12, lr}
  287. subge r2, r2, #0x0c
  288. ldmia sp!, {r4, r7, r8, r9, r10, lr}
  289. Lmemcpy_bl12:
  290. adds r2, r2, #8
  291. blt Lmemcpy_bl4
  292. subs r2, r2, #4
  293. ldrlt r3, [r1, #-4]!
  294. strlt r3, [r0, #-4]!
  295. ldmgedb r1!, {r3, r12}
  296. stmgedb r0!, {r3, r12}
  297. subge r2, r2, #4
  298. Lmemcpy_bl4:
  299. /* less than 4 bytes to go */
  300. adds r2, r2, #4
  301. moveq pc, lr /* done */
  302. /* copy the crud byte at a time */
  303. cmp r2, #2
  304. ldrb r3, [r1, #-1]!
  305. strb r3, [r0, #-1]!
  306. ldrgeb r3, [r1, #-1]!
  307. strgeb r3, [r0, #-1]!
  308. ldrgtb r3, [r1, #-1]!
  309. strgtb r3, [r0, #-1]!
  310. mov pc, lr
  311. /* erg - unaligned destination */
  312. Lmemcpy_bdestul:
  313. cmp r12, #2
  314. /* align destination with byte copies */
  315. ldrb r3, [r1, #-1]!
  316. strb r3, [r0, #-1]!
  317. ldrgeb r3, [r1, #-1]!
  318. strgeb r3, [r0, #-1]!
  319. ldrgtb r3, [r1, #-1]!
  320. strgtb r3, [r0, #-1]!
  321. subs r2, r2, r12
  322. blt Lmemcpy_bl4 /* less than 4 bytes to go */
  323. ands r12, r1, #3
  324. beq Lmemcpy_bt8 /* we have an aligned source */
  325. /* erg - unaligned source */
  326. /* This is where it gets nasty ... */
  327. Lmemcpy_bsrcul:
  328. bic r1, r1, #3
  329. ldr r3, [r1, #0]
  330. cmp r12, #2
  331. blt Lmemcpy_bsrcul1
  332. beq Lmemcpy_bsrcul2
  333. cmp r2, #0x0c
  334. blt Lmemcpy_bsrcul3loop4
  335. sub r2, r2, #0x0c
  336. stmdb sp!, {r4, r5, lr}
  337. Lmemcpy_bsrcul3loop16:
  338. mov lr, r3, lsl #8
  339. ldmdb r1!, {r3-r5, r12}
  340. orr lr, lr, r12, lsr #24
  341. mov r12, r12, lsl #8
  342. orr r12, r12, r5, lsr #24
  343. mov r5, r5, lsl #8
  344. orr r5, r5, r4, lsr #24
  345. mov r4, r4, lsl #8
  346. orr r4, r4, r3, lsr #24
  347. stmdb r0!, {r4, r5, r12, lr}
  348. subs r2, r2, #0x10
  349. bge Lmemcpy_bsrcul3loop16
  350. ldmia sp!, {r4, r5, lr}
  351. adds r2, r2, #0x0c
  352. blt Lmemcpy_bsrcul3l4
  353. Lmemcpy_bsrcul3loop4:
  354. mov r12, r3, lsl #8
  355. ldr r3, [r1, #-4]!
  356. orr r12, r12, r3, lsr #24
  357. str r12, [r0, #-4]!
  358. subs r2, r2, #4
  359. bge Lmemcpy_bsrcul3loop4
  360. Lmemcpy_bsrcul3l4:
  361. add r1, r1, #3
  362. b Lmemcpy_bl4
  363. Lmemcpy_bsrcul2:
  364. cmp r2, #0x0c
  365. blt Lmemcpy_bsrcul2loop4
  366. sub r2, r2, #0x0c
  367. stmdb sp!, {r4, r5, lr}
  368. Lmemcpy_bsrcul2loop16:
  369. mov lr, r3, lsl #16
  370. ldmdb r1!, {r3-r5, r12}
  371. orr lr, lr, r12, lsr #16
  372. mov r12, r12, lsl #16
  373. orr r12, r12, r5, lsr #16
  374. mov r5, r5, lsl #16
  375. orr r5, r5, r4, lsr #16
  376. mov r4, r4, lsl #16
  377. orr r4, r4, r3, lsr #16
  378. stmdb r0!, {r4, r5, r12, lr}
  379. subs r2, r2, #0x10
  380. bge Lmemcpy_bsrcul2loop16
  381. ldmia sp!, {r4, r5, lr}
  382. adds r2, r2, #0x0c
  383. blt Lmemcpy_bsrcul2l4
  384. Lmemcpy_bsrcul2loop4:
  385. mov r12, r3, lsl #16
  386. ldr r3, [r1, #-4]!
  387. orr r12, r12, r3, lsr #16
  388. str r12, [r0, #-4]!
  389. subs r2, r2, #4
  390. bge Lmemcpy_bsrcul2loop4
  391. Lmemcpy_bsrcul2l4:
  392. add r1, r1, #2
  393. b Lmemcpy_bl4
  394. Lmemcpy_bsrcul1:
  395. cmp r2, #0x0c
  396. blt Lmemcpy_bsrcul1loop4
  397. sub r2, r2, #0x0c
  398. stmdb sp!, {r4, r5, lr}
  399. Lmemcpy_bsrcul1loop32:
  400. mov lr, r3, lsl #24
  401. ldmdb r1!, {r3-r5, r12}
  402. orr lr, lr, r12, lsr #8
  403. mov r12, r12, lsl #24
  404. orr r12, r12, r5, lsr #8
  405. mov r5, r5, lsl #24
  406. orr r5, r5, r4, lsr #8
  407. mov r4, r4, lsl #24
  408. orr r4, r4, r3, lsr #8
  409. stmdb r0!, {r4, r5, r12, lr}
  410. subs r2, r2, #0x10
  411. bge Lmemcpy_bsrcul1loop32
  412. ldmia sp!, {r4, r5, lr}
  413. adds r2, r2, #0x0c
  414. blt Lmemcpy_bsrcul1l4
  415. Lmemcpy_bsrcul1loop4:
  416. mov r12, r3, lsl #24
  417. ldr r3, [r1, #-4]!
  418. orr r12, r12, r3, lsr #8
  419. str r12, [r0, #-4]!
  420. subs r2, r2, #4
  421. bge Lmemcpy_bsrcul1loop4
  422. Lmemcpy_bsrcul1l4:
  423. add r1, r1, #1
  424. b Lmemcpy_bl4