string.c 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Copyright (C) 1991,1992,1993,1997,1998,2003, 2005 Free Software Foundation, Inc.
  4. * This file is part of the GNU C Library.
  5. * Copyright (c) 2011 The Chromium OS Authors.
  6. */
  7. /* From glibc-2.14, sysdeps/i386/memset.c */
  8. #include <linux/types.h>
  9. #include <linux/compiler.h>
  10. #include <asm/string.h>
  11. typedef uint32_t op_t;
  12. void *memset(void *dstpp, int c, size_t len)
  13. {
  14. int d0;
  15. unsigned long int dstp = (unsigned long int) dstpp;
  16. /* This explicit register allocation improves code very much indeed. */
  17. register op_t x asm("ax");
  18. x = (unsigned char) c;
  19. /* Clear the direction flag, so filling will move forward. */
  20. asm volatile("cld");
  21. /* This threshold value is optimal. */
  22. if (len >= 12) {
  23. /* Fill X with four copies of the char we want to fill with. */
  24. x |= (x << 8);
  25. x |= (x << 16);
  26. /* Adjust LEN for the bytes handled in the first loop. */
  27. len -= (-dstp) % sizeof(op_t);
  28. /*
  29. * There are at least some bytes to set. No need to test for
  30. * LEN == 0 in this alignment loop.
  31. */
  32. /* Fill bytes until DSTP is aligned on a longword boundary. */
  33. asm volatile(
  34. "rep\n"
  35. "stosb" /* %0, %2, %3 */ :
  36. "=D" (dstp), "=c" (d0) :
  37. "0" (dstp), "1" ((-dstp) % sizeof(op_t)), "a" (x) :
  38. "memory");
  39. /* Fill longwords. */
  40. asm volatile(
  41. "rep\n"
  42. "stosl" /* %0, %2, %3 */ :
  43. "=D" (dstp), "=c" (d0) :
  44. "0" (dstp), "1" (len / sizeof(op_t)), "a" (x) :
  45. "memory");
  46. len %= sizeof(op_t);
  47. }
  48. /* Write the last few bytes. */
  49. asm volatile(
  50. "rep\n"
  51. "stosb" /* %0, %2, %3 */ :
  52. "=D" (dstp), "=c" (d0) :
  53. "0" (dstp), "1" (len), "a" (x) :
  54. "memory");
  55. return dstpp;
  56. }
  57. #define OP_T_THRES 8
  58. #define OPSIZ (sizeof(op_t))
  59. #define BYTE_COPY_FWD(dst_bp, src_bp, nbytes) \
  60. do { \
  61. int __d0; \
  62. asm volatile( \
  63. /* Clear the direction flag, so copying goes forward. */ \
  64. "cld\n" \
  65. /* Copy bytes. */ \
  66. "rep\n" \
  67. "movsb" : \
  68. "=D" (dst_bp), "=S" (src_bp), "=c" (__d0) : \
  69. "0" (dst_bp), "1" (src_bp), "2" (nbytes) : \
  70. "memory"); \
  71. } while (0)
  72. #define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes) \
  73. do { \
  74. int __d0; \
  75. asm volatile( \
  76. /* Clear the direction flag, so copying goes forward. */ \
  77. "cld\n" \
  78. /* Copy longwords. */ \
  79. "rep\n" \
  80. "movsl" : \
  81. "=D" (dst_bp), "=S" (src_bp), "=c" (__d0) : \
  82. "0" (dst_bp), "1" (src_bp), "2" ((nbytes) / 4) : \
  83. "memory"); \
  84. (nbytes_left) = (nbytes) % 4; \
  85. } while (0)
  86. void *memcpy(void *dstpp, const void *srcpp, size_t len)
  87. {
  88. unsigned long int dstp = (long int)dstpp;
  89. unsigned long int srcp = (long int)srcpp;
  90. /* Copy from the beginning to the end. */
  91. /* If there not too few bytes to copy, use word copy. */
  92. if (len >= OP_T_THRES) {
  93. /* Copy just a few bytes to make DSTP aligned. */
  94. len -= (-dstp) % OPSIZ;
  95. BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ);
  96. /* Copy from SRCP to DSTP taking advantage of the known
  97. * alignment of DSTP. Number of bytes remaining is put
  98. * in the third argument, i.e. in LEN. This number may
  99. * vary from machine to machine.
  100. */
  101. WORD_COPY_FWD(dstp, srcp, len, len);
  102. /* Fall out and copy the tail. */
  103. }
  104. /* There are just a few bytes to copy. Use byte memory operations. */
  105. BYTE_COPY_FWD(dstp, srcp, len);
  106. return dstpp;
  107. }
  108. void *memmove(void *dest, const void *src, size_t n)
  109. {
  110. int d0, d1, d2, d3, d4, d5;
  111. char *ret = dest;
  112. __asm__ __volatile__(
  113. /* Handle more 16 bytes in loop */
  114. "cmp $0x10, %0\n\t"
  115. "jb 1f\n\t"
  116. /* Decide forward/backward copy mode */
  117. "cmp %2, %1\n\t"
  118. "jb 2f\n\t"
  119. /*
  120. * movs instruction have many startup latency
  121. * so we handle small size by general register.
  122. */
  123. "cmp $680, %0\n\t"
  124. "jb 3f\n\t"
  125. /* movs instruction is only good for aligned case */
  126. "mov %1, %3\n\t"
  127. "xor %2, %3\n\t"
  128. "and $0xff, %3\n\t"
  129. "jz 4f\n\t"
  130. "3:\n\t"
  131. "sub $0x10, %0\n\t"
  132. /* We gobble 16 bytes forward in each loop */
  133. "3:\n\t"
  134. "sub $0x10, %0\n\t"
  135. "mov 0*4(%1), %3\n\t"
  136. "mov 1*4(%1), %4\n\t"
  137. "mov %3, 0*4(%2)\n\t"
  138. "mov %4, 1*4(%2)\n\t"
  139. "mov 2*4(%1), %3\n\t"
  140. "mov 3*4(%1), %4\n\t"
  141. "mov %3, 2*4(%2)\n\t"
  142. "mov %4, 3*4(%2)\n\t"
  143. "lea 0x10(%1), %1\n\t"
  144. "lea 0x10(%2), %2\n\t"
  145. "jae 3b\n\t"
  146. "add $0x10, %0\n\t"
  147. "jmp 1f\n\t"
  148. /* Handle data forward by movs */
  149. ".p2align 4\n\t"
  150. "4:\n\t"
  151. "mov -4(%1, %0), %3\n\t"
  152. "lea -4(%2, %0), %4\n\t"
  153. "shr $2, %0\n\t"
  154. "rep movsl\n\t"
  155. "mov %3, (%4)\n\t"
  156. "jmp 11f\n\t"
  157. /* Handle data backward by movs */
  158. ".p2align 4\n\t"
  159. "6:\n\t"
  160. "mov (%1), %3\n\t"
  161. "mov %2, %4\n\t"
  162. "lea -4(%1, %0), %1\n\t"
  163. "lea -4(%2, %0), %2\n\t"
  164. "shr $2, %0\n\t"
  165. "std\n\t"
  166. "rep movsl\n\t"
  167. "mov %3,(%4)\n\t"
  168. "cld\n\t"
  169. "jmp 11f\n\t"
  170. /* Start to prepare for backward copy */
  171. ".p2align 4\n\t"
  172. "2:\n\t"
  173. "cmp $680, %0\n\t"
  174. "jb 5f\n\t"
  175. "mov %1, %3\n\t"
  176. "xor %2, %3\n\t"
  177. "and $0xff, %3\n\t"
  178. "jz 6b\n\t"
  179. /* Calculate copy position to tail */
  180. "5:\n\t"
  181. "add %0, %1\n\t"
  182. "add %0, %2\n\t"
  183. "sub $0x10, %0\n\t"
  184. /* We gobble 16 bytes backward in each loop */
  185. "7:\n\t"
  186. "sub $0x10, %0\n\t"
  187. "mov -1*4(%1), %3\n\t"
  188. "mov -2*4(%1), %4\n\t"
  189. "mov %3, -1*4(%2)\n\t"
  190. "mov %4, -2*4(%2)\n\t"
  191. "mov -3*4(%1), %3\n\t"
  192. "mov -4*4(%1), %4\n\t"
  193. "mov %3, -3*4(%2)\n\t"
  194. "mov %4, -4*4(%2)\n\t"
  195. "lea -0x10(%1), %1\n\t"
  196. "lea -0x10(%2), %2\n\t"
  197. "jae 7b\n\t"
  198. /* Calculate copy position to head */
  199. "add $0x10, %0\n\t"
  200. "sub %0, %1\n\t"
  201. "sub %0, %2\n\t"
  202. /* Move data from 8 bytes to 15 bytes */
  203. ".p2align 4\n\t"
  204. "1:\n\t"
  205. "cmp $8, %0\n\t"
  206. "jb 8f\n\t"
  207. "mov 0*4(%1), %3\n\t"
  208. "mov 1*4(%1), %4\n\t"
  209. "mov -2*4(%1, %0), %5\n\t"
  210. "mov -1*4(%1, %0), %1\n\t"
  211. "mov %3, 0*4(%2)\n\t"
  212. "mov %4, 1*4(%2)\n\t"
  213. "mov %5, -2*4(%2, %0)\n\t"
  214. "mov %1, -1*4(%2, %0)\n\t"
  215. "jmp 11f\n\t"
  216. /* Move data from 4 bytes to 7 bytes */
  217. ".p2align 4\n\t"
  218. "8:\n\t"
  219. "cmp $4, %0\n\t"
  220. "jb 9f\n\t"
  221. "mov 0*4(%1), %3\n\t"
  222. "mov -1*4(%1, %0), %4\n\t"
  223. "mov %3, 0*4(%2)\n\t"
  224. "mov %4, -1*4(%2, %0)\n\t"
  225. "jmp 11f\n\t"
  226. /* Move data from 2 bytes to 3 bytes */
  227. ".p2align 4\n\t"
  228. "9:\n\t"
  229. "cmp $2, %0\n\t"
  230. "jb 10f\n\t"
  231. "movw 0*2(%1), %%dx\n\t"
  232. "movw -1*2(%1, %0), %%bx\n\t"
  233. "movw %%dx, 0*2(%2)\n\t"
  234. "movw %%bx, -1*2(%2, %0)\n\t"
  235. "jmp 11f\n\t"
  236. /* Move data for 1 byte */
  237. ".p2align 4\n\t"
  238. "10:\n\t"
  239. "cmp $1, %0\n\t"
  240. "jb 11f\n\t"
  241. "movb (%1), %%cl\n\t"
  242. "movb %%cl, (%2)\n\t"
  243. ".p2align 4\n\t"
  244. "11:"
  245. : "=&c" (d0), "=&S" (d1), "=&D" (d2),
  246. "=r" (d3), "=r" (d4), "=r"(d5)
  247. : "0" (n),
  248. "1" (src),
  249. "2" (dest)
  250. : "memory");
  251. return ret;
  252. }