strncmp.S 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2013 ARM Ltd.
  4. * Copyright (C) 2013 Linaro.
  5. *
  6. * This code is based on glibc cortex strings work originally authored by Linaro
  7. * be found @
  8. *
  9. * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
  10. * files/head:/src/aarch64/
  11. */
  12. #include <linux/linkage.h>
  13. #include <asm/assembler.h>
  14. /*
  15. * compare two strings
  16. *
  17. * Parameters:
  18. * x0 - const string 1 pointer
  19. * x1 - const string 2 pointer
  20. * x2 - the maximal length to be compared
  21. * Returns:
  22. * x0 - an integer less than, equal to, or greater than zero if s1 is found,
  23. * respectively, to be less than, to match, or be greater than s2.
  24. */
  25. #define REP8_01 0x0101010101010101
  26. #define REP8_7f 0x7f7f7f7f7f7f7f7f
  27. #define REP8_80 0x8080808080808080
  28. /* Parameters and result. */
  29. src1 .req x0
  30. src2 .req x1
  31. limit .req x2
  32. result .req x0
  33. /* Internal variables. */
  34. data1 .req x3
  35. data1w .req w3
  36. data2 .req x4
  37. data2w .req w4
  38. has_nul .req x5
  39. diff .req x6
  40. syndrome .req x7
  41. tmp1 .req x8
  42. tmp2 .req x9
  43. tmp3 .req x10
  44. zeroones .req x11
  45. pos .req x12
  46. limit_wd .req x13
  47. mask .req x14
  48. endloop .req x15
  49. SYM_FUNC_START_WEAK_PI(strncmp)
  50. cbz limit, .Lret0
  51. eor tmp1, src1, src2
  52. mov zeroones, #REP8_01
  53. tst tmp1, #7
  54. b.ne .Lmisaligned8
  55. ands tmp1, src1, #7
  56. b.ne .Lmutual_align
  57. /* Calculate the number of full and partial words -1. */
  58. /*
  59. * when limit is mulitply of 8, if not sub 1,
  60. * the judgement of last dword will wrong.
  61. */
  62. sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
  63. lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */
  64. /*
  65. * NUL detection works on the principle that (X - 1) & (~X) & 0x80
  66. * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
  67. * can be done in parallel across the entire word.
  68. */
  69. .Lloop_aligned:
  70. ldr data1, [src1], #8
  71. ldr data2, [src2], #8
  72. .Lstart_realigned:
  73. subs limit_wd, limit_wd, #1
  74. sub tmp1, data1, zeroones
  75. orr tmp2, data1, #REP8_7f
  76. eor diff, data1, data2 /* Non-zero if differences found. */
  77. csinv endloop, diff, xzr, pl /* Last Dword or differences.*/
  78. bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
  79. ccmp endloop, #0, #0, eq
  80. b.eq .Lloop_aligned
  81. /*Not reached the limit, must have found the end or a diff. */
  82. tbz limit_wd, #63, .Lnot_limit
  83. /* Limit % 8 == 0 => all bytes significant. */
  84. ands limit, limit, #7
  85. b.eq .Lnot_limit
  86. lsl limit, limit, #3 /* Bits -> bytes. */
  87. mov mask, #~0
  88. CPU_BE( lsr mask, mask, limit )
  89. CPU_LE( lsl mask, mask, limit )
  90. bic data1, data1, mask
  91. bic data2, data2, mask
  92. /* Make sure that the NUL byte is marked in the syndrome. */
  93. orr has_nul, has_nul, mask
  94. .Lnot_limit:
  95. orr syndrome, diff, has_nul
  96. b .Lcal_cmpresult
  97. .Lmutual_align:
  98. /*
  99. * Sources are mutually aligned, but are not currently at an
  100. * alignment boundary. Round down the addresses and then mask off
  101. * the bytes that precede the start point.
  102. * We also need to adjust the limit calculations, but without
  103. * overflowing if the limit is near ULONG_MAX.
  104. */
  105. bic src1, src1, #7
  106. bic src2, src2, #7
  107. ldr data1, [src1], #8
  108. neg tmp3, tmp1, lsl #3 /* 64 - bits(bytes beyond align). */
  109. ldr data2, [src2], #8
  110. mov tmp2, #~0
  111. sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
  112. /* Big-endian. Early bytes are at MSB. */
  113. CPU_BE( lsl tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */
  114. /* Little-endian. Early bytes are at LSB. */
  115. CPU_LE( lsr tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */
  116. and tmp3, limit_wd, #7
  117. lsr limit_wd, limit_wd, #3
  118. /* Adjust the limit. Only low 3 bits used, so overflow irrelevant.*/
  119. add limit, limit, tmp1
  120. add tmp3, tmp3, tmp1
  121. orr data1, data1, tmp2
  122. orr data2, data2, tmp2
  123. add limit_wd, limit_wd, tmp3, lsr #3
  124. b .Lstart_realigned
  125. /*when src1 offset is not equal to src2 offset...*/
  126. .Lmisaligned8:
  127. cmp limit, #8
  128. b.lo .Ltiny8proc /*limit < 8... */
  129. /*
  130. * Get the align offset length to compare per byte first.
  131. * After this process, one string's address will be aligned.*/
  132. and tmp1, src1, #7
  133. neg tmp1, tmp1
  134. add tmp1, tmp1, #8
  135. and tmp2, src2, #7
  136. neg tmp2, tmp2
  137. add tmp2, tmp2, #8
  138. subs tmp3, tmp1, tmp2
  139. csel pos, tmp1, tmp2, hi /*Choose the maximum. */
  140. /*
  141. * Here, limit is not less than 8, so directly run .Ltinycmp
  142. * without checking the limit.*/
  143. sub limit, limit, pos
  144. .Ltinycmp:
  145. ldrb data1w, [src1], #1
  146. ldrb data2w, [src2], #1
  147. subs pos, pos, #1
  148. ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */
  149. ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
  150. b.eq .Ltinycmp
  151. cbnz pos, 1f /*find the null or unequal...*/
  152. cmp data1w, #1
  153. ccmp data1w, data2w, #0, cs
  154. b.eq .Lstart_align /*the last bytes are equal....*/
  155. 1:
  156. sub result, data1, data2
  157. ret
  158. .Lstart_align:
  159. lsr limit_wd, limit, #3
  160. cbz limit_wd, .Lremain8
  161. /*process more leading bytes to make str1 aligned...*/
  162. ands xzr, src1, #7
  163. b.eq .Lrecal_offset
  164. add src1, src1, tmp3 /*tmp3 is positive in this branch.*/
  165. add src2, src2, tmp3
  166. ldr data1, [src1], #8
  167. ldr data2, [src2], #8
  168. sub limit, limit, tmp3
  169. lsr limit_wd, limit, #3
  170. subs limit_wd, limit_wd, #1
  171. sub tmp1, data1, zeroones
  172. orr tmp2, data1, #REP8_7f
  173. eor diff, data1, data2 /* Non-zero if differences found. */
  174. csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/
  175. bics has_nul, tmp1, tmp2
  176. ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/
  177. b.ne .Lunequal_proc
  178. /*How far is the current str2 from the alignment boundary...*/
  179. and tmp3, tmp3, #7
  180. .Lrecal_offset:
  181. neg pos, tmp3
  182. .Lloopcmp_proc:
  183. /*
  184. * Divide the eight bytes into two parts. First,backwards the src2
  185. * to an alignment boundary,load eight bytes from the SRC2 alignment
  186. * boundary,then compare with the relative bytes from SRC1.
  187. * If all 8 bytes are equal,then start the second part's comparison.
  188. * Otherwise finish the comparison.
  189. * This special handle can garantee all the accesses are in the
  190. * thread/task space in avoid to overrange access.
  191. */
  192. ldr data1, [src1,pos]
  193. ldr data2, [src2,pos]
  194. sub tmp1, data1, zeroones
  195. orr tmp2, data1, #REP8_7f
  196. bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
  197. eor diff, data1, data2 /* Non-zero if differences found. */
  198. csinv endloop, diff, xzr, eq
  199. cbnz endloop, .Lunequal_proc
  200. /*The second part process*/
  201. ldr data1, [src1], #8
  202. ldr data2, [src2], #8
  203. subs limit_wd, limit_wd, #1
  204. sub tmp1, data1, zeroones
  205. orr tmp2, data1, #REP8_7f
  206. eor diff, data1, data2 /* Non-zero if differences found. */
  207. csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/
  208. bics has_nul, tmp1, tmp2
  209. ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/
  210. b.eq .Lloopcmp_proc
  211. .Lunequal_proc:
  212. orr syndrome, diff, has_nul
  213. cbz syndrome, .Lremain8
  214. .Lcal_cmpresult:
  215. /*
  216. * reversed the byte-order as big-endian,then CLZ can find the most
  217. * significant zero bits.
  218. */
  219. CPU_LE( rev syndrome, syndrome )
  220. CPU_LE( rev data1, data1 )
  221. CPU_LE( rev data2, data2 )
  222. /*
  223. * For big-endian we cannot use the trick with the syndrome value
  224. * as carry-propagation can corrupt the upper bits if the trailing
  225. * bytes in the string contain 0x01.
  226. * However, if there is no NUL byte in the dword, we can generate
  227. * the result directly. We can't just subtract the bytes as the
  228. * MSB might be significant.
  229. */
  230. CPU_BE( cbnz has_nul, 1f )
  231. CPU_BE( cmp data1, data2 )
  232. CPU_BE( cset result, ne )
  233. CPU_BE( cneg result, result, lo )
  234. CPU_BE( ret )
  235. CPU_BE( 1: )
  236. /* Re-compute the NUL-byte detection, using a byte-reversed value.*/
  237. CPU_BE( rev tmp3, data1 )
  238. CPU_BE( sub tmp1, tmp3, zeroones )
  239. CPU_BE( orr tmp2, tmp3, #REP8_7f )
  240. CPU_BE( bic has_nul, tmp1, tmp2 )
  241. CPU_BE( rev has_nul, has_nul )
  242. CPU_BE( orr syndrome, diff, has_nul )
  243. /*
  244. * The MS-non-zero bit of the syndrome marks either the first bit
  245. * that is different, or the top bit of the first zero byte.
  246. * Shifting left now will bring the critical information into the
  247. * top bits.
  248. */
  249. clz pos, syndrome
  250. lsl data1, data1, pos
  251. lsl data2, data2, pos
  252. /*
  253. * But we need to zero-extend (char is unsigned) the value and then
  254. * perform a signed 32-bit subtraction.
  255. */
  256. lsr data1, data1, #56
  257. sub result, data1, data2, lsr #56
  258. ret
  259. .Lremain8:
  260. /* Limit % 8 == 0 => all bytes significant. */
  261. ands limit, limit, #7
  262. b.eq .Lret0
  263. .Ltiny8proc:
  264. ldrb data1w, [src1], #1
  265. ldrb data2w, [src2], #1
  266. subs limit, limit, #1
  267. ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */
  268. ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
  269. b.eq .Ltiny8proc
  270. sub result, data1, data2
  271. ret
  272. .Lret0:
  273. mov result, #0
  274. ret
  275. SYM_FUNC_END_PI(strncmp)
  276. EXPORT_SYMBOL_NOKASAN(strncmp)