strlen.S 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2013 ARM Ltd.
  4. * Copyright (C) 2013 Linaro.
  5. *
  6. * This code is based on glibc cortex strings work originally authored by Linaro
  7. * be found @
  8. *
  9. * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
  10. * files/head:/src/aarch64/
  11. */
  12. #include <linux/linkage.h>
  13. #include <asm/assembler.h>
  14. /*
  15. * calculate the length of a string
  16. *
  17. * Parameters:
  18. * x0 - const string pointer
  19. * Returns:
  20. * x0 - the return length of specific string
  21. */
  22. /* Arguments and results. */
  23. srcin .req x0
  24. len .req x0
  25. /* Locals and temporaries. */
  26. src .req x1
  27. data1 .req x2
  28. data2 .req x3
  29. data2a .req x4
  30. has_nul1 .req x5
  31. has_nul2 .req x6
  32. tmp1 .req x7
  33. tmp2 .req x8
  34. tmp3 .req x9
  35. tmp4 .req x10
  36. zeroones .req x11
  37. pos .req x12
  38. #define REP8_01 0x0101010101010101
  39. #define REP8_7f 0x7f7f7f7f7f7f7f7f
  40. #define REP8_80 0x8080808080808080
  41. SYM_FUNC_START_WEAK_PI(strlen)
  42. mov zeroones, #REP8_01
  43. bic src, srcin, #15
  44. ands tmp1, srcin, #15
  45. b.ne .Lmisaligned
  46. /*
  47. * NUL detection works on the principle that (X - 1) & (~X) & 0x80
  48. * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
  49. * can be done in parallel across the entire word.
  50. */
  51. /*
  52. * The inner loop deals with two Dwords at a time. This has a
  53. * slightly higher start-up cost, but we should win quite quickly,
  54. * especially on cores with a high number of issue slots per
  55. * cycle, as we get much better parallelism out of the operations.
  56. */
  57. .Lloop:
  58. ldp data1, data2, [src], #16
  59. .Lrealigned:
  60. sub tmp1, data1, zeroones
  61. orr tmp2, data1, #REP8_7f
  62. sub tmp3, data2, zeroones
  63. orr tmp4, data2, #REP8_7f
  64. bic has_nul1, tmp1, tmp2
  65. bics has_nul2, tmp3, tmp4
  66. ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
  67. b.eq .Lloop
  68. sub len, src, srcin
  69. cbz has_nul1, .Lnul_in_data2
  70. CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/
  71. sub len, len, #8
  72. mov has_nul2, has_nul1
  73. .Lnul_in_data2:
  74. /*
  75. * For big-endian, carry propagation (if the final byte in the
  76. * string is 0x01) means we cannot use has_nul directly. The
  77. * easiest way to get the correct byte is to byte-swap the data
  78. * and calculate the syndrome a second time.
  79. */
  80. CPU_BE( rev data2, data2 )
  81. CPU_BE( sub tmp1, data2, zeroones )
  82. CPU_BE( orr tmp2, data2, #REP8_7f )
  83. CPU_BE( bic has_nul2, tmp1, tmp2 )
  84. sub len, len, #8
  85. rev has_nul2, has_nul2
  86. clz pos, has_nul2
  87. add len, len, pos, lsr #3 /* Bits to bytes. */
  88. ret
  89. .Lmisaligned:
  90. cmp tmp1, #8
  91. neg tmp1, tmp1
  92. ldp data1, data2, [src], #16
  93. lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
  94. mov tmp2, #~0
  95. /* Big-endian. Early bytes are at MSB. */
  96. CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
  97. /* Little-endian. Early bytes are at LSB. */
  98. CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
  99. orr data1, data1, tmp2
  100. orr data2a, data2, tmp2
  101. csinv data1, data1, xzr, le
  102. csel data2, data2, data2a, le
  103. b .Lrealigned
  104. SYM_FUNC_END_PI(strlen)
  105. EXPORT_SYMBOL_NOKASAN(strlen)