memset.S 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (c) 2011, The Linux Foundation. All rights reserved.
  4. */
  5. /* HEXAGON assembly optimized memset */
  6. /* Replaces the standard library function memset */
  7. .macro HEXAGON_OPT_FUNC_BEGIN name
  8. .text
  9. .p2align 4
  10. .globl \name
  11. .type \name, @function
  12. \name:
  13. .endm
  14. .macro HEXAGON_OPT_FUNC_FINISH name
  15. .size \name, . - \name
  16. .endm
  17. /* FUNCTION: memset (v2 version) */
  18. #if __HEXAGON_ARCH__ < 3
  19. HEXAGON_OPT_FUNC_BEGIN memset
  20. {
  21. r6 = #8
  22. r7 = extractu(r0, #3 , #0)
  23. p0 = cmp.eq(r2, #0)
  24. p1 = cmp.gtu(r2, #7)
  25. }
  26. {
  27. r4 = vsplatb(r1)
  28. r8 = r0 /* leave r0 intact for return val */
  29. r9 = sub(r6, r7) /* bytes until double alignment */
  30. if p0 jumpr r31 /* count == 0, so return */
  31. }
  32. {
  33. r3 = #0
  34. r7 = #0
  35. p0 = tstbit(r9, #0)
  36. if p1 jump 2f /* skip byte loop */
  37. }
  38. /* less than 8 bytes to set, so just set a byte at a time and return */
  39. loop0(1f, r2) /* byte loop */
  40. .falign
  41. 1: /* byte loop */
  42. {
  43. memb(r8++#1) = r4
  44. }:endloop0
  45. jumpr r31
  46. .falign
  47. 2: /* skip byte loop */
  48. {
  49. r6 = #1
  50. p0 = tstbit(r9, #1)
  51. p1 = cmp.eq(r2, #1)
  52. if !p0 jump 3f /* skip initial byte store */
  53. }
  54. {
  55. memb(r8++#1) = r4
  56. r3:2 = sub(r3:2, r7:6)
  57. if p1 jumpr r31
  58. }
  59. .falign
  60. 3: /* skip initial byte store */
  61. {
  62. r6 = #2
  63. p0 = tstbit(r9, #2)
  64. p1 = cmp.eq(r2, #2)
  65. if !p0 jump 4f /* skip initial half store */
  66. }
  67. {
  68. memh(r8++#2) = r4
  69. r3:2 = sub(r3:2, r7:6)
  70. if p1 jumpr r31
  71. }
  72. .falign
  73. 4: /* skip initial half store */
  74. {
  75. r6 = #4
  76. p0 = cmp.gtu(r2, #7)
  77. p1 = cmp.eq(r2, #4)
  78. if !p0 jump 5f /* skip initial word store */
  79. }
  80. {
  81. memw(r8++#4) = r4
  82. r3:2 = sub(r3:2, r7:6)
  83. p0 = cmp.gtu(r2, #11)
  84. if p1 jumpr r31
  85. }
  86. .falign
  87. 5: /* skip initial word store */
  88. {
  89. r10 = lsr(r2, #3)
  90. p1 = cmp.eq(r3, #1)
  91. if !p0 jump 7f /* skip double loop */
  92. }
  93. {
  94. r5 = r4
  95. r6 = #8
  96. loop0(6f, r10) /* double loop */
  97. }
  98. /* set bytes a double word at a time */
  99. .falign
  100. 6: /* double loop */
  101. {
  102. memd(r8++#8) = r5:4
  103. r3:2 = sub(r3:2, r7:6)
  104. p1 = cmp.eq(r2, #8)
  105. }:endloop0
  106. .falign
  107. 7: /* skip double loop */
  108. {
  109. p0 = tstbit(r2, #2)
  110. if p1 jumpr r31
  111. }
  112. {
  113. r6 = #4
  114. p0 = tstbit(r2, #1)
  115. p1 = cmp.eq(r2, #4)
  116. if !p0 jump 8f /* skip final word store */
  117. }
  118. {
  119. memw(r8++#4) = r4
  120. r3:2 = sub(r3:2, r7:6)
  121. if p1 jumpr r31
  122. }
  123. .falign
  124. 8: /* skip final word store */
  125. {
  126. p1 = cmp.eq(r2, #2)
  127. if !p0 jump 9f /* skip final half store */
  128. }
  129. {
  130. memh(r8++#2) = r4
  131. if p1 jumpr r31
  132. }
  133. .falign
  134. 9: /* skip final half store */
  135. {
  136. memb(r8++#1) = r4
  137. jumpr r31
  138. }
  139. HEXAGON_OPT_FUNC_FINISH memset
  140. #endif
  141. /* FUNCTION: memset (v3 and higher version) */
  142. #if __HEXAGON_ARCH__ >= 3
  143. HEXAGON_OPT_FUNC_BEGIN memset
  144. {
  145. r7=vsplatb(r1)
  146. r6 = r0
  147. if (r2==#0) jump:nt .L1
  148. }
  149. {
  150. r5:4=combine(r7,r7)
  151. p0 = cmp.gtu(r2,#8)
  152. if (p0.new) jump:nt .L3
  153. }
  154. {
  155. r3 = r0
  156. loop0(.L47,r2)
  157. }
  158. .falign
  159. .L47:
  160. {
  161. memb(r3++#1) = r1
  162. }:endloop0 /* start=.L47 */
  163. jumpr r31
  164. .L3:
  165. {
  166. p0 = tstbit(r0,#0)
  167. if (!p0.new) jump:nt .L8
  168. p1 = cmp.eq(r2, #1)
  169. }
  170. {
  171. r6 = add(r0, #1)
  172. r2 = add(r2,#-1)
  173. memb(r0) = r1
  174. if (p1) jump .L1
  175. }
  176. .L8:
  177. {
  178. p0 = tstbit(r6,#1)
  179. if (!p0.new) jump:nt .L10
  180. }
  181. {
  182. r2 = add(r2,#-2)
  183. memh(r6++#2) = r7
  184. p0 = cmp.eq(r2, #2)
  185. if (p0.new) jump:nt .L1
  186. }
  187. .L10:
  188. {
  189. p0 = tstbit(r6,#2)
  190. if (!p0.new) jump:nt .L12
  191. }
  192. {
  193. r2 = add(r2,#-4)
  194. memw(r6++#4) = r7
  195. p0 = cmp.eq(r2, #4)
  196. if (p0.new) jump:nt .L1
  197. }
  198. .L12:
  199. {
  200. p0 = cmp.gtu(r2,#127)
  201. if (!p0.new) jump:nt .L14
  202. }
  203. r3 = and(r6,#31)
  204. if (r3==#0) jump:nt .L17
  205. {
  206. memd(r6++#8) = r5:4
  207. r2 = add(r2,#-8)
  208. }
  209. r3 = and(r6,#31)
  210. if (r3==#0) jump:nt .L17
  211. {
  212. memd(r6++#8) = r5:4
  213. r2 = add(r2,#-8)
  214. }
  215. r3 = and(r6,#31)
  216. if (r3==#0) jump:nt .L17
  217. {
  218. memd(r6++#8) = r5:4
  219. r2 = add(r2,#-8)
  220. }
  221. .L17:
  222. {
  223. r3 = lsr(r2,#5)
  224. if (r1!=#0) jump:nt .L18
  225. }
  226. {
  227. r8 = r3
  228. r3 = r6
  229. loop0(.L46,r3)
  230. }
  231. .falign
  232. .L46:
  233. {
  234. dczeroa(r6)
  235. r6 = add(r6,#32)
  236. r2 = add(r2,#-32)
  237. }:endloop0 /* start=.L46 */
  238. .L14:
  239. {
  240. p0 = cmp.gtu(r2,#7)
  241. if (!p0.new) jump:nt .L28
  242. r8 = lsr(r2,#3)
  243. }
  244. loop0(.L44,r8)
  245. .falign
  246. .L44:
  247. {
  248. memd(r6++#8) = r5:4
  249. r2 = add(r2,#-8)
  250. }:endloop0 /* start=.L44 */
  251. .L28:
  252. {
  253. p0 = tstbit(r2,#2)
  254. if (!p0.new) jump:nt .L33
  255. }
  256. {
  257. r2 = add(r2,#-4)
  258. memw(r6++#4) = r7
  259. }
  260. .L33:
  261. {
  262. p0 = tstbit(r2,#1)
  263. if (!p0.new) jump:nt .L35
  264. }
  265. {
  266. r2 = add(r2,#-2)
  267. memh(r6++#2) = r7
  268. }
  269. .L35:
  270. p0 = cmp.eq(r2,#1)
  271. if (p0) memb(r6) = r1
  272. .L1:
  273. jumpr r31
  274. .L18:
  275. loop0(.L45,r3)
  276. .falign
  277. .L45:
  278. dczeroa(r6)
  279. {
  280. memd(r6++#8) = r5:4
  281. r2 = add(r2,#-32)
  282. }
  283. memd(r6++#8) = r5:4
  284. memd(r6++#8) = r5:4
  285. {
  286. memd(r6++#8) = r5:4
  287. }:endloop0 /* start=.L45 */
  288. jump .L14
  289. HEXAGON_OPT_FUNC_FINISH memset
  290. #endif