0010-merge-demin-s-mem_xxx-optimization.patch 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382
  1. From 87fc954b961c326c9630df55201320b0d32f66f9 Mon Sep 17 00:00:00 2001
  2. From: "max.ma" <max.ma@starfivetech.com>
  3. Date: Tue, 13 Sep 2022 01:06:07 -0700
  4. Subject: [PATCH 10/19] merge demin's mem_xxx optimization
  5. ---
  6. sysdeps/riscv/rv64/multiarch/Makefile | 2 +-
  7. sysdeps/riscv/rv64/multiarch/memchr_as.S | 150 ++++++++------
  8. sysdeps/riscv/rv64/multiarch/memchr_riscv.S | 6 +-
  9. sysdeps/riscv/rv64/multiarch/memchr_vector.S | 6 +-
  10. sysdeps/riscv/rv64/multiarch/memcmp.c | 2 +
  11. sysdeps/riscv/rv64/multiarch/memcmp_as.S | 150 ++++++++++++++
  12. .../{memcmp_riscv.c => memcmp_riscv.S} | 8 +-
  13. sysdeps/riscv/rv64/multiarch/memcmp_vector.S | 7 +-
  14. sysdeps/riscv/rv64/multiarch/memcpy.c | 1 +
  15. sysdeps/riscv/rv64/multiarch/memcpy_as.S | 196 ++++++++++--------
  16. sysdeps/riscv/rv64/multiarch/memcpy_riscv.S | 2 +-
  17. sysdeps/riscv/rv64/multiarch/memcpy_vector.S | 5 +-
  18. sysdeps/riscv/rv64/multiarch/memmove_as.S | 168 +++++++++++++++
  19. .../{memmove_riscv.c => memmove_riscv.S} | 9 +-
  20. sysdeps/riscv/rv64/multiarch/memmove_vector.S | 6 +-
  21. sysdeps/riscv/rv64/multiarch/memrchr.S | 123 +++++++++++
  22. sysdeps/riscv/rv64/multiarch/memset_as.S | 123 +++++++++++
  23. .../{memset_riscv.c => memset_riscv.S} | 8 +-
  24. sysdeps/riscv/rv64/multiarch/memset_vector.S | 6 +-
  25. .../{rtld-memmove.c => rtld-memmove.S} | 3 +-
  26. sysdeps/riscv/rv64/multiarch/rtld-memrchr.S | 1 +
  27. sysdeps/riscv/rv64/multiarch/rtld-strcmp.S | 2 +-
  28. .../rv64/multiarch/{strcmp_.S => strcmp_as.S} | 0
  29. sysdeps/riscv/rv64/multiarch/strcmp_riscv.S | 6 +-
  30. sysdeps/riscv/rv64/multiarch/strcmp_vector.S | 6 +-
  31. sysdeps/riscv/rv64/multiarch/strlen_riscv.c | 2 +-
  32. sysdeps/riscv/rv64/multiarch/strlen_vector.S | 6 +-
  33. 27 files changed, 799 insertions(+), 205 deletions(-)
  34. create mode 100644 sysdeps/riscv/rv64/multiarch/memcmp_as.S
  35. rename sysdeps/riscv/rv64/multiarch/{memcmp_riscv.c => memcmp_riscv.S} (89%)
  36. create mode 100644 sysdeps/riscv/rv64/multiarch/memmove_as.S
  37. rename sysdeps/riscv/rv64/multiarch/{memmove_riscv.c => memmove_riscv.S} (88%)
  38. create mode 100644 sysdeps/riscv/rv64/multiarch/memrchr.S
  39. create mode 100644 sysdeps/riscv/rv64/multiarch/memset_as.S
  40. rename sysdeps/riscv/rv64/multiarch/{memset_riscv.c => memset_riscv.S} (89%)
  41. rename sysdeps/riscv/rv64/multiarch/{rtld-memmove.c => rtld-memmove.S} (93%)
  42. create mode 100644 sysdeps/riscv/rv64/multiarch/rtld-memrchr.S
  43. rename sysdeps/riscv/rv64/multiarch/{strcmp_.S => strcmp_as.S} (100%)
  44. diff --git a/sysdeps/riscv/rv64/multiarch/Makefile b/sysdeps/riscv/rv64/multiarch/Makefile
  45. index 3349bf4888..7e8bfde544 100644
  46. --- a/sysdeps/riscv/rv64/multiarch/Makefile
  47. +++ b/sysdeps/riscv/rv64/multiarch/Makefile
  48. @@ -1,5 +1,5 @@
  49. ifeq ($(subdir),string)
  50. sysdep_routines += memcpy_vector memcpy_riscv memchr_riscv memchr_vector memcmp_riscv \
  51. memcmp_vector strcmp_riscv strcmp_vector strlen_riscv strlen_vector \
  52. - memmove_vector memmove_riscv memset_vector memset_riscv
  53. + memmove_vector memmove_riscv memset_vector memset_riscv memrchr
  54. endif
  55. diff --git a/sysdeps/riscv/rv64/multiarch/memchr_as.S b/sysdeps/riscv/rv64/multiarch/memchr_as.S
  56. index b8691744f3..e614630a4d 100644
  57. --- a/sysdeps/riscv/rv64/multiarch/memchr_as.S
  58. +++ b/sysdeps/riscv/rv64/multiarch/memchr_as.S
  59. @@ -19,72 +19,90 @@
  60. #include <sysdep.h>
  61. - .p2align 6
  62. -
  63. -ENTRY (memchr)
  64. - zext.b a3,a1
  65. - beqz a2, .L_not_found
  66. - andi a5,a0,7
  67. -.L_not_aligned:
  68. - beqz a5,.L_aligned_8byte
  69. - lbu a5,0(a0)
  70. - addi a2,a2,-1
  71. - beq a5,a3,.L_found
  72. - addi a0,a0,1
  73. - andi a5,a0,7
  74. - bnez a2,.L_not_aligned
  75. -
  76. -.L_not_found:
  77. - li a0,0
  78. -.L_found:
  79. - ret
  80. +.macro chr_8B
  81. + ld a4, 0(a0)
  82. + xor a4, a4, a1
  83. + sub a3, a4, t1
  84. + andn a3, a3, a4
  85. + and a3, a3, a5
  86. + bnez a3, .L_find
  87. +.endm
  88. +.macro gen_pat
  89. + slli a3, a1, 8
  90. + or a1, a1, a3
  91. + slli a3, a1, 16
  92. + or a1, a1, a3
  93. + slli a3, a1, 32
  94. + or a1, a1, a3
  95. -.L_aligned_8byte:
  96. - zext.b a1,a1
  97. - slli a5,a1,0x8
  98. - or a1,a1,a5
  99. - slli a5,a1,0x10
  100. - or a5,a5,a1
  101. - slli a1,a5,0x20
  102. - li a4,7
  103. - or a1,a1,a5
  104. - bgeu a4,a2,.L_less_8bytes
  105. + li a5, 0x80
  106. + slli a3, a5, 8
  107. + or a5, a5, a3
  108. + slli a3, a5, 16
  109. + or a5, a5, a3
  110. + slli a3, a5, 32
  111. + or a5, a5, a3 # 0x8080808080808080
  112. + srli t1, a5, 7 # 0x0101010101010101
  113. +.endm
  114. - ld a7, mask1
  115. - ld a6, mask2
  116. -
  117. - li t1,7
  118. - j .L_8byte_compare_loop
  119. -.L_8byte_compare:
  120. - addi a2,a2,-8
  121. - addi a0,a0,8
  122. - bgeu t1,a2,.L_8byte_compare_exit
  123. -.L_8byte_compare_loop:
  124. - ld a5,0(a0)
  125. - xor a5,a5,a1
  126. - add a4,a5,a7
  127. - not a5,a5
  128. - and a5,a5,a4
  129. - and a5,a5,a6
  130. - beqz a5,.L_8byte_compare
  131. -
  132. -.L_less_8bytes:
  133. - add a2,a2,a0
  134. - j .L_less_8bytes_compare
  135. -.L_less_8bytes_loop:
  136. - addi a0,a0,1
  137. - beq a2,a0,.L_not_found
  138. -.L_less_8bytes_compare:
  139. - lbu a5,0(a0)
  140. - bne a5,a3,.L_less_8bytes_loop
  141. - ret
  142. -.L_8byte_compare_exit:
  143. - bnez a2,.L_less_8bytes
  144. - j .L_not_found
  145. - .align 3
  146. -mask1:
  147. - .dword 0xfefefefefefefeff
  148. -mask2:
  149. - .dword 0x8080808080808080
  150. + .p2align 6
  151. +ENTRY (memchr)
  152. + li a3, 7
  153. + bgtu a2, a3, .L_8_to_16
  154. +.L_0_to_7:
  155. + beqz a2, 1f
  156. +0:
  157. + lbu a4, 0(a0)
  158. + beq a1, a4, 2f
  159. + addi a2, a2, -1
  160. + addi a0, a0, 1
  161. + bnez a2, 0b
  162. +1:
  163. + li a0, 0
  164. + ret
  165. +2:
  166. + ret
  167. +.L_8_to_16:
  168. + gen_pat
  169. + li a3, 16
  170. + bgtu a2, a3, .L_over_16
  171. + addi a2, a2, -8
  172. + chr_8B
  173. + add a0, a0, a2
  174. + chr_8B
  175. + j .L_not_find
  176. +.L_over_16:
  177. + neg t4, a0
  178. + andi t4, t4, 0x7
  179. + beqz t4, .L_dst_aligned
  180. + chr_8B
  181. + sub a2, a2, t4
  182. + add a0, a0, t4
  183. +.L_dst_aligned:
  184. + andi t0, a2, (16-1)
  185. + srli a2, a2, 4
  186. + beqz a2, .L_tail
  187. +.L_loop:
  188. + chr_8B
  189. + add a0, a0, 8
  190. + chr_8B
  191. + addi a2, a2, -1
  192. + add a0, a0, 8
  193. + bnez a2, .L_loop
  194. + beqz t0, .L_not_find
  195. +.L_tail:
  196. + add a0, a0, t0
  197. + addi a0, a0, -16
  198. + chr_8B
  199. + add a0, a0, 8
  200. + chr_8B
  201. +.L_not_find:
  202. + li a0, 0
  203. + ret
  204. +.L_find:
  205. + ctz a3, a3
  206. + srli a3, a3, 3
  207. + add a0, a0, a3
  208. + ret
  209. END (memchr)
  210. -libc_hidden_builtin_def (memchr)
  211. +libc_hidden_builtin_def (memchr)
  212. \ No newline at end of file
  213. diff --git a/sysdeps/riscv/rv64/multiarch/memchr_riscv.S b/sysdeps/riscv/rv64/multiarch/memchr_riscv.S
  214. index ee4394fef9..84b3375f31 100644
  215. --- a/sysdeps/riscv/rv64/multiarch/memchr_riscv.S
  216. +++ b/sysdeps/riscv/rv64/multiarch/memchr_riscv.S
  217. @@ -26,8 +26,10 @@
  218. #include "memchr_as.S"
  219. -#elif !defined __riscv_vector
  220. +#else
  221. #include "memchr_as.S"
  222. -#endif
  223. \ No newline at end of file
  224. +#endif
  225. +
  226. +
  227. diff --git a/sysdeps/riscv/rv64/multiarch/memchr_vector.S b/sysdeps/riscv/rv64/multiarch/memchr_vector.S
  228. index 9441ddb489..87a464f38f 100644
  229. --- a/sysdeps/riscv/rv64/multiarch/memchr_vector.S
  230. +++ b/sysdeps/riscv/rv64/multiarch/memchr_vector.S
  231. @@ -19,12 +19,10 @@
  232. #include <sysdep.h>
  233. /* For __riscv_vector this file defines memchr. */
  234. -#ifdef __riscv_vector
  235. -#ifdef SHARED
  236. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  237. # define memchr __memchr_vector
  238. # undef libc_hidden_builtin_def
  239. # define libc_hidden_builtin_def(a)
  240. -#endif
  241. .p2align 6
  242. ENTRY (memchr)
  243. @@ -51,5 +49,5 @@ ENTRY (memchr)
  244. add a0,zero,zero
  245. ret
  246. END (memchr)
  247. -libc_hidden_builtin_def (memchr)
  248. +
  249. #endif
  250. \ No newline at end of file
  251. diff --git a/sysdeps/riscv/rv64/multiarch/memcmp.c b/sysdeps/riscv/rv64/multiarch/memcmp.c
  252. index 3dab42ea74..a67159346a 100644
  253. --- a/sysdeps/riscv/rv64/multiarch/memcmp.c
  254. +++ b/sysdeps/riscv/rv64/multiarch/memcmp.c
  255. @@ -34,3 +34,5 @@ riscv_libc_ifunc_hidden_def (__redirect_memcmp, memcmp);
  256. #else
  257. # include <string.h>
  258. #endif
  259. +
  260. +
  261. diff --git a/sysdeps/riscv/rv64/multiarch/memcmp_as.S b/sysdeps/riscv/rv64/multiarch/memcmp_as.S
  262. new file mode 100644
  263. index 0000000000..a737172980
  264. --- /dev/null
  265. +++ b/sysdeps/riscv/rv64/multiarch/memcmp_as.S
  266. @@ -0,0 +1,150 @@
  267. +/* The assembly function for memcmp. RISC-V version.
  268. + Copyright (C) 2018 Free Software Foundation, Inc.
  269. + This file is part of the GNU C Library.
  270. +
  271. + The GNU C Library is free software; you can redistribute it and/or
  272. + modify it under the terms of the GNU Lesser General Public
  273. + License as published by the Free Software Foundation; either
  274. + version 2.1 of the License, or (at your option) any later version.
  275. +
  276. + The GNU C Library is distributed in the hope that it will be useful,
  277. + but WITHOUT ANY WARRANTY; without even the implied warranty of
  278. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  279. + Lesser General Public License for more details.
  280. +
  281. + You should have received a copy of the GNU Lesser General Public
  282. + License along with the GNU C Library. If not, see
  283. + <http://www.gnu.org/licenses/>. */
  284. +
  285. +#include <sysdep.h>
  286. +
  287. + .p2align 6
  288. +ENTRY (memcmp)
  289. + li a3, 32
  290. + mv a5, a0
  291. + bgtu a2, a3, .L_over_32
  292. + li a3, 3
  293. + bgtu a2, a3, .L_4_to_8
  294. +.L_0_to_3:
  295. + beqz a2, 2f
  296. +0:
  297. + lbu a3, 0(a1)
  298. + lbu a4, 0(a5)
  299. + bne a3, a4, 1f
  300. + addi a2, a2, -1
  301. + addi a1, a1, 1
  302. + addi a5, a5, 1
  303. + bnez a2, 0b
  304. +1:
  305. + sub a0, a4, a3
  306. + ret
  307. +2:
  308. + li a0, 0
  309. + ret
  310. +.L_4_to_8:
  311. + li a3, 8
  312. + bgtu a2, a3, .L_9_to_16
  313. + lwu a3, 0(a1)
  314. + lwu a0, 0(a5)
  315. + bne a3, a0, .L_end
  316. + add a1, a1, a2
  317. + add a5, a5, a2
  318. + lwu a3, -4(a1)
  319. + lwu a0, -4(a5)
  320. + j .L_end
  321. +.L_9_to_16:
  322. + li a3, 16
  323. + bgtu a2, a3, .L_17_to_32
  324. + addi a2, a2, -8
  325. + ld a3, 0(a1)
  326. + ld a0, 0(a5)
  327. + bne a3, a0, .L_end
  328. + add a1, a1, a2
  329. + add a5, a5, a2
  330. + ld a3, 0(a1)
  331. + ld a0, 0(a5)
  332. + j .L_end
  333. +.L_17_to_32:
  334. + addi a2, a2, -16
  335. + ld a3, 0(a1)
  336. + ld a0, 0(a5)
  337. + bne a3, a0, .L_end
  338. + ld a3, 8(a1)
  339. + ld a0, 8(a5)
  340. + bne a3, a0, .L_end
  341. + add a1, a1, a2
  342. + add a5, a5, a2
  343. + ld a3, 0(a1)
  344. + ld a0, 0(a5)
  345. + bne a3, a0, .L_end
  346. + ld a3, 8(a1)
  347. + ld a0, 8(a5)
  348. + j .L_end
  349. +.L_over_32:
  350. + neg a4, a5
  351. + andi a4, a4, 0x7
  352. + beqz a4, .L_dst_aligned
  353. + ld a3, 0(a1)
  354. + ld a0, 0(a5)
  355. + bne a3, a0, .L_end
  356. + sub a2, a2, a4
  357. + add a5, a5, a4
  358. + add a1, a1, a4
  359. +.L_dst_aligned:
  360. + andi a4, a2, -32
  361. + andi a2, a2, (32-1)
  362. + beqz a4, .L_tail
  363. + add a4, a4, a5
  364. +.L_loop:
  365. + ld a3, 0(a1)
  366. + ld a0, 0(a5)
  367. + bne a3, a0, .L_end
  368. + ld a3, 8(a1)
  369. + ld a0, 8(a5)
  370. + bne a3, a0, .L_end
  371. + ld a3, 16(a1)
  372. + ld a0, 16(a5)
  373. + bne a3, a0, .L_end
  374. + ld a3, 24(a1)
  375. + ld a0, 24(a5)
  376. + bne a3, a0, .L_end
  377. + addi a5, a5, 32
  378. + addi a1, a1, 32
  379. + bltu a5, a4, .L_loop
  380. + beqz a2, .L_end
  381. +.L_tail:
  382. + andi a4, a2, -1
  383. +.L_pc:
  384. + auipc a3, 0
  385. + andi a4, a4, -8
  386. + sub a3, a3, a4
  387. + .equ offset, .L_jmp_end - .L_pc
  388. + jalr x0, a3, %lo(offset)
  389. + ld a3, 16(a1)
  390. + ld a0, 16(a5)
  391. + bne a3, a0, .L_end
  392. + ld a3, 8(a1)
  393. + ld a0, 8(a5)
  394. + bne a3, a0, .L_end
  395. + ld a3, 0(a1)
  396. + ld a0, 0(a5)
  397. + bne a3, a0, .L_end
  398. +.L_jmp_end:
  399. + add a1, a1, a2
  400. + add a5, a5, a2
  401. + ld a3, -8(a1)
  402. + ld a0, -8(a5)
  403. +.L_end:
  404. + bltu a0, a3, 2f
  405. + beq a0, a3, 3f
  406. + li a0, 1
  407. + ret
  408. +2:
  409. + li a0, -1
  410. + ret
  411. +3:
  412. + li a0, 0
  413. + ret
  414. +END (memcmp)
  415. +
  416. +libc_hidden_builtin_def (memcmp)
  417. diff --git a/sysdeps/riscv/rv64/multiarch/memcmp_riscv.c b/sysdeps/riscv/rv64/multiarch/memcmp_riscv.S
  418. similarity index 89%
  419. rename from sysdeps/riscv/rv64/multiarch/memcmp_riscv.c
  420. rename to sysdeps/riscv/rv64/multiarch/memcmp_riscv.S
  421. index 076431132b..32df79f769 100644
  422. --- a/sysdeps/riscv/rv64/multiarch/memcmp_riscv.c
  423. +++ b/sysdeps/riscv/rv64/multiarch/memcmp_riscv.S
  424. @@ -22,11 +22,11 @@
  425. #define libc_hidden_builtin_def(name)
  426. #undef weak_alias
  427. -# define MEMCMP __memcmp_riscv
  428. -# include <string/memcmp.c>
  429. +# define memcmp __memcmp_riscv
  430. +# include "memcmp_as.S"
  431. -#elif !defined __riscv_vector
  432. -# include <string/memcmp.c>
  433. +#else
  434. +# include "memcmp_as.S"
  435. #endif
  436. diff --git a/sysdeps/riscv/rv64/multiarch/memcmp_vector.S b/sysdeps/riscv/rv64/multiarch/memcmp_vector.S
  437. index 4810af026a..89d8a7e08f 100644
  438. --- a/sysdeps/riscv/rv64/multiarch/memcmp_vector.S
  439. +++ b/sysdeps/riscv/rv64/multiarch/memcmp_vector.S
  440. @@ -21,12 +21,10 @@
  441. /* For __riscv_vector this file defines strcmp. */
  442. /* #ifndef __riscv_vector */
  443. -#ifdef __riscv_vector
  444. -#ifdef SHARED
  445. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  446. # define memcmp __memcmp_vector
  447. # undef libc_hidden_builtin_def
  448. # define libc_hidden_builtin_def(a)
  449. -#endif
  450. .p2align 6
  451. ENTRY (memcmp)
  452. @@ -55,6 +53,5 @@ ENTRY (memcmp)
  453. add a0,zero,zero
  454. ret
  455. END (memcmp)
  456. -weak_alias (memcmp, bcmp)
  457. -libc_hidden_builtin_def (memcmp)
  458. +
  459. #endif
  460. diff --git a/sysdeps/riscv/rv64/multiarch/memcpy.c b/sysdeps/riscv/rv64/multiarch/memcpy.c
  461. index 45cf14f52a..4fa4cccd9d 100644
  462. --- a/sysdeps/riscv/rv64/multiarch/memcpy.c
  463. +++ b/sysdeps/riscv/rv64/multiarch/memcpy.c
  464. @@ -34,3 +34,4 @@ riscv_libc_ifunc_hidden_def (__redirect_memcpy, memcpy);
  465. #else
  466. # include <string.h>
  467. #endif
  468. +
  469. diff --git a/sysdeps/riscv/rv64/multiarch/memcpy_as.S b/sysdeps/riscv/rv64/multiarch/memcpy_as.S
  470. index f0a074df13..4a97d75545 100644
  471. --- a/sysdeps/riscv/rv64/multiarch/memcpy_as.S
  472. +++ b/sysdeps/riscv/rv64/multiarch/memcpy_as.S
  473. @@ -19,96 +19,116 @@
  474. #include <sysdep.h>
  475. -# define LABLE_ALIGN \
  476. - .balignl 16, 0x00000013
  477. +.macro copy_32B d, s
  478. + ld a3, 0(\s)
  479. + sd a3, 0(\d)
  480. + ld a3, 8(\s)
  481. + sd a3, 8(\d)
  482. + ld a3, 16(\s)
  483. + sd a3, 16(\d)
  484. + ld a3, 24(\s)
  485. + sd a3, 24(\d)
  486. +.endm
  487. + .p2align 6
  488. ENTRY (memcpy)
  489. -
  490. - /* Test if len less than 8 bytes. */
  491. - mv t6, a0
  492. - sltiu a3, a2, 8
  493. - li t3, 1
  494. - bnez a3, .L_copy_by_byte
  495. -
  496. - andi a3, a0, 7
  497. - li t5, 8
  498. - /* Test if dest is not 8 bytes aligned. */
  499. - bnez a3, .L_dest_not_aligned
  500. -.L_dest_aligned:
  501. - /* If dest is aligned, then copy. */
  502. - srli t4, a2, 6
  503. - /* Test if len less than 32 bytes. */
  504. - beqz t4, .L_len_less_16bytes
  505. - andi a2, a2, 63
  506. -
  507. -.L_len_larger_16bytes:
  508. - ld a4, 0(a1)
  509. - sd a4, 0(a0)
  510. - ld a5, 8(a1)
  511. - sd a5, 8(a0)
  512. - ld a6, 16(a1)
  513. - sd a6, 16(a0)
  514. - ld a7, 24(a1)
  515. - sd a7, 24(a0)
  516. - ld a4, 32(a1)
  517. - sd a4, 32(a0)
  518. - ld a5, 40(a1)
  519. - sd a5, 40(a0)
  520. - ld a6, 48(a1)
  521. - sd a6, 48(a0)
  522. - ld a7, 56(a1)
  523. - sub t4, t4, t3
  524. - addi a1, a1, 64
  525. - sd a7, 56(a0)
  526. - addi a0, a0, 64
  527. - bnez t4, .L_len_larger_16bytes
  528. -
  529. -.L_len_less_16bytes:
  530. - srli t4, a2, 2
  531. - beqz t4, .L_copy_by_byte
  532. - andi a2, a2, 3
  533. -.L_len_less_16bytes_loop:
  534. - lw a4, 0(a1)
  535. - sub t4, t4, t3
  536. - addi a1, a1, 4
  537. - sw a4, 0(a0)
  538. - addi a0, a0, 4
  539. - bnez t4, .L_len_less_16bytes_loop
  540. -
  541. - /* Copy tail. */
  542. -.L_copy_by_byte:
  543. - andi t4, a2, 7
  544. - beqz t4, .L_return
  545. -.L_copy_by_byte_loop:
  546. - lb a4, 0(a1)
  547. - sub t4, t4, t3
  548. - addi a1, a1, 1
  549. - sb a4, 0(a0)
  550. - addi a0, a0, 1
  551. - bnez t4, .L_copy_by_byte_loop
  552. -
  553. -.L_return:
  554. - mv a0, t6
  555. - ret
  556. -
  557. - /* If dest is not aligned, just copying some bytes makes the dest
  558. - align. */
  559. -.L_dest_not_aligned:
  560. - sub a3, t5, a3
  561. - mv t5, a3
  562. -.L_dest_not_aligned_loop:
  563. - /* Makes the dest align. */
  564. - lb a4, 0(a1)
  565. - sub a3, a3, t3
  566. - addi a1, a1, 1
  567. - sb a4, 0(a0)
  568. - addi a0, a0, 1
  569. - bnez a3, .L_dest_not_aligned_loop
  570. - sub a2, a2, t5
  571. - sltiu a3, a2, 4
  572. - bnez a3, .L_copy_by_byte
  573. - /* Check whether the src is aligned. */
  574. - j .L_dest_aligned
  575. + li a3, 32
  576. + mv a5, a0
  577. + bgtu a2, a3, .L_over_32
  578. + li a3, 4
  579. + bgtu a2, a3, .L_5_to_8
  580. +.L_0_to_4:
  581. + beqz a2, 1f
  582. + lb a4, 0(a1)
  583. + sb a4, 0(a5)
  584. + /* process [n-1] */
  585. + add t1, a1, a2
  586. + lb a4, -1(t1)
  587. + add t0, a5, a2
  588. + sb a4, -1(t0)
  589. + li a3, 2
  590. + bleu a2, a3, 1f
  591. + lb a4, 2(a1)
  592. + sb a4, 2(a5)
  593. + lb a4, 1(a1)
  594. + sb a4, 1(a5)
  595. +1:
  596. + ret
  597. +.L_5_to_8:
  598. + li a3, 8
  599. + bgtu a2, a3, .L_9_to_16
  600. + lw a3, 0(a1)
  601. + sw a3, 0(a5)
  602. + add a1, a1, a2
  603. + add a5, a5, a2
  604. + lw a3, -4(a1)
  605. + sw a3, -4(a5)
  606. + ret
  607. +.L_9_to_16:
  608. + li a3, 16
  609. + bgtu a2, a3, .L_17_to_32
  610. + ld a3, 0(a1)
  611. + sd a3, 0(a5)
  612. + add a1, a1, a2
  613. + add a5, a5, a2
  614. + ld a3, -8(a1)
  615. + sd a3, -8(a5)
  616. + ret
  617. +.L_17_to_32:
  618. + addi a2, a2, -16
  619. + ld a3, 0(a1)
  620. + sd a3, 0(a5)
  621. + ld a3, 8(a1)
  622. + sd a3, 8(a5)
  623. + add a1, a1, a2
  624. + add a5, a5, a2
  625. + ld a3, 0(a1)
  626. + sd a3, 0(a5)
  627. + ld a3, 8(a1)
  628. + sd a3, 8(a5)
  629. + ret
  630. +.L_over_32:
  631. + neg a4, a5
  632. + andi a4, a4, 0x7
  633. + beqz a4, .L_dst_aligned
  634. + ld a3, 0(a1)
  635. + sd a3, 0(a5)
  636. + sub a2, a2, a4
  637. + add a5, a5, a4
  638. + add a1, a1, a4
  639. +.L_dst_aligned:
  640. + andi a4, a2, -32
  641. + andi a2, a2, (32-1)
  642. + beqz a4, .L_tail
  643. + add a4, a4, a5
  644. +.L_loop:
  645. + copy_32B a5, a1
  646. + addi a5, a5, 32
  647. + addi a1, a1, 32
  648. + bltu a5, a4, .L_loop
  649. + beqz a2, .L_ret
  650. +.L_tail:
  651. + andi a4, a2, -1
  652. +.L_pc:
  653. + auipc a3, 0
  654. + andi a4, a4, -8
  655. + srli a4, a4, 1
  656. + sub a3, a3, a4
  657. + .equ offset, .L_jmp_end - .L_pc
  658. + jalr x0, a3, %lo(offset)
  659. + ld a3, 16(a1) # offset-12
  660. + sd a3, 16(a5)
  661. + ld a3, 8(a1) # offset-8
  662. + sd a3, 8(a5)
  663. + ld a3, 0(a1) # offset-4
  664. + sd a3, 0(a5)
  665. +.L_jmp_end:
  666. + add a1, a1, a2
  667. + add a5, a5, a2
  668. + ld a3, -8(a1)
  669. + sd a3, -8(a5)
  670. +.L_ret:
  671. + ret
  672. END (memcpy)
  673. libc_hidden_builtin_def (memcpy)
  674. diff --git a/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S b/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S
  675. index 249d4f6067..34b4e981c1 100644
  676. --- a/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S
  677. +++ b/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S
  678. @@ -26,7 +26,7 @@
  679. #include "memcpy_as.S"
  680. -#elif !defined __riscv_vector
  681. +#else
  682. #include "memcpy_as.S"
  683. #endif
  684. diff --git a/sysdeps/riscv/rv64/multiarch/memcpy_vector.S b/sysdeps/riscv/rv64/multiarch/memcpy_vector.S
  685. index d15f4c3954..cba5e00f19 100644
  686. --- a/sysdeps/riscv/rv64/multiarch/memcpy_vector.S
  687. +++ b/sysdeps/riscv/rv64/multiarch/memcpy_vector.S
  688. @@ -19,12 +19,10 @@
  689. #include <sysdep.h>
  690. /* For __riscv_vector this file defines memcpy. */
  691. -#ifdef __riscv_vector
  692. -#ifdef SHARED
  693. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  694. # define memcpy __memcpy_vector
  695. # undef libc_hidden_builtin_def
  696. # define libc_hidden_builtin_def(a)
  697. -#endif
  698. .p2align 6
  699. ENTRY (memcpy)
  700. @@ -52,5 +50,4 @@ ENTRY (memcpy)
  701. ret
  702. END (memcpy)
  703. -libc_hidden_builtin_def (memcpy)
  704. #endif
  705. \ No newline at end of file
  706. diff --git a/sysdeps/riscv/rv64/multiarch/memmove_as.S b/sysdeps/riscv/rv64/multiarch/memmove_as.S
  707. new file mode 100644
  708. index 0000000000..222b87b9c8
  709. --- /dev/null
  710. +++ b/sysdeps/riscv/rv64/multiarch/memmove_as.S
  711. @@ -0,0 +1,168 @@
  712. +/* The assembly function for memmove. RISC-V version.
  713. + Copyright (C) 2018 Free Software Foundation, Inc.
  714. + This file is part of the GNU C Library.
  715. +
  716. + The GNU C Library is free software; you can redistribute it and/or
  717. + modify it under the terms of the GNU Lesser General Public
  718. + License as published by the Free Software Foundation; either
  719. + version 2.1 of the License, or (at your option) any later version.
  720. +
  721. + The GNU C Library is distributed in the hope that it will be useful,
  722. + but WITHOUT ANY WARRANTY; without even the implied warranty of
  723. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  724. + Lesser General Public License for more details.
  725. +
  726. + You should have received a copy of the GNU Lesser General Public
  727. + License along with the GNU C Library. If not, see
  728. + <http://www.gnu.org/licenses/>. */
  729. +
  730. +#include <sysdep.h>
  731. +
  732. +#define memcpy memcpy_internal
  733. +#include "memcpy_as.S"
  734. +
  735. +
  736. + .p2align 6
  737. +
  738. +ENTRY (memmove)
  739. + beq a0, a1, .L_ret_fwd
  740. + /* abs(dst - src) */
  741. + sub a3, a0, a1
  742. + srai a4, a3, 63
  743. + xor a3, a3, a4
  744. + subw a3, a3, a4
  745. +
  746. + bltu a3, a2, .L_overlap
  747. + j memcpy
  748. +.L_overlap:
  749. + li a4, 8
  750. + bltu a1, a0, .L_backward
  751. + mv a5, a0
  752. + bltu a2, a4, .L_byte_fwd
  753. + neg a4, a0
  754. + andi a4, a4, 0x7
  755. + beqz a4, .L_32B_fwd
  756. +.L_byte_head_fwd:
  757. + sub a2, a2, a4
  758. + add a4, a5, a4
  759. +0:
  760. + lb a3, 0(a1)
  761. + sb a3, 0(a5)
  762. + addi a5, a5, 1
  763. + addi a1, a1, 1
  764. + bltu a5, a4, 0b
  765. +.L_32B_fwd:
  766. + andi a4, a2, -32
  767. + andi a2, a2, (32 - 1)
  768. + beqz a4, .L_8B_fwd
  769. + add a4, a4, a5
  770. +0:
  771. + ld a3, 0(a1)
  772. + sd a3, 0(a5)
  773. + ld a3, 8(a1)
  774. + sd a3, 8(a5)
  775. + ld a3, 16(a1)
  776. + sd a3, 16(a5)
  777. + ld a3, 24(a1)
  778. + sd a3, 24(a5)
  779. + addi a5, a5, 32
  780. + addi a1, a1, 32
  781. + bltu a5, a4, 0b
  782. + beqz a2, .L_ret_fwd
  783. +.L_8B_fwd:
  784. + andi a4, a2, -8
  785. + beqz a4, .L_byte_tail_fwd
  786. +.L_pc_fwd:
  787. + auipc a3, 0
  788. + add a1, a1, a4
  789. + add a5, a5, a4
  790. + sub a3, a3, a4
  791. + .equ offset_fwd, .L_jmp_end_fwd - .L_pc_fwd
  792. + jalr x0, a3, %lo(offset_fwd)
  793. + ld a3, -24(a1)
  794. + sd a3, -24(a5)
  795. + ld a3, -16(a1)
  796. + sd a3, -16(a5)
  797. + ld a3, -8(a1)
  798. + sd a3, -8(a5)
  799. +.L_jmp_end_fwd:
  800. +.L_byte_tail_fwd:
  801. + andi a2, a2, (8 - 1)
  802. +.L_byte_fwd:
  803. + beqz a2, .L_ret_fwd
  804. + add a2, a2, a5
  805. +0:
  806. + lb a3, 0(a1)
  807. + sb a3, 0(a5)
  808. + addi a5, a5, 1
  809. + addi a1, a1, 1
  810. + bltu a5, a2, 0b
  811. +.L_ret_fwd:
  812. + ret
  813. +
  814. +.L_backward:
  815. + add a1, a1, a2
  816. + add a5, a0, a2
  817. + bltu a2, a4, .L_byte_bwd
  818. + andi a4, a5, 0x7
  819. + beqz a4, .L_32B_bwd
  820. +.L_byte_head_bwd:
  821. + sub a2, a2, a4
  822. + sub a4, a5, a4
  823. +0:
  824. + addi a5, a5, -1
  825. + addi a1, a1, -1
  826. + lb a3, 0(a1)
  827. + sb a3, 0(a5)
  828. + bltu a4, a5, 0b
  829. +.L_32B_bwd:
  830. + andi a4, a2, -32
  831. + andi a2, a2, (32 - 1)
  832. + beqz a4, .L_8B_bwd
  833. + sub a4, a5, a4
  834. +0:
  835. + addi a5, a5, -32
  836. + addi a1, a1, -32
  837. + ld a3, 24(a1)
  838. + sd a3, 24(a5)
  839. + ld a3, 16(a1)
  840. + sd a3, 16(a5)
  841. + ld a3, 8(a1)
  842. + sd a3, 8(a5)
  843. + ld a3, 0(a1)
  844. + sd a3, 0(a5)
  845. + bltu a4, a5, 0b
  846. + beqz a2, .L_ret_bwd
  847. +.L_8B_bwd:
  848. + andi a4, a2, -8
  849. + beqz a4, .L_byte_tail_bwd
  850. +.L_pc_bwd:
  851. + auipc a3, 0
  852. + sub a1, a1, a4
  853. + sub a5, a5, a4
  854. + srli a4, a4, 1
  855. + sub a3, a3, a4
  856. + .equ offset_bwd, .L_jmp_end_bwd - .L_pc_bwd
  857. + jalr x0, a3, %lo(offset_bwd)
  858. + ld a3, 16(a1)
  859. + sd a3, 16(a5)
  860. + ld a3, 8(a1)
  861. + sd a3, 8(a5)
  862. + ld a3, 0(a1)
  863. + sd a3, 0(a5)
  864. +.L_jmp_end_bwd:
  865. +.L_byte_tail_bwd:
  866. + andi a2, a2, (8 - 1)
  867. +.L_byte_bwd:
  868. + beqz a2, .L_ret_bwd
  869. +0:
  870. + addi a5, a5, -1
  871. + addi a1, a1, -1
  872. + lb a3, 0(a1)
  873. + sb a3, 0(a5)
  874. + bltu a0, a5, 0b
  875. +.L_ret_bwd:
  876. + ret
  877. +END (memmove)
  878. +
  879. +libc_hidden_builtin_def (memmove)
  880. diff --git a/sysdeps/riscv/rv64/multiarch/memmove_riscv.c b/sysdeps/riscv/rv64/multiarch/memmove_riscv.S
  881. similarity index 88%
  882. rename from sysdeps/riscv/rv64/multiarch/memmove_riscv.c
  883. rename to sysdeps/riscv/rv64/multiarch/memmove_riscv.S
  884. index 03708e0deb..14f154e226 100644
  885. --- a/sysdeps/riscv/rv64/multiarch/memmove_riscv.c
  886. +++ b/sysdeps/riscv/rv64/multiarch/memmove_riscv.S
  887. @@ -22,10 +22,11 @@
  888. #define libc_hidden_builtin_def(name)
  889. #undef weak_alias
  890. -# define MEMMOVE __memmove_riscv
  891. -# include <string/memmove.c>
  892. -#elif !defined __riscv_vector
  893. +# define memmove __memmove_riscv
  894. +# include "memmove_as.S"
  895. -# include <string/memmove.c>
  896. +#else
  897. +
  898. +# include "memmove_as.S"
  899. #endif
  900. diff --git a/sysdeps/riscv/rv64/multiarch/memmove_vector.S b/sysdeps/riscv/rv64/multiarch/memmove_vector.S
  901. index af748be5b2..e45137298e 100644
  902. --- a/sysdeps/riscv/rv64/multiarch/memmove_vector.S
  903. +++ b/sysdeps/riscv/rv64/multiarch/memmove_vector.S
  904. @@ -19,12 +19,10 @@
  905. #include <sysdep.h>
  906. /* For __riscv_vector this file defines memmov. */
  907. -#ifdef __riscv_vector
  908. -#ifdef SHARED
  909. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  910. # define memmove __memmove_vector
  911. # undef libc_hidden_builtin_def
  912. # define libc_hidden_builtin_def(a)
  913. -#endif
  914. .p2align 6
  915. ENTRY (memmove)
  916. @@ -54,5 +52,5 @@ ENTRY (memmove)
  917. bltu zero,a2,.L_backward_copy_loop
  918. ret
  919. END (memmove)
  920. -libc_hidden_builtin_def (memmove)
  921. +
  922. #endif
  923. \ No newline at end of file
  924. diff --git a/sysdeps/riscv/rv64/multiarch/memrchr.S b/sysdeps/riscv/rv64/multiarch/memrchr.S
  925. new file mode 100644
  926. index 0000000000..c6db183163
  927. --- /dev/null
  928. +++ b/sysdeps/riscv/rv64/multiarch/memrchr.S
  929. @@ -0,0 +1,123 @@
  930. +
  931. +/* The assembly function for memrchr. RISC-V version.
  932. + Copyright (C) 2018 Free Software Foundation, Inc.
  933. + This file is part of the GNU C Library.
  934. +
  935. + The GNU C Library is free software; you can redistribute it and/or
  936. + modify it under the terms of the GNU Lesser General Public
  937. + License as published by the Free Software Foundation; either
  938. + version 2.1 of the License, or (at your option) any later version.
  939. +
  940. + The GNU C Library is distributed in the hope that it will be useful,
  941. + but WITHOUT ANY WARRANTY; without even the implied warranty of
  942. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  943. + Lesser General Public License for more details.
  944. +
  945. + You should have received a copy of the GNU Lesser General Public
  946. + License along with the GNU C Library. If not, see
  947. + <http://www.gnu.org/licenses/>. */
  948. +
  949. +#include <sysdep.h>
  950. +
  951. +.macro chr_8B
  952. + ld a4, -8(a0)
  953. + xor a4, a4, a1
  954. + sub a3, a4, t1
  955. + andn a3, a3, a4
  956. + and a3, a3, a5
  957. + bnez a3, .L_find
  958. +.endm
  959. +.macro gen_pat
  960. + slli a3, a1, 8
  961. + or a1, a1, a3
  962. + slli a3, a1, 16
  963. + or a1, a1, a3
  964. + slli a3, a1, 32
  965. + or a1, a1, a3
  966. +
  967. + li a5, 0x80
  968. + slli a3, a5, 8
  969. + or a5, a5, a3
  970. + slli a3, a5, 16
  971. + or a5, a5, a3
  972. + slli a3, a5, 32
  973. + or a5, a5, a3 # 0x8080808080808080
  974. + srli t1, a5, 7 # 0x0101010101010101
  975. +.endm
  976. +
  977. + .p2align 6
  978. +#ifndef MEMRCHR
  979. +ENTRY (__memrchr)
  980. +#else
  981. +ENTRY (MEMRCHR)
  982. +#endif
  983. + li a3, 7
  984. + add a0, a0, a2
  985. + bgtu a2, a3, .L_8_to_16
  986. +.L_0_to_7:
  987. + beqz a2, 1f
  988. +0:
  989. + lbu a4, -1(a0)
  990. + beq a1, a4, 2f
  991. + addi a2, a2, -1
  992. + addi a0, a0, -1
  993. + bnez a2, 0b
  994. +1:
  995. + li a0, 0
  996. + ret
  997. +2:
  998. + addi a0, a0, -1
  999. + ret
  1000. +.L_8_to_16:
  1001. + gen_pat
  1002. + li a3, 16
  1003. + bgtu a2, a3, .L_over_16
  1004. + addi a2, a2, -8
  1005. + chr_8B
  1006. + sub a0, a0, a2
  1007. + chr_8B
  1008. + j .L_not_find
  1009. +.L_over_16:
  1010. + and t4, a0, 0x7
  1011. + beqz t4, .L_dst_aligned
  1012. + chr_8B
  1013. + sub a2, a2, t4
  1014. + sub a0, a0, t4
  1015. +.L_dst_aligned:
  1016. + andi t0, a2, (16-1)
  1017. + srli a2, a2, 4
  1018. + beqz a2, .L_tail
  1019. +.L_loop:
  1020. + chr_8B
  1021. + addi a0, a0, -8
  1022. + chr_8B
  1023. + addi a2, a2, -1
  1024. + addi a0, a0, -8
  1025. + bnez a2, .L_loop
  1026. + beqz t0, .L_not_find
  1027. +.L_tail:
  1028. + sub a0, a0, t0
  1029. + addi a0, a0, 16
  1030. + chr_8B
  1031. + addi a0, a0, -8
  1032. + chr_8B
  1033. +.L_not_find:
  1034. + li a0, 0
  1035. + ret
  1036. +.L_find:
  1037. + clz a3, a3
  1038. + addi a0, a0, -1
  1039. + srli a3, a3, 3
  1040. + sub a0, a0, a3
  1041. + ret
  1042. +#ifndef MEMRCHR
  1043. +END (__memrchr)
  1044. +#else
  1045. +END (MEMRCHR)
  1046. +#endif
  1047. +
  1048. +#ifndef MEMRCHR
  1049. +# ifdef weak_alias
  1050. +weak_alias (__memrchr, memrchr)
  1051. +# endif
  1052. +#endif
  1053. diff --git a/sysdeps/riscv/rv64/multiarch/memset_as.S b/sysdeps/riscv/rv64/multiarch/memset_as.S
  1054. new file mode 100644
  1055. index 0000000000..7f67c79215
  1056. --- /dev/null
  1057. +++ b/sysdeps/riscv/rv64/multiarch/memset_as.S
  1058. @@ -0,0 +1,123 @@
  1059. +
  1060. +/* The assembly function for memset. RISC-V version.
  1061. + Copyright (C) 2018 Free Software Foundation, Inc.
  1062. + This file is part of the GNU C Library.
  1063. +
  1064. + The GNU C Library is free software; you can redistribute it and/or
  1065. + modify it under the terms of the GNU Lesser General Public
  1066. + License as published by the Free Software Foundation; either
  1067. + version 2.1 of the License, or (at your option) any later version.
  1068. +
  1069. + The GNU C Library is distributed in the hope that it will be useful,
  1070. + but WITHOUT ANY WARRANTY; without even the implied warranty of
  1071. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  1072. + Lesser General Public License for more details.
  1073. +
  1074. + You should have received a copy of the GNU Lesser General Public
  1075. + License along with the GNU C Library. If not, see
  1076. + <http://www.gnu.org/licenses/>. */
  1077. +
  1078. +#include <sysdep.h>
  1079. +
  1080. + .p2align 6
  1081. +ENTRY (memset)
  1082. + li a3, 32
  1083. + andi a1, a1, 0xff
  1084. + mv a5, a0
  1085. + bgtu a2, a3, .L_over_32
  1086. + li a3, 4
  1087. + bgtu a2, a3, .L_5_to_8
  1088. +.L_0_to_4:
  1089. + beqz a2, 1f
  1090. + sb a1, 0(a5)
  1091. + add t0, a5, a2
  1092. + sb a1, -1(t0)
  1093. + li a3, 2
  1094. + bleu a2, a3, 1f
  1095. + sb a1, 1(a5)
  1096. + sb a1, 2(a5)
  1097. +1:
  1098. + ret
  1099. +.L_5_to_8:
  1100. + slli a3, a1, 8
  1101. + or a1, a1, a3
  1102. + slli a3, a1, 16
  1103. + or a1, a1, a3
  1104. + li a3, 8
  1105. + bgtu a2, a3, .L_9_to_16
  1106. + sw a1, 0(a5)
  1107. + add a5, a5, a2
  1108. + sw a1, -4(a5)
  1109. + ret
  1110. +.L_9_to_16:
  1111. + li a3, 16
  1112. + bgtu a2, a3, .L_17_to_32
  1113. + sw a1, 0(a5)
  1114. + sw a1, 4(a5)
  1115. + add a5, a5, a2
  1116. + sw a1, -8(a5)
  1117. + sw a1, -4(a5)
  1118. + ret
  1119. +.L_17_to_32:
  1120. + slli a3, a1, 32
  1121. + or a1, a1, a3
  1122. + sd a1, 0(a5)
  1123. + sd a1, 8(a5)
  1124. + add a5, a5, a2
  1125. + sd a1, -16(a5)
  1126. + sd a1, -8(a5)
  1127. + ret
  1128. +.L_over_32:
  1129. + slli a3, a1, 8
  1130. + or a1, a1, a3
  1131. + slli a3, a1, 16
  1132. + or a1, a1, a3
  1133. + slli a3, a1, 32
  1134. + or a1, a1, a3
  1135. +
  1136. + neg a4, a5
  1137. + andi a4, a4, 0x7
  1138. + beqz a4, .L_dst_aligned
  1139. + sd a1, 0(a5)
  1140. + sub a2, a2, a4
  1141. + add a5, a5, a4
  1142. +.L_dst_aligned:
  1143. + andi a4, a2, -64
  1144. + andi a2, a2, (64-1)
  1145. + beqz a4, .L_tail
  1146. + add a4, a4, a5
  1147. +.L_loop:
  1148. + sd a1, 0(a5)
  1149. + sd a1, 8(a5)
  1150. + sd a1, 16(a5)
  1151. + sd a1, 24(a5)
  1152. + sd a1, 32(a5)
  1153. + sd a1, 40(a5)
  1154. + sd a1, 48(a5)
  1155. + sd a1, 56(a5)
  1156. + addi a5, a5, 64
  1157. + bltu a5, a4, .L_loop
  1158. + beqz a2, .L_ret
  1159. +.L_tail:
  1160. + andi a4, a2, -1
  1161. +.L_pc:
  1162. + auipc a3, 0
  1163. + andi a4, a4, -8
  1164. + srli a4, a4, 2
  1165. + sub a3, a3, a4
  1166. + .equ offset, .L_jmp_end - .L_pc
  1167. + jalr x0, a3, %lo(offset)
  1168. + sd a1, 48(a5)
  1169. + sd a1, 40(a5)
  1170. + sd a1, 32(a5)
  1171. + sd a1, 24(a5)
  1172. + sd a1, 16(a5)
  1173. + sd a1, 8(a5)
  1174. + sd a1, 0(a5)
  1175. +.L_jmp_end:
  1176. + add a5, a5, a2
  1177. + sd a1, -8(a5)
  1178. + .L_ret:
  1179. + ret
  1180. +END (memset)
  1181. +libc_hidden_builtin_def (memset)
  1182. diff --git a/sysdeps/riscv/rv64/multiarch/memset_riscv.c b/sysdeps/riscv/rv64/multiarch/memset_riscv.S
  1183. similarity index 89%
  1184. rename from sysdeps/riscv/rv64/multiarch/memset_riscv.c
  1185. rename to sysdeps/riscv/rv64/multiarch/memset_riscv.S
  1186. index de631eb187..8585db0228 100644
  1187. --- a/sysdeps/riscv/rv64/multiarch/memset_riscv.c
  1188. +++ b/sysdeps/riscv/rv64/multiarch/memset_riscv.S
  1189. @@ -21,11 +21,11 @@
  1190. #define libc_hidden_builtin_def(name)
  1191. #undef weak_alias
  1192. -# define MEMSET __memset_riscv
  1193. -# include <string/memset.c>
  1194. -#elif !defined __riscv_vector
  1195. +# define memset __memset_riscv
  1196. +# include "memset_as.S"
  1197. +#else
  1198. -# include <string/memset.c>
  1199. +# include "memset_as.S"
  1200. #endif
  1201. diff --git a/sysdeps/riscv/rv64/multiarch/memset_vector.S b/sysdeps/riscv/rv64/multiarch/memset_vector.S
  1202. index dc407630a6..1271b04ba1 100644
  1203. --- a/sysdeps/riscv/rv64/multiarch/memset_vector.S
  1204. +++ b/sysdeps/riscv/rv64/multiarch/memset_vector.S
  1205. @@ -19,12 +19,11 @@
  1206. #include <sysdep.h>
  1207. -#ifdef __riscv_vector
  1208. -#ifdef SHARED
  1209. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  1210. +
  1211. # define memset __memset_vector
  1212. # undef libc_hidden_builtin_def
  1213. # define libc_hidden_builtin_def(a)
  1214. -#endif
  1215. .p2align 6
  1216. ENTRY (memset)
  1217. @@ -42,5 +41,4 @@ ENTRY (memset)
  1218. ret
  1219. END (memset)
  1220. -libc_hidden_builtin_def (memset)
  1221. #endif
  1222. \ No newline at end of file
  1223. diff --git a/sysdeps/riscv/rv64/multiarch/rtld-memmove.c b/sysdeps/riscv/rv64/multiarch/rtld-memmove.S
  1224. similarity index 93%
  1225. rename from sysdeps/riscv/rv64/multiarch/rtld-memmove.c
  1226. rename to sysdeps/riscv/rv64/multiarch/rtld-memmove.S
  1227. index 90e37dbd8b..fb6c2c3748 100644
  1228. --- a/sysdeps/riscv/rv64/multiarch/rtld-memmove.c
  1229. +++ b/sysdeps/riscv/rv64/multiarch/rtld-memmove.S
  1230. @@ -16,4 +16,5 @@
  1231. License along with the GNU C Library; if not, see
  1232. <https://www.gnu.org/licenses/>. */
  1233. -# include <string/memmove.c>
  1234. +//# include <string/memmove.c>
  1235. +#include "memmove_as.S"
  1236. \ No newline at end of file
  1237. diff --git a/sysdeps/riscv/rv64/multiarch/rtld-memrchr.S b/sysdeps/riscv/rv64/multiarch/rtld-memrchr.S
  1238. new file mode 100644
  1239. index 0000000000..b1154d0f2a
  1240. --- /dev/null
  1241. +++ b/sysdeps/riscv/rv64/multiarch/rtld-memrchr.S
  1242. @@ -0,0 +1 @@
  1243. +#include "memrchr_as.S"
  1244. \ No newline at end of file
  1245. diff --git a/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S b/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S
  1246. index eb6ff5f8d3..e59c57f738 100644
  1247. --- a/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S
  1248. +++ b/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S
  1249. @@ -1 +1 @@
  1250. -#include "strcmp_.S"
  1251. \ No newline at end of file
  1252. +#include "strcmp_as.S"
  1253. \ No newline at end of file
  1254. diff --git a/sysdeps/riscv/rv64/multiarch/strcmp_.S b/sysdeps/riscv/rv64/multiarch/strcmp_as.S
  1255. similarity index 100%
  1256. rename from sysdeps/riscv/rv64/multiarch/strcmp_.S
  1257. rename to sysdeps/riscv/rv64/multiarch/strcmp_as.S
  1258. diff --git a/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S b/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S
  1259. index f5be83d5b7..abf2984230 100644
  1260. --- a/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S
  1261. +++ b/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S
  1262. @@ -6,8 +6,8 @@
  1263. # undef libc_hidden_builtin_def
  1264. # define libc_hidden_builtin_def(a)
  1265. -#include "strcmp_.S"
  1266. -#elif !defined __riscv_vector
  1267. +#include "strcmp_as.S"
  1268. +#else
  1269. -#include "strcmp_.S"
  1270. +#include "strcmp_as.S"
  1271. #endif
  1272. diff --git a/sysdeps/riscv/rv64/multiarch/strcmp_vector.S b/sysdeps/riscv/rv64/multiarch/strcmp_vector.S
  1273. index d0b8e316f9..40fc474695 100644
  1274. --- a/sysdeps/riscv/rv64/multiarch/strcmp_vector.S
  1275. +++ b/sysdeps/riscv/rv64/multiarch/strcmp_vector.S
  1276. @@ -19,12 +19,10 @@
  1277. #include <sysdep.h>
  1278. /* For __riscv_vector this file defines strcmp. */
  1279. -#ifdef __riscv_vector
  1280. -#ifdef SHARED
  1281. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  1282. # define strcmp __strcmp_vector
  1283. # undef libc_hidden_builtin_def
  1284. # define libc_hidden_builtin_def(a)
  1285. -#endif
  1286. .p2align 6
  1287. ENTRY (strcmp)
  1288. @@ -51,5 +49,5 @@ ENTRY (strcmp)
  1289. ret
  1290. END (strcmp)
  1291. -libc_hidden_builtin_def (strcmp)
  1292. +
  1293. #endif
  1294. diff --git a/sysdeps/riscv/rv64/multiarch/strlen_riscv.c b/sysdeps/riscv/rv64/multiarch/strlen_riscv.c
  1295. index d605941766..d3bae0fc43 100644
  1296. --- a/sysdeps/riscv/rv64/multiarch/strlen_riscv.c
  1297. +++ b/sysdeps/riscv/rv64/multiarch/strlen_riscv.c
  1298. @@ -24,7 +24,7 @@
  1299. #undef weak_alias
  1300. # define STRLEN __strlen_riscv
  1301. # include <string/strlen.c>
  1302. -#elif !defined __riscv_vector
  1303. +#else
  1304. # include <string/strlen.c>
  1305. #endif
  1306. diff --git a/sysdeps/riscv/rv64/multiarch/strlen_vector.S b/sysdeps/riscv/rv64/multiarch/strlen_vector.S
  1307. index f674de3a26..0b4ae7ba90 100644
  1308. --- a/sysdeps/riscv/rv64/multiarch/strlen_vector.S
  1309. +++ b/sysdeps/riscv/rv64/multiarch/strlen_vector.S
  1310. @@ -19,12 +19,10 @@
  1311. #include <sysdep.h>
  1312. /* For __riscv_vector this file defines strlen. */
  1313. -#ifdef __riscv_vector
  1314. -#ifdef SHARED
  1315. +#if IS_IN (libc) && defined SHARED && defined __riscv_vector
  1316. # define strlen __strlen_vector
  1317. # undef libc_hidden_builtin_def
  1318. # define libc_hidden_builtin_def(a)
  1319. -#endif
  1320. .p2align 6
  1321. ENTRY (strlen)
  1322. @@ -44,5 +42,5 @@ ENTRY (strlen)
  1323. ret
  1324. END (strlen)
  1325. -libc_hidden_builtin_def (strlen)
  1326. +
  1327. #endif
  1328. \ No newline at end of file
  1329. --
  1330. 2.25.1