From 87fc954b961c326c9630df55201320b0d32f66f9 Mon Sep 17 00:00:00 2001 From: "max.ma" Date: Tue, 13 Sep 2022 01:06:07 -0700 Subject: [PATCH 10/19] merge demin's mem_xxx optimization --- sysdeps/riscv/rv64/multiarch/Makefile | 2 +- sysdeps/riscv/rv64/multiarch/memchr_as.S | 150 ++++++++------ sysdeps/riscv/rv64/multiarch/memchr_riscv.S | 6 +- sysdeps/riscv/rv64/multiarch/memchr_vector.S | 6 +- sysdeps/riscv/rv64/multiarch/memcmp.c | 2 + sysdeps/riscv/rv64/multiarch/memcmp_as.S | 150 ++++++++++++++ .../{memcmp_riscv.c => memcmp_riscv.S} | 8 +- sysdeps/riscv/rv64/multiarch/memcmp_vector.S | 7 +- sysdeps/riscv/rv64/multiarch/memcpy.c | 1 + sysdeps/riscv/rv64/multiarch/memcpy_as.S | 196 ++++++++++-------- sysdeps/riscv/rv64/multiarch/memcpy_riscv.S | 2 +- sysdeps/riscv/rv64/multiarch/memcpy_vector.S | 5 +- sysdeps/riscv/rv64/multiarch/memmove_as.S | 168 +++++++++++++++ .../{memmove_riscv.c => memmove_riscv.S} | 9 +- sysdeps/riscv/rv64/multiarch/memmove_vector.S | 6 +- sysdeps/riscv/rv64/multiarch/memrchr.S | 123 +++++++++++ sysdeps/riscv/rv64/multiarch/memset_as.S | 123 +++++++++++ .../{memset_riscv.c => memset_riscv.S} | 8 +- sysdeps/riscv/rv64/multiarch/memset_vector.S | 6 +- .../{rtld-memmove.c => rtld-memmove.S} | 3 +- sysdeps/riscv/rv64/multiarch/rtld-memrchr.S | 1 + sysdeps/riscv/rv64/multiarch/rtld-strcmp.S | 2 +- .../rv64/multiarch/{strcmp_.S => strcmp_as.S} | 0 sysdeps/riscv/rv64/multiarch/strcmp_riscv.S | 6 +- sysdeps/riscv/rv64/multiarch/strcmp_vector.S | 6 +- sysdeps/riscv/rv64/multiarch/strlen_riscv.c | 2 +- sysdeps/riscv/rv64/multiarch/strlen_vector.S | 6 +- 27 files changed, 799 insertions(+), 205 deletions(-) create mode 100644 sysdeps/riscv/rv64/multiarch/memcmp_as.S rename sysdeps/riscv/rv64/multiarch/{memcmp_riscv.c => memcmp_riscv.S} (89%) create mode 100644 sysdeps/riscv/rv64/multiarch/memmove_as.S rename sysdeps/riscv/rv64/multiarch/{memmove_riscv.c => memmove_riscv.S} (88%) create mode 100644 sysdeps/riscv/rv64/multiarch/memrchr.S create mode 100644 sysdeps/riscv/rv64/multiarch/memset_as.S rename sysdeps/riscv/rv64/multiarch/{memset_riscv.c => memset_riscv.S} (89%) rename sysdeps/riscv/rv64/multiarch/{rtld-memmove.c => rtld-memmove.S} (93%) create mode 100644 sysdeps/riscv/rv64/multiarch/rtld-memrchr.S rename sysdeps/riscv/rv64/multiarch/{strcmp_.S => strcmp_as.S} (100%) diff --git a/sysdeps/riscv/rv64/multiarch/Makefile b/sysdeps/riscv/rv64/multiarch/Makefile index 3349bf4888..7e8bfde544 100644 --- a/sysdeps/riscv/rv64/multiarch/Makefile +++ b/sysdeps/riscv/rv64/multiarch/Makefile @@ -1,5 +1,5 @@ ifeq ($(subdir),string) sysdep_routines += memcpy_vector memcpy_riscv memchr_riscv memchr_vector memcmp_riscv \ memcmp_vector strcmp_riscv strcmp_vector strlen_riscv strlen_vector \ - memmove_vector memmove_riscv memset_vector memset_riscv + memmove_vector memmove_riscv memset_vector memset_riscv memrchr endif diff --git a/sysdeps/riscv/rv64/multiarch/memchr_as.S b/sysdeps/riscv/rv64/multiarch/memchr_as.S index b8691744f3..e614630a4d 100644 --- a/sysdeps/riscv/rv64/multiarch/memchr_as.S +++ b/sysdeps/riscv/rv64/multiarch/memchr_as.S @@ -19,72 +19,90 @@ #include - .p2align 6 - -ENTRY (memchr) - zext.b a3,a1 - beqz a2, .L_not_found - andi a5,a0,7 -.L_not_aligned: - beqz a5,.L_aligned_8byte - lbu a5,0(a0) - addi a2,a2,-1 - beq a5,a3,.L_found - addi a0,a0,1 - andi a5,a0,7 - bnez a2,.L_not_aligned - -.L_not_found: - li a0,0 -.L_found: - ret +.macro chr_8B + ld a4, 0(a0) + xor a4, a4, a1 + sub a3, a4, t1 + andn a3, a3, a4 + and a3, a3, a5 + bnez a3, .L_find +.endm +.macro gen_pat + slli a3, a1, 8 + or a1, a1, a3 + slli a3, a1, 16 + or a1, a1, a3 + slli a3, a1, 32 + or a1, a1, a3 -.L_aligned_8byte: - zext.b a1,a1 - slli a5,a1,0x8 - or a1,a1,a5 - slli a5,a1,0x10 - or a5,a5,a1 - slli a1,a5,0x20 - li a4,7 - or a1,a1,a5 - bgeu a4,a2,.L_less_8bytes + li a5, 0x80 + slli a3, a5, 8 + or a5, a5, a3 + slli a3, a5, 16 + or a5, a5, a3 + slli a3, a5, 32 + or a5, a5, a3 # 0x8080808080808080 + srli t1, a5, 7 # 0x0101010101010101 +.endm - ld a7, mask1 - ld a6, mask2 - - li t1,7 - j .L_8byte_compare_loop -.L_8byte_compare: - addi a2,a2,-8 - addi a0,a0,8 - bgeu t1,a2,.L_8byte_compare_exit -.L_8byte_compare_loop: - ld a5,0(a0) - xor a5,a5,a1 - add a4,a5,a7 - not a5,a5 - and a5,a5,a4 - and a5,a5,a6 - beqz a5,.L_8byte_compare - -.L_less_8bytes: - add a2,a2,a0 - j .L_less_8bytes_compare -.L_less_8bytes_loop: - addi a0,a0,1 - beq a2,a0,.L_not_found -.L_less_8bytes_compare: - lbu a5,0(a0) - bne a5,a3,.L_less_8bytes_loop - ret -.L_8byte_compare_exit: - bnez a2,.L_less_8bytes - j .L_not_found - .align 3 -mask1: - .dword 0xfefefefefefefeff -mask2: - .dword 0x8080808080808080 + .p2align 6 +ENTRY (memchr) + li a3, 7 + bgtu a2, a3, .L_8_to_16 +.L_0_to_7: + beqz a2, 1f +0: + lbu a4, 0(a0) + beq a1, a4, 2f + addi a2, a2, -1 + addi a0, a0, 1 + bnez a2, 0b +1: + li a0, 0 + ret +2: + ret +.L_8_to_16: + gen_pat + li a3, 16 + bgtu a2, a3, .L_over_16 + addi a2, a2, -8 + chr_8B + add a0, a0, a2 + chr_8B + j .L_not_find +.L_over_16: + neg t4, a0 + andi t4, t4, 0x7 + beqz t4, .L_dst_aligned + chr_8B + sub a2, a2, t4 + add a0, a0, t4 +.L_dst_aligned: + andi t0, a2, (16-1) + srli a2, a2, 4 + beqz a2, .L_tail +.L_loop: + chr_8B + add a0, a0, 8 + chr_8B + addi a2, a2, -1 + add a0, a0, 8 + bnez a2, .L_loop + beqz t0, .L_not_find +.L_tail: + add a0, a0, t0 + addi a0, a0, -16 + chr_8B + add a0, a0, 8 + chr_8B +.L_not_find: + li a0, 0 + ret +.L_find: + ctz a3, a3 + srli a3, a3, 3 + add a0, a0, a3 + ret END (memchr) -libc_hidden_builtin_def (memchr) +libc_hidden_builtin_def (memchr) \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/memchr_riscv.S b/sysdeps/riscv/rv64/multiarch/memchr_riscv.S index ee4394fef9..84b3375f31 100644 --- a/sysdeps/riscv/rv64/multiarch/memchr_riscv.S +++ b/sysdeps/riscv/rv64/multiarch/memchr_riscv.S @@ -26,8 +26,10 @@ #include "memchr_as.S" -#elif !defined __riscv_vector +#else #include "memchr_as.S" -#endif \ No newline at end of file +#endif + + diff --git a/sysdeps/riscv/rv64/multiarch/memchr_vector.S b/sysdeps/riscv/rv64/multiarch/memchr_vector.S index 9441ddb489..87a464f38f 100644 --- a/sysdeps/riscv/rv64/multiarch/memchr_vector.S +++ b/sysdeps/riscv/rv64/multiarch/memchr_vector.S @@ -19,12 +19,10 @@ #include /* For __riscv_vector this file defines memchr. */ -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector # define memchr __memchr_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (memchr) @@ -51,5 +49,5 @@ ENTRY (memchr) add a0,zero,zero ret END (memchr) -libc_hidden_builtin_def (memchr) + #endif \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/memcmp.c b/sysdeps/riscv/rv64/multiarch/memcmp.c index 3dab42ea74..a67159346a 100644 --- a/sysdeps/riscv/rv64/multiarch/memcmp.c +++ b/sysdeps/riscv/rv64/multiarch/memcmp.c @@ -34,3 +34,5 @@ riscv_libc_ifunc_hidden_def (__redirect_memcmp, memcmp); #else # include #endif + + diff --git a/sysdeps/riscv/rv64/multiarch/memcmp_as.S b/sysdeps/riscv/rv64/multiarch/memcmp_as.S new file mode 100644 index 0000000000..a737172980 --- /dev/null +++ b/sysdeps/riscv/rv64/multiarch/memcmp_as.S @@ -0,0 +1,150 @@ +/* The assembly function for memcmp. RISC-V version. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include + + .p2align 6 +ENTRY (memcmp) + li a3, 32 + mv a5, a0 + bgtu a2, a3, .L_over_32 + li a3, 3 + bgtu a2, a3, .L_4_to_8 +.L_0_to_3: + beqz a2, 2f +0: + lbu a3, 0(a1) + lbu a4, 0(a5) + bne a3, a4, 1f + addi a2, a2, -1 + addi a1, a1, 1 + addi a5, a5, 1 + bnez a2, 0b +1: + sub a0, a4, a3 + ret +2: + li a0, 0 + ret +.L_4_to_8: + li a3, 8 + bgtu a2, a3, .L_9_to_16 + lwu a3, 0(a1) + lwu a0, 0(a5) + bne a3, a0, .L_end + add a1, a1, a2 + add a5, a5, a2 + lwu a3, -4(a1) + lwu a0, -4(a5) + j .L_end +.L_9_to_16: + li a3, 16 + bgtu a2, a3, .L_17_to_32 + addi a2, a2, -8 + ld a3, 0(a1) + ld a0, 0(a5) + bne a3, a0, .L_end + add a1, a1, a2 + add a5, a5, a2 + ld a3, 0(a1) + ld a0, 0(a5) + j .L_end +.L_17_to_32: + addi a2, a2, -16 + ld a3, 0(a1) + ld a0, 0(a5) + bne a3, a0, .L_end + ld a3, 8(a1) + ld a0, 8(a5) + bne a3, a0, .L_end + add a1, a1, a2 + add a5, a5, a2 + ld a3, 0(a1) + ld a0, 0(a5) + bne a3, a0, .L_end + ld a3, 8(a1) + ld a0, 8(a5) + j .L_end +.L_over_32: + neg a4, a5 + andi a4, a4, 0x7 + beqz a4, .L_dst_aligned + ld a3, 0(a1) + ld a0, 0(a5) + bne a3, a0, .L_end + sub a2, a2, a4 + add a5, a5, a4 + add a1, a1, a4 +.L_dst_aligned: + andi a4, a2, -32 + andi a2, a2, (32-1) + beqz a4, .L_tail + add a4, a4, a5 +.L_loop: + ld a3, 0(a1) + ld a0, 0(a5) + bne a3, a0, .L_end + ld a3, 8(a1) + ld a0, 8(a5) + bne a3, a0, .L_end + ld a3, 16(a1) + ld a0, 16(a5) + bne a3, a0, .L_end + ld a3, 24(a1) + ld a0, 24(a5) + bne a3, a0, .L_end + addi a5, a5, 32 + addi a1, a1, 32 + bltu a5, a4, .L_loop + beqz a2, .L_end +.L_tail: + andi a4, a2, -1 +.L_pc: + auipc a3, 0 + andi a4, a4, -8 + sub a3, a3, a4 + .equ offset, .L_jmp_end - .L_pc + jalr x0, a3, %lo(offset) + ld a3, 16(a1) + ld a0, 16(a5) + bne a3, a0, .L_end + ld a3, 8(a1) + ld a0, 8(a5) + bne a3, a0, .L_end + ld a3, 0(a1) + ld a0, 0(a5) + bne a3, a0, .L_end +.L_jmp_end: + add a1, a1, a2 + add a5, a5, a2 + ld a3, -8(a1) + ld a0, -8(a5) +.L_end: + bltu a0, a3, 2f + beq a0, a3, 3f + li a0, 1 + ret +2: + li a0, -1 + ret +3: + li a0, 0 + ret +END (memcmp) + +libc_hidden_builtin_def (memcmp) diff --git a/sysdeps/riscv/rv64/multiarch/memcmp_riscv.c b/sysdeps/riscv/rv64/multiarch/memcmp_riscv.S similarity index 89% rename from sysdeps/riscv/rv64/multiarch/memcmp_riscv.c rename to sysdeps/riscv/rv64/multiarch/memcmp_riscv.S index 076431132b..32df79f769 100644 --- a/sysdeps/riscv/rv64/multiarch/memcmp_riscv.c +++ b/sysdeps/riscv/rv64/multiarch/memcmp_riscv.S @@ -22,11 +22,11 @@ #define libc_hidden_builtin_def(name) #undef weak_alias -# define MEMCMP __memcmp_riscv -# include +# define memcmp __memcmp_riscv +# include "memcmp_as.S" -#elif !defined __riscv_vector -# include +#else +# include "memcmp_as.S" #endif diff --git a/sysdeps/riscv/rv64/multiarch/memcmp_vector.S b/sysdeps/riscv/rv64/multiarch/memcmp_vector.S index 4810af026a..89d8a7e08f 100644 --- a/sysdeps/riscv/rv64/multiarch/memcmp_vector.S +++ b/sysdeps/riscv/rv64/multiarch/memcmp_vector.S @@ -21,12 +21,10 @@ /* For __riscv_vector this file defines strcmp. */ /* #ifndef __riscv_vector */ -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector # define memcmp __memcmp_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (memcmp) @@ -55,6 +53,5 @@ ENTRY (memcmp) add a0,zero,zero ret END (memcmp) -weak_alias (memcmp, bcmp) -libc_hidden_builtin_def (memcmp) + #endif diff --git a/sysdeps/riscv/rv64/multiarch/memcpy.c b/sysdeps/riscv/rv64/multiarch/memcpy.c index 45cf14f52a..4fa4cccd9d 100644 --- a/sysdeps/riscv/rv64/multiarch/memcpy.c +++ b/sysdeps/riscv/rv64/multiarch/memcpy.c @@ -34,3 +34,4 @@ riscv_libc_ifunc_hidden_def (__redirect_memcpy, memcpy); #else # include #endif + diff --git a/sysdeps/riscv/rv64/multiarch/memcpy_as.S b/sysdeps/riscv/rv64/multiarch/memcpy_as.S index f0a074df13..4a97d75545 100644 --- a/sysdeps/riscv/rv64/multiarch/memcpy_as.S +++ b/sysdeps/riscv/rv64/multiarch/memcpy_as.S @@ -19,96 +19,116 @@ #include -# define LABLE_ALIGN \ - .balignl 16, 0x00000013 +.macro copy_32B d, s + ld a3, 0(\s) + sd a3, 0(\d) + ld a3, 8(\s) + sd a3, 8(\d) + ld a3, 16(\s) + sd a3, 16(\d) + ld a3, 24(\s) + sd a3, 24(\d) +.endm + .p2align 6 ENTRY (memcpy) - - /* Test if len less than 8 bytes. */ - mv t6, a0 - sltiu a3, a2, 8 - li t3, 1 - bnez a3, .L_copy_by_byte - - andi a3, a0, 7 - li t5, 8 - /* Test if dest is not 8 bytes aligned. */ - bnez a3, .L_dest_not_aligned -.L_dest_aligned: - /* If dest is aligned, then copy. */ - srli t4, a2, 6 - /* Test if len less than 32 bytes. */ - beqz t4, .L_len_less_16bytes - andi a2, a2, 63 - -.L_len_larger_16bytes: - ld a4, 0(a1) - sd a4, 0(a0) - ld a5, 8(a1) - sd a5, 8(a0) - ld a6, 16(a1) - sd a6, 16(a0) - ld a7, 24(a1) - sd a7, 24(a0) - ld a4, 32(a1) - sd a4, 32(a0) - ld a5, 40(a1) - sd a5, 40(a0) - ld a6, 48(a1) - sd a6, 48(a0) - ld a7, 56(a1) - sub t4, t4, t3 - addi a1, a1, 64 - sd a7, 56(a0) - addi a0, a0, 64 - bnez t4, .L_len_larger_16bytes - -.L_len_less_16bytes: - srli t4, a2, 2 - beqz t4, .L_copy_by_byte - andi a2, a2, 3 -.L_len_less_16bytes_loop: - lw a4, 0(a1) - sub t4, t4, t3 - addi a1, a1, 4 - sw a4, 0(a0) - addi a0, a0, 4 - bnez t4, .L_len_less_16bytes_loop - - /* Copy tail. */ -.L_copy_by_byte: - andi t4, a2, 7 - beqz t4, .L_return -.L_copy_by_byte_loop: - lb a4, 0(a1) - sub t4, t4, t3 - addi a1, a1, 1 - sb a4, 0(a0) - addi a0, a0, 1 - bnez t4, .L_copy_by_byte_loop - -.L_return: - mv a0, t6 - ret - - /* If dest is not aligned, just copying some bytes makes the dest - align. */ -.L_dest_not_aligned: - sub a3, t5, a3 - mv t5, a3 -.L_dest_not_aligned_loop: - /* Makes the dest align. */ - lb a4, 0(a1) - sub a3, a3, t3 - addi a1, a1, 1 - sb a4, 0(a0) - addi a0, a0, 1 - bnez a3, .L_dest_not_aligned_loop - sub a2, a2, t5 - sltiu a3, a2, 4 - bnez a3, .L_copy_by_byte - /* Check whether the src is aligned. */ - j .L_dest_aligned + li a3, 32 + mv a5, a0 + bgtu a2, a3, .L_over_32 + li a3, 4 + bgtu a2, a3, .L_5_to_8 +.L_0_to_4: + beqz a2, 1f + lb a4, 0(a1) + sb a4, 0(a5) + /* process [n-1] */ + add t1, a1, a2 + lb a4, -1(t1) + add t0, a5, a2 + sb a4, -1(t0) + li a3, 2 + bleu a2, a3, 1f + lb a4, 2(a1) + sb a4, 2(a5) + lb a4, 1(a1) + sb a4, 1(a5) +1: + ret +.L_5_to_8: + li a3, 8 + bgtu a2, a3, .L_9_to_16 + lw a3, 0(a1) + sw a3, 0(a5) + add a1, a1, a2 + add a5, a5, a2 + lw a3, -4(a1) + sw a3, -4(a5) + ret +.L_9_to_16: + li a3, 16 + bgtu a2, a3, .L_17_to_32 + ld a3, 0(a1) + sd a3, 0(a5) + add a1, a1, a2 + add a5, a5, a2 + ld a3, -8(a1) + sd a3, -8(a5) + ret +.L_17_to_32: + addi a2, a2, -16 + ld a3, 0(a1) + sd a3, 0(a5) + ld a3, 8(a1) + sd a3, 8(a5) + add a1, a1, a2 + add a5, a5, a2 + ld a3, 0(a1) + sd a3, 0(a5) + ld a3, 8(a1) + sd a3, 8(a5) + ret +.L_over_32: + neg a4, a5 + andi a4, a4, 0x7 + beqz a4, .L_dst_aligned + ld a3, 0(a1) + sd a3, 0(a5) + sub a2, a2, a4 + add a5, a5, a4 + add a1, a1, a4 +.L_dst_aligned: + andi a4, a2, -32 + andi a2, a2, (32-1) + beqz a4, .L_tail + add a4, a4, a5 +.L_loop: + copy_32B a5, a1 + addi a5, a5, 32 + addi a1, a1, 32 + bltu a5, a4, .L_loop + beqz a2, .L_ret +.L_tail: + andi a4, a2, -1 +.L_pc: + auipc a3, 0 + andi a4, a4, -8 + srli a4, a4, 1 + sub a3, a3, a4 + .equ offset, .L_jmp_end - .L_pc + jalr x0, a3, %lo(offset) + ld a3, 16(a1) # offset-12 + sd a3, 16(a5) + ld a3, 8(a1) # offset-8 + sd a3, 8(a5) + ld a3, 0(a1) # offset-4 + sd a3, 0(a5) +.L_jmp_end: + add a1, a1, a2 + add a5, a5, a2 + ld a3, -8(a1) + sd a3, -8(a5) +.L_ret: + ret END (memcpy) libc_hidden_builtin_def (memcpy) diff --git a/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S b/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S index 249d4f6067..34b4e981c1 100644 --- a/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S +++ b/sysdeps/riscv/rv64/multiarch/memcpy_riscv.S @@ -26,7 +26,7 @@ #include "memcpy_as.S" -#elif !defined __riscv_vector +#else #include "memcpy_as.S" #endif diff --git a/sysdeps/riscv/rv64/multiarch/memcpy_vector.S b/sysdeps/riscv/rv64/multiarch/memcpy_vector.S index d15f4c3954..cba5e00f19 100644 --- a/sysdeps/riscv/rv64/multiarch/memcpy_vector.S +++ b/sysdeps/riscv/rv64/multiarch/memcpy_vector.S @@ -19,12 +19,10 @@ #include /* For __riscv_vector this file defines memcpy. */ -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector # define memcpy __memcpy_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (memcpy) @@ -52,5 +50,4 @@ ENTRY (memcpy) ret END (memcpy) -libc_hidden_builtin_def (memcpy) #endif \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/memmove_as.S b/sysdeps/riscv/rv64/multiarch/memmove_as.S new file mode 100644 index 0000000000..222b87b9c8 --- /dev/null +++ b/sysdeps/riscv/rv64/multiarch/memmove_as.S @@ -0,0 +1,168 @@ +/* The assembly function for memmove. RISC-V version. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include + +#define memcpy memcpy_internal +#include "memcpy_as.S" + + + .p2align 6 + +ENTRY (memmove) + beq a0, a1, .L_ret_fwd + /* abs(dst - src) */ + sub a3, a0, a1 + srai a4, a3, 63 + xor a3, a3, a4 + subw a3, a3, a4 + + bltu a3, a2, .L_overlap + j memcpy +.L_overlap: + li a4, 8 + bltu a1, a0, .L_backward + mv a5, a0 + bltu a2, a4, .L_byte_fwd + neg a4, a0 + andi a4, a4, 0x7 + beqz a4, .L_32B_fwd +.L_byte_head_fwd: + sub a2, a2, a4 + add a4, a5, a4 +0: + lb a3, 0(a1) + sb a3, 0(a5) + addi a5, a5, 1 + addi a1, a1, 1 + bltu a5, a4, 0b +.L_32B_fwd: + andi a4, a2, -32 + andi a2, a2, (32 - 1) + beqz a4, .L_8B_fwd + add a4, a4, a5 +0: + ld a3, 0(a1) + sd a3, 0(a5) + ld a3, 8(a1) + sd a3, 8(a5) + ld a3, 16(a1) + sd a3, 16(a5) + ld a3, 24(a1) + sd a3, 24(a5) + addi a5, a5, 32 + addi a1, a1, 32 + bltu a5, a4, 0b + beqz a2, .L_ret_fwd +.L_8B_fwd: + andi a4, a2, -8 + beqz a4, .L_byte_tail_fwd +.L_pc_fwd: + auipc a3, 0 + add a1, a1, a4 + add a5, a5, a4 + sub a3, a3, a4 + .equ offset_fwd, .L_jmp_end_fwd - .L_pc_fwd + jalr x0, a3, %lo(offset_fwd) + ld a3, -24(a1) + sd a3, -24(a5) + ld a3, -16(a1) + sd a3, -16(a5) + ld a3, -8(a1) + sd a3, -8(a5) +.L_jmp_end_fwd: +.L_byte_tail_fwd: + andi a2, a2, (8 - 1) +.L_byte_fwd: + beqz a2, .L_ret_fwd + add a2, a2, a5 +0: + lb a3, 0(a1) + sb a3, 0(a5) + addi a5, a5, 1 + addi a1, a1, 1 + bltu a5, a2, 0b +.L_ret_fwd: + ret + +.L_backward: + add a1, a1, a2 + add a5, a0, a2 + bltu a2, a4, .L_byte_bwd + andi a4, a5, 0x7 + beqz a4, .L_32B_bwd +.L_byte_head_bwd: + sub a2, a2, a4 + sub a4, a5, a4 +0: + addi a5, a5, -1 + addi a1, a1, -1 + lb a3, 0(a1) + sb a3, 0(a5) + bltu a4, a5, 0b +.L_32B_bwd: + andi a4, a2, -32 + andi a2, a2, (32 - 1) + beqz a4, .L_8B_bwd + sub a4, a5, a4 +0: + addi a5, a5, -32 + addi a1, a1, -32 + ld a3, 24(a1) + sd a3, 24(a5) + ld a3, 16(a1) + sd a3, 16(a5) + ld a3, 8(a1) + sd a3, 8(a5) + ld a3, 0(a1) + sd a3, 0(a5) + bltu a4, a5, 0b + beqz a2, .L_ret_bwd +.L_8B_bwd: + andi a4, a2, -8 + beqz a4, .L_byte_tail_bwd +.L_pc_bwd: + auipc a3, 0 + sub a1, a1, a4 + sub a5, a5, a4 + srli a4, a4, 1 + sub a3, a3, a4 + .equ offset_bwd, .L_jmp_end_bwd - .L_pc_bwd + jalr x0, a3, %lo(offset_bwd) + ld a3, 16(a1) + sd a3, 16(a5) + ld a3, 8(a1) + sd a3, 8(a5) + ld a3, 0(a1) + sd a3, 0(a5) +.L_jmp_end_bwd: +.L_byte_tail_bwd: + andi a2, a2, (8 - 1) +.L_byte_bwd: + beqz a2, .L_ret_bwd +0: + addi a5, a5, -1 + addi a1, a1, -1 + lb a3, 0(a1) + sb a3, 0(a5) + bltu a0, a5, 0b +.L_ret_bwd: + ret +END (memmove) + +libc_hidden_builtin_def (memmove) diff --git a/sysdeps/riscv/rv64/multiarch/memmove_riscv.c b/sysdeps/riscv/rv64/multiarch/memmove_riscv.S similarity index 88% rename from sysdeps/riscv/rv64/multiarch/memmove_riscv.c rename to sysdeps/riscv/rv64/multiarch/memmove_riscv.S index 03708e0deb..14f154e226 100644 --- a/sysdeps/riscv/rv64/multiarch/memmove_riscv.c +++ b/sysdeps/riscv/rv64/multiarch/memmove_riscv.S @@ -22,10 +22,11 @@ #define libc_hidden_builtin_def(name) #undef weak_alias -# define MEMMOVE __memmove_riscv -# include -#elif !defined __riscv_vector +# define memmove __memmove_riscv +# include "memmove_as.S" -# include +#else + +# include "memmove_as.S" #endif diff --git a/sysdeps/riscv/rv64/multiarch/memmove_vector.S b/sysdeps/riscv/rv64/multiarch/memmove_vector.S index af748be5b2..e45137298e 100644 --- a/sysdeps/riscv/rv64/multiarch/memmove_vector.S +++ b/sysdeps/riscv/rv64/multiarch/memmove_vector.S @@ -19,12 +19,10 @@ #include /* For __riscv_vector this file defines memmov. */ -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector # define memmove __memmove_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (memmove) @@ -54,5 +52,5 @@ ENTRY (memmove) bltu zero,a2,.L_backward_copy_loop ret END (memmove) -libc_hidden_builtin_def (memmove) + #endif \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/memrchr.S b/sysdeps/riscv/rv64/multiarch/memrchr.S new file mode 100644 index 0000000000..c6db183163 --- /dev/null +++ b/sysdeps/riscv/rv64/multiarch/memrchr.S @@ -0,0 +1,123 @@ + +/* The assembly function for memrchr. RISC-V version. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include + +.macro chr_8B + ld a4, -8(a0) + xor a4, a4, a1 + sub a3, a4, t1 + andn a3, a3, a4 + and a3, a3, a5 + bnez a3, .L_find +.endm +.macro gen_pat + slli a3, a1, 8 + or a1, a1, a3 + slli a3, a1, 16 + or a1, a1, a3 + slli a3, a1, 32 + or a1, a1, a3 + + li a5, 0x80 + slli a3, a5, 8 + or a5, a5, a3 + slli a3, a5, 16 + or a5, a5, a3 + slli a3, a5, 32 + or a5, a5, a3 # 0x8080808080808080 + srli t1, a5, 7 # 0x0101010101010101 +.endm + + .p2align 6 +#ifndef MEMRCHR +ENTRY (__memrchr) +#else +ENTRY (MEMRCHR) +#endif + li a3, 7 + add a0, a0, a2 + bgtu a2, a3, .L_8_to_16 +.L_0_to_7: + beqz a2, 1f +0: + lbu a4, -1(a0) + beq a1, a4, 2f + addi a2, a2, -1 + addi a0, a0, -1 + bnez a2, 0b +1: + li a0, 0 + ret +2: + addi a0, a0, -1 + ret +.L_8_to_16: + gen_pat + li a3, 16 + bgtu a2, a3, .L_over_16 + addi a2, a2, -8 + chr_8B + sub a0, a0, a2 + chr_8B + j .L_not_find +.L_over_16: + and t4, a0, 0x7 + beqz t4, .L_dst_aligned + chr_8B + sub a2, a2, t4 + sub a0, a0, t4 +.L_dst_aligned: + andi t0, a2, (16-1) + srli a2, a2, 4 + beqz a2, .L_tail +.L_loop: + chr_8B + addi a0, a0, -8 + chr_8B + addi a2, a2, -1 + addi a0, a0, -8 + bnez a2, .L_loop + beqz t0, .L_not_find +.L_tail: + sub a0, a0, t0 + addi a0, a0, 16 + chr_8B + addi a0, a0, -8 + chr_8B +.L_not_find: + li a0, 0 + ret +.L_find: + clz a3, a3 + addi a0, a0, -1 + srli a3, a3, 3 + sub a0, a0, a3 + ret +#ifndef MEMRCHR +END (__memrchr) +#else +END (MEMRCHR) +#endif + +#ifndef MEMRCHR +# ifdef weak_alias +weak_alias (__memrchr, memrchr) +# endif +#endif diff --git a/sysdeps/riscv/rv64/multiarch/memset_as.S b/sysdeps/riscv/rv64/multiarch/memset_as.S new file mode 100644 index 0000000000..7f67c79215 --- /dev/null +++ b/sysdeps/riscv/rv64/multiarch/memset_as.S @@ -0,0 +1,123 @@ + +/* The assembly function for memset. RISC-V version. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#include + + .p2align 6 +ENTRY (memset) + li a3, 32 + andi a1, a1, 0xff + mv a5, a0 + bgtu a2, a3, .L_over_32 + li a3, 4 + bgtu a2, a3, .L_5_to_8 +.L_0_to_4: + beqz a2, 1f + sb a1, 0(a5) + add t0, a5, a2 + sb a1, -1(t0) + li a3, 2 + bleu a2, a3, 1f + sb a1, 1(a5) + sb a1, 2(a5) +1: + ret +.L_5_to_8: + slli a3, a1, 8 + or a1, a1, a3 + slli a3, a1, 16 + or a1, a1, a3 + li a3, 8 + bgtu a2, a3, .L_9_to_16 + sw a1, 0(a5) + add a5, a5, a2 + sw a1, -4(a5) + ret +.L_9_to_16: + li a3, 16 + bgtu a2, a3, .L_17_to_32 + sw a1, 0(a5) + sw a1, 4(a5) + add a5, a5, a2 + sw a1, -8(a5) + sw a1, -4(a5) + ret +.L_17_to_32: + slli a3, a1, 32 + or a1, a1, a3 + sd a1, 0(a5) + sd a1, 8(a5) + add a5, a5, a2 + sd a1, -16(a5) + sd a1, -8(a5) + ret +.L_over_32: + slli a3, a1, 8 + or a1, a1, a3 + slli a3, a1, 16 + or a1, a1, a3 + slli a3, a1, 32 + or a1, a1, a3 + + neg a4, a5 + andi a4, a4, 0x7 + beqz a4, .L_dst_aligned + sd a1, 0(a5) + sub a2, a2, a4 + add a5, a5, a4 +.L_dst_aligned: + andi a4, a2, -64 + andi a2, a2, (64-1) + beqz a4, .L_tail + add a4, a4, a5 +.L_loop: + sd a1, 0(a5) + sd a1, 8(a5) + sd a1, 16(a5) + sd a1, 24(a5) + sd a1, 32(a5) + sd a1, 40(a5) + sd a1, 48(a5) + sd a1, 56(a5) + addi a5, a5, 64 + bltu a5, a4, .L_loop + beqz a2, .L_ret +.L_tail: + andi a4, a2, -1 +.L_pc: + auipc a3, 0 + andi a4, a4, -8 + srli a4, a4, 2 + sub a3, a3, a4 + .equ offset, .L_jmp_end - .L_pc + jalr x0, a3, %lo(offset) + sd a1, 48(a5) + sd a1, 40(a5) + sd a1, 32(a5) + sd a1, 24(a5) + sd a1, 16(a5) + sd a1, 8(a5) + sd a1, 0(a5) +.L_jmp_end: + add a5, a5, a2 + sd a1, -8(a5) + .L_ret: + ret +END (memset) +libc_hidden_builtin_def (memset) diff --git a/sysdeps/riscv/rv64/multiarch/memset_riscv.c b/sysdeps/riscv/rv64/multiarch/memset_riscv.S similarity index 89% rename from sysdeps/riscv/rv64/multiarch/memset_riscv.c rename to sysdeps/riscv/rv64/multiarch/memset_riscv.S index de631eb187..8585db0228 100644 --- a/sysdeps/riscv/rv64/multiarch/memset_riscv.c +++ b/sysdeps/riscv/rv64/multiarch/memset_riscv.S @@ -21,11 +21,11 @@ #define libc_hidden_builtin_def(name) #undef weak_alias -# define MEMSET __memset_riscv -# include -#elif !defined __riscv_vector +# define memset __memset_riscv +# include "memset_as.S" +#else -# include +# include "memset_as.S" #endif diff --git a/sysdeps/riscv/rv64/multiarch/memset_vector.S b/sysdeps/riscv/rv64/multiarch/memset_vector.S index dc407630a6..1271b04ba1 100644 --- a/sysdeps/riscv/rv64/multiarch/memset_vector.S +++ b/sysdeps/riscv/rv64/multiarch/memset_vector.S @@ -19,12 +19,11 @@ #include -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector + # define memset __memset_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (memset) @@ -42,5 +41,4 @@ ENTRY (memset) ret END (memset) -libc_hidden_builtin_def (memset) #endif \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/rtld-memmove.c b/sysdeps/riscv/rv64/multiarch/rtld-memmove.S similarity index 93% rename from sysdeps/riscv/rv64/multiarch/rtld-memmove.c rename to sysdeps/riscv/rv64/multiarch/rtld-memmove.S index 90e37dbd8b..fb6c2c3748 100644 --- a/sysdeps/riscv/rv64/multiarch/rtld-memmove.c +++ b/sysdeps/riscv/rv64/multiarch/rtld-memmove.S @@ -16,4 +16,5 @@ License along with the GNU C Library; if not, see . */ -# include +//# include +#include "memmove_as.S" \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/rtld-memrchr.S b/sysdeps/riscv/rv64/multiarch/rtld-memrchr.S new file mode 100644 index 0000000000..b1154d0f2a --- /dev/null +++ b/sysdeps/riscv/rv64/multiarch/rtld-memrchr.S @@ -0,0 +1 @@ +#include "memrchr_as.S" \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S b/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S index eb6ff5f8d3..e59c57f738 100644 --- a/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S +++ b/sysdeps/riscv/rv64/multiarch/rtld-strcmp.S @@ -1 +1 @@ -#include "strcmp_.S" \ No newline at end of file +#include "strcmp_as.S" \ No newline at end of file diff --git a/sysdeps/riscv/rv64/multiarch/strcmp_.S b/sysdeps/riscv/rv64/multiarch/strcmp_as.S similarity index 100% rename from sysdeps/riscv/rv64/multiarch/strcmp_.S rename to sysdeps/riscv/rv64/multiarch/strcmp_as.S diff --git a/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S b/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S index f5be83d5b7..abf2984230 100644 --- a/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S +++ b/sysdeps/riscv/rv64/multiarch/strcmp_riscv.S @@ -6,8 +6,8 @@ # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#include "strcmp_.S" -#elif !defined __riscv_vector +#include "strcmp_as.S" +#else -#include "strcmp_.S" +#include "strcmp_as.S" #endif diff --git a/sysdeps/riscv/rv64/multiarch/strcmp_vector.S b/sysdeps/riscv/rv64/multiarch/strcmp_vector.S index d0b8e316f9..40fc474695 100644 --- a/sysdeps/riscv/rv64/multiarch/strcmp_vector.S +++ b/sysdeps/riscv/rv64/multiarch/strcmp_vector.S @@ -19,12 +19,10 @@ #include /* For __riscv_vector this file defines strcmp. */ -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector # define strcmp __strcmp_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (strcmp) @@ -51,5 +49,5 @@ ENTRY (strcmp) ret END (strcmp) -libc_hidden_builtin_def (strcmp) + #endif diff --git a/sysdeps/riscv/rv64/multiarch/strlen_riscv.c b/sysdeps/riscv/rv64/multiarch/strlen_riscv.c index d605941766..d3bae0fc43 100644 --- a/sysdeps/riscv/rv64/multiarch/strlen_riscv.c +++ b/sysdeps/riscv/rv64/multiarch/strlen_riscv.c @@ -24,7 +24,7 @@ #undef weak_alias # define STRLEN __strlen_riscv # include -#elif !defined __riscv_vector +#else # include #endif diff --git a/sysdeps/riscv/rv64/multiarch/strlen_vector.S b/sysdeps/riscv/rv64/multiarch/strlen_vector.S index f674de3a26..0b4ae7ba90 100644 --- a/sysdeps/riscv/rv64/multiarch/strlen_vector.S +++ b/sysdeps/riscv/rv64/multiarch/strlen_vector.S @@ -19,12 +19,10 @@ #include /* For __riscv_vector this file defines strlen. */ -#ifdef __riscv_vector -#ifdef SHARED +#if IS_IN (libc) && defined SHARED && defined __riscv_vector # define strlen __strlen_vector # undef libc_hidden_builtin_def # define libc_hidden_builtin_def(a) -#endif .p2align 6 ENTRY (strlen) @@ -44,5 +42,5 @@ ENTRY (strlen) ret END (strlen) -libc_hidden_builtin_def (strlen) + #endif \ No newline at end of file -- 2.25.1