riscv: Optimize bitops with Zbb extension
Subsystems (8)
v5 cover riscv: Optimize bitops with Zbb extension
Bitops optimization with specialized instructions is common practice in popular ISAs, this patch set uses RISC-V Zbb extension to optimize four
bitops: __ffs, __fls, ffs and fls.
The first patch rearranges the content in hwcap.h and cpufeature.h, it helps to avoid a cyclic header including issue for patch 2.
The second patch leverages the alternative mechanism to dynamically apply this optimization.
Thanks,
Xiao
v5:
- Fix all the checkpatch complaints from "scripts/checkpatch.pl --strict". (Charlie) There're three kinds of complaints on patch 2/2 code style:
* CHECK: Lines should not end with a '('
* CHECK: spaces preferred around that '-' (ctx:VxV)
* CHECK: Macro argument reuse 'x' - possible side-effects?
The third warning on fls(x) macro is fixed alongside with code style improvement.
- Drop the mistakenly added content in v4. (Charlie)
- Link to v4: https://lore.kernel.org/all/[email protected]/
v4:
- Simplify the asm code in ffs() and fls() by moving general logic into C implementation. (Charlie)
- Add a comment to decorating the large #ifdef block. (Charlie)
- Link to v3: https://lore.kernel.org/all/[email protected]/
v3:
- Fix riscv32 build issue reported by kernel test robot. V3 changes "hwcap.h" to "cpufeature.h" for files where cpu feature detection APIs are used. (Yujie)
- Link to v2: https://lore.kernel.org/all/[email protected]/
v2:
- Remove the "EFI_" prefix from macro name "EFI_NO_ALTERNATIVE" to make it generic. (Ard)
- patch-1 is added, it's based on "RISC-V: Enable cbo.zero in usermode". (Andrew)
- Link to v1: https://lore.kernel.org/all/[email protected]/
Xiao Wang (2):
riscv: Rearrange hwcap.h and cpufeature.h
riscv: Optimize bitops with Zbb extension
arch/riscv/include/asm/bitops.h | 254 +++++++++++++++++++++++++- arch/riscv/include/asm/cpufeature.h | 83 +++++++++ arch/riscv/include/asm/elf.h | 2 +- arch/riscv/include/asm/hwcap.h | 91 --------- arch/riscv/include/asm/pgtable.h | 1 + arch/riscv/include/asm/switch_to.h | 2 +- arch/riscv/include/asm/vector.h | 2 +- arch/riscv/kvm/aia.c | 2 +- arch/riscv/kvm/main.c | 2 +- arch/riscv/kvm/tlb.c | 2 +- arch/riscv/kvm/vcpu_fp.c | 2 +- arch/riscv/kvm/vcpu_onereg.c | 2 +- arch/riscv/kvm/vcpu_vector.c | 2 +- drivers/clocksource/timer-riscv.c | 2 +- drivers/firmware/efi/libstub/Makefile | 2 +- drivers/perf/riscv_pmu_sbi.c | 2 +- 16 files changed, 347 insertions(+), 106 deletions(-)
Signature · 1 line
2.25.1
PATCH v5 1/2 riscv: Rearrange hwcap.h and cpufeature.h
Now hwcap.h and cpufeature.h are mutually including each other, and most of the variable/API declarations in hwcap.h are implemented in cpufeature.c, so, it's better to move them into cpufeature.h and leave only macros for ISA extension logical IDs in hwcap.h.
BTW, the riscv_isa_extension_mask macro is not used now, so this patch removes it.
--- arch/riscv/include/asm/cpufeature.h | 83 ++++++++++++++++++++++++++ arch/riscv/include/asm/elf.h | 2 +- arch/riscv/include/asm/hwcap.h | 91 ----------------------------- arch/riscv/include/asm/pgtable.h | 1 + arch/riscv/include/asm/switch_to.h | 2 +- arch/riscv/include/asm/vector.h | 2 +- arch/riscv/kvm/aia.c | 2 +- arch/riscv/kvm/main.c | 2 +- arch/riscv/kvm/tlb.c | 2 +- arch/riscv/kvm/vcpu_fp.c | 2 +- arch/riscv/kvm/vcpu_onereg.c | 2 +- arch/riscv/kvm/vcpu_vector.c | 2 +- drivers/clocksource/timer-riscv.c | 2 +- drivers/perf/riscv_pmu_sbi.c | 2 +- 14 files changed, 95 insertions(+), 102 deletions(-)
arch/riscv/include/asm/cpufeature.h +83 -0
@@ -7,7 +7,10 @@#define _ASM_CPUFEATURE_H#include <linux/bitmap.h> +#include <linux/jump_label.h> #include <asm/hwcap.h> +#include <asm/alternative-macros.h> +#include <asm/errno.h> /* * These are probed via a device_initcall(), via either the SBI or directly
@@ -33,4 +36,84 @@ extern struct riscv_isainfo hart_isa[NR_CPUS];void check_unaligned_access (int cpu);void riscv_user_isa_enable (void ); +unsigned long riscv_get_elf_hwcap (void ); + +struct riscv_isa_ext_data { +const unsigned int id; +const char *name; +const char *property; +}; + +extern const struct riscv_isa_ext_data riscv_isa_ext[]; +extern const size_t riscv_isa_ext_count; +extern bool riscv_isa_fallback; + +unsigned long riscv_isa_extension_base (const unsigned long *isa_bitmap); + +bool __riscv_isa_extension_available (const unsigned long *isa_bitmap,int bit); +#define riscv_isa_extension_available (isa_bitmap, ext) \ + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext) + +static __always_inline bool +riscv_has_extension_likely (const unsigned long ext) +{ +compiletime_assert (ext <RISCV_ISA_EXT_MAX , +"ext must be < RISCV_ISA_EXT_MAX" ); + +if (IS_ENABLED (CONFIG_RISCV_ALTERNATIVE )) { +asm_volatile_goto ( + ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1) + : + : [ext] "i" (ext) + : + : l_no); + }else { +if (!__riscv_isa_extension_available (NULL , ext)) + goto l_no; + } + +return true; +l_no: +return false; +} + +static __always_inline bool +riscv_has_extension_unlikely (const unsigned long ext) +{ +compiletime_assert (ext <RISCV_ISA_EXT_MAX , +"ext must be < RISCV_ISA_EXT_MAX" ); + +if (IS_ENABLED (CONFIG_RISCV_ALTERNATIVE )) { +asm_volatile_goto ( + ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1) + : + : [ext] "i" (ext) + : + : l_yes); + }else { +if (__riscv_isa_extension_available (NULL , ext)) + goto l_yes; + } + +return false; +l_yes: +return true; +} + +static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext) +{ +if (IS_ENABLED (CONFIG_RISCV_ALTERNATIVE ) &&riscv_has_extension_likely (ext)) +return true; + +return __riscv_isa_extension_available (hart_isa[cpu].isa, ext); +} + +static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext) +{ +if (IS_ENABLED (CONFIG_RISCV_ALTERNATIVE ) &&riscv_has_extension_unlikely (ext)) +return true; + +return __riscv_isa_extension_available (hart_isa[cpu].isa, ext); +} +#endif
arch/riscv/include/asm/elf.h +1 -1
@@ -14,7 +14,7 @@#include <asm/auxvec.h> #include <asm/byteorder.h> #include <asm/cacheinfo.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> /* * These are used to set parameters in the core dumps.
arch/riscv/include/asm/hwcap.h +0 -91
101 lines not on this page · raw message
arch/riscv/include/asm/pgtable.h +1 -0
@@ -291,6 +291,7 @@ static inline pte_t pud_pte(pud_t pud) }#ifdef CONFIG_RISCV_ISA_SVNAPOT +#include <asm/cpufeature.h> static __always_inline bool has_svnapot(void) {
arch/riscv/include/asm/switch_to.h +1 -1
@@ -9,7 +9,7 @@#include <linux/jump_label.h> #include <linux/sched/task_stack.h> #include <asm/vector.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/processor.h> #include <asm/ptrace.h> #include <asm/csr.h>
arch/riscv/include/asm/vector.h +1 -1
@@ -15,7 +15,7 @@#include <linux/sched.h> #include <linux/sched/task_stack.h> #include <asm/ptrace.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/csr.h> #include <asm/asm.h>
arch/riscv/kvm/aia.c +1 -1
@@ -14,7 +14,7 @@#include <linux/kvm_host.h> #include <linux/percpu.h> #include <linux/spinlock.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/kvm_aia_imsic.h> struct aia_hgei_control {
arch/riscv/kvm/main.c +1 -1
@@ -11,7 +11,7 @@#include <linux/module.h> #include <linux/kvm_host.h> #include <asm/csr.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/sbi.h> long kvm_arch_dev_ioctl(struct file *filp,
arch/riscv/kvm/tlb.c +1 -1
@@ -12,7 +12,7 @@#include <linux/kvm_host.h> #include <asm/cacheflush.h> #include <asm/csr.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/insn-def.h> #define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
arch/riscv/kvm/vcpu_fp.c +1 -1
@@ -11,7 +11,7 @@#include <linux/err.h> #include <linux/kvm_host.h> #include <linux/uaccess.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #ifdef CONFIG_FPU void kvm_riscv_vcpu_fp_reset (struct kvm_vcpu *vcpu)
arch/riscv/kvm/vcpu_onereg.c +1 -1
@@ -13,7 +13,7 @@#include <linux/uaccess.h> #include <linux/kvm_host.h> #include <asm/cacheflush.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/kvm_vcpu_vector.h> #include <asm/vector.h>
arch/riscv/kvm/vcpu_vector.c +1 -1
@@ -11,7 +11,7 @@#include <linux/err.h> #include <linux/kvm_host.h> #include <linux/uaccess.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/kvm_vcpu_vector.h> #include <asm/vector.h>
drivers/clocksource/timer-riscv.c +1 -1
@@ -25,7 +25,7 @@#include <linux/limits.h> #include <clocksource/timer-riscv.h> #include <asm/smp.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #include <asm/sbi.h> #include <asm/timex.h>
drivers/perf/riscv_pmu_sbi.c +1 -1
@@ -22,7 +22,7 @@#include <asm/errata_list.h> #include <asm/sbi.h> -#include <asm/hwcap.h> +#include <asm/cpufeature.h> #define SYSCTL_NO_USER_ACCESS 0#define SYSCTL_USER_ACCESS 1
Signature · 1 line
2.25.1
PATCH v5 2/2 riscv: Optimize bitops with Zbb extension
In reply to Xiao WangThis patch leverages the alternative mechanism to dynamically optimize bitops (including __ffs, __fls, ffs, fls) with Zbb instructions. When Zbb ext is not supported by the runtime CPU, legacy implementation is used. If Zbb is supported, then the optimized variants will be selected via alternative patching.
The legacy bitops support is taken from the generic C implementation as fallback.
If the parameter is a build-time constant, we leverage compiler builtin to calculate the result directly, this approach is inspired by x86 bitops implementation.
EFI stub runs before the kernel, so alternative mechanism should not be used there, this patch introduces a macro NO_ALTERNATIVE for this purpose.
--- arch/riscv/include/asm/bitops.h | 254 +++++++++++++++++++++++++- drivers/firmware/efi/libstub/Makefile | 2 +- 2 files changed, 252 insertions(+), 4 deletions(-)
arch/riscv/include/asm/bitops.h +251 -3
@@ -15,13 +15,261 @@#include <asm/barrier.h> #include <asm/bitsperlong.h> +#if !defined(CONFIG_RISCV_ISA_ZBB ) || defined(NO_ALTERNATIVE )#include <asm-generic/bitops/__ffs.h> -#include <asm-generic/bitops/ffz.h> -#include <asm-generic/bitops/fls.h> #include <asm-generic/bitops/__fls.h> +#include <asm-generic/bitops/ffs.h> +#include <asm-generic/bitops/fls.h> + +#else +#include <asm/alternative-macros.h> +#include <asm/hwcap.h> + +#if (BITS_PER_LONG ==64 ) +#define CTZW "ctzw " +#define CLZW "clzw " +#elif (BITS_PER_LONG ==32 ) +#define CTZW "ctz " +#define CLZW "clz " +#else +#error "Unexpected BITS_PER_LONG" +#endif + +static __always_inline unsigned long variable__ffs (unsigned long word) +{ +int num; + +asm_volatile_goto (ALTERNATIVE ("j %l[legacy]" ,"nop" ,0 , +RISCV_ISA_EXT_ZBB ,1 ) + : : : : legacy); + + asmvolatile (".option push\n" +".option arch,+zbb\n" +"ctz %0, %1\n" +".option pop\n" + :"=r" (word) :"r" (word) :); + +return word; + +legacy: + num =0 ; +#if BITS_PER_LONG ==64 +if ((word &0xffffffff ) ==0 ) { + num +=32 ; + word >>=32 ; + } +#endif +if ((word &0xffff ) ==0 ) { + num +=16 ; + word >>=16 ; + } +if ((word &0xff ) ==0 ) { + num +=8 ; + word >>=8 ; + } +if ((word &0xf ) ==0 ) { + num +=4 ; + word >>=4 ; + } +if ((word &0x3 ) ==0 ) { + num +=2 ; + word >>=2 ; + } +if ((word &0x1 ) ==0 ) + num +=1 ; +return num; +} + +/** +* __ffs - find first set bit in a long word +* @word: The word to search +* +* Undefined if no set bit exists, so code should check against 0 first. +*/ +#define __ffs (word) \ + (__builtin_constant_p(word) ? \ + (unsigned long)__builtin_ctzl(word) : \ + variable__ffs(word)) + +static __always_inline unsigned long variable__fls (unsigned long word) +{ +int num; + +asm_volatile_goto (ALTERNATIVE ("j %l[legacy]" ,"nop" ,0 , +RISCV_ISA_EXT_ZBB ,1 ) + : : : : legacy); + + asmvolatile (".option push\n" +".option arch,+zbb\n" +"clz %0, %1\n" +".option pop\n" + :"=r" (word) :"r" (word) :); + +return BITS_PER_LONG -1 - word; + +legacy: + num =BITS_PER_LONG -1 ; +#if BITS_PER_LONG ==64 +if (!(word & (~0ul <<32 ))) { + num -=32 ; + word <<=32 ; + } +#endif +if (!(word & (~0ul << (BITS_PER_LONG -16 )))) { + num -=16 ; + word <<=16 ; + } +if (!(word & (~0ul << (BITS_PER_LONG -8 )))) { + num -=8 ; + word <<=8 ; + } +if (!(word & (~0ul << (BITS_PER_LONG -4 )))) { + num -=4 ; + word <<=4 ; + } +if (!(word & (~0ul << (BITS_PER_LONG -2 )))) { + num -=2 ; + word <<=2 ; + } +if (!(word & (~0ul << (BITS_PER_LONG -1 )))) + num -=1 ; +return num; +} + +/** +* __fls - find last set bit in a long word +* @word: the word to search +* +* Undefined if no set bit exists, so code should check against 0 first. +*/ +#define __fls (word) \ + (__builtin_constant_p(word) ? \ + (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \ + variable__fls(word)) + +static __always_inline int variable_ffs(int x) +{ +int r; + +if (!x) +return 0 ; + +asm_volatile_goto (ALTERNATIVE ("j %l[legacy]" ,"nop" ,0 , +RISCV_ISA_EXT_ZBB ,1 ) + : : : : legacy); + + asmvolatile (".option push\n" +".option arch,+zbb\n" +CTZW "%0, %1\n" +".option pop\n" + :"=r" (r) :"r" (x) :); + +return r +1 ; + +legacy: + r =1 ; +if (!(x &0xffff )) { + x >>=16 ; + r +=16 ; + } +if (!(x &0xff )) { + x >>=8 ; + r +=8 ; + } +if (!(x &0xf )) { + x >>=4 ; + r +=4 ; + } +if (!(x &3 )) { + x >>=2 ; + r +=2 ; + } +if (!(x &1 )) { + x >>=1 ; + r +=1 ; + } +return r; +} + +/** +* ffs - find first set bit in a word +* @x: the word to search +* +* This is defined the same way as the libc and compiler builtin ffs routines. +* +* ffs(value) returns 0 if value is 0 or the position of the first set bit if +* value is nonzero. The first (least significant) bit is at position 1. +*/ +#define ffs (x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x)) + +static __always_inline int variable_fls(unsigned int x) +{ +int r; + +if (!x) +return 0 ; + +asm_volatile_goto (ALTERNATIVE ("j %l[legacy]" ,"nop" ,0 , +RISCV_ISA_EXT_ZBB ,1 ) + : : : : legacy); + + asmvolatile (".option push\n" +".option arch,+zbb\n" +CLZW "%0, %1\n" +".option pop\n" + :"=r" (r) :"r" (x) :); + +return 32 - r; + +legacy: + r =32 ; +if (!(x &0xffff0000u )) { + x <<=16 ; + r -=16 ; + } +if (!(x &0xff000000u )) { + x <<=8 ; + r -=8 ; + } +if (!(x &0xf0000000u )) { + x <<=4 ; + r -=4 ; + } +if (!(x &0xc0000000u )) { + x <<=2 ; + r -=2 ; + } +if (!(x &0x80000000u )) { + x <<=1 ; + r -=1 ; + } +return r; +} + +/** +* fls - find last set bit in a word +* @x: the word to search +* +* This is defined in a similar way as ffs, but returns the position of the most +* significant set bit. +* +* fls(value) returns 0 if value is 0 or the position of the last set bit if +* value is nonzero. The last (most significant) bit is at position 32. +*/ +#define fls (x) \ +({ \ + typeof(x) x_ = (x); \ + __builtin_constant_p(x_) ? \ + (int)((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \ + : \ + variable_fls(x_); \ +}) + +#endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */ + +#include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/fls64.h> #include <asm-generic/bitops/sched.h> -#include <asm-generic/bitops/ffs.h> #include <asm-generic/bitops/hweight.h>
drivers/firmware/efi/libstub/Makefile +1 -1
@@ -28,7 +28,7 @@ cflags-$(CONFIG_ARM) += -DEFI_HAVE_STRLEN -DEFI_HAVE_STRNLEN \ -DEFI_HAVE_MEMCHR -DEFI_HAVE_STRRCHR \ -DEFI_HAVE_STRCMP -fno-builtin -fpic \ $(call cc-option,-mno-single-pic-base) -cflags-$(CONFIG_RISCV) += -fpic +cflags-$(CONFIG_RISCV) += -fpic -DNO_ALTERNATIVE cflags-$(CONFIG_LOONGARCH) += -fpie cflags-$(CONFIG_EFI_PARAMS_FROM_FDT) += -I$(srctree)/scripts/dtc/libfdt
Signature · 1 line
2.25.1
Hi Xiao,
Quoted 13 lines · Xiao Wang
This patch leverages the alternative mechanism to dynamically optimize bitops (including
__ffs,__fls, ffs, fls) with Zbb instructions. When Zbb ext is not supported by the runtime CPU, legacy implementation is used. If Zbb is supported, then the optimized variants will be selected via alternative patching.The legacy bitops support is taken from the generic C implementation as fallback.
If the parameter is a build-time constant, we leverage compiler builtin to calculate the result directly, this approach is inspired by x86 bitops implementation.
Quoted 5 lines
EFI stub runs before the kernel, so alternative mechanism should not be used there, this patch introduces a macro
NO_ALTERNATIVEfor this purpose.Signed-off-by Xiao Wang <[email protected]>Reviewed-by Charlie Jenkins <[email protected]>
Thanks for your patch, which is now commit 457926b253200bd9 ("riscv:
Optimize bitops with Zbb extension") in riscv/for-next.
Quoted 62 lines
arch/riscv/include/asm/bitops.h +52
-2@@ -15,13 +15,261 @@#include <asm/barrier.h> #include <asm/bitsperlong.h> +#if !defined(CONFIG_RISCV_ISA_ZBB ) || defined(NO_ALTERNATIVE )#include <asm-generic/bitops/__ffs.h> -#include <asm-generic/bitops/ffz.h> -#include <asm-generic/bitops/fls.h> #include <asm-generic/bitops/__fls.h> +#include <asm-generic/bitops/ffs.h> +#include <asm-generic/bitops/fls.h> + +#else +#include <asm/alternative-macros.h> +#include <asm/hwcap.h> + +#if (BITS_PER_LONG ==64 ) +#define CTZW "ctzw " +#define CLZW "clzw " +#elif (BITS_PER_LONG ==32 ) +#define CTZW "ctz " +#define CLZW "clz " +#else +#error "Unexpected BITS_PER_LONG" +#endif + +static __always_inline unsigned long variable__ffs (unsigned long word) +{ +int num; + +asm_volatile_goto (ALTERNATIVE ("j %l[legacy]" ,"nop" ,0 , +RISCV_ISA_EXT_ZBB ,1 ) + : : : : legacy); + + asmvolatile (".option push\n" +".option arch,+zbb\n" +"ctz %0, %1\n" +".option pop\n" + :"=r" (word) :"r" (word) :); + +return word; + +legacy: + num =0 ; +#if BITS_PER_LONG ==64 +if ((word &0xffffffff ) ==0 ) { + num +=32 ; + word >>=32 ; + } +#endif +if ((word &0xffff ) ==0 ) { + num +=16 ; + word >>=16 ; + } +if ((word &0xff ) ==0 ) { + num +=8 ; + word >>=8 ; + } +if ((word &0xf ) ==0 ) { + num +=4 ;
arch/riscv/include/asm/bitops.h line 81
Quoted 10 lines
arch/riscv/include/asm/bitops.h +10
-0+ word >>=4 ; + } +if ((word &0x3 ) ==0 ) { + num +=2 ; + word >>=2 ; + } +if ((word &0x1 ) ==0 ) + num +=1 ; +return num; +}
Surely we can do better than duplicating include/asm-generic/bitops/__ffs.h?
E.g. rename the generic implementation to generic___ffs():
-static __always_inline unsigned long __ffs(unsigned long word)
+static __always_inline unsigned long generic__ffs(unsigned long word)
{
...
}
+#ifndef __ffs
+#define __ffs(x) generic__ffs(x)
+#endifand explicitly calling the generic one here?
Same comment for the other functions.
Gr{oetje,eeting}s,
Geert
Signature · 5 lines
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected] In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that. -- Linus Torvalds
Hi Geert,
Quoted 124 lines
-----Original Message-----
From: Geert Uytterhoeven <[email protected]>
Sent: Friday, November 10, 2023 5:25 PM
To: Wang, Xiao W <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Li, Haicheng <[email protected]>; [email protected]; Liu, Yujie <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v5 2/2] riscv: Optimize bitops with Zbb extensionHi Xiao,
Quoted 9 lines · Xiao Wang
This patch leverages the alternative mechanism to dynamically optimize bitops (including
__ffs,__fls, ffs, fls) with Zbb instructions. When Zbb ext is not supported by the runtime CPU, legacy implementation is used. If Zbb is supported, then the optimized variants will be selected via alternative patching.The legacy bitops support is taken from the generic C implementation as fallback.
Quoted 6 lines
If the parameter is a build-time constant, we leverage compiler builtin to calculate the result directly, this approach is inspired by x86 bitops implementation.
EFI stub runs before the kernel, so alternative mechanism should not be used there, this patch introduces a macro
NO_ALTERNATIVEfor thispurpose.
Quoted 3 lines
Signed-off-by Xiao Wang <[email protected]>Reviewed-by Charlie Jenkins <[email protected]>Thanks for your patch, which is now commit
457926b253200bd9("riscv:
Optimize bitops with Zbb extension") in riscv/for-next.Quoted 62 lines
arch/riscv/include/asm/bitops.h +52
-2@@ -15,13 +15,261 @@#include <asm/barrier.h> #include <asm/bitsperlong.h> +#if !defined(CONFIG_RISCV_ISA_ZBB ) || defined(NO_ALTERNATIVE )#include <asm-generic/bitops/__ffs.h> -#include <asm-generic/bitops/ffz.h> -#include <asm-generic/bitops/fls.h> #include <asm-generic/bitops/__fls.h> +#include <asm-generic/bitops/ffs.h> +#include <asm-generic/bitops/fls.h> + +#else +#include <asm/alternative-macros.h> +#include <asm/hwcap.h> + +#if (BITS_PER_LONG ==64 ) +#define CTZW "ctzw " +#define CLZW "clzw " +#elif (BITS_PER_LONG ==32 ) +#define CTZW "ctz " +#define CLZW "clz " +#else +#error "Unexpected BITS_PER_LONG" +#endif + +static __always_inline unsigned long variable__ffs (unsigned long word) +{ +int num; + +asm_volatile_goto (ALTERNATIVE ("j %l[legacy]" ,"nop" ,0 , +RISCV_ISA_EXT_ZBB ,1 ) + : : : : legacy); + + asmvolatile (".option push\n" +".option arch,+zbb\n" +"ctz %0, %1\n" +".option pop\n" + :"=r" (word) :"r" (word) :); + +return word; + +legacy: + num =0 ; +#if BITS_PER_LONG ==64 +if ((word &0xffffffff ) ==0 ) { + num +=32 ; + word >>=32 ; + } +#endif +if ((word &0xffff ) ==0 ) { + num +=16 ; + word >>=16 ; + } +if ((word &0xff ) ==0 ) { + num +=8 ; + word >>=8 ; + } +if ((word &0xf ) ==0 ) { + num +=4 ;arch/riscv/include/asm/bitops.h line 81
Quoted 10 lines
arch/riscv/include/asm/bitops.h +10
-0+ word >>=4 ; + } +if ((word &0x3 ) ==0 ) { + num +=2 ; + word >>=2 ; + } +if ((word &0x1 ) ==0 ) + num +=1 ; +return num; +}Surely we can do better than duplicating
include/asm-generic/bitops/__ffs.h?E.g. rename the generic implementation to
generic___ffs():-static __always_inline unsigned long __ffs(unsigned long word) +static __always_inline unsigned long generic__ffs(unsigned long word) { ... } +#ifndef __ffs +#define __ffs(x) generic__ffs(x) +#endif
Quoted 3 lines
and explicitly calling the generic one here?
Same comment for the other functions.
Thanks for the suggestion. I just tried your above example and got build error of "__ffs" redefinition, I think we can change the macro condition to:
+#ifndef __HAVE_ARCH___FFS
+#define __ffs(word) generic___ffs(word)
+#endif
Besides, adding a "generic_" prefix to __ffs would make it as "generic___ffs". I saw similar API names in include/asm-generic/bitops/generic-non-atomic.h.
I would send a patch for this.
BRs,
Xiao
Quoted 12 lines
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- geert@linux- m68k.orgIn personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
This series was applied to riscv/linux.git (for-next) by Palmer Dabbelt <[email protected]>:
Hello:
This series was applied to riscv/linux.git (for-next) by Palmer Dabbelt <[email protected]>:
Quoted 8 lines · On Tue, 31 Oct 2023 14:45:51 +0800 you wrote:
Bitops optimization with specialized instructions is common practice in popular ISAs, this patch set uses RISC-V Zbb extension to optimize four
bitops:__ffs,__fls, ffs and fls.The first patch rearranges the content in
hwcap.handcpufeature.h, it helps to avoid a cyclic header including issue for patch 2.[...]
Here is the summary with links:
- [v5,1/2] riscv: Rearrange hwcap.h and cpufeature.h
https://git.kernel.org/riscv/c/e72c4333d2f2 - [v5,2/2] riscv: Optimize bitops with Zbb extension https://git.kernel.org/riscv/c/457926b25320
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html