riscv: Optimize crc32 with Zbc extension
Subsystems (2)
PATCH v4 1/1 riscv: Optimize crc32 with Zbc extension
As suggested by the B-ext spec, the Zbc (carry-less multiplication) instructions can be used to accelerate CRC calculations. Currently, the crc32 is the most widely used crc function inside kernel, so this patch focuses on the optimization of just the crc32 APIs.
Compared with the current table-lookup based optimization, Zbc based optimization can also achieve large stride during CRC calculation loop, meantime, it avoids the memory access latency of the table-lookup based implementation and it reduces memory footprint.
If Zbc feature is not supported in a runtime environment, then the table-lookup based implementation would serve as fallback via alternative mechanism.
By inspecting the vmlinux built by gcc v12.2.0 with default optimization level (-O2), we can see below instruction count change for each 8-byte stride in the CRC32 loop:
rv64: crc32_be (54->31), crc32_le (54->13), __crc32c_le (54->13)
rv32: crc32_be (50->32), crc32_le (50->16), __crc32c_le (50->16)
The compile target CPU is little endian, extra effort is needed for byte swapping for the crc32_be API, thus, the instruction count change is not as significant as that in the *_le cases.
This patch is tested on QEMU VM with the kernel CRC32 selftest for both rv64 and rv32.
---
v4:
- Use "asm goto" instead of "asm_volatile_goto". (Charlie)
v3:
- Use Zbc to handle also the data head and tail bytes, instead of calling fallback function.
- Misc changes due to the new design.
v2:
- Fix sparse warnings about type casting. (lkp)
- Add info about instruction count change in commit log. (Andrew)
- Use the min() helper from linux/minmax.h. (Andrew)
- Use "#if __riscv_xlen == 64" macro check to differentiate rv64 and rv32. (Andrew)
- Line up several macro values by tab. (Andrew)
- Make poly_qt as "unsigned long" to unify the code for rv64 and rv32. (David)
- Fix the style of comment wing. (Andrew)
- Add function wrappers for the asm code for the *_le cases. (Andrew)
--- arch/riscv/Kconfig | 23 ++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/crc32.c | 294 ++++++++++++++++++++++++++++++++++++++++ include/linux/crc32.h | 3 + 4 files changed, 321 insertions(+) create mode 100644 arch/riscv/lib/crc32.c
arch/riscv/Kconfig +23 -0
@@ -610,6 +610,29 @@ config RISCV_ISA_ZBB If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBC + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbc) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbc) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + +config RISCV_ISA_ZBC + bool "Zbc extension support for carry-less multiplication instructions" + depends on TOOLCHAIN_HAS_ZBC + depends on MMU + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the Zbc + extension (carry-less multiplication) and enable its usage. + + The Zbc extension could accelerate CRC (cyclic redundancy check) + calculations. + + If you don't know what to do here, say Y. + config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU
arch/riscv/lib/Makefile +1 -0
@@ -13,6 +13,7 @@ endif lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o +lib-$(CONFIG_RISCV_ISA_ZBC) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o lib-$(CONFIG_RISCV_ISA_V) += xor.o
arch/riscv/lib/crc32.c new file +294 -0
294 lines not on this page · raw message
include/linux/crc32.h +3 -0
@@ -9,7 +9,9 @@#include <linux/bitrev.h> u32 __pure crc32_le (u32 crc,unsigned char const *p,size_t len); +u32 __pure crc32_le_base (u32 crc,unsigned char const *p,size_t len);u32 __pure crc32_be (u32 crc,unsigned char const *p,size_t len); +u32 __pure crc32_be_base (u32 crc,unsigned char const *p,size_t len);/** * crc32_le_combine - Combine two crc32 check values into one. For two
@@ -37,6 +39,7 @@ static inline u32 crc32_le_combine(u32 crc1, u32 crc2, size_t len2) }u32 __pure __crc32c_le (u32 crc,unsigned char const *p,size_t len); +u32 __pure __crc32c_le_base (u32 crc,unsigned char const *p,size_t len);/** * __crc32c_le_combine - Combine two crc32c check values into one. For two
Signature · 1 line
2.25.1