lib/crc: make the CPU feature static keys __ro_after_init
Subsystems (9)
PATCH 1/1 lib/crc: make the CPU feature static keys __ro_after_init
From: Eric Biggers <[email protected]>
All of the CRC library's CPU feature static_keys are initialized by initcalls and never change afterwards, so there's no need for them to be in the regular .data section. Put them in .data..ro_after_init instead.
---
I'm planning to take this via the crc tree.
arch/arm/lib/crc-t10dif-glue.c | 4 ++-- arch/arm/lib/crc32-glue.c | 4 ++-- arch/arm64/lib/crc-t10dif-glue.c | 4 ++-- arch/loongarch/lib/crc32-loongarch.c | 2 +- arch/mips/lib/crc32-mips.c | 2 +- arch/powerpc/lib/crc-t10dif-glue.c | 2 +- arch/powerpc/lib/crc32-glue.c | 2 +- arch/s390/lib/crc32-glue.c | 2 +- arch/sparc/lib/crc32_glue.c | 2 +- arch/x86/lib/crc-t10dif-glue.c | 2 +- arch/x86/lib/crc32-glue.c | 4 ++-- arch/x86/lib/crc64-glue.c | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-)
arch/arm/lib/crc-t10dif-glue.c +2 -2
@@ -14,12 +14,12 @@#include <crypto/internal/simd.h> #include <asm/neon.h> #include <asm/simd.h> -static DEFINE_STATIC_KEY_FALSE (have_neon); -static DEFINE_STATIC_KEY_FALSE (have_pmull); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pmull);#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
arch/arm/lib/crc32-glue.c +2 -2
@@ -16,12 +16,12 @@#include <asm/hwcap.h> #include <asm/neon.h> #include <asm/simd.h> -static DEFINE_STATIC_KEY_FALSE (have_crc32); -static DEFINE_STATIC_KEY_FALSE (have_pmull); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pmull);#define PMULL_MIN_LEN 64/* min size of buffer for pmull functions */ asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
arch/arm64/lib/crc-t10dif-glue.c +2 -2
@@ -15,12 +15,12 @@#include <crypto/internal/simd.h> #include <asm/neon.h> #include <asm/simd.h> -static DEFINE_STATIC_KEY_FALSE (have_asimd); -static DEFINE_STATIC_KEY_FALSE (have_pmull); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_asimd); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pmull);#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,u8 out[16 ]);
arch/loongarch/lib/crc32-loongarch.c +1 -1
@@ -24,11 +24,11 @@ do { \ } while (0)#define CRC32 (crc, value, size) _CRC32(crc, value, size, crc)#define CRC32C (crc, value, size) _CRC32(crc, value, size, crcc) -static DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {if (!static_branch_likely (&have_crc32))return crc32_le_base (crc, p, len);
arch/mips/lib/crc32-mips.c +1 -1
@@ -60,11 +60,11 @@ do { \ _CRC32(crc, value, size, crc32)#define CRC32C (crc, value, size) \ _CRC32(crc, value, size, crc32c) -static DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {if (!static_branch_likely (&have_crc32))return crc32_le_base (crc, p, len);
arch/powerpc/lib/crc-t10dif-glue.c +1 -1
@@ -19,11 +19,11 @@#define VMX_ALIGN 16#define VMX_ALIGN_MASK (VMX_ALIGN-1)#define VECTOR_BREAKPOINT 64 -static DEFINE_STATIC_KEY_FALSE (have_vec_crypto); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vec_crypto);u32 __crct10dif_vpmsum (u32 crc,unsigned char const *p,size_t len);u16 crc_t10dif_arch (u16 crci,const u8 *p,size_t len) {
arch/powerpc/lib/crc32-glue.c +1 -1
@@ -11,11 +11,11 @@#define VMX_ALIGN 16#define VMX_ALIGN_MASK (VMX_ALIGN-1)#define VECTOR_BREAKPOINT 512 -static DEFINE_STATIC_KEY_FALSE (have_vec_crypto); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vec_crypto);u32 __crc32c_vpmsum (u32 crc,const u8 *p,size_t len);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {
arch/s390/lib/crc32-glue.c +1 -1
@@ -16,11 +16,11 @@#define VX_MIN_LEN 64#define VX_ALIGNMENT 16L#define VX_ALIGN_MASK (VX_ALIGNMENT - 1) -static DEFINE_STATIC_KEY_FALSE (have_vxrs); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vxrs);/* * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension * * Creates a function to perform a particular CRC-32 computation. Depending
arch/sparc/lib/crc32_glue.c +1 -1
@@ -15,11 +15,11 @@#include <linux/kernel.h> #include <linux/crc32.h> #include <asm/pstate.h> #include <asm/elf.h> -static DEFINE_STATIC_KEY_FALSE (have_crc32c_opcode); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32c_opcode);u32 crc32_le_arch (u32 crc,const u8 *data,size_t len) {return crc32_le_base (crc, data, len); }
arch/x86/lib/crc-t10dif-glue.c +1 -1
@@ -7,11 +7,11 @@#include <linux/crc-t10dif.h> #include <linux/module.h> #include "crc-pclmul-template.h" -static DEFINE_STATIC_KEY_FALSE (have_pclmulqdq); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pclmulqdq);DECLARE_CRC_PCLMUL_FUNCS (crc16_msb, u16);u16 crc_t10dif_arch (u16 crc,const u8 *p,size_t len) {
arch/x86/lib/crc32-glue.c +2 -2
@@ -9,12 +9,12 @@#include <linux/crc32.h> #include <linux/module.h> #include "crc-pclmul-template.h" -static DEFINE_STATIC_KEY_FALSE (have_crc32); -static DEFINE_STATIC_KEY_FALSE (have_pclmulqdq); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pclmulqdq);DECLARE_CRC_PCLMUL_FUNCS (crc32_lsb, u32);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {
arch/x86/lib/crc64-glue.c +1 -1
@@ -7,11 +7,11 @@#include <linux/crc64.h> #include <linux/module.h> #include "crc-pclmul-template.h" -static DEFINE_STATIC_KEY_FALSE (have_pclmulqdq); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pclmulqdq);DECLARE_CRC_PCLMUL_FUNCS (crc64_msb, u64);DECLARE_CRC_PCLMUL_FUNCS (crc64_lsb, u64);u64 crc64_be_arch (u64 crc,const u8 *p,size_t len)
base-commit: e8c24520a1338f938774268a9bafb679ace93b76
Signature · 1 line
2.49.0
Quoted 12 lines · Eric Biggers
From: Eric Biggers <[email protected]>
All of the CRC library's CPU feature
static_keysare initialized by initcalls and never change afterwards, so there's no need for them to be in the regular .data section. Put them in .data..ro_after_init instead.Signed-off-by Eric Biggers <[email protected]>---
I'm planning to take this via the crc tree.
Quoted 14 lines
arch/arm/lib/crc-t10dif-glue.c | 4 ++-- arch/arm/lib/crc32-glue.c | 4 ++-- arch/arm64/lib/crc-t10dif-glue.c | 4 ++-- arch/loongarch/lib/crc32-loongarch.c | 2 +- arch/mips/lib/crc32-mips.c | 2 +- arch/powerpc/lib/crc-t10dif-glue.c | 2 +- arch/powerpc/lib/crc32-glue.c | 2 +- arch/s390/lib/crc32-glue.c | 2 +- arch/sparc/lib/crc32_glue.c | 2 +- arch/x86/lib/crc-t10dif-glue.c | 2 +- arch/x86/lib/crc32-glue.c | 4 ++-- arch/x86/lib/crc64-glue.c | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-)
Quoted 217 lines
arch/arm/lib/crc-t10dif-glue.c +2
-2@@ -14,12 +14,12 @@#include <crypto/internal/simd.h> #include <asm/neon.h> #include <asm/simd.h> -static DEFINE_STATIC_KEY_FALSE (have_neon); -static DEFINE_STATIC_KEY_FALSE (have_pmull); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pmull);#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,arch/arm/lib/crc32-glue.c +2
-2@@ -16,12 +16,12 @@#include <asm/hwcap.h> #include <asm/neon.h> #include <asm/simd.h> -static DEFINE_STATIC_KEY_FALSE (have_crc32); -static DEFINE_STATIC_KEY_FALSE (have_pmull); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pmull);#define PMULL_MIN_LEN 64/* min size of buffer for pmull functions */ asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);arch/arm64/lib/crc-t10dif-glue.c +2
-2@@ -15,12 +15,12 @@#include <crypto/internal/simd.h> #include <asm/neon.h> #include <asm/simd.h> -static DEFINE_STATIC_KEY_FALSE (have_asimd); -static DEFINE_STATIC_KEY_FALSE (have_pmull); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_asimd); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pmull);#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,u8 out[16 ]);arch/loongarch/lib/crc32-loongarch.c +1
-1@@ -24,11 +24,11 @@ do { \ } while (0)#define CRC32 (crc, value, size) _CRC32(crc, value, size, crc)#define CRC32C (crc, value, size) _CRC32(crc, value, size, crcc) -static DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {if (!static_branch_likely (&have_crc32))return crc32_le_base (crc, p, len);arch/mips/lib/crc32-mips.c +1
-1@@ -60,11 +60,11 @@ do { \ _CRC32(crc, value, size, crc32)#define CRC32C (crc, value, size) \ _CRC32(crc, value, size, crc32c) -static DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {if (!static_branch_likely (&have_crc32))return crc32_le_base (crc, p, len);arch/powerpc/lib/crc-t10dif-glue.c +1
-1@@ -19,11 +19,11 @@#define VMX_ALIGN 16#define VMX_ALIGN_MASK (VMX_ALIGN-1)#define VECTOR_BREAKPOINT 64 -static DEFINE_STATIC_KEY_FALSE (have_vec_crypto); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vec_crypto);u32 __crct10dif_vpmsum (u32 crc,unsigned char const *p,size_t len);u16 crc_t10dif_arch (u16 crci,const u8 *p,size_t len) {arch/powerpc/lib/crc32-glue.c +1
-1@@ -11,11 +11,11 @@#define VMX_ALIGN 16#define VMX_ALIGN_MASK (VMX_ALIGN-1)#define VECTOR_BREAKPOINT 512 -static DEFINE_STATIC_KEY_FALSE (have_vec_crypto); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vec_crypto);u32 __crc32c_vpmsum (u32 crc,const u8 *p,size_t len);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {arch/s390/lib/crc32-glue.c +1
-1@@ -16,11 +16,11 @@#define VX_MIN_LEN 64#define VX_ALIGNMENT 16L#define VX_ALIGN_MASK (VX_ALIGNMENT - 1) -static DEFINE_STATIC_KEY_FALSE (have_vxrs); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vxrs);/* * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension * * Creates a function to perform a particular CRC-32 computation. Depending arch/sparc/lib/crc32_glue.c +1
-1@@ -15,11 +15,11 @@#include <linux/kernel.h> #include <linux/crc32.h> #include <asm/pstate.h> #include <asm/elf.h> -static DEFINE_STATIC_KEY_FALSE (have_crc32c_opcode); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32c_opcode);u32 crc32_le_arch (u32 crc,const u8 *data,size_t len) {return crc32_le_base (crc, data, len); }arch/x86/lib/crc-t10dif-glue.c +1
-1@@ -7,11 +7,11 @@#include <linux/crc-t10dif.h> #include <linux/module.h> #include "crc-pclmul-template.h" -static DEFINE_STATIC_KEY_FALSE (have_pclmulqdq); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pclmulqdq);DECLARE_CRC_PCLMUL_FUNCS (crc16_msb, u16);u16 crc_t10dif_arch (u16 crc,const u8 *p,size_t len) {arch/x86/lib/crc32-glue.c +2
-2@@ -9,12 +9,12 @@#include <linux/crc32.h> #include <linux/module.h> #include "crc-pclmul-template.h" -static DEFINE_STATIC_KEY_FALSE (have_crc32); -static DEFINE_STATIC_KEY_FALSE (have_pclmulqdq); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_crc32); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pclmulqdq);DECLARE_CRC_PCLMUL_FUNCS (crc32_lsb, u32);u32 crc32_le_arch (u32 crc,const u8 *p,size_t len) {arch/x86/lib/crc64-glue.c +1
-1@@ -7,11 +7,11 @@#include <linux/crc64.h> #include <linux/module.h> #include "crc-pclmul-template.h" -static DEFINE_STATIC_KEY_FALSE (have_pclmulqdq); +static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_pclmulqdq);DECLARE_CRC_PCLMUL_FUNCS (crc64_msb, u64);DECLARE_CRC_PCLMUL_FUNCS (crc64_lsb, u64);u64 crc64_be_arch (u64 crc,const u8 *p,size_t len)base-commit:
e8c24520a1338f938774268a9bafb679ace93b76--
2.49.0
Eric,
Quoted 4 lines
All of the CRC library's CPU feature
static_keysare initialized by initcalls and never change afterwards, so there's no need for them to be in the regular .data section. Put them in .data..ro_after_init instead.
LGTM.
Signature · 1 line
Martin K. Petersen
Quoted 11 lines · Eric Biggers
From: Eric Biggers <[email protected]>
All of the CRC library's CPU feature
static_keysare initialized by initcalls and never change afterwards, so there's no need for them to be in the regular .data section. Put them in .data..ro_after_init instead.Signed-off-by Eric Biggers <[email protected]>---
I'm planning to take this via the crc tree.
Quoted 8 lines
arch/arm/lib/crc-t10dif-glue.c | 4 ++-- arch/arm/lib/crc32-glue.c | 4 ++-- arch/arm64/lib/crc-t10dif-glue.c | 4 ++-- arch/loongarch/lib/crc32-loongarch.c | 2 +- arch/mips/lib/crc32-mips.c | 2 +- arch/powerpc/lib/crc-t10dif-glue.c | 2 +- arch/powerpc/lib/crc32-glue.c | 2 +- arch/s390/lib/crc32-glue.c | 2 +-
I just realized that we can get rid of the static key in the s390 piece and end up with slightly better code. Could you add the patch below to your tree, please? If this would go via the s390 tree this would result in a merge conflict, which is unnecessary.
From 6c4c0ca6fe87e43acf6192f1afc0a6346db994f4 Mon Sep 17 00:00:00 2001
From: Heiko Carstens <[email protected]>
Date: Thu, 17 Apr 2025 14:30:56 +0200
Subject: [PATCH] s390/crc32: Remove have_vxrs static key
Replace the have_vxrs static key with a cpu_has_vx() call. cpu_has_vx() resolves into a compile time constant (true) if the kernel is compiled for z13 or newer. Otherwise it generates an unconditional one instruction branch, which is patched based on CPU alternatives.
In any case the generated code is at least as good as before and avoids static key handling.
--- arch/s390/lib/crc32-glue.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-)
arch/s390/lib/crc32-glue.c +3 -7
@@ -18,8 +18,6 @@#define VX_ALIGNMENT 16L#define VX_ALIGN_MASK (VX_ALIGNMENT - 1) -static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_vxrs); -/* * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension *
@@ -34,8 +32,7 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vxrs);unsigned long prealign, aligned, remaining; \DECLARE_KERNEL_FPU_ONSTACK16 (vxstate); \ \ -if (datalen <VX_MIN_LEN +VX_ALIGN_MASK || \ - !static_branch_likely (&have_vxrs)) \ +if (datalen <VX_MIN_LEN +VX_ALIGN_MASK || !cpu_has_vx ()) \return ___crc32_sw (crc, data, datalen); \ \if ((unsigned long )data &VX_ALIGN_MASK ) { \
@@ -66,8 +63,6 @@ DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base)static int __init crc32_s390_init (void ) { -if (cpu_have_feature (S390_CPU_FEATURE_VXRS )) -static_branch_enable (&have_vxrs);return 0 ; }arch_initcall (crc32_s390_init);
@@ -79,10 +74,11 @@ module_exit(crc32_s390_exit);u32 crc32_optimizations (void ) { -if (static_key_enabled (&have_vxrs)) +if (cpu_has_vx ()) {return CRC32_LE_OPTIMIZATION |CRC32_BE_OPTIMIZATION |CRC32C_OPTIMIZATION ; + }return 0 ; }EXPORT_SYMBOL (crc32_optimizations);
Signature · 1 line
2.45.2
Quoted 39 lines · Heiko Carstens
Quoted 11 lines · Eric Biggers
From: Eric Biggers <[email protected]>
All of the CRC library's CPU feature
static_keysare initialized by initcalls and never change afterwards, so there's no need for them to be in the regular .data section. Put them in .data..ro_after_init instead.Signed-off-by Eric Biggers <[email protected]>---
I'm planning to take this via the crc tree.
Quoted 8 lines
arch/arm/lib/crc-t10dif-glue.c | 4 ++-- arch/arm/lib/crc32-glue.c | 4 ++-- arch/arm64/lib/crc-t10dif-glue.c | 4 ++-- arch/loongarch/lib/crc32-loongarch.c | 2 +- arch/mips/lib/crc32-mips.c | 2 +- arch/powerpc/lib/crc-t10dif-glue.c | 2 +- arch/powerpc/lib/crc32-glue.c | 2 +- arch/s390/lib/crc32-glue.c | 2 +-I just realized that we can get rid of the static key in the s390 piece and end up with slightly better code. Could you add the patch below to your tree, please? If this would go via the s390 tree this would result in a merge conflict, which is unnecessary.
From
6c4c0ca6fe87e43acf6192f1afc0a6346db994f4Mon Sep 17 00:00:00 2001
From: Heiko Carstens <[email protected]>
Date: Thu, 17 Apr 2025 14:30:56 +0200
Subject: [PATCH] s390/crc32: Removehave_vxrsstatic keyReplace the
have_vxrsstatic key with acpu_has_vx()call.cpu_has_vx()resolves into a compile time constant (true) if the kernel is compiled for z13 or newer. Otherwise it generates an unconditional one instruction branch, which is patched based on CPU alternatives.
Quoted 7 lines
In any case the generated code is at least as good as before and avoids static key handling.
Signed-off-by Heiko Carstens <[email protected]>--- arch/s390/lib/crc32-glue.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-)
Thanks! Applied to
https://web.git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git/log/?h=crc-next
- Eric