SHA-256 cleanup
Subsystems (6)
Eric Biggers
maintainercover SHA-256 cleanup
This series restores the SHA-256 code to have a clean design where the crypto_shash API is implemented on top of the regular library API instead of a special unsafe low-level API. Diffstat is -41 lines.
Eric Biggers (8):
Revert "crypto: sha256 - Use the partial block API" Revert "crypto: lib/sha256 - Use generic block helper" Revert "crypto: x86/sha256 - Add simd block function" Revert "crypto: riscv/sha256 - Add simd block function" Revert "crypto: arm64/sha256 - Add simd block function" Revert "crypto: arm/sha256 - Add simd block function" Revert "crypto: sha256 - Use the partial block API for generic" Revert "crypto: lib/sha256 - Add helpers for block-based shash"
arch/arm/lib/crypto/Kconfig | 1 - arch/arm/lib/crypto/sha256-armv4.pl | 20 ++-- arch/arm/lib/crypto/sha256.c | 14 +-- arch/arm64/crypto/sha512-glue.c | 6 +- arch/arm64/lib/crypto/Kconfig | 1 - arch/arm64/lib/crypto/sha2-armv8.pl | 2 +- arch/arm64/lib/crypto/sha256.c | 14 +-- arch/riscv/lib/crypto/Kconfig | 1 - arch/riscv/lib/crypto/sha256.c | 13 +-- arch/x86/lib/crypto/Kconfig | 1 - arch/x86/lib/crypto/sha256.c | 12 +-- crypto/sha256.c | 142 ++++++++++------------------ include/crypto/internal/sha2.h | 52 ++-------- include/crypto/sha2.h | 7 +- lib/crypto/Kconfig | 8 -- lib/crypto/sha256.c | 97 +++++++++++++++---- 16 files changed, 175 insertions(+), 216 deletions(-)
base-commit: 1bafd82d9a40cf09c6c40f1c09cc35b7050b1a9f
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 1/8 Revert "crypto: sha256 - Use the partial block API"
From: Eric Biggers <[email protected]>
This reverts commit 3bf5337879101166dfacfbc2a780d1a379c288ba which got pushed out despite being nacked.
The library API already has to handle partial blocks, and it makes a lot more sense to just use that.
Signed-off-by Eric Biggers <[email protected]>
--- crypto/sha256.c | 81 ++++++++++++++----------------------------------- 1 file changed, 23 insertions(+), 58 deletions(-)
crypto/sha256.c +23 -58
@@ -50,24 +50,18 @@ static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data, unsigned int len) {return crypto_sha256_update (desc, data, len, true); } -static int crypto_sha256_update_lib (struct shash_desc *desc,const u8 *data, -unsigned int len) -{ -sha256_update (shash_desc_ctx (desc), data, len); -return 0 ; -} -static int crypto_sha256_update_arch (struct shash_desc *desc,const u8 *data,unsigned int len) { -return crypto_sha256_update (desc, data, len, false); +sha256_update (shash_desc_ctx (desc), data, len); +return 0 ; } -static int crypto_sha256_final_lib (struct shash_desc *desc,u8 *out) +static int crypto_sha256_final_arch (struct shash_desc *desc,u8 *out) {sha256_final (shash_desc_ctx (desc), out);return 0 ; }
@@ -97,41 +91,38 @@ static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data, }static int crypto_sha256_finup_arch (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) { -return crypto_sha256_finup (desc, data, len, out, false); +struct sha256_state *sctx =shash_desc_ctx (desc); + +sha256_update (sctx, data, len); +sha256_final (sctx, out); +return 0 ; }static int crypto_sha256_digest_generic (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) {crypto_sha256_init (desc);return crypto_sha256_finup_generic (desc, data, len, out); } -static int crypto_sha256_digest_lib (struct shash_desc *desc,const u8 *data, -unsigned int len,u8 *out) -{ -sha256 (data, len, out); -return 0 ; -} -static int crypto_sha256_digest_arch (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) { -crypto_sha256_init (desc); -return crypto_sha256_finup_arch (desc, data, len, out); +sha256 (data, len, out); +return 0 ; }static int crypto_sha224_init (struct shash_desc *desc) {sha224_block_init (shash_desc_ctx (desc));return 0 ; } -static int crypto_sha224_final_lib (struct shash_desc *desc,u8 *out) +static int crypto_sha224_final_arch (struct shash_desc *desc,u8 *out) {sha224_final (shash_desc_ctx (desc), out);return 0 ; }
@@ -191,79 +182,53 @@ static struct shash_alg algs[] = { .finup = crypto_sha256_finup_generic, .descsize = sizeof(struct crypto_sha256_state), }, { .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-lib", + .base.cra_driver_name = "sha256-" __stringify(ARCH), + .base.cra_priority = 300, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA256_DIGEST_SIZE, .init = crypto_sha256_init, - .update = crypto_sha256_update_lib, - .final = crypto_sha256_final_lib, - .digest = crypto_sha256_digest_lib, + .update = crypto_sha256_update_arch, + .final = crypto_sha256_final_arch, + .finup = crypto_sha256_finup_arch, + .digest = crypto_sha256_digest_arch, .descsize = sizeof(struct sha256_state), .statesize =sizeof (struct crypto_sha256_state ) + SHA256_BLOCK_SIZE + 1, .import = crypto_sha256_import_lib, .export = crypto_sha256_export_lib, }, { .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-lib", + .base.cra_driver_name = "sha224-" __stringify(ARCH), + .base.cra_priority = 300, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA224_DIGEST_SIZE, .init = crypto_sha224_init, - .update = crypto_sha256_update_lib, - .final = crypto_sha224_final_lib, + .update = crypto_sha256_update_arch, + .final = crypto_sha224_final_arch, .descsize = sizeof(struct sha256_state), .statesize =sizeof (struct crypto_sha256_state ) + SHA256_BLOCK_SIZE + 1, .import = crypto_sha256_import_lib, .export = crypto_sha256_export_lib, }, - { - .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-" __stringify(ARCH), - .base.cra_priority = 300, - .base.cra_flags =CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA256_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = SHA256_DIGEST_SIZE, - .init = crypto_sha256_init, - .update = crypto_sha256_update_arch, - .finup = crypto_sha256_finup_arch, - .digest = crypto_sha256_digest_arch, - .descsize = sizeof(struct crypto_sha256_state), - }, - { - .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-" __stringify(ARCH), - .base.cra_priority = 300, - .base.cra_flags =CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA224_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = SHA224_DIGEST_SIZE, - .init = crypto_sha224_init, - .update = crypto_sha256_update_arch, - .finup = crypto_sha256_finup_arch, - .descsize = sizeof(struct crypto_sha256_state), - }, };static unsigned int num_algs;static int __init crypto_sha256_mod_init (void ) {/* register the arch flavours only if they differ from generic */ num_algs =ARRAY_SIZE (algs); -BUILD_BUG_ON (ARRAY_SIZE (algs) <=2 ); +BUILD_BUG_ON (ARRAY_SIZE (algs) %2 !=0 );if (!sha256_is_arch_optimized ()) - num_algs -=2 ; + num_algs /=2 ;return crypto_register_shashes (algs,ARRAY_SIZE (algs)); }module_init (crypto_sha256_mod_init); static void __exit crypto_sha256_mod_exit(void)
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 2/8 Revert "crypto: lib/sha256 - Use generic block helper"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit 3007e90572d0c5fd409c3d2fa8cedcbd5cb06d4b which got pushed out despite being nacked.
BLOCK_HASH_UPDATE_BLOCKS makes the code harder to read and isn't really worth it. The *_generic() functions are needed for shash.
Signed-off-by Eric Biggers <[email protected]>
--- include/crypto/internal/sha2.h | 7 ++++ lib/crypto/sha256.c | 71 +++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-)
include/crypto/internal/sha2.h +7 -0
@@ -8,10 +8,17 @@#include <linux/compiler_attributes.h> #include <linux/string.h> #include <linux/types.h> #include <linux/unaligned.h> +void sha256_update_generic (struct sha256_state *sctx, +const u8 *data,size_t len); +void sha256_final_generic (struct sha256_state *sctx, +u8 out[SHA256_DIGEST_SIZE ]); +void sha224_final_generic (struct sha256_state *sctx, +u8 out[SHA224_DIGEST_SIZE ]); +#if IS_ENABLED (CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256 )bool sha256_is_arch_optimized (void );#else static inline bool sha256_is_arch_optimized (void ) {
lib/crypto/sha256.c +61 -10
123 lines not on this page · raw message
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 3/8 Revert "crypto: x86/sha256 - Add simd block function"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit ee8a720e39ceb7495ab639c1eb6d4987fb6a52bf which got pushed out despite being nacked.
That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode FPU. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd".
Signed-off-by Eric Biggers <[email protected]>
--- arch/x86/lib/crypto/Kconfig | 1 - arch/x86/lib/crypto/sha256.c | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-)
arch/x86/lib/crypto/Kconfig +0 -1
@@ -28,7 +28,6 @@ config CRYPTO_POLY1305_X86_64 config CRYPTO_SHA256_X86_64 tristate depends on 64BIT default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC
arch/x86/lib/crypto/sha256.c +3 -9
@@ -4,10 +4,11 @@* * Copyright 2025 Google LLC */ #include <asm/fpu/api.h> #include <crypto/internal/sha2.h> +#include <crypto/internal/simd.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/static_call.h> asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS],
@@ -21,28 +22,21 @@ asmlinkage void sha256_ni_transform(u32 state[SHA256_STATE_WORDS],static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_sha256_x86);DEFINE_STATIC_CALL (sha256_blocks_x86, sha256_transform_ssse3); -void sha256_blocks_simd (u32 state[SHA256_STATE_WORDS ], +void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks) { -if (static_branch_likely (&have_sha256_x86)) { +if (static_branch_likely (&have_sha256_x86) &&crypto_simd_usable ()) {kernel_fpu_begin ();static_call (sha256_blocks_x86)(state, data, nblocks);kernel_fpu_end (); }else {sha256_blocks_generic (state, data, nblocks); } } -EXPORT_SYMBOL_GPL (sha256_blocks_simd); - -void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ], -const u8 *data,size_t nblocks) -{ -sha256_blocks_generic (state, data, nblocks); -}EXPORT_SYMBOL_GPL (sha256_blocks_arch);bool sha256_is_arch_optimized (void ) {return static_key_enabled (&have_sha256_x86);
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 4/8 Revert "crypto: riscv/sha256 - Add simd block function"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit 491d6024f2820c78216b07cec1cb47c87dcae077 which got pushed out despite being nacked.
That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode vector. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd".
Signed-off-by Eric Biggers <[email protected]>
--- arch/riscv/lib/crypto/Kconfig | 1 - arch/riscv/lib/crypto/sha256.c | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-)
arch/riscv/lib/crypto/Kconfig +0 -1
@@ -10,7 +10,6 @@ config CRYPTO_CHACHA_RISCV64 config CRYPTO_SHA256_RISCV64 tristate depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC
arch/riscv/lib/crypto/sha256.c +4 -9
@@ -7,38 +7,33 @@* * Copyright (C) 2023 SiFive, Inc. * Author: Jerry Shih <[email protected]> */ +#include <asm/simd.h> #include <asm/vector.h> #include <crypto/internal/sha2.h> +#include <crypto/internal/simd.h> #include <linux/kernel.h> #include <linux/module.h> asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks);static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_extensions); -void sha256_blocks_simd (u32 state[SHA256_STATE_WORDS ], +void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks) { -if (static_branch_likely (&have_extensions)) { +if (static_branch_likely (&have_extensions) &&crypto_simd_usable ()) {kernel_vector_begin ();sha256_transform_zvknha_or_zvknhb_zvkb (state, data, nblocks);kernel_vector_end (); }else {sha256_blocks_generic (state, data, nblocks); } } -EXPORT_SYMBOL_GPL (sha256_blocks_simd); - -void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ], -const u8 *data,size_t nblocks) -{ -sha256_blocks_generic (state, data, nblocks); -}EXPORT_SYMBOL_GPL (sha256_blocks_arch);bool sha256_is_arch_optimized (void ) {return static_key_enabled (&have_extensions);
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 5/8 Revert "crypto: arm64/sha256 - Add simd block function"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit adcb9e32e5e28935ec1148e1a314282a7367428d which got pushed out despite being nacked.
That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode NEON. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd".
Signed-off-by Eric Biggers <[email protected]>
--- arch/arm64/crypto/sha512-glue.c | 6 +++--- arch/arm64/lib/crypto/Kconfig | 1 - arch/arm64/lib/crypto/sha2-armv8.pl | 2 +- arch/arm64/lib/crypto/sha256.c | 14 +++++++------- 4 files changed, 11 insertions(+), 12 deletions(-)
arch/arm64/crypto/sha512-glue.c +3 -3
@@ -16,17 +16,17 @@ MODULE_AUTHOR("Andy Polyakov <[email protected]>");MODULE_AUTHOR ("Ard Biesheuvel <[email protected]>" );MODULE_LICENSE ("GPL v2" );MODULE_ALIAS_CRYPTO ("sha384" );MODULE_ALIAS_CRYPTO ("sha512" ); -asmlinkage void sha512_blocks_arch(u64 *digest, const void *data, -unsigned int num_blks); +asmlinkage void sha512_block_data_order(u64 *digest, const void *data, +unsigned int num_blks);static void sha512_arm64_transform (struct sha512_state *sst,u8 const *src,int blocks) { -sha512_blocks_arch (sst->state, src, blocks); +sha512_block_data_order (sst->state, src, blocks); }static int sha512_update (struct shash_desc *desc,const u8 *data,unsigned int len) {
arch/arm64/lib/crypto/Kconfig +0 -1
@@ -15,6 +15,5 @@ config CRYPTO_POLY1305_NEON config CRYPTO_SHA256_ARM64 tristate default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
arch/arm64/lib/crypto/sha2-armv8.pl +1 -1
@@ -93,11 +93,11 @@ if ($output =~ /512/) { @sigma1=(17,19,10); $rounds=64; $reg_t="w"; } -$func="sha${BITS}_blocks_arch"; +$func="sha${BITS}_block_data_order"; ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30)); @X=map("$reg_t$_",(3..15,0..2)); @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
arch/arm64/lib/crypto/sha256.c +7 -7
@@ -4,29 +4,29 @@* * Copyright 2025 Google LLC */ #include <asm/neon.h> #include <crypto/internal/sha2.h> +#include <crypto/internal/simd.h> #include <linux/kernel.h> #include <linux/module.h> -asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], -const u8 *data,size_t nblocks); -EXPORT_SYMBOL_GPL (sha256_blocks_arch); +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], +const u8 *data,size_t nblocks); asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS],const u8 *data,size_t nblocks); asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS],const u8 *data,size_t nblocks);static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_neon);static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_ce); -void sha256_blocks_simd (u32 state[SHA256_STATE_WORDS ], +void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks) {if (IS_ENABLED (CONFIG_KERNEL_MODE_NEON ) && -static_branch_likely (&have_neon)) { +static_branch_likely (&have_neon) &&crypto_simd_usable ()) {if (static_branch_likely (&have_ce)) { do { size_t rem; kernel_neon_begin();
@@ -40,14 +40,14 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS],kernel_neon_begin ();sha256_block_neon (state, data, nblocks);kernel_neon_end (); } }else { -sha256_blocks_arch (state, data, nblocks); +sha256_block_data_order (state, data, nblocks); } } -EXPORT_SYMBOL_GPL (sha256_blocks_simd); +EXPORT_SYMBOL_GPL (sha256_blocks_arch);bool sha256_is_arch_optimized (void ) {/* We always can use at least the ARM64 scalar implementation. */ return true;
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 6/8 Revert "crypto: arm/sha256 - Add simd block function"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit 2e43fc6d79cb24e0fe34aa4c2229ab6c414d852a which got pushed out despite being nacked.
That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode NEON. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd".
Signed-off-by Eric Biggers <[email protected]>
--- arch/arm/lib/crypto/Kconfig | 1 - arch/arm/lib/crypto/sha256-armv4.pl | 20 ++++++++++---------- arch/arm/lib/crypto/sha256.c | 14 +++++++------- 3 files changed, 17 insertions(+), 18 deletions(-)
arch/arm/lib/crypto/Kconfig +0 -1
@@ -26,6 +26,5 @@ config CRYPTO_POLY1305_ARM config CRYPTO_SHA256_ARM tristate depends on !CPU_V7M default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD
arch/arm/lib/crypto/sha256-armv4.pl +10 -10
@@ -202,22 +202,22 @@ K256: .word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .size K256,.-K256 .word 0 @ terminator #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) .LOPENSSL_armcap: -.word OPENSSL_armcap_P-sha256_blocks_arch +.word OPENSSL_armcap_P-sha256_block_data_order #endif .align 5 -.global sha256_blocks_arch -.type sha256_blocks_arch,%function -sha256_blocks_arch: -.Lsha256_blocks_arch: +.global sha256_block_data_order +.type sha256_block_data_order,%function +sha256_block_data_order: +.Lsha256_block_data_order: #if __ARM_ARCH__<7 - sub r3,pc,#8 @ sha256_blocks_arch + sub r3,pc,#8 @ sha256_block_data_order #else - adr r3,.Lsha256_blocks_arch + adr r3,.Lsha256_block_data_order #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap ldr r12,[r3,r12] @ OPENSSL_armcap_P tst r12,#ARMV8_SHA256
@@ -280,11 +280,11 @@ $code.=<<___; ldmia sp!,{r4-r11,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.size sha256_blocks_arch,.-sha256_blocks_arch +.size sha256_block_data_order,.-sha256_block_data_order ___ ###################################################################### # NEON stuff # {{{
@@ -468,12 +468,12 @@ $code.=<<___; sha256_block_data_order_neon: .LNEON: stmdb sp!,{r4-r12,lr} sub $H,sp,#16*4+16 - adr $Ktbl,.Lsha256_blocks_arch - sub $Ktbl,$Ktbl,#.Lsha256_blocks_arch-K256 + adr $Ktbl,.Lsha256_block_data_order + sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256 bic $H,$H,#15 @ align for 128-bit stores mov $t2,sp mov sp,$H @ alloca add $len,$inp,$len,lsl#6 @ len to point at the end of inp
arch/arm/lib/crypto/sha256.c +7 -7
@@ -4,40 +4,40 @@* * Copyright 2025 Google LLC */ #include <asm/neon.h> #include <crypto/internal/sha2.h> +#include <crypto/internal/simd.h> #include <linux/kernel.h> #include <linux/module.h> -asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], -const u8 *data,size_t nblocks); -EXPORT_SYMBOL_GPL (sha256_blocks_arch); +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], +const u8 *data,size_t nblocks); asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS],const u8 *data,size_t nblocks); asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS],const u8 *data,size_t nblocks);static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_neon);static __ro_after_init DEFINE_STATIC_KEY_FALSE (have_ce); -void sha256_blocks_simd (u32 state[SHA256_STATE_WORDS ], +void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks) {if (IS_ENABLED (CONFIG_KERNEL_MODE_NEON ) && -static_branch_likely (&have_neon)) { +static_branch_likely (&have_neon) &&crypto_simd_usable ()) {kernel_neon_begin ();if (static_branch_likely (&have_ce))sha256_ce_transform (state, data, nblocks);else sha256_block_data_order_neon (state, data, nblocks);kernel_neon_end (); }else { -sha256_blocks_arch (state, data, nblocks); +sha256_block_data_order (state, data, nblocks); } } -EXPORT_SYMBOL_GPL (sha256_blocks_simd); +EXPORT_SYMBOL_GPL (sha256_blocks_arch);bool sha256_is_arch_optimized (void ) {/* We always can use at least the ARM scalar implementation. */ return true;
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 7/8 Revert "crypto: sha256 - Use the partial block API for generic"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit ff8f037d394f0900597ba527388a6eb95cd02695 which got pushed out despite being nacked.
The library API already has to handle partial blocks, and it makes a lot more sense to just use that.
Keep sha256_block_init() since drivers/crypto/padlock-sha.c is using it.
Signed-off-by Eric Biggers <[email protected]>
--- crypto/sha256.c | 79 ++++++++++++++++++++----------------------- include/crypto/sha2.h | 7 +--- 2 files changed, 38 insertions(+), 48 deletions(-)
crypto/sha256.c +37 -42
@@ -28,68 +28,48 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { };EXPORT_SYMBOL_GPL (sha256_zero_message_hash);static int crypto_sha256_init (struct shash_desc *desc) { -sha256_block_init (shash_desc_ctx (desc)); +sha256_init (shash_desc_ctx (desc));return 0 ; } -static inline int crypto_sha256_update (struct shash_desc *desc,const u8 *data, -unsigned int len,bool force_generic) -{ -struct crypto_sha256_state *sctx =shash_desc_ctx (desc); -int remain = len %SHA256_BLOCK_SIZE ; - - sctx->count += len - remain; -sha256_choose_blocks (sctx->state, data, len /SHA256_BLOCK_SIZE , - force_generic, !force_generic); -return remain; -} -static int crypto_sha256_update_generic (struct shash_desc *desc,const u8 *data,unsigned int len) { -return crypto_sha256_update (desc, data, len, true); +sha256_update_generic (shash_desc_ctx (desc), data, len); +return 0 ; }static int crypto_sha256_update_arch (struct shash_desc *desc,const u8 *data,unsigned int len) {sha256_update (shash_desc_ctx (desc), data, len);return 0 ; } -static int crypto_sha256_final_arch (struct shash_desc *desc,u8 *out) +static int crypto_sha256_final_generic (struct shash_desc *desc,u8 *out) { -sha256_final (shash_desc_ctx (desc), out); +sha256_final_generic (shash_desc_ctx (desc), out);return 0 ; } -static __always_inline int crypto_sha256_finup(struct shash_desc *desc, -const u8 *data, -unsigned int len,u8 *out, -bool force_generic) +static int crypto_sha256_final_arch (struct shash_desc *desc,u8 *out) { -struct crypto_sha256_state *sctx =shash_desc_ctx (desc); -unsigned int remain = len; -u8 *buf; - -if (len >=SHA256_BLOCK_SIZE ) - remain =crypto_sha256_update (desc, data, len, force_generic); - sctx->count += remain; - buf =memcpy (sctx +1 , data + len - remain, remain); -sha256_finup (sctx, buf, remain, out, -crypto_shash_digestsize (desc->tfm), force_generic, - !force_generic); +sha256_final (shash_desc_ctx (desc), out);return 0 ; }static int crypto_sha256_finup_generic (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) { -return crypto_sha256_finup (desc, data, len, out, true); +struct sha256_state *sctx =shash_desc_ctx (desc); + +sha256_update_generic (sctx, data, len); +sha256_final_generic (sctx, out); +return 0 ; }static int crypto_sha256_finup_arch (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) {
@@ -101,12 +81,16 @@ static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data, }static int crypto_sha256_digest_generic (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) { -crypto_sha256_init (desc); -return crypto_sha256_finup_generic (desc, data, len, out); +struct sha256_state *sctx =shash_desc_ctx (desc); + +sha256_init (sctx); +sha256_update_generic (sctx, data, len); +sha256_final_generic (sctx, out); +return 0 ; }static int crypto_sha256_digest_arch (struct shash_desc *desc,const u8 *data,unsigned int len,u8 *out) {
@@ -114,11 +98,17 @@ static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data,return 0 ; }static int crypto_sha224_init (struct shash_desc *desc) { -sha224_block_init (shash_desc_ctx (desc)); +sha224_init (shash_desc_ctx (desc)); +return 0 ; +} + +static int crypto_sha224_final_generic (struct shash_desc *desc,u8 *out) +{ +sha224_final_generic (shash_desc_ctx (desc), out);return 0 ; }static int crypto_sha224_final_arch (struct shash_desc *desc,u8 *out) {
@@ -155,34 +145,39 @@ static int crypto_sha256_export_lib(struct shash_desc *desc, void *out)static struct shash_alg algs[] = { { .base.cra_name ="sha256" , .base.cra_driver_name ="sha256-generic" , .base.cra_priority =100 , - .base.cra_flags =CRYPTO_AHASH_ALG_BLOCK_ONLY | -CRYPTO_AHASH_ALG_FINUP_MAX , .base.cra_blocksize =SHA256_BLOCK_SIZE , .base.cra_module =THIS_MODULE , .digestsize =SHA256_DIGEST_SIZE , .init = crypto_sha256_init, .update = crypto_sha256_update_generic, + .final = crypto_sha256_final_generic, .finup = crypto_sha256_finup_generic, .digest = crypto_sha256_digest_generic, - .descsize =sizeof (struct crypto_sha256_state ), + .descsize =sizeof (struct sha256_state ), + .statesize =sizeof (struct crypto_sha256_state ) + +SHA256_BLOCK_SIZE +1 , + .import = crypto_sha256_import_lib, + .export = crypto_sha256_export_lib, }, { .base.cra_name ="sha224" , .base.cra_driver_name ="sha224-generic" , .base.cra_priority =100 , - .base.cra_flags =CRYPTO_AHASH_ALG_BLOCK_ONLY | -CRYPTO_AHASH_ALG_FINUP_MAX , .base.cra_blocksize =SHA224_BLOCK_SIZE , .base.cra_module =THIS_MODULE , .digestsize =SHA224_DIGEST_SIZE , .init = crypto_sha224_init, .update = crypto_sha256_update_generic, - .finup = crypto_sha256_finup_generic, - .descsize =sizeof (struct crypto_sha256_state ), + .final = crypto_sha224_final_generic, + .descsize =sizeof (struct sha256_state ), + .statesize =sizeof (struct crypto_sha256_state ) + +SHA256_BLOCK_SIZE +1 , + .import = crypto_sha256_import_lib, + .export = crypto_sha256_export_lib, }, { .base.cra_name ="sha256" , .base.cra_driver_name ="sha256-" __stringify(ARCH ), .base.cra_priority =300 ,
include/crypto/sha2.h +1 -6
@@ -107,11 +107,11 @@ static inline void sha256_init(struct sha256_state *sctx) }void sha256_update (struct sha256_state *sctx,const u8 *data,size_t len);void sha256_final (struct sha256_state *sctx,u8 out[SHA256_DIGEST_SIZE ]);void sha256 (const u8 *data,size_t len,u8 out[SHA256_DIGEST_SIZE ]); -static inline void sha224_block_init (struct crypto_sha256_state *sctx) +static inline void sha224_init (struct sha256_state *sctx) { sctx->state[0 ] =SHA224_H0 ; sctx->state[1 ] =SHA224_H1 ; sctx->state[2 ] =SHA224_H2 ; sctx->state[3 ] =SHA224_H3 ;
@@ -119,14 +119,9 @@ static inline void sha224_block_init(struct crypto_sha256_state *sctx) sctx->state[5 ] =SHA224_H5 ; sctx->state[6 ] =SHA224_H6 ; sctx->state[7 ] =SHA224_H7 ; sctx->count = 0; } - -static inline void sha224_init (struct sha256_state *sctx) -{ -sha224_block_init (&sctx->ctx); -}/* Simply use sha256_update as it is equivalent to sha224_update. */ void sha224_final (struct sha256_state *sctx,u8 out[SHA224_DIGEST_SIZE ]);#endif /* _CRYPTO_SHA2_H */
Signature · 1 line
2.49.0
Eric Biggers
maintainerPATCH 8/8 Revert "crypto: lib/sha256 - Add helpers for block-based shash"
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
This reverts commit 5b90a779bc547939421bfeb333e470658ba94fb6 which got pushed out despite being nacked.
That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using vector or SIMD registers. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd".
Signed-off-by Eric Biggers <[email protected]>
--- include/crypto/internal/sha2.h | 45 ---------------------------------- lib/crypto/Kconfig | 8 ------ lib/crypto/sha256.c | 32 +++++++++++++++++------- 3 files changed, 23 insertions(+), 62 deletions(-)
include/crypto/internal/sha2.h +0 -45
@@ -1,16 +1,11 @@/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef _CRYPTO_INTERNAL_SHA2_H#define _CRYPTO_INTERNAL_SHA2_H -#include <crypto/internal/simd.h> #include <crypto/sha2.h> -#include <linux/compiler_attributes.h> -#include <linux/string.h> -#include <linux/types.h> -#include <linux/unaligned.h> void sha256_update_generic (struct sha256_state *sctx,const u8 *data,size_t len);void sha256_final_generic (struct sha256_state *sctx,u8 out[SHA256_DIGEST_SIZE ]);
@@ -27,47 +22,7 @@ static inline bool sha256_is_arch_optimized(void)#endif void sha256_blocks_generic (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks);void sha256_blocks_arch (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks); -void sha256_blocks_simd (u32 state[SHA256_STATE_WORDS ], -const u8 *data,size_t nblocks); - -static inline void sha256_choose_blocks ( -u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks, -bool force_generic,bool force_simd) -{ -if (!IS_ENABLED (CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256 ) || force_generic) -sha256_blocks_generic (state, data, nblocks); -else if (IS_ENABLED (CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD ) && - (force_simd ||crypto_simd_usable ())) -sha256_blocks_simd (state, data, nblocks); -else -sha256_blocks_arch (state, data, nblocks); -} - -static __always_inline void sha256_finup( -struct crypto_sha256_state *sctx,u8 buf[SHA256_BLOCK_SIZE ], -size_t len,u8 out[SHA256_DIGEST_SIZE ],size_t digest_size, -bool force_generic,bool force_simd) -{ -const size_t bit_offset =SHA256_BLOCK_SIZE -8 ; -__be64 *bits = (__be64 *)&buf[bit_offset]; -int i; - - buf[len++] =0x80 ; -if (len > bit_offset) { -memset (&buf[len],0 ,SHA256_BLOCK_SIZE - len); -sha256_choose_blocks (sctx->state, buf,1 , force_generic, - force_simd); - len =0 ; - } - -memset (&buf[len],0 , bit_offset - len); - *bits =cpu_to_be64 (sctx->count <<3 ); -sha256_choose_blocks (sctx->state, buf,1 , force_generic, force_simd); - -for (i =0 ; i < digest_size; i +=4 ) -put_unaligned_be32 (sctx->state[i /4 ], out + i); -}#endif /* _CRYPTO_INTERNAL_SHA2_H */
lib/crypto/Kconfig +0 -8
@@ -148,18 +148,10 @@ config CRYPTO_ARCH_HAVE_LIB_SHA256 bool help Declares whether the architecture provides an arch-specific accelerated implementation of the SHA-256 library interface. -config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD - bool - help - Declares whether the architecture provides an arch-specific - accelerated implementation of the SHA-256 library interface - that is SIMD-based and therefore not usable in hardirq - context. - config CRYPTO_LIB_SHA256_GENERIC tristate default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 help This symbol can be selected by arch implementations of the SHA-256
lib/crypto/sha256.c +23 -9
@@ -13,30 +13,29 @@#include <crypto/internal/sha2.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/string.h> +#include <linux/unaligned.h> /* * If __DISABLE_EXPORTS is defined, then this file is being compiled for a * pre-boot environment. In that case, ignore the kconfig options, pull the * generic code into the same translation unit, and use that only. */ #ifdef __DISABLE_EXPORTS#include "sha256-generic.c" #endif -static inline bool sha256_purgatory (void ) -{ -return __is_defined (__DISABLE_EXPORTS); -} -static inline void sha256_blocks (u32 state[SHA256_STATE_WORDS ],const u8 *data,size_t nblocks,bool force_generic) { -sha256_choose_blocks (state, data, nblocks, - force_generic ||sha256_purgatory (), false); +#if IS_ENABLED (CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256 ) && !defined(__DISABLE_EXPORTS) +if (!force_generic) +return sha256_blocks_arch (state, data, nblocks); +#endif +sha256_blocks_generic (state, data, nblocks); }static inline void __sha256_update (struct sha256_state *sctx,const u8 *data,size_t len,bool force_generic) {
@@ -78,14 +77,29 @@ void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len)EXPORT_SYMBOL (sha256_update);static inline void __sha256_final (struct sha256_state *sctx,u8 *out,size_t digest_size,bool force_generic) { +const size_t bit_offset =SHA256_BLOCK_SIZE -sizeof (__be64); +__be64 *bits = (__be64 *)&sctx->buf[bit_offset];size_t partial = sctx->count %SHA256_BLOCK_SIZE ; +size_t i; + + sctx->buf[partial++] =0x80 ; +if (partial > bit_offset) { +memset (&sctx->buf[partial],0 ,SHA256_BLOCK_SIZE - partial); +sha256_blocks (sctx->state, sctx->buf,1 , force_generic); + partial =0 ; + } + +memset (&sctx->buf[partial],0 , bit_offset - partial); + *bits =cpu_to_be64 (sctx->count <<3 ); +sha256_blocks (sctx->state, sctx->buf,1 , force_generic); + +for (i =0 ; i < digest_size; i +=4 ) +put_unaligned_be32 (sctx->state[i /4 ], out + i); -sha256_finup (&sctx->ctx, sctx->buf, partial, out, digest_size, - force_generic ||sha256_purgatory (), false);memzero_explicit (sctx,sizeof (*sctx)); }void sha256_final (struct sha256_state *sctx,u8 out[SHA256_DIGEST_SIZE ]) {
Signature · 1 line
2.49.0
kernel test robot
build botRe: [PATCH 8/8] Revert "crypto: lib/sha256 - Add helpers for block-based shash"
kernel test robot noticed the following build errors:
Hi Eric,
kernel test robot noticed the following build errors:
[auto build test ERROR on 1bafd82d9a40cf09c6c40f1c09cc35b7050b1a9f]
url: https://github.com/intel-lab-lkp/linux/commits/Eric-Biggers/Revert-crypto-sha256-Use-the-partial-block-API/20250517-102818 base: 1bafd82d9a40cf09c6c40f1c09cc35b7050b1a9f patch link: https://lore.kernel.org/r/20250517022428.401622-9-ebiggers%40kernel.org patch subject: [PATCH 8/8] Revert "crypto: lib/sha256 - Add helpers for block-based shash" config: arm-s5pv210_defconfig (https://download.01.org/0day-ci/archive/20250523/[email protected]/config) compiler: arm-linux-gnueabi-gcc (GCC) 14.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250523/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
arm-linux-gnueabi-ld: lib/crypto/sha256.o: in function `sha256_blocks':
lib/crypto/sha256.c:36:(.text+0x84): undefined reference to `sha256_blocks_generic'
>> arm-linux-gnueabi-ld: lib/crypto/sha256.c:36:(.text+0xc4): undefined reference to `sha256_blocks_generic'
arm-linux-gnueabi-ld: lib/crypto/sha256.c:36:(.text+0x154): undefined reference to `sha256_blocks_generic'
arm-linux-gnueabi-ld: lib/crypto/sha256.c:36:(.text+0x1ac): undefined reference to `sha256_blocks_generic'
arm-linux-gnueabi-ld: lib/crypto/sha256.c:36:(.text+0x68c): undefined reference to `sha256_blocks_generic'
arm-linux-gnueabi-ld: lib/crypto/sha256.o:lib/crypto/sha256.c:36: more undefined references to `sha256_blocks_generic' follow
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Herbert Xu
maintainerQuoted 33 lines · Eric Biggers
This series restores the SHA-256 code to have a clean design where the
crypto_shashAPI is implemented on top of the regular library API instead of a special unsafe low-level API. Diffstat is -41 lines.Eric Biggers (8):
Revert "crypto: sha256 - Use the partial block API" Revert "crypto: lib/sha256 - Use generic block helper" Revert "crypto: x86/sha256 - Add simd block function" Revert "crypto: riscv/sha256 - Add simd block function" Revert "crypto: arm64/sha256 - Add simd block function" Revert "crypto: arm/sha256 - Add simd block function" Revert "crypto: sha256 - Use the partial block API for generic" Revert "crypto: lib/sha256 - Add helpers for block-based shash"arch/arm/lib/crypto/Kconfig | 1 - arch/arm/lib/crypto/sha256-armv4.pl | 20 ++-- arch/arm/lib/crypto/sha256.c | 14 +-- arch/arm64/crypto/sha512-glue.c | 6 +- arch/arm64/lib/crypto/Kconfig | 1 - arch/arm64/lib/crypto/sha2-armv8.pl | 2 +- arch/arm64/lib/crypto/sha256.c | 14 +-- arch/riscv/lib/crypto/Kconfig | 1 - arch/riscv/lib/crypto/sha256.c | 13 +-- arch/x86/lib/crypto/Kconfig | 1 - arch/x86/lib/crypto/sha256.c | 12 +-- crypto/sha256.c | 142 ++++++++++------------------ include/crypto/internal/sha2.h | 52 ++-------- include/crypto/sha2.h | 7 +- lib/crypto/Kconfig | 8 -- lib/crypto/sha256.c | 97 +++++++++++++++---- 16 files changed, 175 insertions(+), 216 deletions(-)
Quoted 1 line
base-commit:
1bafd82d9a40cf09c6c40f1c09cc35b7050b1a9f
Nack. This breaks the cryptodev tree.
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt