crypto: ARM - run kernel mode NEON with softirqs disabled
Subsystems (3)
cover crypto: ARM - run kernel mode NEON with softirqs disabled
This series is presented in response to the discussion [0] that has been going on on the linux-crypto list regarding the use of SIMD in synchronous skciphers and AEADs, which is problematic if such transforms may be used in softirq context while the SIMD unit is already being used by the kernel in process context as well.
This series proposes a way to work around this restriction. It is mainly intended to elicit discussion, so it is based on ARM not x86, which is probably more instructive, given that ARM does not permit SIMD use in softirq context at all (which is changed by patch #1), and also already carries some of these fallbacks that we should be able to remove if these changes work as intended (patches #2 and #3)
The primary assumption here is that use of skciphers and AEADs is currently only supported in process or softirq context. If this is true, we can avoid the need for dealing with nested use of the SIMD unit (when softirq uses the SIMD unit while it is already being used in process context), by disabling softirq processing entirely when the NEON unit is enabled for kernel mode use.
If this approach works and turns out to be suitable for x86 as well (which will require a separate discussion involving the x86 maintainers), we will probably need to reduce the scope of the kernel_fpu_begin/end blocks to ensure that softirq processing latency is not affected. Given that x86 has recently been updated to drastically reduce the overhead of preserving/ restoring the FPU state, this should not adversely affect performance.
[0] https://lore.kernel.org/linux-crypto/[email protected]/
Ard Biesheuvel (3):
ARM: vfp: allow kernel mode NEON in softirq context
crypto: arm/aes-ce - drop non-SIMD fallbacks and SIMD helper
crypto: arm/aes-neonbs - drop non-SIMD fallbacks and SIMD helper
arch/arm/crypto/aes-ce-glue.c | 84 +------------- arch/arm/crypto/aes-neonbs-glue.c | 119 ++------------------ arch/arm/include/asm/simd.h | 12 ++ arch/arm/vfp/vfpmodule.c | 11 +- 4 files changed, 31 insertions(+), 195 deletions(-) create mode 100644 arch/arm/include/asm/simd.h
Signature · 1 line
2.17.1
RFC PATCH 1/3 ARM: vfp: allow kernel mode NEON in softirq context
Allow kernel mode NEON to be used in softirq context as well as process context. To avoid nested use of the NEON, which would require the kernel mode process context NEON state to be preserved while the NEON is used in softirq context, turn off softirq processing when enabling kernel mode NEON.
--- arch/arm/include/asm/simd.h | 12 ++++++++++++ arch/arm/vfp/vfpmodule.c | 11 +++++++---- 2 files changed, 19 insertions(+), 4 deletions(-)
arch/arm/include/asm/simd.h new file +12 -0
@@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/hardirq.h> + +/* +* may_use_simd - whether it is allowable at this time to issue SIMD +* instructions or access the SIMD register file +*/ +static __must_check inline bool may_use_simd (void ) +{ +return !in_irq () && !irqs_disabled () && !in_nmi (); +}
arch/arm/vfp/vfpmodule.c +7 -4
@@ -21,6 +21,7 @@#include <asm/cp15.h> #include <asm/cputype.h> +#include <asm/simd.h> #include <asm/system_info.h> #include <asm/thread_notify.h> #include <asm/traps.h>
@@ -719,12 +720,13 @@ void kernel_neon_begin(void)u32 fpexc;/* -* Kernel mode NEON is only allowed outside of interrupt context -* with preemption disabled. This will make sure that the kernel -* mode NEON register contents never need to be preserved. +* Kernel mode NEON is only allowed outside of hard interrupt context +* with preemption and softirq processing disabled. This ensures that +* the kernel mode NEON register contents never need to be preserved. */ -BUG_ON (in_interrupt ()); +BUG_ON (!may_use_simd ()); cpu =get_cpu (); +local_bh_disable (); fpexc =fmrx (FPEXC ) |FPEXC_EN ;fmxr (FPEXC , fpexc);
@@ -747,6 +749,7 @@ void kernel_neon_end(void) {/* Disable the NEON/VFP unit. */ fmxr (FPEXC ,fmrx (FPEXC ) & ~FPEXC_EN ); +local_bh_enable ();put_cpu (); }EXPORT_SYMBOL (kernel_neon_end);
Signature · 7 lines
2.17.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
RFC PATCH 2/3 crypto: arm/aes-ce - drop non-SIMD fallbacks and SIMD helper
In reply to Ard BiesheuvelNow that kernel mode NEON is guaranteed to be available both in process and in softirq context, we no longer have a need for the SIMD helper or for non-SIMD fallbacks, given that the skcipher API is not supported in any other context anyway. So drop this code.
--- arch/arm/crypto/aes-ce-glue.c | 84 +------------------- 1 file changed, 3 insertions(+), 81 deletions(-)
arch/arm/crypto/aes-ce-glue.c +3 -81
@@ -7,11 +7,9 @@#include <asm/hwcap.h> #include <asm/neon.h> -#include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/aes.h> #include <crypto/ctr.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h>
@@ -418,29 +416,6 @@ static int ctr_encrypt(struct skcipher_request *req)return err; } -static void ctr_encrypt_one (struct crypto_skcipher *tfm,const u8 *src,u8 *dst) -{ -struct crypto_aes_ctx *ctx =crypto_skcipher_ctx (tfm); -unsigned long flags; - -/* -* Temporarily disable interrupts to avoid races where -* cachelines are evicted when the CPU is interrupted -* to do something else. -*/ -local_irq_save (flags); -aes_encrypt (ctx, dst, src); -local_irq_restore (flags); -} - -static int ctr_encrypt_sync (struct skcipher_request *req) -{ -if (!crypto_simd_usable ()) -return crypto_ctr_encrypt_walk (req, ctr_encrypt_one); - -return ctr_encrypt (req); -} -static int xts_encrypt (struct skcipher_request *req) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req);
@@ -647,25 +622,9 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-ce-sync", - .base.cra_priority = 300 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .chunksize = AES_BLOCK_SIZE, - .setkey = ce_aes_setkey, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-ce", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-ce", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .base.cra_module = THIS_MODULE,
@@ -679,51 +638,14 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE (aes_algs)]; -static void aes_exit (void ) { -int i; - -for (i =0 ; i <ARRAY_SIZE (aes_simd_algs) && aes_simd_algs[i]; i++) -simd_skcipher_free (aes_simd_algs[i]); -crypto_unregister_skciphers (aes_algs,ARRAY_SIZE (aes_algs)); }static int __init aes_init (void ) { -struct simd_skcipher_alg *simd; -const char *basename; -const char *algname; -const char *drvname; -int err; -int i; - - err =crypto_register_skciphers (aes_algs,ARRAY_SIZE (aes_algs)); -if (err) -return err; - -for (i =0 ; i <ARRAY_SIZE (aes_algs); i++) { -if (!(aes_algs[i].base.cra_flags &CRYPTO_ALG_INTERNAL )) -continue ; - - algname = aes_algs[i].base.cra_name +2 ; - drvname = aes_algs[i].base.cra_driver_name +2 ; - basename = aes_algs[i].base.cra_driver_name; - simd =simd_skcipher_create_compat (algname, drvname, basename); - err =PTR_ERR (simd); -if (IS_ERR (simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - -return 0 ; - -unregister_simds: -aes_exit (); -return err; +return crypto_register_skciphers (aes_algs,ARRAY_SIZE (aes_algs)); }module_cpu_feature_match (AES , aes_init);
Signature · 1 line
2.17.1
RFC PATCH 3/3 crypto: arm/aes-neonbs - drop non-SIMD fallbacks and SIMD helper
In reply to Ard BiesheuvelNow that kernel mode NEON is guaranteed to be available both in process and in softirq context, we no longer have a need for the SIMD helper or for non-SIMD fallbacks, given that the skcipher API is not supported in any other context anyway. So drop this code.
--- arch/arm/crypto/aes-neonbs-glue.c | 119 ++------------------ 1 file changed, 9 insertions(+), 110 deletions(-)
arch/arm/crypto/aes-neonbs-glue.c +9 -110
@@ -6,11 +6,9 @@*/ #include <asm/neon.h> -#include <asm/simd.h> #include <crypto/aes.h> #include <crypto/ctr.h> #include <crypto/internal/cipher.h> -#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <crypto/xts.h>
@@ -212,25 +210,6 @@ static void cbc_exit(struct crypto_skcipher *tfm)crypto_free_skcipher (ctx->enc_tfm); } -static int aesbs_ctr_setkey_sync (struct crypto_skcipher *tfm,const u8 *in_key, -unsigned int key_len) -{ -struct aesbs_ctr_ctx *ctx =crypto_skcipher_ctx (tfm); -int err; - - err =aes_expandkey (&ctx->fallback, in_key, key_len); -if (err) -return err; - - ctx->key.rounds =6 + key_len /4 ; - -kernel_neon_begin (); -aesbs_convert_key (ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); -kernel_neon_end (); - -return 0 ; -} -static int ctr_encrypt (struct skcipher_request *req) {struct crypto_skcipher *tfm =crypto_skcipher_reqtfm (req);
@@ -273,29 +252,6 @@ static int ctr_encrypt(struct skcipher_request *req)return err; } -static void ctr_encrypt_one (struct crypto_skcipher *tfm,const u8 *src,u8 *dst) -{ -struct aesbs_ctr_ctx *ctx =crypto_skcipher_ctx (tfm); -unsigned long flags; - -/* -* Temporarily disable interrupts to avoid races where -* cachelines are evicted when the CPU is interrupted -* to do something else. -*/ -local_irq_save (flags); -aes_encrypt (&ctx->fallback, dst, src); -local_irq_restore (flags); -} - -static int ctr_encrypt_sync (struct skcipher_request *req) -{ -if (!crypto_simd_usable ()) -return crypto_ctr_encrypt_walk (req, ctr_encrypt_one); - -return ctr_encrypt (req); -} -static int aesbs_xts_setkey (struct crypto_skcipher *tfm,const u8 *in_key,unsigned int key_len) {
@@ -423,13 +379,12 @@ static int xts_decrypt(struct skcipher_request *req) }static struct skcipher_alg aes_algs[] = { { - .base.cra_name ="__ecb(aes)" , - .base.cra_driver_name ="__ecb-aes-neonbs" , + .base.cra_name ="ecb(aes)" , + .base.cra_driver_name ="ecb-aes-neonbs" , .base.cra_priority =250 , .base.cra_blocksize =AES_BLOCK_SIZE , .base.cra_ctxsize =sizeof (struct aesbs_ctx ), .base.cra_module =THIS_MODULE , - .base.cra_flags =CRYPTO_ALG_INTERNAL , .min_keysize =AES_MIN_KEY_SIZE , .max_keysize =AES_MAX_KEY_SIZE ,
@@ -438,13 +393,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-neonbs", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE,
@@ -456,13 +410,12 @@ static struct skcipher_alg aes_algs[] = { { .init = cbc_init, .exit = cbc_exit, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-neonbs", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE,
@@ -473,29 +426,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-neonbs-sync", - .base.cra_priority = 250 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .chunksize = AES_BLOCK_SIZE, - .walksize = 8 * AES_BLOCK_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_ctr_setkey_sync, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-neonbs", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE,
@@ -508,54 +444,17 @@ static struct skcipher_alg aes_algs[] = { { .exit = xts_exit, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE (aes_algs)]; -static void aes_exit (void ) { -int i; - -for (i =0 ; i <ARRAY_SIZE (aes_simd_algs); i++) -if (aes_simd_algs[i]) -simd_skcipher_free (aes_simd_algs[i]); -crypto_unregister_skciphers (aes_algs,ARRAY_SIZE (aes_algs)); }static int __init aes_init (void ) { -struct simd_skcipher_alg *simd; -const char *basename; -const char *algname; -const char *drvname; -int err; -int i; -if (!(elf_hwcap &HWCAP_NEON ))return -ENODEV ; - err =crypto_register_skciphers (aes_algs,ARRAY_SIZE (aes_algs)); -if (err) -return err; - -for (i =0 ; i <ARRAY_SIZE (aes_algs); i++) { -if (!(aes_algs[i].base.cra_flags &CRYPTO_ALG_INTERNAL )) -continue ; - - algname = aes_algs[i].base.cra_name +2 ; - drvname = aes_algs[i].base.cra_driver_name +2 ; - basename = aes_algs[i].base.cra_driver_name; - simd =simd_skcipher_create_compat (algname, drvname, basename); - err =PTR_ERR (simd); -if (IS_ERR (simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } -return 0 ; - -unregister_simds: -aes_exit (); -return err; +return crypto_register_skciphers (aes_algs,ARRAY_SIZE (aes_algs)); }late_initcall (aes_init);
Signature · 1 line
2.17.1