crypto: x86/aes - drop the avx10_256 AES-XTS and AES-CTR code
Subsystems (2)
PATCH v2 1/1 crypto: x86/aes - drop the avx10_256 AES-XTS and AES-CTR code
From: Eric Biggers <[email protected]>
Intel made a late change to the AVX10 specification that removes support for a 256-bit maximum vector length and enumeration of the maximum vector length. AVX10 will imply a maximum vector length of 512 bits. I.e. there won't be any such thing as AVX10/256 or AVX10/512; there will just be AVX10, and it will essentially just consolidate AVX512 features.
As a result of this new development, my strategy of providing both *_avx10_256 and *_avx10_512 functions didn't turn out to be that useful. The only remaining motivation for the 256-bit AVX512 / AVX10 functions is to avoid downclocking on older Intel CPUs. But in the case of AES-XTS and AES-CTR, I already wrote *_avx2 code too (primarily to support CPUs without AVX512), which performs almost as well as *_avx10_256. So we should just use that.
Therefore, remove the *_avx10_256 AES-XTS and AES-CTR functions and algorithms, and rename the *_avx10_512 AES-XTS and AES-CTR functions and algorithms to *_avx512. Make Ice Lake and Tiger Lake use *_avx2 instead of *_avx10_256 which they previously used.
I've left AES-GCM unchanged for now. There is no VAES+AVX2 optimized AES-GCM in the kernel yet, so the path forward for that is not as clear. However, I did write a VAES+AVX2 optimized AES-GCM for BoringSSL. So one option is to port that to the kernel and then do the same cleanup.
Signed-off-by Eric Biggers <[email protected]>
---
v2: fixed a comment in aes-xts-avx-x86_64.S, and
fixed a typo in the commit message.
arch/x86/crypto/aes-ctr-avx-x86_64.S | 47 ++++------- arch/x86/crypto/aes-xts-avx-x86_64.S | 118 ++++++++++++--------------- arch/x86/crypto/aesni-intel_glue.c | 30 +++---- 3 files changed, 74 insertions(+), 121 deletions(-)
arch/x86/crypto/aes-ctr-avx-x86_64.S +14 -33
@@ -46,12 +46,11 @@ // // This file contains x86_64 assembly implementations of AES-CTR and AES-XCTR // using the following sets of CPU features: // - AES-NI && AVX // - VAES && AVX2 -// - VAES && (AVX10/256 || (AVX512BW && AVX512VL)) && BMI2 -// - VAES && (AVX10/512 || (AVX512BW && AVX512VL)) && BMI2 +// - VAES && AVX512BW && AVX512VL && BMI2 // // See the function definitions at the bottom of the file for more information. #include <linux/linkage.h> #include <linux/cfi_types.h>
@@ -74,31 +73,29 @@ .quad 4, 0 .text // Move a vector between memory and a register. -// The register operand must be in the first 16 vector registers. .macro _vmovdqu src, dst .if VL < 64 vmovdqu \src, \dst .else vmovdqu8 \src, \dst .endif .endm // Move a vector between registers. -// The registers must be in the first 16 vector registers. .macro _vmovdqa src, dst .if VL < 64 vmovdqa \src, \dst .else vmovdqa64 \src, \dst .endif .endm // Broadcast a 128-bit value from memory to all 128-bit lanes of a vector -// register. The register operand must be in the first 16 vector registers. +// register. .macro _vbroadcast128 src, dst .if VL == 16 vmovdqu \src, \dst .elseif VL == 32 vbroadcasti128 \src, \dst
@@ -106,11 +103,10 @@ vbroadcasti32x4 \src, \dst .endif .endm // XOR two vectors together. -// Any register operands must be in the first 16 vector registers. .macro _vpxor src1, src2, dst .if VL < 64 vpxor \src1, \src2, \dst .else vpxord \src1, \src2, \dst
@@ -197,20 +193,20 @@ // Prepare the next two vectors of AES inputs in AESDATA\i0 and AESDATA\i1, and // XOR each with the zero-th round key. Also update LE_CTR if !\final. .macro _prepare_2_ctr_vecs is_xctr, i0, i1, final=0 .if \is_xctr - .if USE_AVX10 - _vmovdqa LE_CTR, AESDATA\i0 + .if USE_AVX512 + vmovdqa64 LE_CTR, AESDATA\i0 vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i0 .else vpxor XCTR_IV, LE_CTR, AESDATA\i0 vpxor RNDKEY0, AESDATA\i0, AESDATA\i0 .endif vpaddq LE_CTR_INC1, LE_CTR, AESDATA\i1 - .if USE_AVX10 + .if USE_AVX512 vpternlogd $0x96, XCTR_IV, RNDKEY0, AESDATA\i1 .else vpxor XCTR_IV, AESDATA\i1, AESDATA\i1 vpxor RNDKEY0, AESDATA\i1, AESDATA\i1 .endif
@@ -479,22 +475,16 @@ _vmovdqa AESDATA3, AESDATA0 .Lxor_tail_partial_vec_0\@: // XOR the remaining 1 <= LEN < VL bytes. It's easy if masked // loads/stores are available; otherwise it's a bit harder... -.if USE_AVX10 - .if VL <= 32 - mov $-1, %eax - bzhi LEN, %eax, %eax - kmovd %eax, %k1 - .else +.if USE_AVX512 mov $-1, %rax bzhi LEN64, %rax, %rax kmovq %rax, %k1 - .endif vmovdqu8 (SRC), AESDATA1{%k1}{z} - _vpxor AESDATA1, AESDATA0, AESDATA0 + vpxord AESDATA1, AESDATA0, AESDATA0 vmovdqu8 AESDATA0, (DST){%k1} .else .if VL == 32 cmp $16, LEN jl 1f
@@ -552,41 +542,32 @@ // with HCTR2" (https://eprint.iacr.org/2021/1441.pdf). XCTR is an // easier-to-implement variant of CTR that uses little endian byte order and // eliminates carries. |ctr| is the per-message block counter starting at 1. .set VL, 16 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx) _aes_ctr_crypt 0 SYM_FUNC_END(aes_ctr64_crypt_aesni_avx) SYM_TYPED_FUNC_START(aes_xctr_crypt_aesni_avx) _aes_ctr_crypt 1 SYM_FUNC_END(aes_xctr_crypt_aesni_avx) #if defined(CONFIG_AS_VAES) && defined(CONFIG_AS_VPCLMULQDQ) .set VL, 32 -.set USE_AVX10, 0 +.set USE_AVX512, 0 SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2) _aes_ctr_crypt 0 SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2) SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2) _aes_ctr_crypt 1 SYM_FUNC_END(aes_xctr_crypt_vaes_avx2) -.set VL, 32 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx10_256) - _aes_ctr_crypt 0 -SYM_FUNC_END(aes_ctr64_crypt_vaes_avx10_256) -SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx10_256) - _aes_ctr_crypt 1 -SYM_FUNC_END(aes_xctr_crypt_vaes_avx10_256) - .set VL, 64 -.set USE_AVX10, 1 -SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx10_512) +.set USE_AVX512, 1 +SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512) _aes_ctr_crypt 0 -SYM_FUNC_END(aes_ctr64_crypt_vaes_avx10_512) -SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx10_512) +SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512) +SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512) _aes_ctr_crypt 1 -SYM_FUNC_END(aes_xctr_crypt_vaes_avx10_512) +SYM_FUNC_END(aes_xctr_crypt_vaes_avx512) #endif // CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ
arch/x86/crypto/aes-xts-avx-x86_64.S +50 -68
336 lines not on this page · raw message
arch/x86/crypto/aesni-intel_glue.c +10 -20
@@ -842,12 +842,11 @@ static struct simd_skcipher_alg * \ simd_skcipher_algs_##suffix[ARRAY_SIZE(skcipher_algs_##suffix)]DEFINE_AVX_SKCIPHER_ALGS (aesni_avx,"aesni-avx" ,500 );#if defined(CONFIG_AS_VAES ) && defined(CONFIG_AS_VPCLMULQDQ )DEFINE_AVX_SKCIPHER_ALGS (vaes_avx2,"vaes-avx2" ,600 ); -DEFINE_AVX_SKCIPHER_ALGS (vaes_avx10_256,"vaes-avx10_256" ,700 ); -DEFINE_AVX_SKCIPHER_ALGS (vaes_avx10_512,"vaes-avx10_512" ,800 ); +DEFINE_AVX_SKCIPHER_ALGS (vaes_avx512,"vaes-avx512" ,800 );#endif /* The common part of the x86_64 AES-GCM key struct */ struct aes_gcm_key {/* Expanded AES key and the AES key length in bytes */
@@ -1610,33 +1609,28 @@ static int __init register_avx_algs(void) !boot_cpu_has (X86_FEATURE_BMI2 ) || !cpu_has_xfeatures (XFEATURE_MASK_SSE |XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL)) return 0; - err =simd_register_skciphers_compat (skcipher_algs_vaes_avx10_256, -ARRAY_SIZE (skcipher_algs_vaes_avx10_256), - simd_skcipher_algs_vaes_avx10_256); -if (err) -return err; err =simd_register_aeads_compat (aes_gcm_algs_vaes_avx10_256,ARRAY_SIZE (aes_gcm_algs_vaes_avx10_256), aes_gcm_simdalgs_vaes_avx10_256);if (err)return err;if (x86_match_cpu (zmm_exclusion_list)) {int i; -for (i =0 ; i <ARRAY_SIZE (skcipher_algs_vaes_avx10_512); i++) - skcipher_algs_vaes_avx10_512[i].base.cra_priority =1 ; +for (i =0 ; i <ARRAY_SIZE (skcipher_algs_vaes_avx512); i++) + skcipher_algs_vaes_avx512[i].base.cra_priority =1 ;for (i =0 ; i <ARRAY_SIZE (aes_gcm_algs_vaes_avx10_512); i++) aes_gcm_algs_vaes_avx10_512[i].base.cra_priority =1 ; } - err =simd_register_skciphers_compat (skcipher_algs_vaes_avx10_512, -ARRAY_SIZE (skcipher_algs_vaes_avx10_512), - simd_skcipher_algs_vaes_avx10_512); + err =simd_register_skciphers_compat (skcipher_algs_vaes_avx512, +ARRAY_SIZE (skcipher_algs_vaes_avx512), + simd_skcipher_algs_vaes_avx512);if (err)return err; err =simd_register_aeads_compat (aes_gcm_algs_vaes_avx10_512,ARRAY_SIZE (aes_gcm_algs_vaes_avx10_512), aes_gcm_simdalgs_vaes_avx10_512);
@@ -1659,22 +1653,18 @@ static void unregister_avx_algs(void)#if defined(CONFIG_AS_VAES ) && defined(CONFIG_AS_VPCLMULQDQ )if (simd_skcipher_algs_vaes_avx2[0 ])simd_unregister_skciphers (skcipher_algs_vaes_avx2,ARRAY_SIZE (skcipher_algs_vaes_avx2), simd_skcipher_algs_vaes_avx2); -if (simd_skcipher_algs_vaes_avx10_256[0 ]) -simd_unregister_skciphers (skcipher_algs_vaes_avx10_256, -ARRAY_SIZE (skcipher_algs_vaes_avx10_256), - simd_skcipher_algs_vaes_avx10_256);if (aes_gcm_simdalgs_vaes_avx10_256[0 ])simd_unregister_aeads (aes_gcm_algs_vaes_avx10_256,ARRAY_SIZE (aes_gcm_algs_vaes_avx10_256), aes_gcm_simdalgs_vaes_avx10_256); -if (simd_skcipher_algs_vaes_avx10_512[0 ]) -simd_unregister_skciphers (skcipher_algs_vaes_avx10_512, -ARRAY_SIZE (skcipher_algs_vaes_avx10_512), - simd_skcipher_algs_vaes_avx10_512); +if (simd_skcipher_algs_vaes_avx512[0 ]) +simd_unregister_skciphers (skcipher_algs_vaes_avx512, +ARRAY_SIZE (skcipher_algs_vaes_avx512), + simd_skcipher_algs_vaes_avx512);if (aes_gcm_simdalgs_vaes_avx10_512[0 ])simd_unregister_aeads (aes_gcm_algs_vaes_avx10_512,ARRAY_SIZE (aes_gcm_algs_vaes_avx10_512), aes_gcm_simdalgs_vaes_avx10_512);#endif
base-commit: d2d072a313c1817a0d72d7b8301eaf29ce7f83fc
Signature · 1 line
2.49.0