Skip to contentSkip to conversation

lib/crypto: x86/chacha: Use bzhi to simplify mask generation

PATCH Applied Eric Biggers 2 participants 3 messages Nov 3 – Nov 7

Subsystems (2)
Crypto API Crypto library
lore
Share & tools
lore

Eric Biggers

author maintainer
#0 lore raw

PATCH 1/1 lib/crypto: x86/chacha: Use bzhi to simplify mask generation

When XOR'ing the bytes from a partial register at the end, use the bzhi instruction (from BMI2) to generate the needed mask a bit more efficiently. All real processors with AVX-512 also have BMI2, so we might as well make use of it. This follows the example of the AVX-512 optimized AES-GCM code which already does this.

Signed-off-by Eric Biggers <[email protected]>
---
 lib/crypto/x86/chacha-avx512-x86_64.S | 30 +++++++++++----------------
 lib/crypto/x86/chacha.h               |  3 ++-
 2 files changed, 14 insertions(+), 19 deletions(-)
lib/crypto/x86/chacha-avx512-x86_64.S +12 -18
index 92babb1d3844..422479631d5b 100644
@@ -189,15 +189,13 @@ SYM_FUNC_START(chacha_2block_xor_avx512vl)
 
 .Lxorpart2:
 	# xor remaining bytes from partial register into output
-	mov		%rcx,%rax
+	mov		%rcx,%r9
+	and		$~0xf,%r9
 	and		$0xf,%rcx
 	jz		.Ldone2
-	mov		%rax,%r9
-	and		$~0xf,%r9
 
-	mov		$1,%rax
-	shld		%cl,%rax,%rax
-	sub		$1,%rax
+	mov		$-1,%rax
+	bzhi		%rcx,%rax,%rax
 	kmovq		%rax,%k1
 
 	vmovdqu8	(%rdx,%r9),%xmm1{%k1}{z}

@@ -455,15 +453,13 @@ SYM_FUNC_START(chacha_4block_xor_avx512vl)
 
 .Lxorpart4:
 	# xor remaining bytes from partial register into output
-	mov		%rcx,%rax
+	mov		%rcx,%r9
+	and		$~0xf,%r9
 	and		$0xf,%rcx
 	jz		.Ldone4
-	mov		%rax,%r9
-	and		$~0xf,%r9
 
-	mov		$1,%rax
-	shld		%cl,%rax,%rax
-	sub		$1,%rax
+	mov		$-1,%rax
+	bzhi		%rcx,%rax,%rax
 	kmovq		%rax,%k1
 
 	vmovdqu8	(%rdx,%r9),%xmm1{%k1}{z}

@@ -977,15 +973,13 @@ SYM_FUNC_END(chacha_4block_xor_avx512vl)
 
 .Lxorpart\@:
 	# xor remaining bytes from partial register into output
-	mov		%rcx,%rax
+	mov		%rcx,%r9
+	and		$~(VL - 1),%r9
 	and		$(VL - 1),%rcx
 	jz		.Ldone\@
-	mov		%rax,%r9
-	and		$~(VL - 1),%r9
 
-	mov		$1,%rax
-	shld		%cl,%rax,%rax
-	sub		$1,%rax
+	mov		$-1,%rax
+	bzhi		%rcx,%rax,%rax
 	kmovq		%rax,%k1
 
 	vmovdqu8	(%rdx,%r9),V1{%k1}{z}
lib/crypto/x86/chacha.h +2 -1
index fdf01b07742e..ed39b3fdda84 100644
@@ -191,7 +191,8 @@ static void chacha_mod_init_arch(void)
 		static_branch_enable(&chacha_use_avx2);
 
 		if (boot_cpu_has(X86_FEATURE_AVX512VL) &&
-		    boot_cpu_has(X86_FEATURE_AVX512BW)) { /* kmovq */
+		    boot_cpu_has(X86_FEATURE_AVX512BW) &&
+		    boot_cpu_has(X86_FEATURE_BMI2)) {
 			static_branch_enable(&chacha_use_avx512vl);
 
 			if (!boot_cpu_has(X86_FEATURE_PREFER_YMM))

base-commit: 9fbbbaefd5f95551da18426c200cf41e1b4255a1

Signature · 1 line
2.56.0
Tested-by on v1 1/1
#1 lore raw

Hi,

Quoted 7 lines · Eric Biggers

When XOR'ing the bytes from a partial register at the end, use the bzhi instruction (from BMI2) to generate the needed mask a bit more efficiently. All real processors with AVX-512 also have BMI2, so we might as well make use of it. This follows the example of the AVX-512 optimized AES-GCM code which already does this.

Signed-off-by Eric Biggers <[email protected]>

KUnit test is happy and my tests (on Zen 5) too :)

Tested-by Antonio Quartulli <[email protected]>
Signature · 5 lines
Antonio Quartulli

CEO and Co-Founder
Mandelbit Srl
https://www.mandelbit.com
author maintainer Applied
#2 lore raw
In reply to Eric Biggers
Quoted 11 lines · Eric Biggers

When XOR'ing the bytes from a partial register at the end, use the bzhi instruction (from BMI2) to generate the needed mask a bit more efficiently. All real processors with AVX-512 also have BMI2, so we might as well make use of it. This follows the example of the AVX-512 optimized AES-GCM code which already does this.

Signed-off-by Eric Biggers <[email protected]>
---
 lib/crypto/x86/chacha-avx512-x86_64.S | 30 +++++++++++----------------
 lib/crypto/x86/chacha.h               |  3 ++-
 2 files changed, 14 insertions(+), 19 deletions(-)

Applied to https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git/log/?h=libcrypto-next

- Eric