Skip to contentSkip to conversation

lib/crypto: x86/chacha: Drop redundant moves from AVX2 code

PATCH Applied Eric Biggers 2 participants 3 messages Nov 11 – Nov 14

Subsystems (2)
lore
Share & tools
lore

Eric Biggers

author maintainer
#0 lore raw

PATCH 1/1 lib/crypto: x86/chacha: Drop redundant moves from AVX2 code

The AVX2 optimized ChaCha code has many instances of moving one register to another, then left-shifting the destination register with 'vpslld'. This pattern likely was copied from the SSSE3 code, which uses 'pslld'. However, like all the other AVX instructions, 'vpslld' has separate source and destination registers. Just utilize that instead.

---
 lib/crypto/x86/chacha-avx2-x86_64.S | 36 ++++++++++-------------------
 1 file changed, 12 insertions(+), 24 deletions(-)
lib/crypto/x86/chacha-avx2-x86_64.S +12 -24
index 18ae03e05074f..8aa3227f7302b 100644
@@ -74,8 +74,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm6
-	vpslld		$12,%ymm6,%ymm6
+	vpslld		$12,%ymm1,%ymm6
 	vpsrld		$20,%ymm1,%ymm1
 	vpor		%ymm6,%ymm1,%ymm1
 
@@ -87,8 +86,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm7
-	vpslld		$7,%ymm7,%ymm7
+	vpslld		$7,%ymm1,%ymm7
 	vpsrld		$25,%ymm1,%ymm1
 	vpor		%ymm7,%ymm1,%ymm1
 
@@ -107,8 +105,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm6
-	vpslld		$12,%ymm6,%ymm6
+	vpslld		$12,%ymm1,%ymm6
 	vpsrld		$20,%ymm1,%ymm1
 	vpor		%ymm6,%ymm1,%ymm1
 
@@ -120,8 +117,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm7
-	vpslld		$7,%ymm7,%ymm7
+	vpslld		$7,%ymm1,%ymm7
 	vpsrld		$25,%ymm1,%ymm1
 	vpor		%ymm7,%ymm1,%ymm1
 
@@ -251,15 +247,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$12,%ymm10,%ymm10
+	vpslld		$12,%ymm1,%ymm10
 	vpsrld		$20,%ymm1,%ymm1
 	vpor		%ymm10,%ymm1,%ymm1
 
 	vpaddd		%ymm7,%ymm6,%ymm6
 	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$12,%ymm10,%ymm10
+	vpslld		$12,%ymm5,%ymm10
 	vpsrld		$20,%ymm5,%ymm5
 	vpor		%ymm10,%ymm5,%ymm5
 
@@ -275,15 +269,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$7,%ymm10,%ymm10
+	vpslld		$7,%ymm1,%ymm10
 	vpsrld		$25,%ymm1,%ymm1
 	vpor		%ymm10,%ymm1,%ymm1
 
 	vpaddd		%ymm7,%ymm6,%ymm6
 	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$7,%ymm10,%ymm10
+	vpslld		$7,%ymm5,%ymm10
 	vpsrld		$25,%ymm5,%ymm5
 	vpor		%ymm10,%ymm5,%ymm5
 
@@ -309,15 +301,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 12)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$12,%ymm10,%ymm10
+	vpslld		$12,%ymm1,%ymm10
 	vpsrld		$20,%ymm1,%ymm1
 	vpor		%ymm10,%ymm1,%ymm1
 
 	vpaddd		%ymm7,%ymm6,%ymm6
 	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$12,%ymm10,%ymm10
+	vpslld		$12,%ymm5,%ymm10
 	vpsrld		$20,%ymm5,%ymm5
 	vpor		%ymm10,%ymm5,%ymm5
 
@@ -333,15 +323,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2)
 	# x2 += x3, x1 = rotl32(x1 ^ x2, 7)
 	vpaddd		%ymm3,%ymm2,%ymm2
 	vpxor		%ymm2,%ymm1,%ymm1
-	vmovdqa		%ymm1,%ymm10
-	vpslld		$7,%ymm10,%ymm10
+	vpslld		$7,%ymm1,%ymm10
 	vpsrld		$25,%ymm1,%ymm1
 	vpor		%ymm10,%ymm1,%ymm1
 
 	vpaddd		%ymm7,%ymm6,%ymm6
 	vpxor		%ymm6,%ymm5,%ymm5
-	vmovdqa		%ymm5,%ymm10
-	vpslld		$7,%ymm10,%ymm10
+	vpslld		$7,%ymm5,%ymm10
 	vpsrld		$25,%ymm5,%ymm5
 	vpor		%ymm10,%ymm5,%ymm5
 

base-commit: 33b4f31b1aea3f62b61c892daefc0e6a77859032

Signature · 1 line
2.56.1
Reviewed-by Tested-by on v1 1/1
#1 lore raw

Hi,

Quoted 5 lines · Eric Biggers

The AVX2 optimized ChaCha code has many instances of moving one register to another, then left-shifting the destination register with 'vpslld'. This pattern likely was copied from the SSSE3 code, which uses 'pslld'. However, like all the other AVX instructions, 'vpslld' has separate source and destination registers. Just utilize that instead.

Tested on a Ryzen 9 9950X in a KVM guest with AVX-512 hidden, so that the AVX2 code is used: the ChaCha and ChaCha20-Poly1305 KUnit tests pass.
The ChaCha benchmark shows no difference beyond noise (+/-0.6% at every length, 3 runs each).

Signature · 5 lines
Antonio Quartulli

CEO and Co-Founder
Mandelbit Srl
https://www.mandelbit.com
author maintainer Applied
#2 lore raw
In reply to Eric Biggers
Quoted 10 lines · Eric Biggers

The AVX2 optimized ChaCha code has many instances of moving one register to another, then left-shifting the destination register with 'vpslld'. This pattern likely was copied from the SSSE3 code, which uses 'pslld'. However, like all the other AVX instructions, 'vpslld' has separate source and destination registers. Just utilize that instead.

---
 lib/crypto/x86/chacha-avx2-x86_64.S | 36 ++++++++++-------------------
 1 file changed, 12 insertions(+), 24 deletions(-)

Applied to https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git/log/?h=libcrypto-next

- Eric