lib/crypto: x86/chacha: Drop redundant moves from AVX2 code
Subsystems (2)
Eric Biggers
maintainerPATCH 1/1 lib/crypto: x86/chacha: Drop redundant moves from AVX2 code
The AVX2 optimized ChaCha code has many instances of moving one register to another, then left-shifting the destination register with 'vpslld'. This pattern likely was copied from the SSSE3 code, which uses 'pslld'. However, like all the other AVX instructions, 'vpslld' has separate source and destination registers. Just utilize that instead.
--- lib/crypto/x86/chacha-avx2-x86_64.S | 36 ++++++++++------------------- 1 file changed, 12 insertions(+), 24 deletions(-)
lib/crypto/x86/chacha-avx2-x86_64.S +12 -24
@@ -74,8 +74,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 12) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm6 - vpslld $12,%ymm6,%ymm6 + vpslld $12,%ymm1,%ymm6 vpsrld $20,%ymm1,%ymm1 vpor %ymm6,%ymm1,%ymm1
@@ -87,8 +86,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 7) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm7 - vpslld $7,%ymm7,%ymm7 + vpslld $7,%ymm1,%ymm7 vpsrld $25,%ymm1,%ymm1 vpor %ymm7,%ymm1,%ymm1
@@ -107,8 +105,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 12) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm6 - vpslld $12,%ymm6,%ymm6 + vpslld $12,%ymm1,%ymm6 vpsrld $20,%ymm1,%ymm1 vpor %ymm6,%ymm1,%ymm1
@@ -120,8 +117,7 @@ SYM_FUNC_START(chacha_1to2blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 7) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm7 - vpslld $7,%ymm7,%ymm7 + vpslld $7,%ymm1,%ymm7 vpsrld $25,%ymm1,%ymm1 vpor %ymm7,%ymm1,%ymm1
@@ -251,15 +247,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 12) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm10 - vpslld $12,%ymm10,%ymm10 + vpslld $12,%ymm1,%ymm10 vpsrld $20,%ymm1,%ymm1 vpor %ymm10,%ymm1,%ymm1 vpaddd %ymm7,%ymm6,%ymm6 vpxor %ymm6,%ymm5,%ymm5 - vmovdqa %ymm5,%ymm10 - vpslld $12,%ymm10,%ymm10 + vpslld $12,%ymm5,%ymm10 vpsrld $20,%ymm5,%ymm5 vpor %ymm10,%ymm5,%ymm5
@@ -275,15 +269,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 7) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm10 - vpslld $7,%ymm10,%ymm10 + vpslld $7,%ymm1,%ymm10 vpsrld $25,%ymm1,%ymm1 vpor %ymm10,%ymm1,%ymm1 vpaddd %ymm7,%ymm6,%ymm6 vpxor %ymm6,%ymm5,%ymm5 - vmovdqa %ymm5,%ymm10 - vpslld $7,%ymm10,%ymm10 + vpslld $7,%ymm5,%ymm10 vpsrld $25,%ymm5,%ymm5 vpor %ymm10,%ymm5,%ymm5
@@ -309,15 +301,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 12) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm10 - vpslld $12,%ymm10,%ymm10 + vpslld $12,%ymm1,%ymm10 vpsrld $20,%ymm1,%ymm1 vpor %ymm10,%ymm1,%ymm1 vpaddd %ymm7,%ymm6,%ymm6 vpxor %ymm6,%ymm5,%ymm5 - vmovdqa %ymm5,%ymm10 - vpslld $12,%ymm10,%ymm10 + vpslld $12,%ymm5,%ymm10 vpsrld $20,%ymm5,%ymm5 vpor %ymm10,%ymm5,%ymm5
@@ -333,15 +323,13 @@ SYM_FUNC_START(chacha_2to4blocks_avx2) # x2 += x3, x1 = rotl32(x1 ^ x2, 7) vpaddd %ymm3,%ymm2,%ymm2 vpxor %ymm2,%ymm1,%ymm1 - vmovdqa %ymm1,%ymm10 - vpslld $7,%ymm10,%ymm10 + vpslld $7,%ymm1,%ymm10 vpsrld $25,%ymm1,%ymm1 vpor %ymm10,%ymm1,%ymm1 vpaddd %ymm7,%ymm6,%ymm6 vpxor %ymm6,%ymm5,%ymm5 - vmovdqa %ymm5,%ymm10 - vpslld $7,%ymm10,%ymm10 + vpslld $7,%ymm5,%ymm10 vpsrld $25,%ymm5,%ymm5 vpor %ymm10,%ymm5,%ymm5
base-commit: 33b4f31b1aea3f62b61c892daefc0e6a77859032
Signature · 1 line
2.56.1
Hi,
Quoted 5 lines · Eric Biggers
The AVX2 optimized ChaCha code has many instances of moving one register to another, then left-shifting the destination register with 'vpslld'. This pattern likely was copied from the SSSE3 code, which uses 'pslld'. However, like all the other AVX instructions, 'vpslld' has separate source and destination registers. Just utilize that instead.
Tested on a Ryzen 9 9950X in a KVM guest with AVX-512 hidden, so that the AVX2 code is used: the ChaCha and ChaCha20-Poly1305 KUnit tests pass.
The ChaCha benchmark shows no difference beyond noise (+/-0.6% at every length, 3 runs each).
Signature · 5 lines
Antonio Quartulli CEO and Co-Founder Mandelbit Srl https://www.mandelbit.com
Quoted 10 lines · Eric Biggers
The AVX2 optimized ChaCha code has many instances of moving one register to another, then left-shifting the destination register with 'vpslld'. This pattern likely was copied from the SSSE3 code, which uses 'pslld'. However, like all the other AVX instructions, 'vpslld' has separate source and destination registers. Just utilize that instead.
Signed-off-by Eric Biggers <[email protected]>--- lib/crypto/x86/chacha-avx2-x86_64.S | 36 ++++++++++------------------- 1 file changed, 12 insertions(+), 24 deletions(-)
Applied to https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git/log/?h=libcrypto-next
- Eric