lib/crypto: x86/chacha: Avoid overly-narrow keystream XORs
Subsystems (2)
Eric Biggers
maintainerPATCH 1/1 lib/crypto: x86/chacha: Avoid overly-narrow keystream XORs
chacha_2block_xor_avx2(), chacha_4block_xor_avx2(), chacha_2block_xor_avx512vl(), and chacha_4block_xor_avx512vl() currently XOR the data with the keystream 128 bits at a time. This is unnecessarily narrow; the same functions already use 256-bit vectors elsewhere. Do it 256 bits at a time instead, which is more efficient.
Signed-off-by Eric Biggers <[email protected]>
--- lib/crypto/x86/chacha-avx2-x86_64.S | 220 +++++++++----------------- lib/crypto/x86/chacha-avx512-x86_64.S | 220 +++++++++----------------- 2 files changed, 150 insertions(+), 290 deletions(-)
lib/crypto/x86/chacha-avx2-x86_64.S +75 -145
@@ -135,59 +135,37 @@ SYM_FUNC_START(chacha_2block_xor_avx2) sub $2,%r8d jnz .Ldoubleround - # o0 = i0 ^ (x0 + s0) - vpaddd %ymm8,%ymm0,%ymm7 - cmp $0x10,%rax - jl .Lxorpart2 - vpxor 0x00(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x00(%rsi) - vextracti128 $1,%ymm7,%xmm0 - # o1 = i1 ^ (x1 + s1) - vpaddd %ymm9,%ymm1,%ymm7 - cmp $0x20,%rax - jl .Lxorpart2 - vpxor 0x10(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x10(%rsi) - vextracti128 $1,%ymm7,%xmm1 - # o2 = i2 ^ (x2 + s2) - vpaddd %ymm10,%ymm2,%ymm7 - cmp $0x30,%rax - jl .Lxorpart2 - vpxor 0x20(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x20(%rsi) - vextracti128 $1,%ymm7,%xmm2 - # o3 = i3 ^ (x3 + s3) - vpaddd %ymm11,%ymm3,%ymm7 - cmp $0x40,%rax - jl .Lxorpart2 - vpxor 0x30(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x30(%rsi) - vextracti128 $1,%ymm7,%xmm3 + # x0 += s0, x1 += s1, x2 += s2, x3 += s3 + vpaddd %ymm8,%ymm0,%ymm0 + vpaddd %ymm9,%ymm1,%ymm1 + vpaddd %ymm10,%ymm2,%ymm2 + vpaddd %ymm11,%ymm3,%ymm3 - # xor and write second block - vmovdqa %xmm0,%xmm7 - cmp $0x50,%rax + # xor the data with the keystream + + vperm2i128 $0x20, %ymm1, %ymm0, %ymm4 + cmp $1*32,%rax jl .Lxorpart2 - vpxor 0x40(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x40(%rsi) + vpxor 0*32(%rdx),%ymm4,%ymm4 + vmovdqu %ymm4,0*32(%rsi) - vmovdqa %xmm1,%xmm7 - cmp $0x60,%rax + vperm2i128 $0x20, %ymm3, %ymm2, %ymm4 + cmp $2*32,%rax jl .Lxorpart2 - vpxor 0x50(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x50(%rsi) + vpxor 1*32(%rdx),%ymm4,%ymm4 + vmovdqu %ymm4,1*32(%rsi) - vmovdqa %xmm2,%xmm7 - cmp $0x70,%rax + vperm2i128 $0x31, %ymm1, %ymm0, %ymm4 + cmp $3*32,%rax jl .Lxorpart2 - vpxor 0x60(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x60(%rsi) + vpxor 2*32(%rdx),%ymm4,%ymm4 + vmovdqu %ymm4,2*32(%rsi) - vmovdqa %xmm3,%xmm7 - cmp $0x80,%rax + vperm2i128 $0x31, %ymm3, %ymm2, %ymm4 + cmp $4*32,%rax jl .Lxorpart2 - vpxor 0x70(%rdx),%xmm7,%xmm6 - vmovdqu %xmm6,0x70(%rsi) + vpxor 3*32(%rdx),%ymm4,%ymm4 + vmovdqu %ymm4,3*32(%rsi) .Ldone2: vzeroupper
@@ -196,14 +174,14 @@ SYM_FUNC_START(chacha_2block_xor_avx2) .Lxorpart2: # xor remaining bytes from partial register into output mov %rax,%r9 - and $0x0f,%r9 + and $31,%r9 jz .Ldone2 - and $~0x0f,%rax + and $~31,%rax mov %rsi,%r11 lea 8(%rsp),%r10 - sub $0x10,%rsp + sub $32,%rsp and $~31,%rsp lea (%rdx,%rax),%rsi
@@ -211,8 +189,8 @@ SYM_FUNC_START(chacha_2block_xor_avx2) mov %r9,%rcx rep movsb - vpxor 0x00(%rsp),%xmm7,%xmm7 - vmovdqa %xmm7,0x00(%rsp) + vpxor 0x00(%rsp),%ymm4,%ymm4 + vmovdqa %ymm4,0x00(%rsp) mov %rsp,%rsi lea (%r11,%rax),%rdi
@@ -384,113 +362,65 @@ SYM_FUNC_START(chacha_4block_xor_avx2) sub $2,%r8d jnz .Ldoubleround4 - # o0 = i0 ^ (x0 + s0), first block - vpaddd %ymm11,%ymm0,%ymm10 - cmp $0x10,%rax - jl .Lxorpart4 - vpxor 0x00(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x00(%rsi) - vextracti128 $1,%ymm10,%xmm0 - # o1 = i1 ^ (x1 + s1), first block - vpaddd %ymm12,%ymm1,%ymm10 - cmp $0x20,%rax - jl .Lxorpart4 - vpxor 0x10(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x10(%rsi) - vextracti128 $1,%ymm10,%xmm1 - # o2 = i2 ^ (x2 + s2), first block - vpaddd %ymm13,%ymm2,%ymm10 - cmp $0x30,%rax - jl .Lxorpart4 - vpxor 0x20(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x20(%rsi) - vextracti128 $1,%ymm10,%xmm2 - # o3 = i3 ^ (x3 + s3), first block - vpaddd %ymm14,%ymm3,%ymm10 - cmp $0x40,%rax - jl .Lxorpart4 - vpxor 0x30(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x30(%rsi) - vextracti128 $1,%ymm10,%xmm3 + # x0 += s0, x1 += s1, x2 += s2, x3 += s3 + vpaddd %ymm11,%ymm0,%ymm0 + vpaddd %ymm12,%ymm1,%ymm1 + vpaddd %ymm13,%ymm2,%ymm2 + vpaddd %ymm14,%ymm3,%ymm3 + vpaddd %ymm11,%ymm4,%ymm4 + vpaddd %ymm12,%ymm5,%ymm5 + vpaddd %ymm13,%ymm6,%ymm6 + vpaddd %ymm15,%ymm7,%ymm7 - # xor and write second block - vmovdqa %xmm0,%xmm10 - cmp $0x50,%rax - jl .Lxorpart4 - vpxor 0x40(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x40(%rsi) + # xor the data with the keystream - vmovdqa %xmm1,%xmm10 - cmp $0x60,%rax + vperm2i128 $0x20, %ymm1, %ymm0, %ymm8 + cmp $1*32,%rax jl .Lxorpart4 - vpxor 0x50(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x50(%rsi) + vpxor 0*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,0*32(%rsi) - vmovdqa %xmm2,%xmm10 - cmp $0x70,%rax + vperm2i128 $0x20, %ymm3, %ymm2, %ymm8 + cmp $2*32,%rax jl .Lxorpart4 - vpxor 0x60(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x60(%rsi) + vpxor 1*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,1*32(%rsi) - vmovdqa %xmm3,%xmm10 - cmp $0x80,%rax + vperm2i128 $0x31, %ymm1, %ymm0, %ymm8 + cmp $3*32,%rax jl .Lxorpart4 - vpxor 0x70(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x70(%rsi) + vpxor 2*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,2*32(%rsi) - # o0 = i0 ^ (x0 + s0), third block - vpaddd %ymm11,%ymm4,%ymm10 - cmp $0x90,%rax - jl .Lxorpart4 - vpxor 0x80(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x80(%rsi) - vextracti128 $1,%ymm10,%xmm4 - # o1 = i1 ^ (x1 + s1), third block - vpaddd %ymm12,%ymm5,%ymm10 - cmp $0xa0,%rax - jl .Lxorpart4 - vpxor 0x90(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0x90(%rsi) - vextracti128 $1,%ymm10,%xmm5 - # o2 = i2 ^ (x2 + s2), third block - vpaddd %ymm13,%ymm6,%ymm10 - cmp $0xb0,%rax - jl .Lxorpart4 - vpxor 0xa0(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0xa0(%rsi) - vextracti128 $1,%ymm10,%xmm6 - # o3 = i3 ^ (x3 + s3), third block - vpaddd %ymm15,%ymm7,%ymm10 - cmp $0xc0,%rax + vperm2i128 $0x31, %ymm3, %ymm2, %ymm8 + cmp $4*32,%rax jl .Lxorpart4 - vpxor 0xb0(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0xb0(%rsi) - vextracti128 $1,%ymm10,%xmm7 + vpxor 3*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,3*32(%rsi) - # xor and write fourth block - vmovdqa %xmm4,%xmm10 - cmp $0xd0,%rax + vperm2i128 $0x20, %ymm5, %ymm4, %ymm8 + cmp $5*32,%rax jl .Lxorpart4 - vpxor 0xc0(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0xc0(%rsi) + vpxor 4*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,4*32(%rsi) - vmovdqa %xmm5,%xmm10 - cmp $0xe0,%rax + vperm2i128 $0x20, %ymm7, %ymm6, %ymm8 + cmp $6*32,%rax jl .Lxorpart4 - vpxor 0xd0(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0xd0(%rsi) + vpxor 5*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,5*32(%rsi) - vmovdqa %xmm6,%xmm10 - cmp $0xf0,%rax + vperm2i128 $0x31, %ymm5, %ymm4, %ymm8 + cmp $7*32,%rax jl .Lxorpart4 - vpxor 0xe0(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0xe0(%rsi) + vpxor 6*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,6*32(%rsi) - vmovdqa %xmm7,%xmm10 - cmp $0x100,%rax + vperm2i128 $0x31, %ymm7, %ymm6, %ymm8 + cmp $8*32,%rax jl .Lxorpart4 - vpxor 0xf0(%rdx),%xmm10,%xmm9 - vmovdqu %xmm9,0xf0(%rsi) + vpxor 7*32(%rdx),%ymm8,%ymm8 + vmovdqu %ymm8,7*32(%rsi) .Ldone4: vzeroupper
@@ -499,14 +429,14 @@ SYM_FUNC_START(chacha_4block_xor_avx2) .Lxorpart4: # xor remaining bytes from partial register into output mov %rax,%r9 - and $0x0f,%r9 + and $31,%r9 jz .Ldone4 - and $~0x0f,%rax + and $~31,%rax mov %rsi,%r11 lea 8(%rsp),%r10 - sub $0x10,%rsp + sub $32,%rsp and $~31,%rsp lea (%rdx,%rax),%rsi
@@ -514,8 +444,8 @@ SYM_FUNC_START(chacha_4block_xor_avx2) mov %r9,%rcx rep movsb - vpxor 0x00(%rsp),%xmm10,%xmm10 - vmovdqa %xmm10,0x00(%rsp) + vpxor 0x00(%rsp),%ymm8,%ymm8 + vmovdqa %ymm8,0x00(%rsp) mov %rsp,%rsi lea (%r11,%rax),%rdi
lib/crypto/x86/chacha-avx512-x86_64.S +75 -145
281 lines not on this page · raw message
base-commit: 9fbbbaefd5f95551da18426c200cf41e1b4255a1
prerequisite-patch-id: 890b33260038bff2e3adfe7867af5919666677fc
prerequisite-patch-id: 36cadf8c358a9778d745637c00368df82a93ce0d
prerequisite-patch-id: b9ef92f4abcdfe8946d645db12187f6293a46381
Signature · 1 line
2.56.0
Tested on Zen 5 (Ryzen 9 9950X) in a KVM guest, on top of the bzhi, vzeroupper and interleaving patches. KUnit chacha suite + benchmark (64..16384 B, 3 runs each), compared with the tree without this patch:
path CPU setup KUnit perf ------------------- --------------------- ----- ------------ AVX-512, zmm native pass -1.8..+2.8% AVX-512VL, ymm only spoofed SKX (6/0x55) pass -0.2..+1.1% AVX2 clearcpuid=avx512f pass -1.2..+0.6%
Small but consistent gains at 192..1420 B on the AVX-512 paths; AVX2 is neutral.
Tested-by Antonio Quartulli <[email protected]>
Signature · 5 lines
Antonio Quartulli CEO and Co-Founder Mandelbit Srl https://www.mandelbit.com
Quoted 11 lines · Eric Biggers
chacha_2block_xor_avx2(),chacha_4block_xor_avx2(),chacha_2block_xor_avx512vl(), andchacha_4block_xor_avx512vl()currently XOR the data with the keystream 128 bits at a time. This is unnecessarily narrow; the same functions already use 256-bit vectors elsewhere. Do it 256 bits at a time instead, which is more efficient.Signed-off-by Eric Biggers <[email protected]>--- lib/crypto/x86/chacha-avx2-x86_64.S | 220 +++++++++----------------- lib/crypto/x86/chacha-avx512-x86_64.S | 220 +++++++++----------------- 2 files changed, 150 insertions(+), 290 deletions(-)
Applied to https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git/log/?h=libcrypto-next
- Eric