lib/raid: opt out of initializing MXCSR and 387 state
Subsystems (3)
cover lib/raid: opt out of initializing MXCSR and 387 state
This series slightly reduces the overhead of the x86 kernel-mode "FPU" sections in the RAID code by using kernel_fpu_begin_mask(0) instead of kernel_fpu_begin(), thus opting out of the 'ldmxcsr' (and on 32-bit kernels, also 'fninit'). MXCSR is irrelevant for integer code.
This mirrors similar optimizations for the crypto and CRC code
(https://lore.kernel.org/linux-crypto/[email protected]/).
It's more important there than in the RAID code, since short lengths are common in the x86-optimized crypto and CRC functions. However, I think it's worth updating the RAID code too for consistency.
Unlike the crypto and CRC code, the RAID code is built for user-mode Linux. Therefore, as a prerequisite for this change, patch 1 also defines a stub for kernel_fpu_begin_mask() on user-mode Linux.
This series is targeting mm/mm-nonmm-unstable.
Eric Biggers (3):
um: Define kernel_fpu_begin_mask()
xor: opt out of initializing MXCSR and 387 state
raid6: opt out of initializing MXCSR and 387 state
arch/um/include/asm/fpu/api.h | 1 + lib/raid/raid6/x86/avx2.c | 12 ++++++------ lib/raid/raid6/x86/avx512.c | 12 ++++++------ lib/raid/raid6/x86/recov_avx2.c | 4 ++-- lib/raid/raid6/x86/recov_avx512.c | 4 ++-- lib/raid/raid6/x86/recov_ssse3.c | 4 ++-- lib/raid/raid6/x86/sse2.c | 12 ++++++------ lib/raid/xor/x86/xor-avx.c | 2 +- lib/raid/xor/x86/xor-avx512.c | 2 +- lib/raid/xor/x86/xor-sse.c | 4 ++-- 10 files changed, 29 insertions(+), 28 deletions(-)
base-commit: d0ad02b9d5dd7ae551f450cfd233f3e28bbf59d3
Signature · 1 line
2.56.1
PATCH 1/3 um: Define kernel_fpu_begin_mask()
Define a stub for kernel_fpu_begin_mask() on user-mode Linux so that lib/raid/ can switch from kernel_fpu_begin() to kernel_fpu_begin_mask().
Note: this causes a checkpatch error due to the macro's value not having outer parentheses, but it's left as-is for consistency with the existing kernel_fpu_begin() and kernel_fpu_end() definitions right next to it.
--- arch/um/include/asm/fpu/api.h | 1 + 1 file changed, 1 insertion(+)
arch/um/include/asm/fpu/api.h +1 -0
@@ -11,6 +11,7 @@* UML code tree. */ #define kernel_fpu_begin () (void)0 +#define kernel_fpu_begin_mask (mask) (void)(mask)#define kernel_fpu_end () (void)0static inline bool irq_fpu_usable (void )
Signature · 1 line
2.56.1
Johannes Berg
maintainerRe: [PATCH 1/3] um: Define kernel_fpu_begin_mask()
Quoted 7 lines · Eric Biggers
Define a stub for
kernel_fpu_begin_mask()on user-mode Linux so thatlib/raid/can switch fromkernel_fpu_begin()tokernel_fpu_begin_mask().Note: this causes a checkpatch error due to the macro's value not having outer parentheses, but it's left as-is for consistency with the existing
kernel_fpu_begin()andkernel_fpu_end()definitions right next to it.
Probably should've CCed linux-um (done now), but the change seems fine. There's no FPU handling needed in UM since the host kernel deals with it.
johannes
PATCH 2/3 xor: opt out of initializing MXCSR and 387 state
In reply to Eric BiggersThe XOR code using "kernel-mode FPU" is integer code that does not use any MMX or x87 instructions. (MMX optimized code previously existed but was recently removed.) To the extent that float-domain instructions are used at all, they are only moves, shuffles, and XORs, none of which raises SIMD floating-point exceptions or depends on MXCSR.
Therefore, there's no need to initialize MXCSR or the 387 FPU state.
Opt out of this initialization by using kernel_fpu_begin_mask(0) instead of kernel_fpu_begin(). This eliminates an 'ldmxcsr' instruction in every kernel-mode FPU section, reducing overhead, which matters most for short input lengths. On 32-bit kernels it also eliminates 'fninit'.
This will make the RAID code consistent with the crypto and CRC code, which are also switching to kernel_fpu_begin_mask(0)
(https://lore.kernel.org/linux-crypto/[email protected]/).
Another precedent for this optimization can be found in commit a58db7ad80e8 ("netfilter: nft_set_pipapo_avx2: Skip LDMXCSR, we don't need a valid MXCSR state").
--- lib/raid/xor/x86/xor-avx.c | 2 +- lib/raid/xor/x86/xor-avx512.c | 2 +- lib/raid/xor/x86/xor-sse.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-)
lib/raid/xor/x86/xor-avx.c +1 -1
@@ -145,7 +145,7 @@ DO_XOR_BLOCKS(avx_inner, xor_avx_2, xor_avx_3, xor_avx_4, xor_avx_5);static void xor_gen_avx (void *dest,void **srcs,unsigned int src_cnt,unsigned int bytes) { -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 );xor_gen_avx_inner (dest, srcs, src_cnt, bytes); asmvolatile ("vzeroupper" );kernel_fpu_end ();
lib/raid/xor/x86/xor-avx512.c +1 -1
@@ -110,7 +110,7 @@ DO_XOR_BLOCKS(avx512_inner, xor_avx512_2, xor_avx512_3, xor_avx512_4,static void xor_gen_avx512 (void *dest,void **srcs,unsigned int src_cnt,unsigned int bytes) { -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 );xor_gen_avx512_inner (dest, srcs, src_cnt, bytes); asmvolatile ("vzeroupper" );kernel_fpu_end ();
lib/raid/xor/x86/xor-sse.c +2 -2
@@ -432,7 +432,7 @@ DO_XOR_BLOCKS(sse_inner, xor_sse_2, xor_sse_3, xor_sse_4, xor_sse_5);static void xor_gen_sse (void *dest,void **srcs,unsigned int src_cnt,unsigned int bytes) { -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 );xor_gen_sse_inner (dest, srcs, src_cnt, bytes);kernel_fpu_end (); }
@@ -448,7 +448,7 @@ DO_XOR_BLOCKS(sse_pf64_inner, xor_sse_2_pf64, xor_sse_3_pf64, xor_sse_4_pf64,static void xor_gen_sse_pf64 (void *dest,void **srcs,unsigned int src_cnt,unsigned int bytes) { -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 );xor_gen_sse_pf64_inner (dest, srcs, src_cnt, bytes);kernel_fpu_end (); }
Signature · 1 line
2.56.1
PATCH 3/3 raid6: opt out of initializing MXCSR and 387 state
In reply to Eric BiggersThe RAID6 code using "kernel-mode FPU" is integer code that does not use any MMX or x87 instructions. (MMX optimized code previously existed but was recently removed.) To the extent that float-domain instructions are used at all, they are only moves, shuffles, and XORs, none of which raises SIMD floating-point exceptions or depends on MXCSR.
Therefore, there's no need to initialize MXCSR or the 387 FPU state.
Opt out of this initialization by using kernel_fpu_begin_mask(0) instead of kernel_fpu_begin(). This eliminates an 'ldmxcsr' instruction in every kernel-mode FPU section, reducing overhead, which matters most for short input lengths. On 32-bit kernels it also eliminates 'fninit'.
This will make the RAID code consistent with the crypto and CRC code, which are also switching to kernel_fpu_begin_mask(0)
(https://lore.kernel.org/linux-crypto/[email protected]/).
Another precedent for this optimization can be found in commit a58db7ad80e8 ("netfilter: nft_set_pipapo_avx2: Skip LDMXCSR, we don't need a valid MXCSR state").
--- lib/raid/raid6/x86/avx2.c | 12 ++++++------ lib/raid/raid6/x86/avx512.c | 12 ++++++------ lib/raid/raid6/x86/recov_avx2.c | 4 ++-- lib/raid/raid6/x86/recov_avx512.c | 4 ++-- lib/raid/raid6/x86/recov_ssse3.c | 4 ++-- lib/raid/raid6/x86/sse2.c | 12 ++++++------ 6 files changed, 24 insertions(+), 24 deletions(-)
lib/raid/raid6/x86/avx2.c +6 -6
@@ -32,7 +32,7 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa %0,%%ymm0" : :"m" (raid6_avx2_constants.x1d[0 ])); asmvolatile ("vpxor %ymm3,%ymm3,%ymm3" );/* Zero temp */
@@ -82,7 +82,7 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa %0,%%ymm0" : :"m" (raid6_avx2_constants.x1d[0 ]));
@@ -139,7 +139,7 @@ static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa %0,%%ymm0" : :"m" (raid6_avx2_constants.x1d[0 ])); asmvolatile ("vpxor %ymm1,%ymm1,%ymm1" );/* Zero temp */
@@ -192,7 +192,7 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa %0,%%ymm0" : :"m" (raid6_avx2_constants.x1d[0 ]));
@@ -271,7 +271,7 @@ static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa %0,%%ymm0" : :"m" (raid6_avx2_constants.x1d[0 ])); asmvolatile ("vpxor %ymm1,%ymm1,%ymm1" );/* Zero temp */
@@ -353,7 +353,7 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa %0,%%ymm0" ::"m" (raid6_avx2_constants.x1d[0 ]));
lib/raid/raid6/x86/avx512.c +6 -6
@@ -34,7 +34,7 @@ static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa64 %0,%%zmm0\n\t" "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
@@ -93,7 +93,7 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa64 %0,%%zmm0" : :"m" (raid6_avx512_constants.x1d[0 ]));
@@ -161,7 +161,7 @@ static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa64 %0,%%zmm0\n\t" "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
@@ -225,7 +225,7 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa64 %0,%%zmm0" : :"m" (raid6_avx512_constants.x1d[0 ]));
@@ -320,7 +320,7 @@ static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa64 %0,%%zmm0\n\t" "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t" /* Zero temp */
@@ -415,7 +415,7 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vmovdqa64 %0,%%zmm0" ::"m" (raid6_avx512_constants.x1d[0 ]));
lib/raid/raid6/x86/recov_avx2.c +2 -2
@@ -43,7 +43,7 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 );/* ymm0 = x0f[16] */ asmvolatile ("vpbroadcastb %0, %%ymm7" : :"m" (x0f));
@@ -205,7 +205,7 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,/* Now, pick the proper data tables */ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vpbroadcastb %0, %%ymm7" : :"m" (x0f));
lib/raid/raid6/x86/recov_avx512.c +2 -2
@@ -48,7 +48,7 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 );/* zmm0 = x0f[16] */ asmvolatile ("vpbroadcastb %0, %%zmm7" : :"m" (x0f));
@@ -243,7 +243,7 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,/* Now, pick the proper data tables */ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("vpbroadcastb %0, %%zmm7" : :"m" (x0f));
lib/raid/raid6/x86/recov_ssse3.c +2 -2
@@ -44,7 +44,7 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm7" : :"m" (x0f[0 ]));
@@ -210,7 +210,7 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,/* Now, pick the proper data tables */ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0, %%xmm7" : :"m" (x0f[0 ]));
lib/raid/raid6/x86/sse2.c +6 -6
@@ -28,7 +28,7 @@ static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm0" : :"m" (raid6_sse_constants.x1d[0 ])); asmvolatile ("pxor %xmm5,%xmm5" );/* Zero temp */
@@ -80,7 +80,7 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm0" : :"m" (raid6_sse_constants.x1d[0 ]));
@@ -136,7 +136,7 @@ static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm0" : :"m" (raid6_sse_constants.x1d[0 ])); asmvolatile ("pxor %xmm5,%xmm5" );/* Zero temp */
@@ -189,7 +189,7 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm0" : :"m" (raid6_sse_constants.x1d[0 ]));
@@ -266,7 +266,7 @@ static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs) p = dptr[z0+1 ];/* XOR parity */ q = dptr[z0+2 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm0" ::"m" (raid6_sse_constants.x1d[0 ])); asmvolatile ("pxor %xmm2,%xmm2" );/* P[0] */
@@ -353,7 +353,7 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop, p = dptr[disks-2 ];/* XOR parity */ q = dptr[disks-1 ];/* RS syndrome */ -kernel_fpu_begin (); +kernel_fpu_begin_mask (0 ); asmvolatile ("movdqa %0,%%xmm0" ::"m" (raid6_sse_constants.x1d[0 ]));
Signature · 1 line
2.56.1
Quoted 4 lines · Eric Biggers
This series slightly reduces the overhead of the x86 kernel-mode "FPU" sections in the RAID code by using
kernel_fpu_begin_mask(0)instead ofkernel_fpu_begin(), thus opting out of the 'ldmxcsr' (and on 32-bit kernels, also 'fninit'). MXCSR is irrelevant for integer code.
Maybe we really want a kernel_avx_{begin,end} after all :)
I'd still prefer that, but in the meantime the change looks fine: