x86: branchless clear_page()
PATCH 1/1 x86: branchless clear_page()
Apply alternatives at the call site instead of function body. Save branch per clean page.
Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.
---
arch/x86/include/asm/page_64.h | 16 +++++++++++++++- arch/x86/lib/clear_page_64.S | 18 ++++++------------ 2 files changed, 21 insertions(+), 13 deletions(-)
arch/x86/include/asm/page_64.h +15 -1
@@ -4,6 +4,7 @@#include <asm/page_64_types.h> #ifndef __ASSEMBLY__ +#include <asm/alternative.h> /* duplicated to the one in bootmem.h */ extern unsigned long max_pfn;
@@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long);#define pfn_valid (pfn) ((pfn) < max_pfn)#endif -void clear_page (void *page); +void clear_page_mov (void *page); +void clear_page_rep_stosq (void *page); +void clear_page_rep_stosb (void *page); +static __always_inline void clear_page(void *page) +{ +alternative_call_2 ( + clear_page_mov, + clear_page_rep_stosq,X86_FEATURE_REP_GOOD , + clear_page_rep_stosb,X86_FEATURE_ERMS , +"=D" (page), + "0" (page) + : "rax", "rcx", "memory" + ); +}void copy_page (void *to,void *from);#endif /* !__ASSEMBLY__ */
arch/x86/lib/clear_page_64.S +6 -12
@@ -1,6 +1,4 @@ #include <linux/linkage.h> -#include <asm/cpufeatures.h> -#include <asm/alternative-asm.h> /* * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is
@@ -13,18 +11,14 @@ * Zero a page. * %rdi - page */ -ENTRY(clear_page) - - ALTERNATIVE_2 "jmp clear_page_orig", "", X86_FEATURE_REP_GOOD, \ - "jmp clear_page_c_e", X86_FEATURE_ERMS - +ENTRY(clear_page_rep_stosq) movl $4096/8,%ecx xorl %eax,%eax rep stosq ret -ENDPROC(clear_page) +ENDPROC(clear_page_rep_stosq) -ENTRY(clear_page_orig) +ENTRY(clear_page_mov) xorl %eax,%eax movl $4096/64,%ecx
@@ -44,11 +38,11 @@ ENTRY(clear_page_orig) jnz .Lloop nop ret -ENDPROC(clear_page_orig) +ENDPROC(clear_page_mov) -ENTRY(clear_page_c_e) +ENTRY(clear_page_rep_stosb) movl $4096,%ecx xorl %eax,%eax rep stosb ret -ENDPROC(clear_page_c_e) +ENDPROC(clear_page_rep_stosb)
Quoted 5 lines · Alexey Dobriyan
Apply alternatives at the call site instead of function body. Save branch per clean page.
Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.
Rename individual clear_page() functions closer to implementation details (_c_e suffix sucks).
Borislav Petkov
maintainerQuoted 2 lines · Alexey Dobriyan
Apply alternatives at the call site instead of function body. Save branch per clean page.
Any size increase to an allyesconfig vmlinux and increase to alternatives application time?
Booting with "debug-alternative" should help.
Signature · 5 lines
Regards/Gruss,
Boris.
ECO tip #101: Trim your mails when you reply.
--Quoted 8 lines · Borislav Petkov
Quoted 2 lines · Alexey Dobriyan
Apply alternatives at the call site instead of function body. Save branch per clean page.
Any size increase to an allyesconfig vmlinux and increase to alternatives application time?
Booting with "debug-alternative" should help.
x86_64 allyesconfig kernel lists whole 56 calls to clear_page(). My usual kernel lists 8.
There are tons copy_from_user/copy_to_user entries and POPCNT and CLAC/STAC, so I think those several dozens clear pages don't matter.
Quoted 81 lines · Alexey Dobriyan
Apply alternatives at the call site instead of function body. Save branch per clean page.
Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.
Signed-off-by Alexey Dobriyan <[email protected]>---
arch/x86/include/asm/page_64.h | 16 +++++++++++++++- arch/x86/lib/clear_page_64.S | 18 ++++++------------ 2 files changed, 21 insertions(+), 13 deletions(-)arch/x86/include/asm/page_64.h +15
-1@@ -4,6 +4,7 @@#include <asm/page_64_types.h> #ifndef __ASSEMBLY__ +#include <asm/alternative.h> /* duplicated to the one in bootmem.h */ extern unsigned long max_pfn;@@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long);#define pfn_valid (pfn) ((pfn) < max_pfn)#endif -void clear_page (void *page); +void clear_page_mov (void *page); +void clear_page_rep_stosq (void *page); +void clear_page_rep_stosb (void *page); +static __always_inline void clear_page(void *page) +{ +alternative_call_2 ( + clear_page_mov, + clear_page_rep_stosq,X86_FEATURE_REP_GOOD , + clear_page_rep_stosb,X86_FEATURE_ERMS , +"=D" (page), + "0" (page) + : "rax", "rcx", "memory" + ); +}void copy_page (void *to,void *from);#endif /* !__ASSEMBLY__ */ arch/x86/lib/clear_page_64.S +3
-10@@ -1,6 +1,4 @@ #include <linux/linkage.h> -#include <asm/cpufeatures.h> -#include <asm/alternative-asm.h> /* * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is@@ -13,18 +11,14 @@ * Zero a page. * %rdi - page */ -ENTRY(clear_page) - - ALTERNATIVE_2 "jmp clear_page_orig", "", X86_FEATURE_REP_GOOD, \ - "jmp clear_page_c_e", X86_FEATURE_ERMS - +ENTRY(clear_page_rep_stosq) movl $4096/8,%ecx xorl %eax,%eax rep stosq ret -ENDPROC(clear_page) +ENDPROC(clear_page_rep_stosq) -ENTRY(clear_page_orig) +ENTRY(clear_page_mov) xorl %eax,%eax movl $4096/64,%ecx@@ -44,11 +38,11 @@ ENTRY(clear_page_orig) jnz .Lloop nop ret -ENDPROC(clear_page_orig)
arch/x86/lib/clear_page_64.S ENTRY(clear_page_orig) · line 48
Quoted 10 lines
arch/x86/lib/clear_page_64.S +3
-2+ENDPROC(clear_page_mov) -ENTRY(clear_page_c_e) +ENTRY(clear_page_rep_stosb) movl $4096,%ecx xorl %eax,%eax rep stosb ret -ENDPROC(clear_page_c_e) +ENDPROC(clear_page_rep_stosb)
I like this idea, but does it make sense to take it a step further and inline the string instruction alternatives to avoid a call altogether?
Also, 32-bit should be converted to do the same thing as 64-bit.
--
Brian Gerst
Quoted 11 lines · Brian Gerst
arch/x86/lib/clear_page_64.S ENTRY(clear_page_orig) · line 48
Quoted 8 lines
arch/x86/lib/clear_page_64.S +2
-2-ENTRY(clear_page_c_e) +ENTRY(clear_page_rep_stosb) movl $4096,%ecx xorl %eax,%eax rep stosb ret -ENDPROC(clear_page_c_e) +ENDPROC(clear_page_rep_stosb)I like this idea, but does it make sense to take it a step further and inline the string instruction alternatives to avoid a call altogether?
It is easy but you can't do runtime patching then.
Quoted 1 line
Also, 32-bit should be converted to do the same thing as 64-bit.
I don't run 32-bit. :-)
PATCH 1/3 x86: branchless clear_page()
In reply to Alexey DobriyanApply alternatives at the call site instead of function body. Save branch per clean page.
Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.
Rename individual clear_page() functions closer to implementation details.
---
Export individual functions.
arch/x86/include/asm/page_64.h | 16 +++++++++++++++- arch/x86/kernel/x8664_ksyms_64.c | 4 +++- arch/x86/lib/clear_page_64.S | 18 ++++++------------ 3 files changed, 24 insertions(+), 14 deletions(-)
arch/x86/include/asm/page_64.h +15 -1
@@ -4,6 +4,7 @@#include <asm/page_64_types.h> #ifndef __ASSEMBLY__ +#include <asm/alternative.h> /* duplicated to the one in bootmem.h */ extern unsigned long max_pfn;
@@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long);#define pfn_valid (pfn) ((pfn) < max_pfn)#endif -void clear_page (void *page); +void clear_page_mov (void *page); +void clear_page_rep_stosq (void *page); +void clear_page_rep_stosb (void *page); +static __always_inline void clear_page(void *page) +{ +alternative_call_2 ( + clear_page_mov, + clear_page_rep_stosq,X86_FEATURE_REP_GOOD , + clear_page_rep_stosb,X86_FEATURE_ERMS , +"=D" (page), + "0" (page) + : "rax", "rcx", "memory" + ); +}void copy_page (void *to,void *from);#endif /* !__ASSEMBLY__ */
arch/x86/kernel/x8664_ksyms_64.c +3 -1
@@ -41,7 +41,9 @@ EXPORT_SYMBOL(_copy_to_user);EXPORT_SYMBOL_GPL (memcpy_mcsafe);EXPORT_SYMBOL (copy_page); -EXPORT_SYMBOL (clear_page); +EXPORT_SYMBOL (clear_page_mov); +EXPORT_SYMBOL (clear_page_rep_stosq); +EXPORT_SYMBOL (clear_page_rep_stosb);EXPORT_SYMBOL (csum_partial);
arch/x86/lib/clear_page_64.S +6 -12
@@ -1,6 +1,4 @@ #include <linux/linkage.h> -#include <asm/cpufeatures.h> -#include <asm/alternative-asm.h> /* * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is
@@ -13,18 +11,14 @@ * Zero a page. * %rdi - page */ -ENTRY(clear_page) - - ALTERNATIVE_2 "jmp clear_page_orig", "", X86_FEATURE_REP_GOOD, \ - "jmp clear_page_c_e", X86_FEATURE_ERMS - +ENTRY(clear_page_rep_stosq) movl $4096/8,%ecx xorl %eax,%eax rep stosq ret -ENDPROC(clear_page) +ENDPROC(clear_page_rep_stosq) -ENTRY(clear_page_orig) +ENTRY(clear_page_mov) xorl %eax,%eax movl $4096/64,%ecx
@@ -44,11 +38,11 @@ ENTRY(clear_page_orig) jnz .Lloop nop ret -ENDPROC(clear_page_orig) +ENDPROC(clear_page_mov) -ENTRY(clear_page_c_e) +ENTRY(clear_page_rep_stosb) movl $4096,%ecx xorl %eax,%eax rep stosb ret -ENDPROC(clear_page_c_e) +ENDPROC(clear_page_rep_stosb)
PATCH 2/3 x86: support REP MOVSB copy_page()
Microbenchmark shows that "REP MOVSB" copy_page() is faster than "REP MOVSQ" version on Intel i5-something Haswell REP_GOOD/ERMS capable CPU.
N=1<<27
rep movsq: 6.758841901 ± 0.04%
rep movsb: 6.253927309 ± 0.02%
-----------------------------------
-7.5%
---
arch/x86/lib/copy_page_64.S | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
arch/x86/lib/copy_page_64.S +10 -1
@@ -12,12 +12,21 @@ */ ALIGN ENTRY(copy_page) - ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD + ALTERNATIVE_2 "jmp copy_page_regs", \ + "", X86_FEATURE_REP_GOOD, \ + "jmp copy_page_rep_movsb", X86_FEATURE_ERMS + movl $4096/8, %ecx rep movsq ret ENDPROC(copy_page) +ENTRY(copy_page_rep_movsb) + mov $4096, %ecx + rep movsb + ret +ENDPROC(copy_page_rep_movsb) + ENTRY(copy_page_regs) subq $2*8, %rsp movq %rbx, (%rsp)
PATCH 3/3 x86: branchless copy_page()
Apply alternatives at the call site instead of function body. Save branch per copied page.
Rename individual function, show immediately in profiles which method is being used.
RBX and R12 are saved and restored by unrolled-mov implementation, don't clobber them explicitly.
---
arch/x86/include/asm/page_64.h | 15 ++++++++++++++- arch/x86/kernel/x8664_ksyms_64.c | 4 +++- arch/x86/lib/copy_page_64.S | 14 ++++---------- 3 files changed, 21 insertions(+), 12 deletions(-)
arch/x86/include/asm/page_64.h +14 -1
@@ -49,8 +49,21 @@ static __always_inline void clear_page(void *page) : "rax", "rcx", "memory" ); } -void copy_page (void *to,void *from); +void copy_page_mov (void *to,void *from); +void copy_page_rep_movsq (void *to,void *from); +void copy_page_rep_movsb (void *to,void *from); +static __always_inline void copy_page(void *to, void *from) +{ +alternative_call_2 ( + copy_page_mov, + copy_page_rep_movsq,X86_FEATURE_REP_GOOD , + copy_page_rep_movsb,X86_FEATURE_ERMS , +ASM_OUTPUT2 ("=D" (to),"=S" (from)), + "0" (to), "1" (from) + : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "memory" + ); +}#endif /* !__ASSEMBLY__ */ #ifdef CONFIG_X86_VSYSCALL_EMULATION
arch/x86/kernel/x8664_ksyms_64.c +3 -1
@@ -40,7 +40,9 @@ EXPORT_SYMBOL(_copy_to_user);EXPORT_SYMBOL_GPL (memcpy_mcsafe); -EXPORT_SYMBOL (copy_page); +EXPORT_SYMBOL (copy_page_mov); +EXPORT_SYMBOL (copy_page_rep_movsq); +EXPORT_SYMBOL (copy_page_rep_movsb);EXPORT_SYMBOL (clear_page_mov);EXPORT_SYMBOL (clear_page_rep_stosq);EXPORT_SYMBOL (clear_page_rep_stosb);
arch/x86/lib/copy_page_64.S +4 -10
@@ -1,8 +1,6 @@ /* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */ #include <linux/linkage.h> -#include <asm/cpufeatures.h> -#include <asm/alternative-asm.h> /* * Some CPUs run faster using the string copy instructions (sane microcode).
@@ -11,15 +9,11 @@ * prefetch distance based on SMP/UP. */ ALIGN -ENTRY(copy_page) - ALTERNATIVE_2 "jmp copy_page_regs", \ - "", X86_FEATURE_REP_GOOD, \ - "jmp copy_page_rep_movsb", X86_FEATURE_ERMS - +ENTRY(copy_page_rep_movsq) movl $4096/8, %ecx rep movsq ret -ENDPROC(copy_page) +ENDPROC(copy_page_rep_movsq) ENTRY(copy_page_rep_movsb) mov $4096, %ecx
@@ -27,7 +21,7 @@ ENTRY(copy_page_rep_movsb) ret ENDPROC(copy_page_rep_movsb) -ENTRY(copy_page_regs) +ENTRY(copy_page_mov) subq $2*8, %rsp movq %rbx, (%rsp) movq %r12, 1*8(%rsp)
@@ -92,4 +86,4 @@ ENTRY(copy_page_regs) movq 1*8(%rsp), %r12 addq $2*8, %rsp ret -ENDPROC(copy_page_regs) +ENDPROC(copy_page_mov)