Skip to contentSkip to conversation

x86: branchless clear_page()

PATCH In review Alexey Dobriyan 3 participants 9 messages Aug 22, 2016 – Aug 23, 2016 x86

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86: branchless clear_page()

Apply alternatives at the call site instead of function body. Save branch per clean page.

Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.

---

 arch/x86/include/asm/page_64.h |   16 +++++++++++++++-
 arch/x86/lib/clear_page_64.S   |   18 ++++++------------
 2 files changed, 21 insertions(+), 13 deletions(-)
arch/x86/include/asm/page_64.h +15 -1
@@ -4,6 +4,7 @@
 #include <asm/page_64_types.h>
 
 #ifndef __ASSEMBLY__
+#include <asm/alternative.h>
 
 /* duplicated to the one in bootmem.h */
 extern unsigned long max_pfn;
@@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long);
 #define pfn_valid(pfn)          ((pfn) < max_pfn)
 #endif
 
-void clear_page(void *page);
+void clear_page_mov(void *page);
+void clear_page_rep_stosq(void *page);
+void clear_page_rep_stosb(void *page);
+static __always_inline void clear_page(void *page)
+{
+	alternative_call_2(
+		clear_page_mov,
+		clear_page_rep_stosq, X86_FEATURE_REP_GOOD,
+		clear_page_rep_stosb, X86_FEATURE_ERMS,
+		"=D" (page),
+		"0" (page)
+		: "rax", "rcx", "memory"
+	);
+}
 void copy_page(void *to, void *from);
 
 #endif	/* !__ASSEMBLY__ */
arch/x86/lib/clear_page_64.S +6 -12
@@ -1,6 +1,4 @@
 #include <linux/linkage.h>
-#include <asm/cpufeatures.h>
-#include <asm/alternative-asm.h>
 
 /*
  * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is
@@ -13,18 +11,14 @@
  * Zero a page.
  * %rdi	- page
  */
-ENTRY(clear_page)
-
-	ALTERNATIVE_2 "jmp clear_page_orig", "", X86_FEATURE_REP_GOOD, \
-		      "jmp clear_page_c_e", X86_FEATURE_ERMS
-
+ENTRY(clear_page_rep_stosq)
 	movl $4096/8,%ecx
 	xorl %eax,%eax
 	rep stosq
 	ret
-ENDPROC(clear_page)
+ENDPROC(clear_page_rep_stosq)
 
-ENTRY(clear_page_orig)
+ENTRY(clear_page_mov)
 
 	xorl   %eax,%eax
 	movl   $4096/64,%ecx
@@ -44,11 +38,11 @@ ENTRY(clear_page_orig)
 	jnz	.Lloop
 	nop
 	ret
-ENDPROC(clear_page_orig)
+ENDPROC(clear_page_mov)
 
-ENTRY(clear_page_c_e)
+ENTRY(clear_page_rep_stosb)
 	movl $4096,%ecx
 	xorl %eax,%eax
 	rep stosb
 	ret
-ENDPROC(clear_page_c_e)
+ENDPROC(clear_page_rep_stosb)
#1 lore raw
Quoted 5 lines · Alexey Dobriyan

Apply alternatives at the call site instead of function body. Save branch per clean page.

Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.

Rename individual clear_page() functions closer to implementation details (_c_e suffix sucks).

maintainer
#2 lore raw
In reply to Alexey Dobriyan
Quoted 2 lines · Alexey Dobriyan

Apply alternatives at the call site instead of function body. Save branch per clean page.

Any size increase to an allyesconfig vmlinux and increase to alternatives application time?

Booting with "debug-alternative" should help.

Signature · 5 lines
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--
#3 lore raw
Quoted 8 lines · Borislav Petkov
Quoted 2 lines · Alexey Dobriyan

Apply alternatives at the call site instead of function body. Save branch per clean page.

Any size increase to an allyesconfig vmlinux and increase to alternatives application time?

Booting with "debug-alternative" should help.

x86_64 allyesconfig kernel lists whole 56 calls to clear_page(). My usual kernel lists 8.

There are tons copy_from_user/copy_to_user entries and POPCNT and CLAC/STAC, so I think those several dozens clear pages don't matter.

#4 lore raw
In reply to Alexey Dobriyan
Quoted 81 lines · Alexey Dobriyan

Apply alternatives at the call site instead of function body. Save branch per clean page.

Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.

---

 arch/x86/include/asm/page_64.h |   16 +++++++++++++++-
 arch/x86/lib/clear_page_64.S   |   18 ++++++------------
 2 files changed, 21 insertions(+), 13 deletions(-)
arch/x86/include/asm/page_64.h +15 -1
@@ -4,6 +4,7 @@
 #include <asm/page_64_types.h>
 
 #ifndef __ASSEMBLY__
+#include <asm/alternative.h>
 
 /* duplicated to the one in bootmem.h */
 extern unsigned long max_pfn;
@@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long);
 #define pfn_valid(pfn)          ((pfn) < max_pfn)
 #endif
 
-void clear_page(void *page);
+void clear_page_mov(void *page);
+void clear_page_rep_stosq(void *page);
+void clear_page_rep_stosb(void *page);
+static __always_inline void clear_page(void *page)
+{
+       alternative_call_2(
+               clear_page_mov,
+               clear_page_rep_stosq, X86_FEATURE_REP_GOOD,
+               clear_page_rep_stosb, X86_FEATURE_ERMS,
+               "=D" (page),
+               "0" (page)
+               : "rax", "rcx", "memory"
+       );
+}
 void copy_page(void *to, void *from);
 
 #endif /* !__ASSEMBLY__ */
arch/x86/lib/clear_page_64.S +3 -10
@@ -1,6 +1,4 @@
 #include <linux/linkage.h>
-#include <asm/cpufeatures.h>
-#include <asm/alternative-asm.h>
 
 /*
  * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is
@@ -13,18 +11,14 @@
  * Zero a page.
  * %rdi        - page
  */
-ENTRY(clear_page)
-
-       ALTERNATIVE_2 "jmp clear_page_orig", "", X86_FEATURE_REP_GOOD, \
-                     "jmp clear_page_c_e", X86_FEATURE_ERMS
-
+ENTRY(clear_page_rep_stosq)
        movl $4096/8,%ecx
        xorl %eax,%eax
        rep stosq
        ret
-ENDPROC(clear_page)
+ENDPROC(clear_page_rep_stosq)
 
-ENTRY(clear_page_orig)
+ENTRY(clear_page_mov)
 
        xorl   %eax,%eax
        movl   $4096/64,%ecx
@@ -44,11 +38,11 @@ ENTRY(clear_page_orig)
        jnz     .Lloop
        nop
        ret
-ENDPROC(clear_page_orig)

arch/x86/lib/clear_page_64.S ENTRY(clear_page_orig) · line 48

Quoted 10 lines
arch/x86/lib/clear_page_64.S +3 -2
+ENDPROC(clear_page_mov)
 
-ENTRY(clear_page_c_e)
+ENTRY(clear_page_rep_stosb)
        movl $4096,%ecx
        xorl %eax,%eax
        rep stosb
        ret
-ENDPROC(clear_page_c_e)
+ENDPROC(clear_page_rep_stosb)

I like this idea, but does it make sense to take it a step further and inline the string instruction alternatives to avoid a call altogether?

Also, 32-bit should be converted to do the same thing as 64-bit.

--
Brian Gerst

#5 lore raw
Quoted 11 lines · Brian Gerst

arch/x86/lib/clear_page_64.S ENTRY(clear_page_orig) · line 48

Quoted 8 lines
arch/x86/lib/clear_page_64.S +2 -2
-ENTRY(clear_page_c_e)
+ENTRY(clear_page_rep_stosb)
        movl $4096,%ecx
        xorl %eax,%eax
        rep stosb
        ret
-ENDPROC(clear_page_c_e)
+ENDPROC(clear_page_rep_stosb)

I like this idea, but does it make sense to take it a step further and inline the string instruction alternatives to avoid a call altogether?

It is easy but you can't do runtime patching then.

Quoted 1 line

Also, 32-bit should be converted to do the same thing as 64-bit.

I don't run 32-bit. :-)

#6 lore raw

PATCH 1/3 x86: branchless clear_page()

In reply to Alexey Dobriyan

Apply alternatives at the call site instead of function body. Save branch per clean page.

Bonus: tell gcc to not flush whole shebang of registers, just RDI, RAX, RCX.

Rename individual clear_page() functions closer to implementation details.

---

Export individual functions.

 arch/x86/include/asm/page_64.h   |   16 +++++++++++++++-
 arch/x86/kernel/x8664_ksyms_64.c |    4 +++-
 arch/x86/lib/clear_page_64.S     |   18 ++++++------------
 3 files changed, 24 insertions(+), 14 deletions(-)
arch/x86/include/asm/page_64.h +15 -1
@@ -4,6 +4,7 @@
 #include <asm/page_64_types.h>
 
 #ifndef __ASSEMBLY__
+#include <asm/alternative.h>
 
 /* duplicated to the one in bootmem.h */
 extern unsigned long max_pfn;
@@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long);
 #define pfn_valid(pfn)          ((pfn) < max_pfn)
 #endif
 
-void clear_page(void *page);
+void clear_page_mov(void *page);
+void clear_page_rep_stosq(void *page);
+void clear_page_rep_stosb(void *page);
+static __always_inline void clear_page(void *page)
+{
+	alternative_call_2(
+		clear_page_mov,
+		clear_page_rep_stosq, X86_FEATURE_REP_GOOD,
+		clear_page_rep_stosb, X86_FEATURE_ERMS,
+		"=D" (page),
+		"0" (page)
+		: "rax", "rcx", "memory"
+	);
+}
 void copy_page(void *to, void *from);
 
 #endif	/* !__ASSEMBLY__ */
arch/x86/kernel/x8664_ksyms_64.c +3 -1
@@ -41,7 +41,9 @@ EXPORT_SYMBOL(_copy_to_user);
 EXPORT_SYMBOL_GPL(memcpy_mcsafe);
 
 EXPORT_SYMBOL(copy_page);
-EXPORT_SYMBOL(clear_page);
+EXPORT_SYMBOL(clear_page_mov);
+EXPORT_SYMBOL(clear_page_rep_stosq);
+EXPORT_SYMBOL(clear_page_rep_stosb);
 
 EXPORT_SYMBOL(csum_partial);
 
arch/x86/lib/clear_page_64.S +6 -12
@@ -1,6 +1,4 @@
 #include <linux/linkage.h>
-#include <asm/cpufeatures.h>
-#include <asm/alternative-asm.h>
 
 /*
  * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is
@@ -13,18 +11,14 @@
  * Zero a page.
  * %rdi	- page
  */
-ENTRY(clear_page)
-
-	ALTERNATIVE_2 "jmp clear_page_orig", "", X86_FEATURE_REP_GOOD, \
-		      "jmp clear_page_c_e", X86_FEATURE_ERMS
-
+ENTRY(clear_page_rep_stosq)
 	movl $4096/8,%ecx
 	xorl %eax,%eax
 	rep stosq
 	ret
-ENDPROC(clear_page)
+ENDPROC(clear_page_rep_stosq)
 
-ENTRY(clear_page_orig)
+ENTRY(clear_page_mov)
 
 	xorl   %eax,%eax
 	movl   $4096/64,%ecx
@@ -44,11 +38,11 @@ ENTRY(clear_page_orig)
 	jnz	.Lloop
 	nop
 	ret
-ENDPROC(clear_page_orig)
+ENDPROC(clear_page_mov)
 
-ENTRY(clear_page_c_e)
+ENTRY(clear_page_rep_stosb)
 	movl $4096,%ecx
 	xorl %eax,%eax
 	rep stosb
 	ret
-ENDPROC(clear_page_c_e)
+ENDPROC(clear_page_rep_stosb)
#7 lore raw

PATCH 2/3 x86: support REP MOVSB copy_page()

Microbenchmark shows that "REP MOVSB" copy_page() is faster than "REP MOVSQ" version on Intel i5-something Haswell REP_GOOD/ERMS capable CPU.

N=1<<27
rep movsq: 6.758841901 ± 0.04%
rep movsb: 6.253927309 ± 0.02%
-----------------------------------

			-7.5%

---

 arch/x86/lib/copy_page_64.S |   11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
arch/x86/lib/copy_page_64.S +10 -1
@@ -12,12 +12,21 @@
  */
 	ALIGN
 ENTRY(copy_page)
-	ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD
+	ALTERNATIVE_2 "jmp copy_page_regs",	\
+		"", X86_FEATURE_REP_GOOD,	\
+		"jmp copy_page_rep_movsb", X86_FEATURE_ERMS
+
 	movl	$4096/8, %ecx
 	rep	movsq
 	ret
 ENDPROC(copy_page)
 
+ENTRY(copy_page_rep_movsb)
+	mov	$4096, %ecx
+	rep movsb
+	ret
+ENDPROC(copy_page_rep_movsb)
+
 ENTRY(copy_page_regs)
 	subq	$2*8,	%rsp
 	movq	%rbx,	(%rsp)
#8 lore raw

PATCH 3/3 x86: branchless copy_page()

Apply alternatives at the call site instead of function body. Save branch per copied page.

Rename individual function, show immediately in profiles which method is being used.

RBX and R12 are saved and restored by unrolled-mov implementation, don't clobber them explicitly.

---

 arch/x86/include/asm/page_64.h   |   15 ++++++++++++++-
 arch/x86/kernel/x8664_ksyms_64.c |    4 +++-
 arch/x86/lib/copy_page_64.S      |   14 ++++----------
 3 files changed, 21 insertions(+), 12 deletions(-)
arch/x86/include/asm/page_64.h +14 -1
@@ -49,8 +49,21 @@ static __always_inline void clear_page(void *page)
 		: "rax", "rcx", "memory"
 	);
 }
-void copy_page(void *to, void *from);
 
+void copy_page_mov(void *to, void *from);
+void copy_page_rep_movsq(void *to, void *from);
+void copy_page_rep_movsb(void *to, void *from);
+static __always_inline void copy_page(void *to, void *from)
+{
+	alternative_call_2(
+		copy_page_mov,
+		copy_page_rep_movsq, X86_FEATURE_REP_GOOD,
+		copy_page_rep_movsb, X86_FEATURE_ERMS,
+		ASM_OUTPUT2("=D" (to), "=S" (from)),
+		"0" (to), "1" (from)
+		: "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "memory"
+	);
+}
 #endif	/* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_VSYSCALL_EMULATION
arch/x86/kernel/x8664_ksyms_64.c +3 -1
@@ -40,7 +40,9 @@ EXPORT_SYMBOL(_copy_to_user);
 
 EXPORT_SYMBOL_GPL(memcpy_mcsafe);
 
-EXPORT_SYMBOL(copy_page);
+EXPORT_SYMBOL(copy_page_mov);
+EXPORT_SYMBOL(copy_page_rep_movsq);
+EXPORT_SYMBOL(copy_page_rep_movsb);
 EXPORT_SYMBOL(clear_page_mov);
 EXPORT_SYMBOL(clear_page_rep_stosq);
 EXPORT_SYMBOL(clear_page_rep_stosb);
arch/x86/lib/copy_page_64.S +4 -10
@@ -1,8 +1,6 @@
 /* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */
 
 #include <linux/linkage.h>
-#include <asm/cpufeatures.h>
-#include <asm/alternative-asm.h>
 
 /*
  * Some CPUs run faster using the string copy instructions (sane microcode).
@@ -11,15 +9,11 @@
  * prefetch distance based on SMP/UP.
  */
 	ALIGN
-ENTRY(copy_page)
-	ALTERNATIVE_2 "jmp copy_page_regs",	\
-		"", X86_FEATURE_REP_GOOD,	\
-		"jmp copy_page_rep_movsb", X86_FEATURE_ERMS
-
+ENTRY(copy_page_rep_movsq)
 	movl	$4096/8, %ecx
 	rep	movsq
 	ret
-ENDPROC(copy_page)
+ENDPROC(copy_page_rep_movsq)
 
 ENTRY(copy_page_rep_movsb)
 	mov	$4096, %ecx
@@ -27,7 +21,7 @@ ENTRY(copy_page_rep_movsb)
 	ret
 ENDPROC(copy_page_rep_movsb)
 
-ENTRY(copy_page_regs)
+ENTRY(copy_page_mov)
 	subq	$2*8,	%rsp
 	movq	%rbx,	(%rsp)
 	movq	%r12,	1*8(%rsp)
@@ -92,4 +86,4 @@ ENTRY(copy_page_regs)
 	movq	1*8(%rsp), %r12
 	addq	$2*8, %rsp
 	ret
-ENDPROC(copy_page_regs)
+ENDPROC(copy_page_mov)