x86_64: use REP MOVSB in copy_page()

Patch changes market-ish acronyms like ERMS and chatty names to consistent and shorter versions:

xxx_mov
xxx_rep_stosq xxx_rep_movsq
xxx_rep_stosb xxx_rep_movsb

Signed-off-by Alexey Dobriyan <[email protected]>

---

 arch/x86/include/asm/page_64.h    |   12 ++++++------
 arch/x86/include/asm/uaccess_64.h |   18 +++++++++---------
 arch/x86/lib/clear_page_64.S      |   18 +++++++++---------
 arch/x86/lib/copy_user_64.S       |   20 ++++++++++----------
 tools/perf/ui/browsers/annotate.c |    2 +-
 5 files changed, 35 insertions(+), 35 deletions(-)

arch/x86/include/asm/page_64.h +6 -6

@@ -35,15 +35,15 @@ extern unsigned long __phys_addr_symbol(unsigned long);
 #define pfn_valid(pfn)          ((pfn) < max_pfn)
 #endif
 
-void clear_page_orig(void *page);
-void clear_page_rep(void *page);
-void clear_page_erms(void *page);
+void clear_page_mov(void *page);
+void clear_page_rep_stosq(void *page);
+void clear_page_rep_stosb(void *page);
 
 static inline void clear_page(void *page)
 {
-	alternative_call_2(clear_page_orig,
-			   clear_page_rep, X86_FEATURE_REP_GOOD,
-			   clear_page_erms, X86_FEATURE_ERMS,
+	alternative_call_2(clear_page_mov,
+			   clear_page_rep_stosq, X86_FEATURE_REP_GOOD,
+			   clear_page_rep_stosb, X86_FEATURE_ERMS,
 			   "=D" (page),
 			   "0" (page)
 			   : "memory", "rax", "rcx");

arch/x86/include/asm/uaccess_64.h +2 -2

 __must_check unsigned long
-copy_user_generic_string(void *to, const void *from, unsigned len);
+copy_user_rep_movsq(void *to, const void *from, unsigned len);
 __must_check unsigned long
-copy_user_generic_unrolled(void *to, const void *from, unsigned len);
+copy_user_mov(void *to, const void *from, unsigned len);
 
 static __always_inline __must_check unsigned long
 copy_user_generic(void *to, const void *from, unsigned len)

arch/x86/include/asm/uaccess_64.h +4 -3

+	 * Otherwise, use copy_user_mov.
 	 */
-	alternative_call_2(copy_user_generic_unrolled,
-			 copy_user_generic_string,
+	alternative_call_2(copy_user_mov,
+			 copy_user_rep_movsq,
 			 X86_FEATURE_REP_GOOD,
-			 copy_user_enhanced_fast_string,
+			 copy_user_rep_movsb,
 			 X86_FEATURE_ERMS,
 			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
 				     "=d" (len)),

arch/x86/lib/clear_page_64.S +4 -4

@@ -14,15 +14,15 @@
  * Zero a page.
  * %rdi	- page
  */
-ENTRY(clear_page_rep)
+ENTRY(clear_page_rep_stosq)
 	movl $4096/8,%ecx
 	xorl %eax,%eax
 	rep stosq
 	ret
-ENDPROC(clear_page_rep)
-EXPORT_SYMBOL_GPL(clear_page_rep)
+ENDPROC(clear_page_rep_stosq)
+EXPORT_SYMBOL_GPL(clear_page_rep_stosq)
 
-ENTRY(clear_page_orig)
+ENTRY(clear_page_mov)
 	xorl   %eax,%eax
 	movl   $4096/64,%ecx
 	.p2align 4

arch/x86/lib/clear_page_64.S +3 -3

-ENTRY(clear_page_erms)
+ENTRY(clear_page_rep_stosb)
 	movl $4096,%ecx
 	xorl %eax,%eax
 	rep stosb
 	ret
-ENDPROC(clear_page_erms)
-EXPORT_SYMBOL_GPL(clear_page_erms)
+ENDPROC(clear_page_rep_stosb)
+EXPORT_SYMBOL_GPL(clear_page_rep_stosb)

arch/x86/lib/copy_user_64.S +0 -0

  * This version is for CPUs like P4 that don't have efficient micro
  * code for rep movsq
  *

Avoid unconditional branch at every copy_page() call by using alternatives and calling optimal variant directly.

Rename individual versions to immediately show which one is used in profiles, etc.

RBX and R12 aren't clobbered because generic version restores them and both REP versions don't touch them.

Signed-off-by Alexey Dobriyan <[email protected]>

---

 arch/x86/include/asm/page_64.h |   16 ++++++++++++++--
 arch/x86/lib/copy_page_64.S    |   17 +++++++----------
 2 files changed, 21 insertions(+), 12 deletions(-)

arch/x86/include/asm/page_64.h +6 -0

+		copy_page_rep_movsb, X86_FEATURE_ERMS,
+		ASM_OUTPUT2("=D" (to), "=S" (from)),
+		"0" (to), "1" (from)
+		: "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "cc", "memory"
+	);
+}
 #endif	/* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_VSYSCALL_EMULATION

arch/x86/include/asm/page_64.h static inline void clear_page(void *page) · line 57

Quoted 3 lines · Alexey Dobriyan

arch/x86/include/asm/page_64.h +3 -0

+static __always_inline void copy_page(void *to, void *from)
+{
+     alternative_call_2(

Please align at the opening brace, like clear_page() above it:

        alternative_call_2(copy_page_mov,
                           copy_page_rep_movsq, X86_FEATURE_REP_GOOD,
                           ...

Quoted 9 lines

arch/x86/include/asm/page_64.h +6 -0

+             copy_page_rep_movsb, X86_FEATURE_ERMS,
+             ASM_OUTPUT2("=D" (to), "=S" (from)),
+             "0" (to), "1" (from)
+             : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "cc", "memory"
+     );
+}
 #endif       /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_VSYSCALL_EMULATION

...

Quoted 7 lines

ENTRY(copy_page_rep_movsb)

      mov     $4096, %ecx
      rep movsb
      ret

ENDPROC(copy_page_rep_movsb)
+EXPORT_SYMBOL(copy_page_rep_movsb)

arch/x86/lib/copy_page_64.S line 27

Quoted 2 lines

arch/x86/lib/copy_page_64.S +1 -1

-ENTRY(copy_page_regs)
+ENTRY(copy_page_mov)

copy_page_regs() is a better name IMO. copy_page_mov() doesn't tell me anything - all three use "mov". copy_page_unrolled() sounds ok too.

It says unambiguously which instruction does the actual copying.