Skip to contentSkip to conversation

x86: memcpy() / memset() (non-)ERMS flavors plus fallout

PATCH Superseded Jan Beulich 4 participants 18 messages Jun 5, 2025 – Jun 10, 2025 RISC-V

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v5 cover x86: memcpy() / memset() (non-)ERMS flavors plus fallout

While the performance varies quite a bit on older (pre-ERMS) and newer (ERMS) hardware, so far we've been going with just a single flavor of these two functions, and oddly enough with ones not consistent with one another. Using plain memcpy() / memset() on MMIO (video frame buffer) is generally okay, but the ERMS variant of memcpy() turned out to regress (boot) performance in a way easily visible to the human eye (meanwhile addressed by using ioremap_wc() there).

01: x86: suppress ERMS for internal use when MISC_ENABLE.FAST_STRING is clear 02: x86: re-work memset()
03: x86: re-work memcpy()
04: x86: control memset() and memcpy() inlining
05: x86: introduce "hot" and "cold" page clearing functions
06: mm: allow page scrubbing routine(s) to be arch controlled

Jan

#1 lore raw

PATCH v5 1/6 x86: suppress ERMS for internal use when MISC_ENABLE.FAST_STRING is clear

Before we start actually adjusting behavior when ERMS is available, follow Linux commit 161ec53c702c ("x86, mem, intel: Initialize Enhanced REP MOVSB/STOSB") and zap the CPUID-derived feature flag when the MSR bit is clear. Don't extend the artificial clearing to guest view,
though: Guests can take their own decision in this regard, as they can read (most of) MISC_ENABLE.

---
TBD: Would be nice if "cpuid=no-erms" propagated to guest view (for
"cpuid=" generally meaning to affect guests as well as Xen), but since both disabling paths use setup_clear_cpu_cap() they're indistinguishable in guest_common_feature_adjustments(). A separate boolean could take care of this, but would look clumsy to me.
---
v5: Correct guest_common_max_feature_adjustments() addition.
v4: Also adjust guest_common_max_feature_adjustments().
v3: New.

xen/arch/x86/cpu/intel.c +10 -0
@@ -366,8 +366,18 @@ static void cf_check early_init_intel(st
 		paddr_bits = 36;
 
 	if (c == &boot_cpu_data) {
+		uint64_t misc_enable;
+
 		check_memory_type_self_snoop_errata();
 
+		/*
+		 * If fast string is not enabled in IA32_MISC_ENABLE for any reason,
+		 * clear the enhanced fast string CPU capability.
+		 */
+		rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
+		if (!(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING))
+			setup_clear_cpu_cap(X86_FEATURE_ERMS);
+
 		intel_init_levelling();
 	}
 
xen/arch/x86/cpu-policy.c +16 -0
@@ -487,6 +487,12 @@ static void __init guest_common_max_feat
      */
     if ( test_bit(X86_FEATURE_RTM, fs) )
         __set_bit(X86_FEATURE_RTM_ALWAYS_ABORT, fs);
+
+    /*
+     * We expose MISC_ENABLE to guests, so our internal clearing of ERMS when
+     * FAST_STRING is not set should not affect the view of migrating-in guests.
+     */
+    __set_bit(X86_FEATURE_ERMS, fs);
 }
 
 static void __init guest_common_default_feature_adjustments(uint32_t *fs)
@@ -567,6 +573,16 @@ static void __init guest_common_default_
         __clear_bit(X86_FEATURE_RTM, fs);
         __set_bit(X86_FEATURE_RTM_ALWAYS_ABORT, fs);
     }
+
+    /*
+     * We expose MISC_ENABLE to guests, so our internal clearing of ERMS when
+     * FAST_STRING is not set should not propagate to guest view.  Guests can
+     * judge on their own whether to ignore the CPUID bit when the MSR bit is
+     * clear.  The bit being uniformly set in the max policies, we only need
+     * to clear it here (if hardware doesn't have it).
+     */
+    if ( !raw_cpu_policy.feat.erms )
+        __clear_bit(X86_FEATURE_ERMS, fs);
 }
 
 static void __init guest_common_feature_adjustments(uint32_t *fs)
xen/arch/x86/include/asm/msr-index.h +1 -0
@@ -493,6 +493,7 @@
 #define MSR_IA32_THERM_INTERRUPT	0x0000019b
 #define MSR_IA32_THERM_STATUS		0x0000019c
 #define MSR_IA32_MISC_ENABLE		0x000001a0
+#define MSR_IA32_MISC_ENABLE_FAST_STRING  (1<<0)
 #define MSR_IA32_MISC_ENABLE_PERF_AVAIL   (1<<7)
 #define MSR_IA32_MISC_ENABLE_BTS_UNAVAIL  (1<<11)
 #define MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL (1<<12)
#2 lore raw

PATCH v5 2/6 x86: re-work memset()

In reply to Jan Beulich

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

For RET to be usable in an alternative's replacement code, extend the CALL/JMP patching to cover the case of "JMP __x86_return_thunk" coming last in replacement code.

---
We may want to consider branching over the REP STOSQ as well, if the number of qwords turns out to be zero.
We may also want to consider using non-REP STOS{L,W,B} for the tail.
---
v5: Re-base.
v4: Use %r8 instead of %rsi in a few places.
v3: Re-base.

xen/arch/x86/Makefile +1 -0
@@ -47,6 +47,7 @@ obj-$(CONFIG_RETURN_THUNK) += indirect-t
 obj-$(CONFIG_PV) += ioport_emulate.o
 obj-y += irq.o
 obj-$(CONFIG_KEXEC) += machine_kexec.o
+obj-y += memset.o
 obj-y += mm.o x86_64/mm.o
 obj-$(CONFIG_VM_EVENT) += monitor.o
 obj-y += mpparse.o
xen/arch/x86/alternative.c +6 -0
@@ -346,6 +346,12 @@ static int init_or_livepatch _apply_alte
         /* 0xe8/0xe9 are relative branches; fix the offset. */
         if ( a->repl_len >= 5 && (*buf & 0xfe) == 0xe8 )
             *(int32_t *)(buf + 1) += repl - orig;
+        else if ( IS_ENABLED(CONFIG_RETURN_THUNK) &&
+                  a->repl_len > 5 && buf[a->repl_len - 5] == 0xe9 &&
+                  ((long)repl + a->repl_len +
+                   *(int32_t *)(buf + a->repl_len - 4) ==
+                   (long)__x86_return_thunk) )
+            *(int32_t *)(buf + a->repl_len - 4) += repl - orig;
 
         a->priv = 1;
 
xen/arch/x86/memset.S new file +30 -0
@@ -0,0 +1,30 @@
+#include <asm/asm_defns.h>
+
+.macro memset
+        and     $7, %edx
+        shr     $3, %rcx
+        movzbl  %sil, %esi
+        mov     $0x0101010101010101, %rax
+        imul    %rsi, %rax
+        mov     %rdi, %r8
+        rep stosq
+        or      %edx, %ecx
+        jz      0f
+        rep stosb
+0:
+        mov     %r8, %rax
+        RET
+.endm
+
+.macro memset_erms
+        mov     %esi, %eax
+        mov     %rdi, %r8
+        rep stosb
+        mov     %r8, %rax
+        RET
+.endm
+
+FUNC(memset)
+        mov     %rdx, %rcx
+        ALTERNATIVE memset, memset_erms, X86_FEATURE_ERMS
+END(memset)
xen/arch/x86/string.c +0 -13
@@ -22,19 +22,6 @@ void *(memcpy)(void *dest, const void *s
     return dest;
 }
 
-void *(memset)(void *s, int c, size_t n)
-{
-    long d0, d1;
-
-    asm volatile (
-        "rep stosb"
-        : "=&c" (d0), "=&D" (d1)
-        : "a" (c), "1" (s), "0" (n)
-        : "memory");
-
-    return s;
-}
-
 void *(memmove)(void *dest, const void *src, size_t n)
 {
     long d0, d1, d2;
#3 lore raw

Re: [PATCH v5 2/6] x86: re-work memset()

Le 05/06/2025 à 12:26, Jan Beulich a écrit :

Quoted 33 lines

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

For RET to be usable in an alternative's replacement code, extend the CALL/JMP patching to cover the case of "JMP __x86_return_thunk" coming last in replacement code.

---
We may want to consider branching over the REP STOSQ as well, if the number of qwords turns out to be zero.
We may also want to consider using non-REP STOS{L,W,B} for the tail.
---
v5: Re-base.
v4: Use %r8 instead of %rsi in a few places.
v3: Re-base.

xen/arch/x86/Makefile +1 -0
@@ -47,6 +47,7 @@ obj-$(CONFIG_RETURN_THUNK) += indirect-t
  obj-$(CONFIG_PV) += ioport_emulate.o
  obj-y += irq.o
  obj-$(CONFIG_KEXEC) += machine_kexec.o
+obj-y += memset.o
  obj-y += mm.o x86_64/mm.o
  obj-$(CONFIG_VM_EVENT) += monitor.o
  obj-y += mpparse.o

xen/arch/x86/alternative.c static int init_or_livepatch _apply_alte · line 354

Quoted 9 lines
xen/arch/x86/alternative.c +6 -0
@@ -346,6 +346,12 @@ static int init_or_livepatch _apply_alte
          /* 0xe8/0xe9 are relative branches; fix the offset. */
          if ( a->repl_len >= 5 && (*buf & 0xfe) == 0xe8 )
              *(int32_t *)(buf + 1) += repl - orig;
+        else if ( IS_ENABLED(CONFIG_RETURN_THUNK) &&
+                  a->repl_len > 5 && buf[a->repl_len - 5] == 0xe9 &&
+                  ((long)repl + a->repl_len +
+                   *(int32_t *)(buf + a->repl_len - 4) ==
+                   (long)__x86_return_thunk) )
+            *(int32_t *)(buf + a->repl_len - 4) += repl - orig;

That looks a bit confusing, to me that probably some comment explaining what instructions transform it's looking to make.

xen/arch/x86/memset.S line 2

Quoted 8 lines
xen/arch/x86/alternative.c +0 -0
          a->priv = 1;
 
xen/arch/x86/memset.S new file +2 -0
@@ -0,0 +1,30 @@
+#include <asm/asm_defns.h>
+

It would be nice to have a reminder of the calling convention (i.e what register maps to what memset parameter) as it will definitely help future readers.

If I understand correctly here :
- rdi: destination (s)
- rsi: byte to write (c)
- rdx: number of bytes to write (n)
- eventually that rcx = rdx (mov %rdx, %rcx) in memset/memset_erms

Quoted 14 lines
xen/arch/x86/memset.S new file +14 -0
+.macro memset
+        and     $7, %edx
+        shr     $3, %rcx
+        movzbl  %sil, %esi
+        mov     $0x0101010101010101, %rax
+        imul    %rsi, %rax
+        mov     %rdi, %r8
+        rep stosq
+        or      %edx, %ecx
+        jz      0f
+        rep stosb
+0:
+        mov     %r8, %rax
+        RET

xen/arch/x86/memset.S line 26

Quoted 10 lines
xen/arch/x86/memset.S new file +10 -0
+.endm
+
+.macro memset_erms
+        mov     %esi, %eax
+        mov     %rdi, %r8
+        rep stosb
+        mov     %r8, %rax
+        RET
+.endm
+

Overall, I am a bit confused on the mixing of 32-bits (edx, esi, ...) and 64-bits registers (rax, ...). But it looks ok to me.

Quoted 18 lines
xen/arch/x86/memset.S new file +4 -0
+FUNC(memset)
+        mov     %rdx, %rcx
+        ALTERNATIVE memset, memset_erms, X86_FEATURE_ERMS
+END(memset)
xen/arch/x86/string.c +0 -6
@@ -22,19 +22,6 @@ void *(memcpy)(void *dest, const void *s
      return dest;
  }
 
-void *(memset)(void *s, int c, size_t n)
-{
-    long d0, d1;
-
-    asm volatile (
-        "rep stosb"

xen/arch/x86/string.c void *(memcpy)(void *dest, const void *s · line 27

Quoted 10 lines
xen/arch/x86/string.c +0 -7
-        : "=&c" (d0), "=&D" (d1)
-        : "a" (c), "1" (s), "0" (n)
-        : "memory");
-
-    return s;
-}
-
  void *(memmove)(void *dest, const void *src, size_t n)
  {
      long d0, d1, d2;

Teddy

Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech

#4 lore raw

Re: [PATCH v5 2/6] x86: re-work memset()

Quoted 16 lines · Teddy Astie

Le 05/06/2025 à 12:26, Jan Beulich a écrit :

xen/arch/x86/alternative.c static int init_or_livepatch _apply_alte · line 354

Quoted 12 lines
xen/arch/x86/alternative.c +6 -0
@@ -346,6 +346,12 @@ static int init_or_livepatch _apply_alte
          /* 0xe8/0xe9 are relative branches; fix the offset. */
          if ( a->repl_len >= 5 && (*buf & 0xfe) == 0xe8 )
              *(int32_t *)(buf + 1) += repl - orig;
+        else if ( IS_ENABLED(CONFIG_RETURN_THUNK) &&
+                  a->repl_len > 5 && buf[a->repl_len - 5] == 0xe9 &&
+                  ((long)repl + a->repl_len +
+                   *(int32_t *)(buf + a->repl_len - 4) ==
+                   (long)__x86_return_thunk) )
+            *(int32_t *)(buf + a->repl_len - 4) += repl - orig;

That looks a bit confusing, to me that probably some comment explaining what instructions transform it's looking to make.

It's still the same comment ahead of the if() that applies here. This will all become easier with Andrew's decode-light, at which point we'll be able to spot CALL/JMP anywhere in a blob.

Quoted 10 lines

xen/arch/x86/memset.S line 2

Quoted 5 lines
xen/arch/x86/memset.S new file +2 -0
@@ -0,0 +1,30 @@
+#include <asm/asm_defns.h>
+

It would be nice to have a reminder of the calling convention (i.e what register maps to what memset parameter) as it will definitely help future readers.

Quoted 4 lines

If I understand correctly here :
- rdi: destination (s)
- rsi: byte to write (c)
- rdx: number of bytes to write (n)

I don't think the (default) ABI needs re-stating for every function in every assembly file.

Quoted 19 lines
Quoted 19 lines
xen/arch/x86/memset.S new file +19 -0
+.macro memset
+        and     $7, %edx
+        shr     $3, %rcx
+        movzbl  %sil, %esi
+        mov     $0x0101010101010101, %rax
+        imul    %rsi, %rax
+        mov     %rdi, %r8
+        rep stosq
+        or      %edx, %ecx
+        jz      0f
+        rep stosb
+0:
+        mov     %r8, %rax
+        RET
+.endm
+
+.macro memset_erms
+        mov     %esi, %eax
+        mov     %rdi, %r8
Quoted 8 lines

xen/arch/x86/memset.S line 18

Quoted 5 lines
xen/arch/x86/memset.S new file +5 -0
+        rep stosb
+        mov     %r8, %rax
+        RET
+.endm
+

Overall, I am a bit confused on the mixing of 32-bits (edx, esi, ...) and 64-bits registers (rax, ...). But it looks ok to me.

Since 64-bit forms require REX, 32-bit ones are used wherever that's sufficient for the purpose.

Jan

#5 lore raw

PATCH v5 3/6 x86: re-work memcpy()

In reply to Jan Beulich

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

Alternatives patching, however, requires an extra precaution: It uses memcpy() itself, and hence the function may patch itself. Luckily the patched-in code only replaces the prolog of the original function. Make sure this remains this way.

Additionally alternatives patching, while supposedly safe via enforcing a control flow change when modifying already prefetched code, may not really be. Afaict a request is pending to drop the first of the two options in the SDM's "Handling Self- and Cross-Modifying Code" section. Insert a serializing instruction there.

---
We may want to consider branching over the REP MOVSQ as well, if the number of qwords turns out to be zero.
We may also want to consider using non-REP MOVS{L,W,B} for the tail.

TBD: We may further need a workaround similar to Linux'es 8ca97812c3c8
("x86/mce: Work around an erratum on fast string copy instructions").

TBD: Some older AMD CPUs have an issue with REP MOVS when source and
destination are misaligned with one another (modulo 32?), which may require a separate memcpy() flavor.
---
v5: Re-base.
v4: Use CR2 write as serializing insn, and limit its use to boot time.
v3: Re-base.

xen/arch/x86/Makefile +1 -0
@@ -47,6 +47,7 @@ obj-$(CONFIG_RETURN_THUNK) += indirect-t
 obj-$(CONFIG_PV) += ioport_emulate.o
 obj-y += irq.o
 obj-$(CONFIG_KEXEC) += machine_kexec.o
+obj-y += memcpy.o
 obj-y += memset.o
 obj-y += mm.o x86_64/mm.o
 obj-$(CONFIG_VM_EVENT) += monitor.o
xen/arch/x86/alternative.c +5 -1
@@ -195,12 +195,16 @@ void *place_ret(void *ptr)
  * executing.
  *
  * "noinline" to cause control flow change and thus invalidate I$ and
- * cause refetch after modification.
+ * cause refetch after modification.  While the SDM continues to suggest this
+ * is sufficient, it may not be - issue a serializing insn afterwards as well,
+ * unless this is for live-patching.
  */
 static void init_or_livepatch noinline
 text_poke(void *addr, const void *opcode, size_t len)
 {
     memcpy(addr, opcode, len);
+    if ( system_state < SYS_STATE_active )
+        asm volatile ( "mov %%rax, %%cr2" ::: "memory" );
 }
 
 extern void *const __initdata_cf_clobber_start[];
xen/arch/x86/memcpy.S new file +20 -0
@@ -0,0 +1,20 @@
+#include <asm/asm_defns.h>
+
+FUNC(memcpy)
+        mov     %rdx, %rcx
+        mov     %rdi, %rax
+        /*
+         * We need to be careful here: memcpy() is involved in alternatives
+         * patching, so the code doing the actual copying (i.e. past setting
+         * up registers) may not be subject to patching (unless further
+         * precautions were taken).
+         */
+        ALTERNATIVE "and $7, %edx; shr $3, %rcx", \
+                    STR(rep movsb; RET), X86_FEATURE_ERMS
+        rep movsq
+        or      %edx, %ecx
+        jz      1f
+        rep movsb
+1:
+        RET
+END(memcpy)
xen/arch/x86/string.c +0 -15
@@ -7,21 +7,6 @@
 
 #include <xen/lib.h>
 
-void *(memcpy)(void *dest, const void *src, size_t n)
-{
-    long d0, d1, d2;
-
-    asm volatile (
-        "   rep ; movs"__OS" ; "
-        "   mov %k4,%k3      ; "
-        "   rep ; movsb        "
-        : "=&c" (d0), "=&D" (d1), "=&S" (d2)
-        : "0" (n/BYTES_PER_LONG), "r" (n%BYTES_PER_LONG), "1" (dest), "2" (src)
-        : "memory" );
-
-    return dest;
-}
-
 void *(memmove)(void *dest, const void *src, size_t n)
 {
     long d0, d1, d2;
Reviewed-by on v5 3/6
#6 lore raw

Re: [PATCH v5 3/6] x86: re-work memcpy()

Le 05/06/2025 à 12:27, Jan Beulich a écrit :

Quoted 11 lines

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

Alternatives patching, however, requires an extra precaution: It uses memcpy() itself, and hence the function may patch itself. Luckily the patched-in code only replaces the prolog of the original function. Make sure this remains this way.

We can probably workaround that by using a separate memcpy for alternatives patching. So it wouldn't end up patching itself.

Quoted 93 lines

Additionally alternatives patching, while supposedly safe via enforcing a control flow change when modifying already prefetched code, may not really be. Afaict a request is pending to drop the first of the two options in the SDM's "Handling Self- and Cross-Modifying Code" section. Insert a serializing instruction there.

---
We may want to consider branching over the REP MOVSQ as well, if the number of qwords turns out to be zero.
We may also want to consider using non-REP MOVS{L,W,B} for the tail.

TBD: We may further need a workaround similar to Linux'es 8ca97812c3c8
("x86/mce: Work around an erratum on fast string copy instructions").

TBD: Some older AMD CPUs have an issue with REP MOVS when source and
destination are misaligned with one another (modulo 32?), which may require a separate memcpy() flavor.
---
v5: Re-base.
v4: Use CR2 write as serializing insn, and limit its use to boot time.
v3: Re-base.

xen/arch/x86/Makefile +1 -0
@@ -47,6 +47,7 @@ obj-$(CONFIG_RETURN_THUNK) += indirect-t
  obj-$(CONFIG_PV) += ioport_emulate.o
  obj-y += irq.o
  obj-$(CONFIG_KEXEC) += machine_kexec.o
+obj-y += memcpy.o
  obj-y += memset.o
  obj-y += mm.o x86_64/mm.o
  obj-$(CONFIG_VM_EVENT) += monitor.o
xen/arch/x86/alternative.c +5 -1
@@ -195,12 +195,16 @@ void *place_ret(void *ptr)
   * executing.
   *
   * "noinline" to cause control flow change and thus invalidate I$ and
- * cause refetch after modification.
+ * cause refetch after modification.  While the SDM continues to suggest this
+ * is sufficient, it may not be - issue a serializing insn afterwards as well,
+ * unless this is for live-patching.
   */
  static void init_or_livepatch noinline
  text_poke(void *addr, const void *opcode, size_t len)
  {
      memcpy(addr, opcode, len);
+    if ( system_state < SYS_STATE_active )
+        asm volatile ( "mov %%rax, %%cr2" ::: "memory" );
  }
 
  extern void *const __initdata_cf_clobber_start[];
xen/arch/x86/memcpy.S new file +20 -0
@@ -0,0 +1,20 @@
+#include <asm/asm_defns.h>
+
+FUNC(memcpy)
+        mov     %rdx, %rcx
+        mov     %rdi, %rax
+        /*
+         * We need to be careful here: memcpy() is involved in alternatives
+         * patching, so the code doing the actual copying (i.e. past setting
+         * up registers) may not be subject to patching (unless further
+         * precautions were taken).
+         */
+        ALTERNATIVE "and $7, %edx; shr $3, %rcx", \
+                    STR(rep movsb; RET), X86_FEATURE_ERMS
+        rep movsq
+        or      %edx, %ecx
+        jz      1f
+        rep movsb
+1:
+        RET
+END(memcpy)
xen/arch/x86/string.c +0 -8
@@ -7,21 +7,6 @@
 
  #include <xen/lib.h>
 
-void *(memcpy)(void *dest, const void *src, size_t n)
-{
-    long d0, d1, d2;
-
-    asm volatile (
-        "   rep ; movs"__OS" ; "
-        "   mov %k4,%k3      ; "
-        "   rep ; movsb        "

xen/arch/x86/string.c line 12

Quoted 10 lines
xen/arch/x86/string.c +0 -7
-        : "=&c" (d0), "=&D" (d1), "=&S" (d2)
-        : "0" (n/BYTES_PER_LONG), "r" (n%BYTES_PER_LONG), "1" (dest), "2" (src)
-        : "memory" );
-
-    return dest;
-}
-
  void *(memmove)(void *dest, const void *src, size_t n)
  {
      long d0, d1, d2;

Aside that:

Teddy

Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech

#7 lore raw

Re: [PATCH v5 3/6] x86: re-work memcpy()

Quoted 7 lines · Teddy Astie

Le 05/06/2025 à 12:27, Jan Beulich a écrit :

Quoted 6 lines

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

Quoted 7 lines
Quoted 5 lines

Alternatives patching, however, requires an extra precaution: It uses memcpy() itself, and hence the function may patch itself. Luckily the patched-in code only replaces the prolog of the original function. Make sure this remains this way.

We can probably workaround that by using a separate memcpy for alternatives patching. So it wouldn't end up patching itself.

Not memcpy() you can't.

The compiler is permitted to invent calls to memset()/memcpy() out of thin air, e.g. struct big_foo = {}; as a local on the stack.

This is the same reason why it's impossible to do speculation safety in C; you cannot guarantee to get protections ahead of the first RET.

~Andrew

#8 lore raw

Re: [PATCH v5 3/6] x86: re-work memcpy()

In reply to Teddy Astie
Quoted 7 lines · Teddy Astie

Le 05/06/2025 à 12:27, Jan Beulich a écrit :

Quoted 6 lines

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

Quoted 7 lines
Quoted 4 lines

Alternatives patching, however, requires an extra precaution: It uses memcpy() itself, and hence the function may patch itself. Luckily the patched-in code only replaces the prolog of the original function. Make sure this remains this way.

We can probably workaround that by using a separate memcpy for alternatives patching. So it wouldn't end up patching itself.

We could, yes, but imo we better wouldn't.

Quoted 2 lines

Aside that:

Please clarify whether this applies without that suggestion of yours taken care of.

Jan

#9 lore raw

Re: [PATCH v5 3/6] x86: re-work memcpy()

Le 06/06/2025 à 11:13, Jan Beulich a écrit :

Quoted 13 lines · Teddy Astie
Quoted 12 lines

Le 05/06/2025 à 12:27, Jan Beulich a écrit :

Quoted 10 lines

Move the function to its own assembly file. Having it in C just for the entire body to be an asm() isn't really helpful. Then have two flavors:
A "basic" version using qword steps for the bulk of the operation, and an ERMS version for modern hardware, to be substituted in via alternatives patching.

Alternatives patching, however, requires an extra precaution: It uses memcpy() itself, and hence the function may patch itself. Luckily the patched-in code only replaces the prolog of the original function. Make sure this remains this way.

Quoted 5 lines
Quoted 2 lines

We can probably workaround that by using a separate memcpy for alternatives patching. So it wouldn't end up patching itself.

We could, yes, but imo we better wouldn't.

As Andrew pointed out that it's not that simple to use a separate memcpy. So should probably keep the current approach.

Quoted 6 lines
Quoted 2 lines

Aside that:

Please clarify whether this applies without that suggestion of yours taken care of.

Yes.

Quoted 1 line

Jan

Teddy

Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech

#10 lore raw

Re: [PATCH v5 3/6] x86: re-work memcpy()

In reply to Jan Beulich
Quoted 8 lines · Jan Beulich
xen/arch/x86/alternative.c +1 -1
@@ -195,12 +195,16 @@ void *place_ret(void *ptr)
  * executing.
  *
  * "noinline" to cause control flow change and thus invalidate I$ and
- * cause refetch after modification.
+ * cause refetch after modification.  While the SDM continues to suggest this

xen/arch/x86/alternative.c void *place_ret(void *ptr) · line 208

Quoted 10 lines
xen/arch/x86/alternative.c +4 -0
+ * is sufficient, it may not be - issue a serializing insn afterwards as well,
+ * unless this is for live-patching.
  */
 static void init_or_livepatch noinline
 text_poke(void *addr, const void *opcode, size_t len)
 {
     memcpy(addr, opcode, len);
+    if ( system_state < SYS_STATE_active )
+        asm volatile ( "mov %%rax, %%cr2" ::: "memory" );
 }

This hunk wants pulling out separately.  It's not really related to memcpy(), and probably ought to be backported.

Architecturally, both the APM and SDM say you're ok doing nothing on 64bit capable CPUs.

However, there are errata, and at least one recent AMD CPU needs serialisation for self modifying code.  (Not sure if the rev guide has been updated yet, and I can't remember offhand which CPU it is.)

You should also discuss the choice of serialising instruction in a comment.  Mov to %cr2 is serialising on everything more modern than the 486, and least likely to be intercepted under virt (== most performant).

You also need to explain that we only do true SMC during boot.  After boot for livepatch, it's prior to addr going live.

Finally, you're loading rubble into %cr2.  It's not even reliably 'addr' because of the transformations the compiler is permitted to make on memcpy().  I really think you should be reliably feeding in 0.  We're doing self-modifying code here with serialisation; an extra xor won't be measurable.

~Andrew

P.S. We should drop the noinline.  That's only applicable to the P5(?) and earlier where you had to use branches to flush the prefetch queue. It's irrelevant on 64bit capable CPUs.

#11 lore raw

PATCH v5 4/6 x86: control memset() and memcpy() inlining

In reply to Jan Beulich

Stop the compiler from inlining non-trivial memset() and memcpy() (for memset() see e.g. map_vcpu_info() or kimage_load_segments() for examples). This way we even keep the compiler from using REP STOSQ / REP MOVSQ when we'd prefer REP STOSB / REP MOVSB (when ERMS is available).

With gcc10 this yields a modest .text size reduction (release build) of around 2k.

Unfortunately these options aren't understood by the clang versions I have readily available for testing with; I'm unaware of equivalents.

Note also that using cc-option-add is not an option here, or at least I couldn't make things work with it (in case the option was not supported by the compiler): The embedded comma in the option looks to be getting in the way.

---
v3: Re-base.
v2: New.
---
The boundary values are of course up for discussion - I wasn't really certain whether to use 16 or 32; I'd be less certain about using yet larger values.

Similarly whether to permit the compiler to emit REP STOSQ / REP MOVSQ for known size, properly aligned blocks is up for discussion.

xen/arch/x86/arch.mk +3 -0
@@ -58,6 +58,9 @@ endif
 $(call cc-option-add,CFLAGS_stack_boundary,CC,-mpreferred-stack-boundary=3)
 export CFLAGS_stack_boundary
 
+CFLAGS += $(call cc-option,$(CC),-mmemcpy-strategy=unrolled_loop:16:noalign$(comma)libcall:-1:noalign)
+CFLAGS += $(call cc-option,$(CC),-mmemset-strategy=unrolled_loop:16:noalign$(comma)libcall:-1:noalign)
+
 ifeq ($(CONFIG_UBSAN),y)
 # Don't enable alignment sanitisation.  x86 has efficient unaligned accesses,
 # and various things (ACPI tables, hypercall pages, stubs, etc) are wont-fix.
#12 lore raw

Re: [PATCH v5 4/6] x86: control memset() and memcpy() inlining

Le 05/06/2025 à 12:28, Jan Beulich a écrit :

Quoted 6 lines

Stop the compiler from inlining non-trivial memset() and memcpy() (for memset() see e.g. map_vcpu_info() or kimage_load_segments() for examples). This way we even keep the compiler from using REP STOSQ / REP MOVSQ when we'd prefer REP STOSB / REP MOVSB (when ERMS is available).

If the size is known and constant, and the compiler is able to generate a trivial rep movs/stos (usually with a mov $x, %ecx before). I don't see a reason to prevent it or forcing it to make a function call, as I suppose it is very likely that the plain inline rep movs/stos will perform better than a function call (even if it is not the prefered rep movsb/stosb), eventually also being smaller.

I wonder if it is possible to only generate inline rep movs/stos for "trivial cases" (i.e preceded with a plain mov $x, %ecx), and rely on either inline movs or function call in other cases (non-trivial ones).

Quoted 29 lines

With gcc10 this yields a modest .text size reduction (release build) of around 2k.

Unfortunately these options aren't understood by the clang versions I have readily available for testing with; I'm unaware of equivalents.

Note also that using cc-option-add is not an option here, or at least I couldn't make things work with it (in case the option was not supported by the compiler): The embedded comma in the option looks to be getting in the way.

---
v3: Re-base.
v2: New.
---
The boundary values are of course up for discussion - I wasn't really certain whether to use 16 or 32; I'd be less certain about using yet larger values.

Similarly whether to permit the compiler to emit REP STOSQ / REP MOVSQ for known size, properly aligned blocks is up for discussion.

xen/arch/x86/arch.mk endif · line 66

Quoted 9 lines
xen/arch/x86/arch.mk +3 -0
@@ -58,6 +58,9 @@ endif
  $(call cc-option-add,CFLAGS_stack_boundary,CC,-mpreferred-stack-boundary=3)
  export CFLAGS_stack_boundary
 
+CFLAGS += $(call cc-option,$(CC),-mmemcpy-strategy=unrolled_loop:16:noalign$(comma)libcall:-1:noalign)
+CFLAGS += $(call cc-option,$(CC),-mmemset-strategy=unrolled_loop:16:noalign$(comma)libcall:-1:noalign)
+
  ifeq ($(CONFIG_UBSAN),y)
  # Don't enable alignment sanitisation.  x86 has efficient unaligned accesses,
  # and various things (ACPI tables, hypercall pages, stubs, etc) are wont-fix.

Teddy

Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech

#13 lore raw

Re: [PATCH v5 4/6] x86: control memset() and memcpy() inlining

Quoted 15 lines · Teddy Astie

Le 05/06/2025 à 12:28, Jan Beulich a écrit :

Quoted 6 lines

Stop the compiler from inlining non-trivial memset() and memcpy() (for memset() see e.g. map_vcpu_info() or kimage_load_segments() for examples). This way we even keep the compiler from using REP STOSQ / REP MOVSQ when we'd prefer REP STOSB / REP MOVSB (when ERMS is available).

If the size is known and constant, and the compiler is able to generate a trivial rep movs/stos (usually with a mov $x, %ecx before). I don't see a reason to prevent it or forcing it to make a function call, as I suppose it is very likely that the plain inline rep movs/stos will perform better than a function call (even if it is not the prefered rep movsb/stosb), eventually also being smaller.

Quoted 3 lines

I wonder if it is possible to only generate inline rep movs/stos for "trivial cases" (i.e preceded with a plain mov $x, %ecx), and rely on either inline movs or function call in other cases (non-trivial ones).

Note how the description starts with "Stop the compiler from inlining non-trivial ...", which indeed remain unaffected according to my observations (back at the time).

Jan

#14 lore raw

Re: [PATCH v5 4/6] x86: control memset() and memcpy() inlining

Le 06/06/2025 à 11:21, Jan Beulich a écrit :

Quoted 16 lines · Teddy Astie
Quoted 15 lines

Le 05/06/2025 à 12:28, Jan Beulich a écrit :

Quoted 6 lines

Stop the compiler from inlining non-trivial memset() and memcpy() (for memset() see e.g. map_vcpu_info() or kimage_load_segments() for examples). This way we even keep the compiler from using REP STOSQ / REP MOVSQ when we'd prefer REP STOSB / REP MOVSB (when ERMS is available).

If the size is known and constant, and the compiler is able to generate a trivial rep movs/stos (usually with a mov $x, %ecx before). I don't see a reason to prevent it or forcing it to make a function call, as I suppose it is very likely that the plain inline rep movs/stos will perform better than a function call (even if it is not the prefered rep movsb/stosb), eventually also being smaller.

Quoted 7 lines
Quoted 3 lines

I wonder if it is possible to only generate inline rep movs/stos for "trivial cases" (i.e preceded with a plain mov $x, %ecx), and rely on either inline movs or function call in other cases (non-trivial ones).

Note how the description starts with "Stop the compiler from inlining non-trivial ...", which indeed remain unaffected according to my observations (back at the time).

Yes, at least it is what appears to happen when testing using GCC 14.2.0 where non-trivial memset/memcpy are replaced with explicit functions call, and some trivial ones still use rep movsb/l/q.

Though,

Quoted 1 line

unrolled_loop:16:noalign,libcall:-1:noalign

to me sounds like :
- use a inline unrolled loop for 0-16 memcpy/memset
- call memset/memcpy for other cases
(thus no rep-prefix shall be used)

The align/noalign meaning being vaguely documented in GCC documentation, so it's unclear if it only affects "non-aligned" copies, or potentially all of them.

Quoted 3 lines

Jan

Teddy

Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech

#15 lore raw

PATCH v5 5/6 x86: introduce "hot" and "cold" page clearing functions

In reply to Jan Beulich

The present clear_page_sse2() is useful in case a page isn't going to get touched again soon, or if we want to limit churn on the caches. Amend it by alternatively using CLZERO, which has been found to be quite a bit faster on Zen2 hardware at least. Note that to use CLZERO, we need to know the cache line size, and hence a feature dependency on CLFLUSH gets introduced.

For cases where latency is the most important aspect, or when it is expected that sufficiently large parts of a page will get accessed again soon after the clearing, introduce a "hot" alternative. Again use alternatives patching to select between a "legacy" and an ERMS variant.

Don't switch any callers just yet - this will be the subject of subsequent changes.

---
v5: Re-base.
v3: Re-base.
v2: New.
---
Note: Ankur indicates that for ~L3-size or larger regions MOVNT/CLZERO
is better even latency-wise.

xen/arch/x86/clear_page.S +40 -2
@@ -5,7 +5,7 @@
 #include <asm/asm_defns.h>
 #include <asm/page.h>
 
-FUNC(clear_page_sse2)
+        .macro clear_page_sse2
         mov     $PAGE_SIZE/32, %ecx
         xor     %eax,%eax
 
@@ -19,4 +19,42 @@ FUNC(clear_page_sse2)
 
         sfence
         RET
-END(clear_page_sse2)
+        .endm
+
+        .macro clear_page_clzero
+        mov     %rdi, %rax
+        mov     $PAGE_SIZE/64, %ecx
+        .globl clear_page_clzero_post_count
+clear_page_clzero_post_count:
+
+0:      clzero
+        sub     $-64, %rax
+        .globl clear_page_clzero_post_neg_size
+clear_page_clzero_post_neg_size:
+        sub     $1, %ecx
+        jnz     0b
+
+        sfence
+        RET
+        .endm
+
+FUNC(clear_page_cold)
+        ALTERNATIVE clear_page_sse2, clear_page_clzero, X86_FEATURE_CLZERO
+END(clear_page_cold)
+
+        .macro clear_page_stosb
+        mov     $PAGE_SIZE, %ecx
+        xor     %eax,%eax
+        rep stosb
+        .endm
+
+        .macro clear_page_stosq
+        mov     $PAGE_SIZE/8, %ecx
+        xor     %eax, %eax
+        rep stosq
+        .endm
+
+FUNC(clear_page_hot)
+        ALTERNATIVE clear_page_stosq, clear_page_stosb, X86_FEATURE_ERMS
+        RET
+END(clear_page_hot)
xen/arch/x86/cpu/common.c +35 -2
@@ -60,6 +60,9 @@ DEFINE_PER_CPU(bool, full_gdt_loaded);
 
 DEFINE_PER_CPU(uint32_t, pkrs);
 
+extern uint32_t clear_page_clzero_post_count[];
+extern int8_t clear_page_clzero_post_neg_size[];
+
 void __init setup_clear_cpu_cap(unsigned int cap)
 {
 	const uint32_t *dfs;
@@ -357,8 +360,38 @@ void __init early_cpu_init(bool verbose)
 
 	edx &= ~cleared_caps[FEATURESET_1d];
 	ecx &= ~cleared_caps[FEATURESET_1c];
-	if (edx & cpufeat_mask(X86_FEATURE_CLFLUSH))
-		c->x86_cache_alignment = ((ebx >> 8) & 0xff) * 8;
+	if (edx & cpufeat_mask(X86_FEATURE_CLFLUSH)) {
+		unsigned int size = ((ebx >> 8) & 0xff) * 8;
+
+		c->x86_cache_alignment = size;
+
+		/*
+		 * Patch in parameters of clear_page_cold()'s CLZERO
+		 * alternative. Note that for now we cap this at 128 bytes.
+		 * Larger cache line sizes would still be dealt with
+		 * correctly, but would cause redundant work done.
+		 */
+		if (size > 128)
+			size = 128;
+		if (size && !(size & (size - 1))) {
+			/*
+			 * Need to play some games to keep the compiler from
+			 * recognizing the negative array index as being out
+			 * of bounds. The labels in assembler code really are
+			 * _after_ the locations to be patched, so the
+			 * negative index is intentional.
+			 */
+			uint32_t *pcount = clear_page_clzero_post_count;
+			int8_t *neg_size = clear_page_clzero_post_neg_size;
+
+			OPTIMIZER_HIDE_VAR(pcount);
+			OPTIMIZER_HIDE_VAR(neg_size);
+			pcount[-1] = PAGE_SIZE / size;
+			neg_size[-1] = -size;
+		}
+		else
+			setup_clear_cpu_cap(X86_FEATURE_CLZERO);
+	}
 	/* Leaf 0x1 capabilities filled in early for Xen. */
 	c->x86_capability[FEATURESET_1d] = edx;
 	c->x86_capability[FEATURESET_1c] = ecx;
xen/arch/x86/include/asm/asm-defns.h +4 -0
@@ -1,5 +1,9 @@
 #include <asm/page-bits.h>
 
+.macro clzero
+    .byte 0x0f, 0x01, 0xfc
+.endm
+
 /*
  * Call a noreturn function.  This could be JMP, but CALL results in a more
  * helpful backtrace.  BUG is to catch functions which do decide to return...
xen/arch/x86/include/asm/page.h +3 -2
@@ -219,10 +219,11 @@ typedef struct { u64 pfn; } pagetable_t;
 #define pagetable_from_paddr(p) pagetable_from_pfn((p)>>PAGE_SHIFT)
 #define pagetable_null()        pagetable_from_pfn(0)
 
-void clear_page_sse2(void *pg);
+void clear_page_hot(void *pg);
+void clear_page_cold(void *pg);
 void copy_page_sse2(void *to, const void *from);
 
-#define clear_page(_p)      clear_page_sse2(_p)
+#define clear_page(_p)      clear_page_cold(_p)
 #define copy_page(_t, _f)   copy_page_sse2(_t, _f)
 
 /* Convert between Xen-heap virtual addresses and machine addresses. */
xen/tools/gen-cpuid.py +4 -0
@@ -212,6 +212,10 @@ def crunch_numbers(state):
         # the first place.
         APIC: [X2APIC, TSC_DEADLINE, EXTAPIC],
 
+        # The CLZERO insn requires a means to determine the cache line size,
+        # which is tied to the CLFLUSH insn.
+        CLFLUSH: [CLZERO],
+
         # AMD built MMXExtentions and 3DNow as extentions to MMX.
         MMX: [MMXEXT, _3DNOW],
 
#16 lore raw

PATCH v5 6/6 mm: allow page scrubbing routine(s) to be arch controlled

In reply to Jan Beulich

Especially when dealing with large amounts of memory, memset() may not be very efficient; this can be bad enough that even for debug builds a custom function is warranted. We additionally want to distinguish "hot" and "cold" cases (with, as initial heuristic, "hot" being for any allocations a domain does for itself, assuming that in all other cases the page wouldn't be accessed [again] soon). The goal is for accesses of "cold" pages to not disturb caches (albeit finding a good balance between this and the higher latency looks to be difficult).

Keep the default fallback to clear_page_*() in common code; this may want to be revisited down the road.

---
v4: Re-base.
v3: Re-base.
v2: New.
---
The choice between hot and cold in scrub_one_page()'s callers is certainly up for discussion / improvement.

xen/arch/arm/include/asm/page.h +6 -0
@@ -144,6 +144,12 @@ extern size_t dcache_line_bytes;
 
 #define copy_page(dp, sp) memcpy(dp, sp, PAGE_SIZE)
 
+#define clear_page_hot  clear_page
+#define clear_page_cold clear_page
+
+#define scrub_page_hot(page) memset(page, SCRUB_BYTE_PATTERN, PAGE_SIZE)
+#define scrub_page_cold      scrub_page_hot
+
 static inline size_t read_dcache_line_bytes(void)
 {
     register_t ctr;
xen/arch/ppc/include/asm/page.h +6 -0
@@ -188,6 +188,12 @@ static inline void invalidate_icache(voi
 #define clear_page(page) memset(page, 0, PAGE_SIZE)
 #define copy_page(dp, sp) memcpy(dp, sp, PAGE_SIZE)
 
+#define clear_page_hot  clear_page
+#define clear_page_cold clear_page
+
+#define scrub_page_hot(page) memset(page, SCRUB_BYTE_PATTERN, PAGE_SIZE)
+#define scrub_page_cold      scrub_page_hot
+
 /* TODO: Flush the dcache for an entire page. */
 static inline void flush_page_to_ram(unsigned long mfn, bool sync_icache)
 {
xen/arch/riscv/include/asm/page.h +6 -0
@@ -198,6 +198,12 @@ static inline void invalidate_icache(voi
 #define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
 #define copy_page(dp, sp) memcpy(dp, sp, PAGE_SIZE)
 
+#define clear_page_hot  clear_page
+#define clear_page_cold clear_page
+
+#define scrub_page_hot(page) memset(page, SCRUB_BYTE_PATTERN, PAGE_SIZE)
+#define scrub_page_cold      scrub_page_hot
+
 static inline void flush_page_to_ram(unsigned long mfn, bool sync_icache)
 {
     const void *v = map_domain_page(_mfn(mfn));
xen/arch/x86/Makefile +1 -0
@@ -58,6 +58,7 @@ obj-y += pci.o
 obj-y += physdev.o
 obj-$(CONFIG_COMPAT) += x86_64/physdev.o
 obj-$(CONFIG_X86_PSR) += psr.o
+obj-bin-$(CONFIG_DEBUG) += scrub_page.o
 obj-y += setup.o
 obj-y += shutdown.o
 obj-y += smp.o
xen/arch/x86/include/asm/page.h +5 -0
@@ -226,6 +226,11 @@ void copy_page_sse2(void *to, const void
 #define clear_page(_p)      clear_page_cold(_p)
 #define copy_page(_t, _f)   copy_page_sse2(_t, _f)
 
+#ifdef CONFIG_DEBUG
+void scrub_page_hot(void *);
+void scrub_page_cold(void *);
+#endif
+
 /* Convert between Xen-heap virtual addresses and machine addresses. */
 #define __pa(x)             (virt_to_maddr(x))
 #define __va(x)             (maddr_to_virt(x))
xen/arch/x86/scrub_page.S new file +39 -0
@@ -0,0 +1,39 @@
+        .file __FILE__
+
+#include <asm/asm_defns.h>
+#include <xen/page-size.h>
+#include <xen/scrub.h>
+
+FUNC(scrub_page_cold)
+        mov     $PAGE_SIZE/32, %ecx
+        mov     $SCRUB_PATTERN, %rax
+
+0:      movnti  %rax,   (%rdi)
+        movnti  %rax,  8(%rdi)
+        movnti  %rax, 16(%rdi)
+        movnti  %rax, 24(%rdi)
+        add     $32, %rdi
+        sub     $1, %ecx
+        jnz     0b
+
+        sfence
+        ret
+END(scrub_page_cold)
+
+        .macro scrub_page_stosb
+        mov     $PAGE_SIZE, %ecx
+        mov     $SCRUB_BYTE_PATTERN, %eax
+        rep stosb
+        ret
+        .endm
+
+        .macro scrub_page_stosq
+        mov     $PAGE_SIZE/8, %ecx
+        mov     $SCRUB_PATTERN, %rax
+        rep stosq
+        ret
+        .endm
+
+FUNC(scrub_page_hot)
+        ALTERNATIVE scrub_page_stosq, scrub_page_stosb, X86_FEATURE_ERMS
+END(scrub_page_hot)
xen/common/page_alloc.c +27 -20

100 lines not on this page · raw message

xen/include/xen/scrub.h new file +24 -0
@@ -0,0 +1,24 @@
+#ifndef __XEN_SCRUB_H__
+#define __XEN_SCRUB_H__
+
+#include <xen/const.h>
+
+/* SCRUB_PATTERN needs to be a repeating series of bytes. */
+#ifdef CONFIG_DEBUG
+# define SCRUB_PATTERN       _AC(0xc2c2c2c2c2c2c2c2,ULL)
+#else
+# define SCRUB_PATTERN       _AC(0,ULL)
+#endif
+#define SCRUB_BYTE_PATTERN   (SCRUB_PATTERN & 0xff)
+
+#endif /* __XEN_SCRUB_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
#17 lore raw

Re: [PATCH v5 6/6] mm: allow page scrubbing routine(s) to be arch controlled

Quoted 55 lines · Jan Beulich

Especially when dealing with large amounts of memory, memset() may not be very efficient; this can be bad enough that even for debug builds a custom function is warranted. We additionally want to distinguish "hot" and "cold" cases (with, as initial heuristic, "hot" being for any allocations a domain does for itself, assuming that in all other cases the page wouldn't be accessed [again] soon). The goal is for accesses of "cold" pages to not disturb caches (albeit finding a good balance between this and the higher latency looks to be difficult).

Keep the default fallback to clear_page_*() in common code; this may want to be revisited down the road.

---
v4: Re-base.
v3: Re-base.
v2: New.
---
The choice between hot and cold in scrub_one_page()'s callers is certainly up for discussion / improvement.

xen/arch/arm/include/asm/page.h +6 -0
@@ -144,6 +144,12 @@ extern size_t dcache_line_bytes;
 
  #define copy_page(dp, sp) memcpy(dp, sp, PAGE_SIZE)
 
+#define clear_page_hot  clear_page
+#define clear_page_cold clear_page
+
+#define scrub_page_hot(page) memset(page, SCRUB_BYTE_PATTERN, PAGE_SIZE)
+#define scrub_page_cold      scrub_page_hot
+
  static inline size_t read_dcache_line_bytes(void)
  {
      register_t ctr;
xen/arch/ppc/include/asm/page.h +6 -0
@@ -188,6 +188,12 @@ static inline void invalidate_icache(voi
  #define clear_page(page) memset(page, 0, PAGE_SIZE)
  #define copy_page(dp, sp) memcpy(dp, sp, PAGE_SIZE)
 
+#define clear_page_hot  clear_page
+#define clear_page_cold clear_page
+
+#define scrub_page_hot(page) memset(page, SCRUB_BYTE_PATTERN, PAGE_SIZE)
+#define scrub_page_cold      scrub_page_hot
+
  /* TODO: Flush the dcache for an entire page. */
  static inline void flush_page_to_ram(unsigned long mfn, bool sync_icache)
  {

xen/arch/riscv/include/asm/page.h static inline void invalidate_icache(voi · line 206

Quoted 9 lines
xen/arch/riscv/include/asm/page.h +6 -0
@@ -198,6 +198,12 @@ static inline void invalidate_icache(voi
  #define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
  #define copy_page(dp, sp) memcpy(dp, sp, PAGE_SIZE)
 
+#define clear_page_hot  clear_page
+#define clear_page_cold clear_page
+
+#define scrub_page_hot(page) memset(page, SCRUB_BYTE_PATTERN, PAGE_SIZE)
+#define scrub_page_cold      scrub_page_hot
+

LGTM: Reviewed-by: Oleksii Kurochko<[email protected]>

Thanks.

~ Oleksii

Quoted 206 lines
xen/arch/riscv/include/asm/page.h +0 -0
  static inline void flush_page_to_ram(unsigned long mfn, bool sync_icache)
  {
      const void *v = map_domain_page(_mfn(mfn));
xen/arch/x86/Makefile +1 -0
@@ -58,6 +58,7 @@ obj-y += pci.o
  obj-y += physdev.o
  obj-$(CONFIG_COMPAT) += x86_64/physdev.o
  obj-$(CONFIG_X86_PSR) += psr.o
+obj-bin-$(CONFIG_DEBUG) += scrub_page.o
  obj-y += setup.o
  obj-y += shutdown.o
  obj-y += smp.o
xen/arch/x86/include/asm/page.h +5 -0
@@ -226,6 +226,11 @@ void copy_page_sse2(void *to, const void
  #define clear_page(_p)      clear_page_cold(_p)
  #define copy_page(_t, _f)   copy_page_sse2(_t, _f)
 
+#ifdef CONFIG_DEBUG
+void scrub_page_hot(void *);
+void scrub_page_cold(void *);
+#endif
+
  /* Convert between Xen-heap virtual addresses and machine addresses. */
  #define __pa(x)             (virt_to_maddr(x))
  #define __va(x)             (maddr_to_virt(x))
xen/arch/x86/scrub_page.S new file +39 -0
@@ -0,0 +1,39 @@
+        .file __FILE__
+
+#include <asm/asm_defns.h>
+#include <xen/page-size.h>
+#include <xen/scrub.h>
+
+FUNC(scrub_page_cold)
+        mov     $PAGE_SIZE/32, %ecx
+        mov     $SCRUB_PATTERN, %rax
+
+0:      movnti  %rax,   (%rdi)
+        movnti  %rax,  8(%rdi)
+        movnti  %rax, 16(%rdi)
+        movnti  %rax, 24(%rdi)
+        add     $32, %rdi
+        sub     $1, %ecx
+        jnz     0b
+
+        sfence
+        ret
+END(scrub_page_cold)
+
+        .macro scrub_page_stosb
+        mov     $PAGE_SIZE, %ecx
+        mov     $SCRUB_BYTE_PATTERN, %eax
+        rep stosb
+        ret
+        .endm
+
+        .macro scrub_page_stosq
+        mov     $PAGE_SIZE/8, %ecx
+        mov     $SCRUB_PATTERN, %rax
+        rep stosq
+        ret
+        .endm
+
+FUNC(scrub_page_hot)
+        ALTERNATIVE scrub_page_stosq, scrub_page_stosb, X86_FEATURE_ERMS
+END(scrub_page_hot)
xen/common/page_alloc.c +27 -20
@@ -135,6 +135,7 @@
  #include <xen/pfn.h>
  #include <xen/types.h>
  #include <xen/sched.h>
+#include <xen/scrub.h>
  #include <xen/sections.h>
  #include <xen/softirq.h>
  #include <xen/spinlock.h>
@@ -779,27 +780,31 @@ static void page_list_add_scrub(struct p
          page_list_add(pg, &heap(node, zone, order));
  }
 
-/* SCRUB_PATTERN needs to be a repeating series of bytes. */
-#ifndef NDEBUG
-#define SCRUB_PATTERN        0xc2c2c2c2c2c2c2c2ULL
-#else
-#define SCRUB_PATTERN        0ULL
+/*
+ * While in debug builds we want callers to avoid relying on allocations
+ * returning zeroed pages, for a production build, clear_page_*() is the
+ * fastest way to scrub.
+ */
+#ifndef CONFIG_DEBUG
+# undef  scrub_page_hot
+# define scrub_page_hot clear_page_hot
+# undef  scrub_page_cold
+# define scrub_page_cold clear_page_cold
  #endif
-#define SCRUB_BYTE_PATTERN   (SCRUB_PATTERN & 0xff)
 
-static void scrub_one_page(const struct page_info *pg)
+static void scrub_one_page(const struct page_info *pg, bool cold)
  {
+    void *ptr;
+
      if ( unlikely(pg->count_info & PGC_broken) )
          return;
 
-#ifndef NDEBUG
-    /* Avoid callers relying on allocations returning zeroed pages. */
-    unmap_domain_page(memset(__map_domain_page(pg),
-                             SCRUB_BYTE_PATTERN, PAGE_SIZE));
-#else
-    /* For a production build, clear_page() is the fastest way to scrub. */
-    clear_domain_page(_mfn(page_to_mfn(pg)));
-#endif
+    ptr = __map_domain_page(pg);
+    if ( cold )
+        scrub_page_cold(ptr);
+    else
+        scrub_page_hot(ptr);
+    unmap_domain_page(ptr);
  }
 
  static void poison_one_page(struct page_info *pg)
@@ -1079,12 +1084,14 @@ static struct page_info *alloc_heap_page
      if ( first_dirty != INVALID_DIRTY_IDX ||
           (scrub_debug && !(memflags & MEMF_no_scrub)) )
      {
+        bool cold = d && d != current->domain;
+
          for ( i = 0; i < (1U << order); i++ )
          {
              if ( test_and_clear_bit(_PGC_need_scrub, &pg[i].count_info) )
              {
                  if ( !(memflags & MEMF_no_scrub) )
-                    scrub_one_page(&pg[i]);
+                    scrub_one_page(&pg[i], cold);
 
                  dirty_cnt++;
              }
@@ -1349,7 +1356,7 @@ bool scrub_free_pages(void)
                  {
                      if ( test_bit(_PGC_need_scrub, &pg[i].count_info) )
                      {
-                        scrub_one_page(&pg[i]);
+                        scrub_one_page(&pg[i], true);
                          /*
                           * We can modify count_info without holding heap
                           * lock since we effectively locked this buddy by
@@ -2074,7 +2081,7 @@ static struct page_info *alloc_color_hea
      if ( !(memflags & MEMF_no_scrub) )
      {
          if ( need_scrub )
-            scrub_one_page(pg);
+            scrub_one_page(pg, d != current->domain);
          else
              check_one_page(pg);
      }
@@ -2225,7 +2232,7 @@ static void __init cf_check smp_scrub_he
          if ( !mfn_valid(_mfn(mfn)) || !page_state_is(pg, free) )
              continue;
 
-        scrub_one_page(pg);
+        scrub_one_page(pg, true);
      }
  }
 
@@ -2930,7 +2937,7 @@ void unprepare_staticmem_pages(struct pa
          if ( need_scrub )
          {
              /* TODO: asynchronous scrubbing for pages of static memory. */
-            scrub_one_page(pg);
+            scrub_one_page(pg, true);
          }
 
          pg[i].count_info |= PGC_static;
xen/include/xen/scrub.h new file +24 -0
@@ -0,0 +1,24 @@
+#ifndef __XEN_SCRUB_H__
+#define __XEN_SCRUB_H__
+
+#include <xen/const.h>
+
+/* SCRUB_PATTERN needs to be a repeating series of bytes. */
+#ifdef CONFIG_DEBUG
+# define SCRUB_PATTERN       _AC(0xc2c2c2c2c2c2c2c2,ULL)
+#else
+# define SCRUB_PATTERN       _AC(0,ULL)
+#endif
+#define SCRUB_BYTE_PATTERN   (SCRUB_PATTERN & 0xff)
+
+#endif /* __XEN_SCRUB_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */