Skip to contentSkip to conversation

Better backtraces for leaf functions

PATCH Applied Mathias Krause 3 participants 22 messages Sep 15, 2025 – Dec 19, 2025

Subsystems (3)

v2 of 4 Applied Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover Better backtraces for leaf functions

This is v2 of [1], trying to enhance backtraces involving leaf functions.

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

v2 fixes this by introducing the concept of "late CFLAGS" that get evaluated in the top-level Makefile once all other optional flags have been added to $(CFLAGS), which is needed for x86's version at least.

Please apply!

Thanks,
Mathias

[1] https://lore.kernel.org/kvm/[email protected]/

Mathias Krause (4):
Makefile: Provide a concept of late CFLAGS
x86: Better backtraces for leaf functions
arm64: Better backtraces for leaf functions
arm: Fix backtraces involving leaf functions

 Makefile            |  4 ++++
 arm/Makefile.arm    |  8 ++++++++
 arm/Makefile.arm64  |  6 ++++++
 x86/Makefile.common | 11 +++++++++++
 lib/arm/stack.c     | 18 ++++++++++++++++--
 5 files changed, 45 insertions(+), 2 deletions(-)
Signature · 1 line
2.47.3
#1 lore raw

PATCH v2 1/4 Makefile: Provide a concept of late CFLAGS

Allow architectures to provide CFLAGS that should be added only after all other optional CFLAGS have been evaluated.

This will be useful for flags that depend on other, generic ones.

To allow 'LATE_CFLAGS' to make use of the $(cc-option ...) helper, assume it'll be a lazily evaluated variable. To further ensure the $(cc-option ...) compiler invocation overhead won't be per-use of $(CFLAGS), enforce its evaluation prior to extending CFLAGS.

---
 Makefile | 4 ++++
 1 file changed, 4 insertions(+)
Makefile +4 -0
index 9dc5d2234e2a..0ce0813bf124 100644
@@ -95,6 +95,10 @@ CFLAGS += $(wmissing_parameter_type)
 CFLAGS += $(wold_style_declaration)
 CFLAGS += -Woverride-init -Wmissing-prototypes -Wstrict-prototypes
 
+# Evaluate and add late cflags last -- they may depend on previous flags
+LATE_CFLAGS := $(LATE_CFLAGS)
+CFLAGS += $(LATE_CFLAGS)
+
 autodepend-flags = -MMD -MP -MF $(dir $*).$(notdir $*).d
 
 LDFLAGS += -nostdlib $(no_pie) -z noexecstack
Signature · 1 line
2.47.3
#2 lore raw

PATCH v2 2/4 x86: Better backtraces for leaf functions

In reply to Mathias Krause

Leaf functions are problematic for backtraces as they lack the frame pointer setup epilogue. If such a function causes a fault, the original caller won't be part of the backtrace. That's problematic if, for example, memcpy() is failing because it got passed a bad pointer. The generated backtrace will look like this, providing no clue what the issue may be:

	STACK: @401b31 4001ad
  0x0000000000401b31: memcpy at lib/string.c:136 (discriminator 3)
        	for (i = 0; i < n; ++i)
      > 		a[i] = b[i];

0x00000000004001ac: gdt32_end at x86/cstart64.S:127
lea __environ(%rip), %rdx
> call main
mov %eax, %edi

By abusing profiling, we can force the compiler to emit a frame pointer setup epilogue even for leaf functions, making the above backtrace change like this:

	STACK: @401c21 400512 4001ad
  0x0000000000401c21: memcpy at lib/string.c:136 (discriminator 3)
        	for (i = 0; i < n; ++i)
      > 		a[i] = b[i];

0x0000000000400511: main at x86/hypercall.c:91 (discriminator 24)

      > 	memcpy((void *)~0xbadc0de, (void *)0xdeadbeef, 42);

0x00000000004001ac: gdt32_end at x86/cstart64.S:127
lea __environ(%rip), %rdx
> call main
mov %eax, %edi

Above backtrace includes the failing memcpy() call, making it much easier to spot the bug.

Enable "fake profiling" if supported by the compiler to get better backtraces. The runtime overhead should be negligible for the gained debugability as the profiling call is actually a NOP.

---
One may argure that the "ifneq ($(KEEP_FRAME_POINTER),) ... endif" wrapping isn't needed, and that's true. However, it simplifies toggling that variable, if there'll ever be a need for it.

 x86/Makefile.common | 11 +++++++++++
 1 file changed, 11 insertions(+)
x86/Makefile.common +11 -0
index 5663a65d3df4..be18a77a779e 100644
@@ -43,6 +43,17 @@ COMMON_CFLAGS += -O1
 # stack.o relies on frame pointers.
 KEEP_FRAME_POINTER := y
 
+ifneq ($(KEEP_FRAME_POINTER),)
+# Fake profiling to force the compiler to emit a frame pointer setup also in
+# leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately).
+#
+# Note:
+# We need to defer the cc-option test until -fno-pic or -no-pie have been
+# added to CFLAGS as -mnop-mcount needs it. The lazy evaluation of CFLAGS
+# during compilation makes this do "The Right Thing."
+LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "")
+endif
+
 FLATLIBS = lib/libcflat.a
 
 ifeq ($(CONFIG_EFI),y)
Signature · 1 line
2.47.3
#3 lore raw

Re: [kvm-unit-tests PATCH v2 2/4] x86: Better backtraces for leaf functions

Quoted 63 lines · Mathias Krause

Leaf functions are problematic for backtraces as they lack the frame pointer setup epilogue. If such a function causes a fault, the original caller won't be part of the backtrace. That's problematic if, for example, memcpy() is failing because it got passed a bad pointer. The generated backtrace will look like this, providing no clue what the issue may be:

	STACK: @401b31 4001ad
  0x0000000000401b31: memcpy at lib/string.c:136 (discriminator 3)
        	for (i = 0; i < n; ++i)
      > 		a[i] = b[i];

0x00000000004001ac: gdt32_end at x86/cstart64.S:127
lea __environ(%rip), %rdx
> call main
mov %eax, %edi

By abusing profiling, we can force the compiler to emit a frame pointer setup epilogue even for leaf functions, making the above backtrace change like this:

	STACK: @401c21 400512 4001ad
  0x0000000000401c21: memcpy at lib/string.c:136 (discriminator 3)
        	for (i = 0; i < n; ++i)
      > 		a[i] = b[i];

0x0000000000400511: main at x86/hypercall.c:91 (discriminator 24)

      > 	memcpy((void *)~0xbadc0de, (void *)0xdeadbeef, 42);

0x00000000004001ac: gdt32_end at x86/cstart64.S:127
lea __environ(%rip), %rdx
> call main
mov %eax, %edi

Above backtrace includes the failing memcpy() call, making it much easier to spot the bug.

Enable "fake profiling" if supported by the compiler to get better backtraces. The runtime overhead should be negligible for the gained debugability as the profiling call is actually a NOP.

---
One may argure that the "ifneq ($(KEEP_FRAME_POINTER),) ... endif" wrapping isn't needed, and that's true. However, it simplifies toggling that variable, if there'll ever be a need for it.

 x86/Makefile.common | 11 +++++++++++
 1 file changed, 11 insertions(+)
x86/Makefile.common +4 -0
index 5663a65d3df4..be18a77a779e 100644
@@ -43,6 +43,17 @@ COMMON_CFLAGS += -O1
 # stack.o relies on frame pointers.
 KEEP_FRAME_POINTER := y
 
+ifneq ($(KEEP_FRAME_POINTER),)
+# Fake profiling to force the compiler to emit a frame pointer setup also in
+# leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately).
+#

x86/Makefile.common COMMON_CFLAGS += -O1 · line 59

Quoted 10 lines
x86/Makefile.common +7 -0
+# Note:
+# We need to defer the cc-option test until -fno-pic or -no-pie have been
+# added to CFLAGS as -mnop-mcount needs it. The lazy evaluation of CFLAGS
+# during compilation makes this do "The Right Thing."
+LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "")
+endif
+
 FLATLIBS = lib/libcflat.a
 
 ifeq ($(CONFIG_EFI),y)

Paolo, can you please comment on this one, so the fixes for ARM and AArch64 are no longer blocked and, ideally, this series can be merged?

Thanks,
Mathias

#4 lore raw

PATCH v2 3/4 arm64: Better backtraces for leaf functions

In reply to Mathias Krause

Similar to x86 before, ARM64 skips the stack frame setup for leaf functions, making backtraces miss frames.

Fortunately, gcc supports forcing generating stack frames for these via -mno-omit-leaf-frame-pointer.

---
 arm/Makefile.arm64 | 6 ++++++
 1 file changed, 6 insertions(+)
arm/Makefile.arm64 +6 -0
index bf7ea2a36d3a..a40c830df20f 100644
@@ -67,5 +67,11 @@ tests += $(TEST_DIR)/mte.$(exe)
 
 include $(SRCDIR)/$(TEST_DIR)/Makefile.common
 
+ifneq ($(KEEP_FRAME_POINTER),)
+# Force the generation of a regular stack frame even for leaf functions to make
+# stack walking reliable.
+LATE_CFLAGS += $(call cc-option, -mno-omit-leaf-frame-pointer, "")
+endif
+
 arch_clean: arm_clean
 	$(RM) lib/arm64/.*.d
Signature · 1 line
2.47.3
#5 lore raw

PATCH v2 4/4 arm: Fix backtraces involving leaf functions

In reply to Mathias Krause

When backtracing starts in a leaf function, it'll cause the code to go off the rials as the stack frame for leaf functions is incomplete -- it lacks the return address, likely just causing recursive exception handling by trying to follow an invalid pointer chain.

Unfortunately, -mno-omit-leaf-frame-pointer isn't supported for the ARM target as an easy fix. Make use of -mapcs-frame instead to force the generation of an APCS stack frame layout[1] that can be traversed reliably.

As Clang doesn't support -mapcs-frame, make the stack walking code handle this by using the (old) more compact standard format as a fall-back.

---
I failed to build KUT with Clang for ARM for various reasons, the code is clearly lacking Clang support for ARM, so I doubt this fall-back will be needed / used anytime soon.

 arm/Makefile.arm |  8 ++++++++
 lib/arm/stack.c  | 18 ++++++++++++++++--
 2 files changed, 24 insertions(+), 2 deletions(-)
arm/Makefile.arm +8 -0
index d6250b7fb686..7734e17fe583 100644
@@ -39,4 +39,12 @@ tests =
 
 include $(SRCDIR)/$(TEST_DIR)/Makefile.common
 
+ifneq ($(KEEP_FRAME_POINTER),)
+# Force the generation of an APCS stack frame layout to be able to reliably
+# walk the stack. Otherwise the compiler may omit saving LR on the stack for
+# leaf functions and, unfortunately, -mno-omit-leaf-frame-pointer isn't
+# supported on ARM :(
+LATE_CFLAGS += $(call cc-option, -mapcs-frame -DAPCS_FRAMES, "")
+endif
+
 arch_clean: arm_clean
lib/arm/stack.c +16 -2
index 66d18b47ea53..b2384d8eb4c1 100644
@@ -8,6 +8,20 @@
 #include <libcflat.h>
 #include <stack.h>
 
+/*
+ * APCS stack frames are generated by code like this:
+ * | mov  ip, sp
+ * | push {..., fp, ip, lr, pc}
+ * | sub  fp, ip, #4
+ */
+#ifdef APCS_FRAMES
+# define FP_IDX -3
+# define LR_IDX -1
+#else
+# define FP_IDX -1
+# define LR_IDX 0
+#endif
+
 int arch_backtrace_frame(const void *frame, const void **return_addrs,
 			 int max_depth, bool current_frame)
 {
@@ -27,10 +41,10 @@ int arch_backtrace_frame(const void *frame, const void **return_addrs,
 	for (depth = 0; depth < max_depth; depth++) {
 		if (!fp)
 			break;
-		return_addrs[depth] = (void *)fp[0];
+		return_addrs[depth] = (void *)fp[LR_IDX];
 		if (return_addrs[depth] == 0)
 			break;
-		fp = (unsigned long *)fp[-1];
+		fp = (unsigned long *)fp[FP_IDX];
 	}
 
 	walking = 0;
Signature · 1 line
2.47.3
Reviewed-by Tested-by on the v2 series
#6 lore raw
In reply to Mathias Krause
Quoted 7 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

Hi Mathias,

Thank you for the arm/arm64 fixes!

For the arm/arm64 patches

Thanks,
drew

Quoted 28 lines

v2 fixes this by introducing the concept of "late CFLAGS" that get evaluated in the top-level Makefile once all other optional flags have been added to $(CFLAGS), which is needed for x86's version at least.

Please apply!

Thanks,
Mathias

[1] https://lore.kernel.org/kvm/[email protected]/

Mathias Krause (4):
Makefile: Provide a concept of late CFLAGS
x86: Better backtraces for leaf functions
arm64: Better backtraces for leaf functions
arm: Fix backtraces involving leaf functions

 Makefile            |  4 ++++
 arm/Makefile.arm    |  8 ++++++++
 arm/Makefile.arm64  |  6 ++++++
 x86/Makefile.common | 11 +++++++++++
 lib/arm/stack.c     | 18 ++++++++++++++++--
 5 files changed, 45 insertions(+), 2 deletions(-)
Signature · 1 line
2.47.3
#7 lore raw
In reply to Mathias Krause

+Sean

Sean, while you're currently in KUT maintainer mode, can you please take a look at this? The ARM bits have already been ack'ed by Andrew and Paolo, apparently, has little time for KUT as well. But as you currently seem to be looking through the pending KUT patches, maybe this one can get some attention as well?

Thanks,
Mathias

Quoted 32 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

v2 fixes this by introducing the concept of "late CFLAGS" that get evaluated in the top-level Makefile once all other optional flags have been added to $(CFLAGS), which is needed for x86's version at least.

Please apply!

Thanks,
Mathias

[1] https://lore.kernel.org/kvm/[email protected]/

Mathias Krause (4):
Makefile: Provide a concept of late CFLAGS
x86: Better backtraces for leaf functions
arm64: Better backtraces for leaf functions
arm: Fix backtraces involving leaf functions

 Makefile            |  4 ++++
 arm/Makefile.arm    |  8 ++++++++
 arm/Makefile.arm64  |  6 ++++++
 x86/Makefile.common | 11 +++++++++++
 lib/arm/stack.c     | 18 ++++++++++++++++--
 5 files changed, 45 insertions(+), 2 deletions(-)
#8 lore raw
Quoted 7 lines · Mathias Krause

+Sean

Sean, while you're currently in KUT maintainer mode, can you please take a look at this? The ARM bits have already been ack'ed by Andrew and Paolo, apparently, has little time for KUT as well. But as you currently seem to be looking through the pending KUT patches, maybe this one can get some attention as well?

Got 'em. Thanks for the heads up, this wasn't on my radar (I saw that Drew had responded to v1 and assumed he had taken them).

reviewer Applied
#9 lore raw
In reply to Mathias Krause
Quoted 9 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

[...]

Applied to kvm-x86 next, thanks!

P.S. This also prompted me to get pretty_print_stacks.py working in my
environment, so double thanks!

[1/4] Makefile: Provide a concept of late CFLAGS

      https://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed
[2/4] x86: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
[3/4] arm64: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/da1804215c8e
[4/4] arm: Fix backtraces involving leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/c885c94f523e

--
https://github.com/kvm-x86/kvm-unit-tests/tree/next

#10 lore raw
Quoted 4 lines · Sean Christopherson
Quoted 3 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

Quoted 8 lines
Quoted 6 lines

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

[...]

Applied to kvm-x86 next, thanks!

Thanks a lot, Sean!

Quoted 2 lines

P.S. This also prompted me to get pretty_print_stacks.py working in my
environment, so double thanks!

Haha, you're welcome! :D

Quoted 12 lines

[1/4] Makefile: Provide a concept of late CFLAGS

      https://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed
[2/4] x86: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
[3/4] arm64: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/da1804215c8e
[4/4] arm: Fix backtraces involving leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/c885c94f523e

--
https://github.com/kvm-x86/kvm-unit-tests/tree/next

#11 lore raw
Quoted 12 lines · Mathias Krause
Quoted 11 lines · Sean Christopherson
Quoted 9 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

[...]

Quoted 14 lines
Quoted 1 line

Applied to kvm-x86 next, thanks!

Thanks a lot, Sean!

Quoted 2 lines

P.S. This also prompted me to get pretty_print_stacks.py working in my
environment, so double thanks!

Haha, you're welcome! :D

Quoted 5 lines

[1/4] Makefile: Provide a concept of late CFLAGS

      https://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed
[2/4] x86: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a

Spoke too soon :-(

The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].

I can't for the life of me figure out how to get Makefile variables to do what I want, so for now I'm going to drop the x86 change so as not to block the rest of the stuff I've got applied.

I'll keep the rest applied, so just resubmit the x86 patch against kvm-x86/next.

FWIW, conceptually I think we want something like this:

x86/Makefile.common +3 -0
index be18a77a..65e41578 100644
@@ -44,6 +44,7 @@ COMMON_CFLAGS += -O1
 KEEP_FRAME_POINTER := y
 
 ifneq ($(KEEP_FRAME_POINTER),)
+ifneq ($(no_profiling),y)
 # Fake profiling to force the compiler to emit a frame pointer setup also in
 # leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately).
 #
@@ -53,6 +54,7 @@ ifneq ($(KEEP_FRAME_POINTER),)
 # during compilation makes this do "The Right Thing."
 LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "")
 endif
+endif
 
 FLATLIBS = lib/libcflat.a
 
@@ -120,6 +122,7 @@ $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode.
        $(LD) -m elf_i386 -nostdlib -o $@ \
              -T $(SRCDIR)/$(TEST_DIR)/realmode.lds $(filter %.o, $^)
 
+$(TEST_DIR)/realmode.o: no_profiling = y
 $(TEST_DIR)/realmode.o: bits = $(realmode_bits)
 
 $(TEST_DIR)/access_test.$(bin): $(TEST_DIR)/access.o
Quoted 8 lines
Quoted 7 lines

[3/4] arm64: Better backtraces for leaf functions

      https://github.com/kvm-x86/kvm-unit-tests/commit/da1804215c8e
[4/4] arm: Fix backtraces involving leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/c885c94f523e

--
https://github.com/kvm-x86/kvm-unit-tests/tree/next

#12 lore raw
Quoted 28 lines · Sean Christopherson
Quoted 12 lines · Mathias Krause
Quoted 11 lines · Sean Christopherson
Quoted 9 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

[...]

Quoted 14 lines
Quoted 1 line

Applied to kvm-x86 next, thanks!

Thanks a lot, Sean!

Quoted 2 lines

P.S. This also prompted me to get pretty_print_stacks.py working in my
environment, so double thanks!

Haha, you're welcome! :D

Quoted 5 lines

[1/4] Makefile: Provide a concept of late CFLAGS

      https://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed
[2/4] x86: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
Quoted 4 lines

Spoke too soon :-(

The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].

Gee! 16bit code at its finest.

So I looked at it and this seems to be a bug in gcc because it emits a 5-byte NOP to make room for a 3-byte call. Actually, it doesn't even emit a NOP, it just spits out some bytes:

minipli@bell:~$ diff -u pg.s nop_mcount.s
--- pg.s	2025-11-18 01:34:34.884472417 +0100
+++ nop_mcount.s	2025-11-18 01:34:05.716096290 +0100
@@ -11,7 +11,7 @@
 	.cfi_offset 5, -8
 	movl	%esp, %ebp
 	.cfi_def_cfa_register 5
-1:	call	mcount
+1:	.byte	0x0f, 0x1f, 0x44, 0x00, 0x00
 	movl	8(%ebp), %eax
 	addl	%eax, %eax
 	popl	%ebp

The issue is, while that is a perfectly fine NOPL for 32-bit and 64-bit, it's not for 16-bit. The 16-bit version gets treated as a 4-byte NOPW with a left-over zero byte which messes with all the following opcodes and causes issues like you observed.

However, gcc is probably not fully to blame, as it cannot know that it's generating 16-bit code. That piece is hidden in a asm(".code16gcc") statement.

Quoted 32 lines

I can't for the life of me figure out how to get Makefile variables to do what I want, so for now I'm going to drop the x86 change so as not to block the rest of the stuff I've got applied.

I'll keep the rest applied, so just resubmit the x86 patch against kvm-x86/next.

FWIW, conceptually I think we want something like this:

x86/Makefile.common +2 -0
index be18a77a..65e41578 100644
@@ -44,6 +44,7 @@ COMMON_CFLAGS += -O1
 KEEP_FRAME_POINTER := y
 
 ifneq ($(KEEP_FRAME_POINTER),)
+ifneq ($(no_profiling),y)
 # Fake profiling to force the compiler to emit a frame pointer setup also in
 # leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately).
 #
@@ -53,6 +54,7 @@ ifneq ($(KEEP_FRAME_POINTER),)
 # during compilation makes this do "The Right Thing."
 LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "")
 endif
+endif
 
 FLATLIBS = lib/libcflat.a
 

x86/Makefile.common $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode. · line 126

Quoted 5 lines
x86/Makefile.common +1 -0
@@ -120,6 +122,7 @@ $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode.
        $(LD) -m elf_i386 -nostdlib -o $@ \
              -T $(SRCDIR)/$(TEST_DIR)/realmode.lds $(filter %.o, $^)
 
+$(TEST_DIR)/realmode.o: no_profiling = y
 $(TEST_DIR)/realmode.o: bits = $(realmode_bits)

Kind of, yes. Just with the $(no_profiling) part embedded into LATE_CFLAGS. However, as that gets forcibly evaluated early (on purpose!), that can't work.

I tried to play with $(filter-out ...,$(CFLAGS)) but that didn't work out either. What does work, however, is to disable -mnop-mcount and make it a call again. This just needs a stub mcount() and the realmode test is working again.

I see that kvm-x86/next as of now (e403d30a8210 ("x86/emulator: Treat DR6_BUS_LOCK as writable if CPU has BUS_LOCK_DETECT")) still has f01ea38a385a ("x86: Better backtraces for leaf functions"), so I just created a fixup commit (attached). Feel free to squash it into f01ea38a385a or yell at if you want something else.

Thanks,
Mathias

From d2ac15881b794a4e3932c49a97278d0426f0aa67 Mon Sep 17 00:00:00 2001
From: Mathias Krause <[email protected]>
Date: Tue, 18 Nov 2025 02:14:27 +0100
Subject: [kvm-unit-tests PATCH] x86: Prevent realmode test code
instrumentation with nop-mcount

Commit f01ea38a385a ("x86: Better backtraces for leaf functions") made use of '-pg -mnop-mcount' to provide a lightweight way to force leaf functions to emit a proper prologue for the backtracing code. However, -mnop-mcount doesn't play well with 16-bit code generation for C code. gcc happily emits a 5-byte NOP that transmutes to a 4-byte NOP followed by a zero byte when executed in real mode, wrecking all code that follows.

Fix that by selectively disabling '-mnop-mcount' for realmode.c, making it call mcount(), which is provided as a stub function.

Fixes f01ea38a385a ("x86: Better backtraces for leaf functions")
---
 x86/Makefile.common | 5 ++++-
 x86/realmode.c      | 3 +++
 2 files changed, 7 insertions(+), 1 deletion(-)
x86/Makefile.common +4 -1
index be18a77a779e..6b59f26a3d2c 100644
@@ -51,7 +51,9 @@ ifneq ($(KEEP_FRAME_POINTER),)
 # We need to defer the cc-option test until -fno-pic or -no-pie have been
 # added to CFLAGS as -mnop-mcount needs it. The lazy evaluation of CFLAGS
 # during compilation makes this do "The Right Thing."
-LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "")
+NOP_PGFLAGS := -pg -mnop-mcount
+LATE_CFLAGS += $(call cc-option, $(NOP_PGFLAGS), "")
+NO_NOP_MCOUNT = $(if $(filter $(NOP_PGFLAGS),$(LATE_CFLAGS)),-mno-nop-mcount)
 endif
 
 FLATLIBS = lib/libcflat.a
@@ -121,6 +123,7 @@ $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode.
 	      -T $(SRCDIR)/$(TEST_DIR)/realmode.lds $(filter %.o, $^)
 
 $(TEST_DIR)/realmode.o: bits = $(realmode_bits)
+$(TEST_DIR)/realmode.o: CFLAGS += $(NO_NOP_MCOUNT)
 
 $(TEST_DIR)/access_test.$(bin): $(TEST_DIR)/access.o
 
x86/realmode.c +3 -0
index 7a4423ec6098..0a7104d4cff4 100644
@@ -23,6 +23,9 @@ void test_function(void);
 asm(
 	"test_function: \n\t"
 	"mov $0x1234, %eax \n\t"
+	"ret\n\t"
+	/* mcount() stub */
+	"mcount:\n\t"
 	"ret"
    );
 
Signature · 1 line
2.47.3
#13 lore raw
Quoted 29 lines · Mathias Krause
Quoted 28 lines · Sean Christopherson
Quoted 26 lines · Mathias Krause
Quoted 4 lines · Sean Christopherson
Quoted 3 lines · Mathias Krause

This is v2 of [1], trying to enhance backtraces involving leaf functions.

Quoted 8 lines
Quoted 6 lines

This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.

[...]

Applied to kvm-x86 next, thanks!

Thanks a lot, Sean!

Quoted 2 lines

P.S. This also prompted me to get pretty_print_stacks.py working in my
environment, so double thanks!

Haha, you're welcome! :D

Quoted 5 lines

[1/4] Makefile: Provide a concept of late CFLAGS

      https://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed
[2/4] x86: Better backtraces for leaf functions
      https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
Quoted 4 lines
Quoted 4 lines

Spoke too soon :-(

The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].

Bleh, I just noticed, f01ea38a385a ("x86: Better backtraces for leaf functions") broke vmx_sipi_signal_test too :(

Looking into it!

Mathias

#14 lore raw
Quoted 11 lines · Mathias Krause
Quoted 5 lines · Mathias Krause
Quoted 4 lines · Sean Christopherson

Spoke too soon :-(

The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].

Bleh, I just noticed, f01ea38a385a ("x86: Better backtraces for leaf functions") broke vmx_sipi_signal_test too :(

Looking into it!

Can't see it. The issue is, the call to hypercall(1) does return in x86/vmx.c:guest_entry() with f01ea38a385a applied but not w/o it. But that makes no sense to me whatsoever. :/

Below debug diff shows the issue:

x86/vmx.c +1 -0
index c803eaa67ac6..e49668f5ff95 100644
@@ -806,4 +806,5 @@ asm(
        "       mov $1, %edi\n\t"
        "       call hypercall\n\t"
+       "       ud2"
 );
 

It also explains why the test fails: it's executing random code that follows, in my case, __this_cpu_has().

Catching some sleep!

Mathias

#15 lore raw
Quoted 8 lines · Mathias Krause
Quoted 7 lines · Mathias Krause
Quoted 5 lines · Mathias Krause
Quoted 4 lines · Sean Christopherson

Spoke too soon :-(

The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].

Quoted 8 lines
Quoted 4 lines

Bleh, I just noticed, f01ea38a385a ("x86: Better backtraces for leaf functions") broke vmx_sipi_signal_test too :(

Looking into it!

Can't see it. The issue is, the call to hypercall(1) does return in x86/vmx.c:guest_entry() with f01ea38a385a applied but not w/o it. But that makes no sense to me whatsoever. :/

After some sleep it makes lot more sense. Well, not really, but it's a race that f01ea38a385a, apparently, makes easier to hit.

x86/vmx.c asm( · line 809

Quoted 12 lines

Below debug diff shows the issue:

x86/vmx.c +1 -0
index c803eaa67ac6..e49668f5ff95 100644
@@ -806,4 +806,5 @@ asm(
        "       mov $1, %edi\n\t"
        "       call hypercall\n\t"
+       "       ud2"
 );

That hypercall call shouldn't return, as it's HYPERCALL_VMEXIT which makes vmx_run() return and not re-enter the guest. But, apparently, it does so and it does so because the guest/VMM shared variable 'hypercall_field' to communicate to the VMM what to do can also be modified concurrently by the other CPU/guest #2 after it was written by CPU/guest #1 but before read by the VMM. Bummer!

I'll send a fix in a few.

Thanks,
Mathias

#16 lore raw
Quoted 2 lines · Mathias Krause

On 18.11.25 05:04, Mathias Krause wrote:
[...]> After some sleep it makes lot more sense. Well, not really, but

it's a

Quoted 5 lines

race that f01ea38a385a, apparently, makes easier to hit.

Quoted 3 lines

Below debug diff shows the issue:

Quoted 15 lines

x86/vmx.c asm( · line 809

Quoted 9 lines
x86/vmx.c +1 -0
index c803eaa67ac6..e49668f5ff95 100644
@@ -806,4 +806,5 @@ asm(
        "       mov $1, %edi\n\t"
        "       call hypercall\n\t"
+       "       ud2"
 );

That hypercall call shouldn't return, as it's HYPERCALL_VMEXIT which makes vmx_run() return and not re-enter the guest. But, apparently, it does so and it does so because the guest/VMM shared variable 'hypercall_field' to communicate to the VMM what to do can also be modified concurrently by the other CPU/guest #2 after it was written by CPU/guest #1 but before read by the VMM. Bummer!

Bleh, I messed up my test setup and it's not it. Back to square one.

#17 lore raw
In reply to Mathias Krause
Quoted 6 lines · Mathias Krause

On 18.11.25 02:33, Mathias Krause wrote:
[...]
Bleh, I just noticed, f01ea38a385a ("x86: Better backtraces for leaf functions") broke vmx_sipi_signal_test too :(

Looking into it!

Finally found it. It's register corruption within both host and guest.

It's not related to f01ea38a385a at all but, apparently, it actually exposes it, likely because of the enforced stack frame setup, making the code rely on (a corrupted) RBP instead of the properly restored (because VMCS managed) RSP.

The core issue is, 'regs' being a singleton, used by multiple CPUs, so all SMP VMX tests concurrently making use of vmx_enter_guest() are potentially affected.

When the first vCPU calls vmx_enter_guest() to launch a guest, it'll use 'regs' to load the guest registers but also store its host register state. Now, if while that vCPU is running, another vCPU gets launched via vmx_enter_guest(), it'll load the previous vCPU's host register values as guest register state and store its host registers in 'regs'.

Depending on which vCPU returns first, it'll either load the other vCPU's host registers effectively "switching threads" or, if it's the vCPU that called vmx_enter_guest() last, it'll resume just fine. Either way, the next vCPU returning will run with the guest register values.

The latter is what happens with vmx_sipi_signal_test, causing the crash.

I read a lot of vmx.c and vmx_test.c in the last few days and it's really not meant to be used concurrently by multiple guests. vmx_test.c has quite some hacks to work around obvious limitations (allocating dedicated stacks for APs) but state variables like 'launched', 'in_guest', 'guest_finished', 'hypercall_field' and 'regs' are shared but really meant to be used only by a single thread.

I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct vmx_test already has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM in vmx_enter_guest() to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)

To see the register corruption, one could start the vmx_sipi_signal_test test with -s -S, attach gdb to it and add a watch for regs.rax. Stepping through the test will clearly show how 'regs' get overwritten wrongly.

In one shell:

$ ./x86-run x86/vmx.flat -append vmx_sipi_signal_test \
	-cpu max,+vmx -smp 2 -s -S

In another:

$ gdb -q -ex 'target remote :1234' x86/vmx.elf
Reading symbols from x86/vmx.elf...
Remote debugging using :1234
0x000000000000fff0 in ?? ()
(gdb) watch regs.rax
Hardware watchpoint 1: regs.rax
(gdb) c
Continuing.

Thread 1 hit Hardware watchpoint 1: regs.rax

Old value = 0
New value = 7589640
0x00000000004006b5 in vmx_enter_guest (result=result@entry=0x73cf08) at x86/vmx.c:1754
1754 asm volatile (
(gdb)
Continuing.
[Switching to Thread 1.2]

Thread 2 hit Hardware watchpoint 1: regs.rax

Old value = 7589640
New value = 7577392
0x00000000004006b5 in vmx_enter_guest (result=result@entry=0x739f30) at x86/vmx.c:1754
1754 asm volatile (
(gdb)
Continuing.

Interrupting the test like this even makes it hang for me but above already shows that vCPU 1 makes use of 'regs' to load and store register state, followed by vCPU 2, doing the same, destroying vCPU 1's stored host register state. Bummer!

Now, the question is, how to properly fix that. (No, not my hack! :P)

Thanks,
Mathias

From 3fb07081769ae6680ac732168aab23cf45eaedc6 Mon Sep 17 00:00:00 2001
From: Mathias Krause <[email protected]>
Date: Fri, 21 Nov 2025 17:34:19 +0100
Subject: [kvm-unit-tests PATCH] x86: hacky attempt to make 'regs' per-cpu

The current concurrent use of globals like 'regs' causes register corruption when multiple CPUs are executing vmx_enter_guest() in parallel.

---
 x86/vmx.h       | 121 +++++++++++++++++++++++++++++++++--------------
 x86/vmx.c       |  70 ++++++++++++++++++---------
 x86/vmx_tests.c | 122 ++++++++++++++++++++++++------------------------
 3 files changed, 196 insertions(+), 117 deletions(-)
x86/vmx.h +85 -36
index 33373bd1a2a9..3dfd5d0c2e71 100644
@@ -120,7 +120,7 @@ struct vmx_test {
 	const char *name;
 	int (*init)(struct vmcs *vmcs);
 	void (*guest_main)(void);
-	int (*exit_handler)(union exit_reason exit_reason);
+	int (*exit_handler)(struct regs *regs, union exit_reason exit_reason);
 	void (*syscall_handler)(u64 syscall_no);
 	struct regs guest_regs;
 	int (*entry_failure_handler)(struct vmentry_result *result);
@@ -588,43 +588,92 @@ enum vm_entry_failure_code {
 	ENTRY_FAIL_VMCS_LINK_PTR	= 4,
 };
 
-#define SAVE_GPR				\
-	"xchg %rax, regs\n\t"			\
-	"xchg %rcx, regs+0x8\n\t"		\
-	"xchg %rdx, regs+0x10\n\t"		\
-	"xchg %rbx, regs+0x18\n\t"		\
-	"xchg %rbp, regs+0x28\n\t"		\
-	"xchg %rsi, regs+0x30\n\t"		\
-	"xchg %rdi, regs+0x38\n\t"		\
-	"xchg %r8, regs+0x40\n\t"		\
-	"xchg %r9, regs+0x48\n\t"		\
-	"xchg %r10, regs+0x50\n\t"		\
-	"xchg %r11, regs+0x58\n\t"		\
-	"xchg %r12, regs+0x60\n\t"		\
-	"xchg %r13, regs+0x68\n\t"		\
-	"xchg %r14, regs+0x70\n\t"		\
-	"xchg %r15, regs+0x78\n\t"
+#define SAVE_GPR(regs)				\
+	"xchg %rax, " regs "\n\t"		\
+	"xchg %rcx, " regs "+0x8\n\t"		\
+	"xchg %rdx, " regs "+0x10\n\t"		\
+	"xchg %rbx, " regs "+0x18\n\t"		\
+	"xchg %rbp, " regs "+0x28\n\t"		\
+	"xchg %rsi, " regs "+0x30\n\t"		\
+	"xchg %rdi, " regs "+0x38\n\t"		\
+	"xchg %r8,  " regs "+0x40\n\t"		\
+	"xchg %r9,  " regs "+0x48\n\t"		\
+	"xchg %r10, " regs "+0x50\n\t"		\
+	"xchg %r11, " regs "+0x58\n\t"		\
+	"xchg %r12, " regs "+0x60\n\t"		\
+	"xchg %r13, " regs "+0x68\n\t"		\
+	"xchg %r14, " regs "+0x70\n\t"		\
+	"xchg %r15, " regs "+0x78\n\t"
 
 #define LOAD_GPR	SAVE_GPR
 
-#define SAVE_GPR_C				\
-	"xchg %%rax, regs\n\t"			\
-	"xchg %%rcx, regs+0x8\n\t"		\
-	"xchg %%rdx, regs+0x10\n\t"		\
-	"xchg %%rbx, regs+0x18\n\t"		\
-	"xchg %%rbp, regs+0x28\n\t"		\
-	"xchg %%rsi, regs+0x30\n\t"		\
-	"xchg %%rdi, regs+0x38\n\t"		\
-	"xchg %%r8, regs+0x40\n\t"		\
-	"xchg %%r9, regs+0x48\n\t"		\
-	"xchg %%r10, regs+0x50\n\t"		\
-	"xchg %%r11, regs+0x58\n\t"		\
-	"xchg %%r12, regs+0x60\n\t"		\
-	"xchg %%r13, regs+0x68\n\t"		\
-	"xchg %%r14, regs+0x70\n\t"		\
-	"xchg %%r15, regs+0x78\n\t"
+#define PUSH_GPR_C				\
+	"push %%rax\n\t"			\
+	"push %%rcx\n\t"			\
+	"push %%rdx\n\t"			\
+	"push %%rbx\n\t"			\
+	"push %%rbp\n\t"			\
+	"push %%rsi\n\t"			\
+	"push %%rdi\n\t"			\
+	"push %%r8\n\t"				\
+	"push %%r9\n\t"				\
+	"push %%r10\n\t"			\
+	"push %%r11\n\t"			\
+	"push %%r12\n\t"			\
+	"push %%r13\n\t"			\
+	"push %%r14\n\t"			\
+	"push %%r15\n\t"
 
-#define LOAD_GPR_C	SAVE_GPR_C
+#define LOAD_GPR_C(regs)			\
+	"mov  " regs ", %%rax\n\t"		\
+	"mov  " regs "+0x8, %%rcx\n\t"		\
+	"mov  " regs "+0x10, %%rdx\n\t"		\
+	"mov  " regs "+0x18, %%rbx\n\t"		\
+	"mov  " regs "+0x28, %%rbp\n\t"		\
+	"mov  " regs "+0x30, %%rsi\n\t"		\
+	"mov  " regs "+0x38, %%rdi\n\t"		\
+	"mov  " regs "+0x40, %%r8\n\t"		\
+	"mov  " regs "+0x48, %%r9\n\t"		\
+	"mov  " regs "+0x50, %%r10\n\t"		\
+	"mov  " regs "+0x58, %%r11\n\t"		\
+	"mov  " regs "+0x60, %%r12\n\t"		\
+	"mov  " regs "+0x68, %%r13\n\t"		\
+	"mov  " regs "+0x70, %%r14\n\t"		\
+	"mov  " regs "+0x78, %%r15\n\t"
+
+#define SAFE_GPR_C(regs)			\
+	"mov  %%rax, " regs "\n\t"		\
+	"mov  %%rcx, " regs "+0x8\n\t"		\
+	"mov  %%rdx, " regs "+0x10\n\t"		\
+	"mov  %%rbx, " regs "+0x18\n\t"		\
+	"mov  %%rbp, " regs "+0x28\n\t"		\
+	"mov  %%rsi, " regs "+0x30\n\t"		\
+	"mov  %%rdi, " regs "+0x38\n\t"		\
+	"mov  %%r8,  " regs "+0x40\n\t"		\
+	"mov  %%r9,  " regs "+0x48\n\t"		\
+	"mov  %%r10, " regs "+0x50\n\t"		\
+	"mov  %%r11, " regs "+0x58\n\t"		\
+	"mov  %%r12, " regs "+0x60\n\t"		\
+	"mov  %%r13, " regs "+0x68\n\t"		\
+	"mov  %%r14, " regs "+0x70\n\t"		\
+	"mov  %%r15, " regs "+0x78\n\t"
+
+#define POP_GPR_C				\
+	"pop  %%r15\n\t"			\
+	"pop  %%r14\n\t"			\
+	"pop  %%r13\n\t"			\
+	"pop  %%r12\n\t"			\
+	"pop  %%r11\n\t"			\
+	"pop  %%r10\n\t"			\
+	"pop  %%r9\n\t"				\
+	"pop  %%r8\n\t"				\
+	"pop  %%rdi\n\t"			\
+	"pop  %%rsi\n\t"			\
+	"pop  %%rbp\n\t"			\
+	"pop  %%rbx\n\t"			\
+	"pop  %%rdx\n\t"			\
+	"pop  %%rcx\n\t"			\
+	"pop  %%rax\n\t"
 
 #define VMX_IO_SIZE_MASK	0x7
 #define _VMX_IO_BYTE		0
@@ -755,7 +804,7 @@ enum vm_entry_failure_code {
 #define VMCS_FIELD_RESERVED_SHIFT	(15)
 #define VMCS_FIELD_BIT_SIZE		(BITS_PER_LONG)
 
-extern struct regs regs;
+struct regs *guest_regs(void);
 
 extern union vmx_basic_msr basic_msr;
 extern union vmx_ctrl_msr ctrl_pin_rev;
@@ -1017,7 +1066,7 @@ void init_vmx(u64 *vmxon_region);
 int init_vmcs(struct vmcs **vmcs);
 
 const char *exit_reason_description(u64 reason);
-void print_vmexit_info(union exit_reason exit_reason);
+void print_vmexit_info(struct regs *regs, union exit_reason exit_reason);
 void print_vmentry_failure_info(struct vmentry_result *result);
 void install_ept_entry(unsigned long *pml4, int pte_level,
 		unsigned long guest_addr, unsigned long pte,
x86/vmx.c +49 -21

171 lines not on this page · raw message

x86/vmx_tests.c +62 -60

411 lines not on this page · raw message

Signature · 1 line
2.47.3
#18 lore raw
Quoted 10 lines · Mathias Krause
Quoted 6 lines · Mathias Krause

On 18.11.25 02:33, Mathias Krause wrote:
[...]
Bleh, I just noticed, f01ea38a385a ("x86: Better backtraces for leaf functions") broke vmx_sipi_signal_test too :(

Looking into it!

Finally found it. It's register corruption within both host and guest.

Quoted 8 lines

It's not related to f01ea38a385a at all but, apparently, it actually exposes it, likely because of the enforced stack frame setup, making the code rely on (a corrupted) RBP instead of the properly restored (because VMCS managed) RSP.

The core issue is, 'regs' being a singleton, used by multiple CPUs, so all SMP VMX tests concurrently making use of vmx_enter_guest() are potentially affected.

*sigh*

I'm not going to type out the first dozen words that escaped my mouth when reading this. I happen to like my job :-)

Quoted 11 lines

When the first vCPU calls vmx_enter_guest() to launch a guest, it'll use 'regs' to load the guest registers but also store its host register state. Now, if while that vCPU is running, another vCPU gets launched via vmx_enter_guest(), it'll load the previous vCPU's host register values as guest register state and store its host registers in 'regs'.

Depending on which vCPU returns first, it'll either load the other vCPU's host registers effectively "switching threads" or, if it's the vCPU that called vmx_enter_guest() last, it'll resume just fine. Either way, the next vCPU returning will run with the guest register values.

Quoted 8 lines

The latter is what happens with vmx_sipi_signal_test, causing the crash.

I read a lot of vmx.c and vmx_test.c in the last few days and it's really not meant to be used concurrently by multiple guests. vmx_test.c has quite some hacks to work around obvious limitations (allocating dedicated stacks for APs) but state variables like 'launched', 'in_guest', 'guest_finished', 'hypercall_field' and 'regs' are shared but really meant to be used only by a single thread.

You're much more generous than me in your description.

Quoted 8 lines

I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct vmx_test already has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM in vmx_enter_guest() to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)

So, much to my chagrin, I started coding away before reading your entire mail (I got through the paragraph about 'regs' getting clobbered, and then came back to the rest later; that was a mistake).

I had the exact same idea about making regs per-CPU, but without the quotes. Of course, because I didn't read your entire mail, converting only regs to be per-CPU didn't help. It actually made things worse (TIL the INIT test shares a VMCS between CPUs... WTF).

After making launch and in_guest per-CPU, and using RAX to communicate hypercalls, I've got everything passing (module one unsolvable SIPI wart; see below). I need to write changelogs and squash a few fixups, but overall it ended up being a nice cleanuped (de-duplicating the VMX vs. SVM GPR handling drops a lot of code). I'm tempted to delete a blank lines just to get to net negative LoC :-D

 lib/x86/smp.h       |  32 ++++++++++++++++++++++++++++++++
 lib/x86/virt.h      |  61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 x86/Makefile.common |  14 ++++++++++++++
 x86/realmode.c      |   3 +++
 x86/svm.c           |  19 ++++++++-----------
 x86/svm.h           |  61 +++++++++----------------------------------------------------
 x86/svm_tests.c     |   5 +++--
 x86/vmx.c           | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------------------------------------------------
 x86/vmx.h           |  72 +++---------------------------------------------------------------------
 x86/vmx_tests.c     | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------
 10 files changed, 247 insertions(+), 246 deletions(-)
Quoted 3 lines

To see the register corruption, one could start the vmx_sipi_signal_test test with -s -S, attach gdb to it and add a watch for regs.rax. Stepping through the test will clearly show how 'regs' get overwritten wrongly.

The test itself is also flawed. On top of this race that you spotted:

@@ -9985,11 +9985,11 @@ static void vmx_sipi_signal_test(void)

       /* update CR3 on AP */                                                  
       on_cpu(1, update_cr3, (void *)read_cr3());                              

+ vmx_set_test_stage(0);
+

       /* start AP */                                                          
       on_cpu_async(1, sipi_test_ap_thread, NULL);                             

- vmx_set_test_stage(0);
-

       /* BSP enter guest */                                                   
       enter_guest();                                                          

}

This snippet is also broken:

	vmx_set_test_stage(1);

	/* AP enter guest */
	enter_guest();

because the BSP can think the AP has entered WFS before it has even attempted VMLAUNCH. It's "fine" so long as there are host CPUs available, but the test fails 100% for me if I run KUT in a VM with e.g. j<number of CPUs>, and on an Ivybridge server CPU, it fails pretty consistently. No idea why, maybe a slower nested VM-Enter path? E.g. the test passes on EMR even if I run with j<2x CPUs>.

Unfortunately, I can't think of any way to fix that problem. To recognize the SIPI, the AP needs to do VM-Enter with a WFS activity state, and I'm struggling to think of a way to atomically write software-visible state at the time of VM-Enter. E.g. in theory, the test could peek at the LAUNCHED field in the VMCS, but that would require reverse engineering the VMCS layout for every CPU. If KVM emulated any VM-wide MSRs, maybe we could throw one in the MSR load list? But all the ones I can think of, e.g. Hyper-V's partition-wide MSRs, aren't guaranteed to be available.

Anywho, now that I know what to look for, I can ignore false failures easily enough. If someone cares enough to come up with a clever fix, then yay. Otherwise, I'll just deal with the intermittent failures (or maybe nuke and pave).

As for this patch, I'll write changelogs and get a series posted with the backtrace and realmode patch at the end.

#19 lore raw

Hi Sean,

Quoted 10 lines · Sean Christopherson
Quoted 4 lines · Mathias Krause

On 18.11.25 02:47, Mathias Krause wrote:

Finally found it. It's register corruption within both host and guest. [...]

*sigh*

I'm not going to type out the first dozen words that escaped my mouth when reading this. I happen to like my job :-)

Me too, that's why I had to switch tasks to gain back some sanity ;)

Quoted 10 lines

[...]

Quoted 8 lines

I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct vmx_test already has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM in vmx_enter_guest() to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)

Quoted 3 lines

So, much to my chagrin, I started coding away before reading your entire mail (I got through the paragraph about 'regs' getting clobbered, and then came back to the rest later; that was a mistake).

A common mistake of mine: writing lengthy explanations that don't get read. Maybe I should start included some tl;dr section for such cases ;)

Quoted 1 line

I had the exact same idea about making regs per-CPU, but without the quotes.

Actually, I wanted something quick to test, so I went the easy route and used a global array. However, my plan for a proper fix was to do the real per-cpu thing, as that would still fit the "offsetable absolute memory reference" needs for inline asm, as GS/FS-prefixed addresses would be just fine.

Quoted 3 lines

Of course, because I didn't read your entire mail, converting only regs to be per-CPU didn't help. It actually made things worse (TIL the INIT test shares a VMCS between CPUs... WTF).

Hah! This rabbit hole has way to many branches!

Quoted 2 lines

After making launch and in_guest per-CPU, and using RAX to communicate hypercalls,

Nice! The use of a global 'hypercall_field' for communicating the intended hypercall was just puzzling to me. Using registers is such a more natural way of doing it.

Quoted 5 lines

I've got everything passing (module one unsolvable SIPI wart; see below). I need to write changelogs and squash a few fixups, but overall it ended up being a nice cleanuped (de-duplicating the VMX vs. SVM GPR handling drops a lot of code). I'm tempted to delete a blank lines just to get to net negative LoC :-D

Quoted 11 lines
 lib/x86/smp.h       |  32 ++++++++++++++++++++++++++++++++
 lib/x86/virt.h      |  61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 x86/Makefile.common |  14 ++++++++++++++
 x86/realmode.c      |   3 +++
 x86/svm.c           |  19 ++++++++-----------
 x86/svm.h           |  61 +++++++++----------------------------------------------------
 x86/svm_tests.c     |   5 +++--
 x86/vmx.c           | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------------------------------------------------
 x86/vmx.h           |  72 +++---------------------------------------------------------------------
 x86/vmx_tests.c     | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------
 10 files changed, 247 insertions(+), 246 deletions(-)

Not too bad!

Quoted 29 lines
Quoted 4 lines

To see the register corruption, one could start the vmx_sipi_signal_test test with -s -S, attach gdb to it and add a watch for regs.rax. Stepping through the test will clearly show how 'regs' get overwritten wrongly.

The test itself is also flawed. On top of this race that you spotted:

@@ -9985,11 +9985,11 @@ static void vmx_sipi_signal_test(void)

       /* update CR3 on AP */                                                  
       on_cpu(1, update_cr3, (void *)read_cr3());                              

+ vmx_set_test_stage(0);
+

       /* start AP */                                                          
       on_cpu_async(1, sipi_test_ap_thread, NULL);                             

- vmx_set_test_stage(0);
-

       /* BSP enter guest */                                                   
       enter_guest();                                                          

}

This snippet is also broken:

	vmx_set_test_stage(1);

	/* AP enter guest */
	enter_guest();
Quoted 5 lines

because the BSP can think the AP has entered WFS before it has even attempted VMLAUNCH. It's "fine" so long as there are host CPUs available, but the test fails 100% for me if I run KUT in a VM with e.g. j<number of CPUs>, and on an Ivybridge server CPU, it fails pretty consistently. No idea why, maybe a slower nested VM-Enter path? E.g. the test passes on EMR even if I run with j<2x CPUs>.

Yeah, I noticed the failing synchronization between the vCPUs too. I even tried to fix them by introducing a few more states (simply more steps). However, then I ended up in deadlocks, as the AP re-enters vmx_sipi_test_guest() multiple times (well, twice). It was likely related to the register corruption, I only noticed later. So I believed those magic sleeps (delay(SIPI_SIGNAL_TEST_DELAY)) are for the lack of synchronization and should account for the time span of the AP setting a new step value but not yet having switched to the guest.
It might be worth revisiting my attempts of explicitly adding new step states, now with the register corrupting being handled...

Quoted 8 lines

Unfortunately, I can't think of any way to fix that problem. To recognize the SIPI, the AP needs to do VM-Enter with a WFS activity state, and I'm struggling to think of a way to atomically write software-visible state at the time of VM-Enter. E.g. in theory, the test could peek at the LAUNCHED field in the VMCS, but that would require reverse engineering the VMCS layout for every CPU. If KVM emulated any VM-wide MSRs, maybe we could throw one in the MSR load list? But all the ones I can think of, e.g. Hyper-V's partition-wide MSRs, aren't guaranteed to be available.

Ahh, that explains the deadlock I observed. So adding more states won't help. Yeah, it needs some trickery... or those magic sleeps :D

Quoted 8 lines

Anywho, now that I know what to look for, I can ignore false failures easily enough. If someone cares enough to come up with a clever fix, then yay. Otherwise, I'll just deal with the intermittent failures (or maybe nuke and pave).

As for this patch, I'll write changelogs and get a series posted with the backtrace and realmode patch at the end.

Thanks a lot for picking up the work! For me, KUT is far from my daily work, so spending time on it cuts on the time budget what I'm actually supposed to do. After having spent nearly a week in chasing that bug made me realize, the work to fix it properly would be, at least for me, yet another week and I didn't had the time for it back then.

Of course, if changes of mine cause regressions, I feel strongly obligated to fix these -- even if it isn't my code's fault after all. However, in this case, I just didn't found the time to do so yet. So thanks again for working on this!

I'll be off for the season from tomorrow on, so may not provide instant feedback for the series, in case you post it in the next couple of days. But I'm looking forward reviewing it! :)

One more thing. I was able to land the gcc bug fix[1], so the realmode patch could be reworded to mention that only "pre gcc-16" is broken but the patch is still needed as gcc 16 isn't even released yet and far from being the minimal supported version for KUT.

Thanks,
Mathias

[1] https://gcc.gnu.org/git/?p=gcc.git;a=commitdiff;h=114a19fae9bd

#20 lore raw

VICTORY IS MINE!!!!!!!

Quoted 24 lines · Mathias Krause
Quoted 10 lines · Sean Christopherson
Quoted 4 lines · Mathias Krause

On 18.11.25 02:47, Mathias Krause wrote:

Finally found it. It's register corruption within both host and guest. [...]

*sigh*

I'm not going to type out the first dozen words that escaped my mouth when reading this. I happen to like my job :-)

Me too, that's why I had to switch tasks to gain back some sanity ;)

Quoted 10 lines

[...]

Quoted 8 lines

I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct vmx_test already has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM in vmx_enter_guest() to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)

Quoted 6 lines
Quoted 3 lines

So, much to my chagrin, I started coding away before reading your entire mail (I got through the paragraph about 'regs' getting clobbered, and then came back to the rest later; that was a mistake).

A common mistake of mine: writing lengthy explanations that don't get read. Maybe I should start included some tl;dr section for such cases ;)

FWIW, I like the lengthy, detailed explanations. I just got excited about having a clever idea for addressing the regs.

Quoted 50 lines
Quoted 24 lines

The test itself is also flawed. On top of this race that you spotted:

@@ -9985,11 +9985,11 @@ static void vmx_sipi_signal_test(void)

       /* update CR3 on AP */                                                  
       on_cpu(1, update_cr3, (void *)read_cr3());                              

+ vmx_set_test_stage(0);
+

       /* start AP */                                                          
       on_cpu_async(1, sipi_test_ap_thread, NULL);                             

- vmx_set_test_stage(0);
-

       /* BSP enter guest */                                                   
       enter_guest();                                                          

}

This snippet is also broken:

	vmx_set_test_stage(1);

	/* AP enter guest */
	enter_guest();
Quoted 5 lines

because the BSP can think the AP has entered WFS before it has even attempted VMLAUNCH. It's "fine" so long as there are host CPUs available, but the test fails 100% for me if I run KUT in a VM with e.g. j<number of CPUs>, and on an Ivybridge server CPU, it fails pretty consistently. No idea why, maybe a slower nested VM-Enter path? E.g. the test passes on EMR even if I run with j<2x CPUs>.

Yeah, I noticed the failing synchronization between the vCPUs too. I even tried to fix them by introducing a few more states (simply more steps). However, then I ended up in deadlocks, as the AP re-enters vmx_sipi_test_guest() multiple times (well, twice). It was likely related to the register corruption, I only noticed later. So I believed those magic sleeps (delay(SIPI_SIGNAL_TEST_DELAY)) are for the lack of synchronization and should account for the time span of the AP setting a new step value but not yet having switched to the guest.
It might be worth revisiting my attempts of explicitly adding new step states, now with the register corrupting being handled...

Quoted 8 lines

Unfortunately, I can't think of any way to fix that problem. To recognize the SIPI, the AP needs to do VM-Enter with a WFS activity state, and I'm struggling to think of a way to atomically write software-visible state at the time of VM-Enter. E.g. in theory, the test could peek at the LAUNCHED field in the VMCS, but that would require reverse engineering the VMCS layout for every CPU. If KVM emulated any VM-wide MSRs, maybe we could throw one in the MSR load list? But all the ones I can think of, e.g. Hyper-V's partition-wide MSRs, aren't guaranteed to be available.

Quoted 2 lines

Ahh, that explains the deadlock I observed. So adding more states won't help. Yeah, it needs some trickery... or those magic sleeps :D

Trickery obtained! MSR_KVM_WALL_CLOCK_NEW to the rescue. Even if with a delay on the AP but not the BSP:

x86/vmx_tests.c +2 -1
index 6bfd6280..047eb1f5 100644
@@ -9869,7 +9869,7 @@ static void vmx_sipi_test_guest(void)
                /* wait AP enter guest with activity=WAIT_SIPI */
                while (vmx_get_test_stage() != 1)
                        ;
-               delay(SIPI_SIGNAL_TEST_DELAY);
+               // delay(SIPI_SIGNAL_TEST_DELAY);
 
                /* First SIPI signal */
                apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_STARTUP | APIC_INT_ASSERT, id_map[1]);
@@ -9938,6 +9938,7 @@ static void sipi_test_ap_thread(void *data)
        vmcs_write(GUEST_ACTV_STATE, ACTV_WAIT_SIPI);
 
        vmx_set_test_stage(1);
+       delay(SIPI_SIGNAL_TEST_DELAY);
 
        /* AP enter guest */
        enter_guest();

Writing MSR_KVM_WALL_CLOCK_NEW via the AP's VM-Entry load list passes with.

x86/vmx_tests.c +29 -3
index 510454a6..2d140ee5 100644
@@ -6,6 +6,7 @@
 
 #include <asm/debugreg.h>
 
+#include "kvmclock.h"
 #include "vmx.h"
 #include "msr.h"
 #include "processor.h"
@@ -1967,7 +1968,7 @@ struct vmx_msr_entry {
        u32 index;
        u32 reserved;
        u64 value;
-} __attribute__((packed));
+} __attribute__((packed)) __attribute__((aligned(16)));
 
 #define MSR_MAGIC 0x31415926
 struct vmx_msr_entry *exit_msr_store, *entry_msr_load, *exit_msr_load;
@@ -9861,6 +9862,8 @@ static void vmx_init_signal_test(void)
         */
 }
 
+static bool use_kvm_wall_clock;
+
 #define SIPI_SIGNAL_TEST_DELAY 100000000ULL
 
 static void vmx_sipi_test_guest(void)
@@ -9869,7 +9872,7 @@ static void vmx_sipi_test_guest(void)
                /* wait AP enter guest with activity=WAIT_SIPI */
                while (vmx_get_test_stage() != 1)
                        ;
-               delay(SIPI_SIGNAL_TEST_DELAY);
+               // delay(SIPI_SIGNAL_TEST_DELAY);
 
                /* First SIPI signal */
                apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_STARTUP | APIC_INT_ASSERT, id_map[1]);
@@ -9903,6 +9906,12 @@ static void vmx_sipi_test_guest(void)
        }
 }
 
+static const struct vmx_msr_entry msr_load_wall_clock = {
+       .index = MSR_KVM_WALL_CLOCK_NEW,
+       .reserved = 0,
+       .value = 1,
+};
+
 static void sipi_test_ap_thread(void *data)
 {
        struct guest_regs *regs = this_cpu_guest_regs();
@@ -9937,7 +9946,15 @@ static void sipi_test_ap_thread(void *data)
        /* Set guest activity state to wait-for-SIPI state */
        vmcs_write(GUEST_ACTV_STATE, ACTV_WAIT_SIPI);
 
-       vmx_set_test_stage(1);
+       if (use_kvm_wall_clock) {
+               wrmsr(MSR_KVM_WALL_CLOCK_NEW, 0);
+               vmcs_write(ENT_MSR_LD_CNT, 1);
+               vmcs_write(ENTER_MSR_LD_ADDR, virt_to_phys(&msr_load_wall_clock));
+       } else {
+               vmx_set_test_stage(1);
+       }
+
+       delay(SIPI_SIGNAL_TEST_DELAY);
 
        /* AP enter guest */
        enter_guest();
@@ -9980,6 +9997,8 @@ static void vmx_sipi_signal_test(void)
        u64 cpu_ctrl_0 = CPU_SECONDARY;
        u64 cpu_ctrl_1 = 0;
 
+       use_kvm_wall_clock = this_cpu_has_kvm() && this_cpu_has(KVM_FEATURE_CLOCKSOURCE2);
+
        /* passthrough lapic to L2 */
        disable_intercept_for_x2apic_msrs();
        vmcs_write(PIN_CONTROLS, vmcs_read(PIN_CONTROLS) & ~PIN_EXTINT);
@@ -9996,6 +10015,13 @@ static void vmx_sipi_signal_test(void)
        /* start AP */
        on_cpu_async(1, sipi_test_ap_thread, NULL);
 
+       if (use_kvm_wall_clock) {
+               while (rdmsr(MSR_KVM_WALL_CLOCK_NEW) != 1)
+                       cpu_relax();
+
+               vmx_set_test_stage(1);
+       }
+
        /* BSP enter guest */
        enter_guest();
 }
#21 lore raw
Quoted 1 line · Sean Christopherson

VICTORY IS MINE!!!!!!!

🏆

Quoted 86 lines

[...]
Writing MSR_KVM_WALL_CLOCK_NEW via the AP's VM-Entry load list passes with.

x86/vmx_tests.c +22 -3
index 510454a6..2d140ee5 100644
@@ -6,6 +6,7 @@
 
 #include <asm/debugreg.h>
 
+#include "kvmclock.h"
 #include "vmx.h"
 #include "msr.h"
 #include "processor.h"
@@ -1967,7 +1968,7 @@ struct vmx_msr_entry {
        u32 index;
        u32 reserved;
        u64 value;
-} __attribute__((packed));
+} __attribute__((packed)) __attribute__((aligned(16)));
 
 #define MSR_MAGIC 0x31415926
 struct vmx_msr_entry *exit_msr_store, *entry_msr_load, *exit_msr_load;
@@ -9861,6 +9862,8 @@ static void vmx_init_signal_test(void)
         */
 }
 
+static bool use_kvm_wall_clock;
+
 #define SIPI_SIGNAL_TEST_DELAY 100000000ULL
 
 static void vmx_sipi_test_guest(void)
@@ -9869,7 +9872,7 @@ static void vmx_sipi_test_guest(void)
                /* wait AP enter guest with activity=WAIT_SIPI */
                while (vmx_get_test_stage() != 1)
                        ;
-               delay(SIPI_SIGNAL_TEST_DELAY);
+               // delay(SIPI_SIGNAL_TEST_DELAY);
 
                /* First SIPI signal */
                apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_STARTUP | APIC_INT_ASSERT, id_map[1]);
@@ -9903,6 +9906,12 @@ static void vmx_sipi_test_guest(void)
        }
 }
 
+static const struct vmx_msr_entry msr_load_wall_clock = {
+       .index = MSR_KVM_WALL_CLOCK_NEW,
+       .reserved = 0,
+       .value = 1,
+};
+
 static void sipi_test_ap_thread(void *data)
 {
        struct guest_regs *regs = this_cpu_guest_regs();
@@ -9937,7 +9946,15 @@ static void sipi_test_ap_thread(void *data)
        /* Set guest activity state to wait-for-SIPI state */
        vmcs_write(GUEST_ACTV_STATE, ACTV_WAIT_SIPI);
 
-       vmx_set_test_stage(1);
+       if (use_kvm_wall_clock) {
+               wrmsr(MSR_KVM_WALL_CLOCK_NEW, 0);
+               vmcs_write(ENT_MSR_LD_CNT, 1);
+               vmcs_write(ENTER_MSR_LD_ADDR, virt_to_phys(&msr_load_wall_clock));
+       } else {
+               vmx_set_test_stage(1);
+       }
+
+       delay(SIPI_SIGNAL_TEST_DELAY);
 
        /* AP enter guest */
        enter_guest();
@@ -9980,6 +9997,8 @@ static void vmx_sipi_signal_test(void)
        u64 cpu_ctrl_0 = CPU_SECONDARY;
        u64 cpu_ctrl_1 = 0;
 
+       use_kvm_wall_clock = this_cpu_has_kvm() && this_cpu_has(KVM_FEATURE_CLOCKSOURCE2);
+
        /* passthrough lapic to L2 */
        disable_intercept_for_x2apic_msrs();
        vmcs_write(PIN_CONTROLS, vmcs_read(PIN_CONTROLS) & ~PIN_EXTINT);
@@ -9996,6 +10015,13 @@ static void vmx_sipi_signal_test(void)
        /* start AP */
        on_cpu_async(1, sipi_test_ap_thread, NULL);
 

x86/vmx_tests.c static void vmx_sipi_signal_test(void) · line 10027

Quoted 10 lines
x86/vmx_tests.c +7 -0
+       if (use_kvm_wall_clock) {
+               while (rdmsr(MSR_KVM_WALL_CLOCK_NEW) != 1)
+                       cpu_relax();
+
+               vmx_set_test_stage(1);
+       }
+
        /* BSP enter guest */
        enter_guest();
 }

What a hack! I like it :D

Happy holidays!

Thanks,
Mathias