Better backtraces for leaf functions
v2 cover Better backtraces for leaf functions
This is v2 of [1], trying to enhance backtraces involving leaf functions.
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
v2 fixes this by introducing the concept of "late CFLAGS" that get evaluated in the top-level Makefile once all other optional flags have been added to $(CFLAGS), which is needed for x86's version at least.
Please apply!
Thanks,
Mathias
[1] https://lore.kernel.org/kvm/[email protected]/
Mathias Krause (4):
Makefile: Provide a concept of late CFLAGS
x86: Better backtraces for leaf functions
arm64: Better backtraces for leaf functions
arm: Fix backtraces involving leaf functions
Makefile | 4 ++++ arm/Makefile.arm | 8 ++++++++ arm/Makefile.arm64 | 6 ++++++ x86/Makefile.common | 11 +++++++++++ lib/arm/stack.c | 18 ++++++++++++++++-- 5 files changed, 45 insertions(+), 2 deletions(-)
Signature · 1 line
2.47.3
PATCH v2 1/4 Makefile: Provide a concept of late CFLAGS
Allow architectures to provide CFLAGS that should be added only after all other optional CFLAGS have been evaluated.
This will be useful for flags that depend on other, generic ones.
To allow 'LATE_CFLAGS' to make use of the $(cc-option ...) helper, assume it'll be a lazily evaluated variable. To further ensure the $(cc-option ...) compiler invocation overhead won't be per-use of $(CFLAGS), enforce its evaluation prior to extending CFLAGS.
--- Makefile | 4 ++++ 1 file changed, 4 insertions(+)
Makefile +4 -0
@@ -95,6 +95,10 @@ CFLAGS += $(wmissing_parameter_type) CFLAGS += $(wold_style_declaration) CFLAGS += -Woverride-init -Wmissing-prototypes -Wstrict-prototypes +# Evaluate and add late cflags last -- they may depend on previous flags +LATE_CFLAGS := $(LATE_CFLAGS) +CFLAGS += $(LATE_CFLAGS) + autodepend-flags = -MMD -MP -MF $(dir $*).$(notdir $*).d LDFLAGS += -nostdlib $(no_pie) -z noexecstack
Signature · 1 line
2.47.3
PATCH v2 2/4 x86: Better backtraces for leaf functions
In reply to Mathias KrauseLeaf functions are problematic for backtraces as they lack the frame pointer setup epilogue. If such a function causes a fault, the original caller won't be part of the backtrace. That's problematic if, for example, memcpy() is failing because it got passed a bad pointer. The generated backtrace will look like this, providing no clue what the issue may be:
STACK: @401b31 4001ad
0x0000000000401b31: memcpy at lib/string.c:136 (discriminator 3)
for (i = 0; i < n; ++i)
> a[i] = b[i]; 0x00000000004001ac: gdt32_end at x86/cstart64.S:127
lea __environ(%rip), %rdx
> call main
mov %eax, %edi
By abusing profiling, we can force the compiler to emit a frame pointer setup epilogue even for leaf functions, making the above backtrace change like this:
STACK: @401c21 400512 4001ad
0x0000000000401c21: memcpy at lib/string.c:136 (discriminator 3)
for (i = 0; i < n; ++i)
> a[i] = b[i]; 0x0000000000400511: main at x86/hypercall.c:91 (discriminator 24)
> memcpy((void *)~0xbadc0de, (void *)0xdeadbeef, 42);
0x00000000004001ac: gdt32_end at x86/cstart64.S:127
lea __environ(%rip), %rdx
> call main
mov %eax, %edi
Above backtrace includes the failing memcpy() call, making it much easier to spot the bug.
Enable "fake profiling" if supported by the compiler to get better backtraces. The runtime overhead should be negligible for the gained debugability as the profiling call is actually a NOP.
---
One may argure that the "ifneq ($(KEEP_FRAME_POINTER),) ... endif" wrapping isn't needed, and that's true. However, it simplifies toggling that variable, if there'll ever be a need for it.
x86/Makefile.common | 11 +++++++++++ 1 file changed, 11 insertions(+)
x86/Makefile.common +11 -0
@@ -43,6 +43,17 @@ COMMON_CFLAGS += -O1 # stack.o relies on frame pointers. KEEP_FRAME_POINTER := y +ifneq ($(KEEP_FRAME_POINTER),) +# Fake profiling to force the compiler to emit a frame pointer setup also in +# leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately). +# +# Note: +# We need to defer the cc-option test until -fno-pic or -no-pie have been +# added to CFLAGS as -mnop-mcount needs it. The lazy evaluation of CFLAGS +# during compilation makes this do "The Right Thing." +LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "") +endif + FLATLIBS = lib/libcflat.a ifeq ($(CONFIG_EFI),y)
Signature · 1 line
2.47.3
Re: [kvm-unit-tests PATCH v2 2/4] x86: Better backtraces for leaf functions
Quoted 63 lines · Mathias Krause
Leaf functions are problematic for backtraces as they lack the frame pointer setup epilogue. If such a function causes a fault, the original caller won't be part of the backtrace. That's problematic if, for example,
memcpy()is failing because it got passed a bad pointer. The generated backtrace will look like this, providing no clue what the issue may be:STACK: @401b31 4001ad 0x0000000000401b31: memcpy at lib/string.c:136 (discriminator 3) for (i = 0; i < n; ++i) > a[i] = b[i];
0x00000000004001ac:gdt32_endatx86/cstart64.S:127
lea__environ(%rip), %rdx
> call main
mov %eax, %ediBy abusing profiling, we can force the compiler to emit a frame pointer setup epilogue even for leaf functions, making the above backtrace change like this:
STACK: @401c21 400512 4001ad 0x0000000000401c21: memcpy at lib/string.c:136 (discriminator 3) for (i = 0; i < n; ++i) > a[i] = b[i];
0x0000000000400511: main atx86/hypercall.c:91(discriminator 24)> memcpy((void *)~0xbadc0de, (void *)0xdeadbeef, 42);
0x00000000004001ac:gdt32_endatx86/cstart64.S:127
lea__environ(%rip), %rdx
> call main
mov %eax, %ediAbove backtrace includes the failing
memcpy()call, making it much easier to spot the bug.Enable "fake profiling" if supported by the compiler to get better backtraces. The runtime overhead should be negligible for the gained debugability as the profiling call is actually a NOP.
Signed-off-by Mathias Krause <[email protected]>---
One may argure that the "ifneq ($(KEEP_FRAME_POINTER),) ... endif" wrapping isn't needed, and that's true. However, it simplifies toggling that variable, if there'll ever be a need for it.x86/Makefile.common | 11 +++++++++++ 1 file changed, 11 insertions(+)x86/Makefile.common +4
-0@@ -43,6 +43,17 @@ COMMON_CFLAGS += -O1 # stack.o relies on frame pointers. KEEP_FRAME_POINTER := y +ifneq ($(KEEP_FRAME_POINTER),) +# Fake profiling to force the compiler to emit a frame pointer setup also in +# leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately). +#
x86/Makefile.common COMMON_CFLAGS += -O1 · line 59
Quoted 10 lines
x86/Makefile.common +7
-0+# Note: +# We need to defer the cc-option test until -fno-pic or -no-pie have been +# added to CFLAGS as -mnop-mcount needs it. The lazy evaluation of CFLAGS +# during compilation makes this do "The Right Thing." +LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "") +endif + FLATLIBS = lib/libcflat.a ifeq ($(CONFIG_EFI),y)
Paolo, can you please comment on this one, so the fixes for ARM and AArch64 are no longer blocked and, ideally, this series can be merged?
Thanks,
Mathias
PATCH v2 3/4 arm64: Better backtraces for leaf functions
In reply to Mathias KrauseSimilar to x86 before, ARM64 skips the stack frame setup for leaf functions, making backtraces miss frames.
Fortunately, gcc supports forcing generating stack frames for these via -mno-omit-leaf-frame-pointer.
--- arm/Makefile.arm64 | 6 ++++++ 1 file changed, 6 insertions(+)
arm/Makefile.arm64 +6 -0
@@ -67,5 +67,11 @@ tests += $(TEST_DIR)/mte.$(exe) include $(SRCDIR)/$(TEST_DIR)/Makefile.common +ifneq ($(KEEP_FRAME_POINTER),) +# Force the generation of a regular stack frame even for leaf functions to make +# stack walking reliable. +LATE_CFLAGS += $(call cc-option, -mno-omit-leaf-frame-pointer, "") +endif + arch_clean: arm_clean $(RM) lib/arm64/.*.d
Signature · 1 line
2.47.3
PATCH v2 4/4 arm: Fix backtraces involving leaf functions
In reply to Mathias KrauseWhen backtracing starts in a leaf function, it'll cause the code to go off the rials as the stack frame for leaf functions is incomplete -- it lacks the return address, likely just causing recursive exception handling by trying to follow an invalid pointer chain.
Unfortunately, -mno-omit-leaf-frame-pointer isn't supported for the ARM target as an easy fix. Make use of -mapcs-frame instead to force the generation of an APCS stack frame layout[1] that can be traversed reliably.
As Clang doesn't support -mapcs-frame, make the stack walking code handle this by using the (old) more compact standard format as a fall-back.
---
I failed to build KUT with Clang for ARM for various reasons, the code is clearly lacking Clang support for ARM, so I doubt this fall-back will be needed / used anytime soon.
arm/Makefile.arm | 8 ++++++++ lib/arm/stack.c | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-)
arm/Makefile.arm +8 -0
@@ -39,4 +39,12 @@ tests = include $(SRCDIR)/$(TEST_DIR)/Makefile.common +ifneq ($(KEEP_FRAME_POINTER),) +# Force the generation of an APCS stack frame layout to be able to reliably +# walk the stack. Otherwise the compiler may omit saving LR on the stack for +# leaf functions and, unfortunately, -mno-omit-leaf-frame-pointer isn't +# supported on ARM :( +LATE_CFLAGS += $(call cc-option, -mapcs-frame -DAPCS_FRAMES, "") +endif + arch_clean: arm_clean
lib/arm/stack.c +16 -2
@@ -8,6 +8,20 @@#include <libcflat.h> #include <stack.h> +/* +* APCS stack frames are generated by code like this: +* | mov ip, sp +* | push {..., fp, ip, lr, pc} +* | sub fp, ip, #4 +*/ +#ifdef APCS_FRAMES +# define FP_IDX -3 +# define LR_IDX -1 +#else +# define FP_IDX -1 +# define LR_IDX 0 +#endif +int arch_backtrace_frame (const void *frame,const void **return_addrs,int max_depth,bool current_frame) {
@@ -27,10 +41,10 @@ int arch_backtrace_frame(const void *frame, const void **return_addrs,for (depth =0 ; depth < max_depth; depth++) {if (!fp)break ; - return_addrs[depth] = (void *)fp[0 ]; + return_addrs[depth] = (void *)fp[LR_IDX ];if (return_addrs[depth] ==0 )break ; - fp = (unsigned long *)fp[-1 ]; + fp = (unsigned long *)fp[FP_IDX ]; } walking =0 ;
Signature · 1 line
2.47.3
Quoted 7 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
Hi Mathias,
Thank you for the arm/arm64 fixes!
For the arm/arm64 patches
Thanks,
drew
Quoted 28 lines
v2 fixes this by introducing the concept of "late CFLAGS" that get evaluated in the top-level
Makefileonce all other optional flags have been added to $(CFLAGS), which is needed for x86's version at least.Please apply!
Thanks,
Mathias[1] https://lore.kernel.org/kvm/[email protected]/
Mathias Krause (4):
Makefile: Provide a concept of late CFLAGS
x86: Better backtraces for leaf functions
arm64: Better backtraces for leaf functions
arm: Fix backtraces involving leaf functionsMakefile | 4 ++++ arm/Makefile.arm | 8 ++++++++ arm/Makefile.arm64 | 6 ++++++ x86/Makefile.common | 11 +++++++++++ lib/arm/stack.c | 18 ++++++++++++++++-- 5 files changed, 45 insertions(+), 2 deletions(-)Signature · 1 line
2.47.3
+Sean
Sean, while you're currently in KUT maintainer mode, can you please take a look at this? The ARM bits have already been ack'ed by Andrew and Paolo, apparently, has little time for KUT as well. But as you currently seem to be looking through the pending KUT patches, maybe this one can get some attention as well?
Thanks,
Mathias
Quoted 32 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
v2 fixes this by introducing the concept of "late CFLAGS" that get evaluated in the top-level
Makefileonce all other optional flags have been added to $(CFLAGS), which is needed for x86's version at least.Please apply!
Thanks,
Mathias[1] https://lore.kernel.org/kvm/[email protected]/
Mathias Krause (4):
Makefile: Provide a concept of late CFLAGS
x86: Better backtraces for leaf functions
arm64: Better backtraces for leaf functions
arm: Fix backtraces involving leaf functionsMakefile | 4 ++++ arm/Makefile.arm | 8 ++++++++ arm/Makefile.arm64 | 6 ++++++ x86/Makefile.common | 11 +++++++++++ lib/arm/stack.c | 18 ++++++++++++++++-- 5 files changed, 45 insertions(+), 2 deletions(-)
Sean Christopherson
reviewerQuoted 7 lines · Mathias Krause
+Sean
Sean, while you're currently in KUT maintainer mode, can you please take a look at this? The ARM bits have already been ack'ed by Andrew and Paolo, apparently, has little time for KUT as well. But as you currently seem to be looking through the pending KUT patches, maybe this one can get some attention as well?
Got 'em. Thanks for the heads up, this wasn't on my radar (I saw that Drew had responded to v1 and assumed he had taken them).
Quoted 9 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
[...]
Applied to kvm-x86 next, thanks!
P.S. This also prompted me to get pretty_print_stacks.py working in my
environment, so double thanks!
[1/4] Makefile: Provide a concept of late CFLAGS
https://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed [2/4] x86: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a [3/4] arm64: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/da1804215c8e [4/4] arm: Fix backtraces involving leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/c885c94f523e
Quoted 4 lines · Sean Christopherson
Quoted 3 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
Quoted 8 lines
Quoted 6 lines
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
[...]
Applied to kvm-x86 next, thanks!
Thanks a lot, Sean!
Quoted 2 lines
P.S. This also prompted me to getpretty_print_stacks.pyworking in my
environment, so double thanks!
Haha, you're welcome! :D
Quoted 12 lines
[1/4]
Makefile: Provide a concept of late CFLAGShttps://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed [2/4] x86: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a [3/4] arm64: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/da1804215c8e [4/4] arm: Fix backtraces involving leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/c885c94f523e
Sean Christopherson
reviewerQuoted 12 lines · Mathias Krause
Quoted 11 lines · Sean Christopherson
Quoted 9 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
[...]
Quoted 14 lines
Quoted 1 line
Applied to kvm-x86 next, thanks!
Thanks a lot, Sean!
Quoted 2 lines
P.S. This also prompted me to getpretty_print_stacks.pyworking in my
environment, so double thanks!Haha, you're welcome! :D
Quoted 5 lines
[1/4]
Makefile: Provide a concept of late CFLAGShttps://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed [2/4] x86: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
Spoke too soon :-(
The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].
I can't for the life of me figure out how to get Makefile variables to do what I want, so for now I'm going to drop the x86 change so as not to block the rest of the stuff I've got applied.
I'll keep the rest applied, so just resubmit the x86 patch against kvm-x86/next.
FWIW, conceptually I think we want something like this:
x86/Makefile.common +3 -0
@@ -44,6 +44,7 @@ COMMON_CFLAGS += -O1 KEEP_FRAME_POINTER := y ifneq ($(KEEP_FRAME_POINTER),) +ifneq ($(no_profiling),y) # Fake profiling to force the compiler to emit a frame pointer setup also in # leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately). #
@@ -53,6 +54,7 @@ ifneq ($(KEEP_FRAME_POINTER),) # during compilation makes this do "The Right Thing." LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "") endif +endif FLATLIBS = lib/libcflat.a
@@ -120,6 +122,7 @@ $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode. $(LD) -m elf_i386 -nostdlib -o $@ \ -T $(SRCDIR)/$(TEST_DIR)/realmode.lds $(filter %.o, $^) +$(TEST_DIR)/realmode.o: no_profiling = y $(TEST_DIR)/realmode.o: bits = $(realmode_bits) $(TEST_DIR)/access_test.$(bin): $(TEST_DIR)/access.o
Quoted 8 lines
Quoted 7 lines
[3/4] arm64: Better backtraces for leaf functions
https://github.com/kvm-x86/kvm-unit-tests/commit/da1804215c8e [4/4] arm: Fix backtraces involving leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/c885c94f523e
Quoted 28 lines · Sean Christopherson
Quoted 12 lines · Mathias Krause
Quoted 11 lines · Sean Christopherson
Quoted 9 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
[...]
Quoted 14 lines
Quoted 1 line
Applied to kvm-x86 next, thanks!
Thanks a lot, Sean!
Quoted 2 lines
P.S. This also prompted me to getpretty_print_stacks.pyworking in my
environment, so double thanks!Haha, you're welcome! :D
Quoted 5 lines
[1/4]
Makefile: Provide a concept of late CFLAGShttps://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed [2/4] x86: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
Quoted 4 lines
Spoke too soon :-(
The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].
Gee! 16bit code at its finest.
So I looked at it and this seems to be a bug in gcc because it emits a 5-byte NOP to make room for a 3-byte call. Actually, it doesn't even emit a NOP, it just spits out some bytes:
minipli@bell:~$ diff -u pg.s nop_mcount.s --- pg.s 2025-11-18 01:34:34.884472417 +0100 +++ nop_mcount.s 2025-11-18 01:34:05.716096290 +0100 @@ -11,7 +11,7 @@ .cfi_offset 5, -8 movl %esp, %ebp .cfi_def_cfa_register 5 -1: call mcount +1: .byte 0x0f, 0x1f, 0x44, 0x00, 0x00 movl 8(%ebp), %eax addl %eax, %eax popl %ebp
The issue is, while that is a perfectly fine NOPL for 32-bit and 64-bit, it's not for 16-bit. The 16-bit version gets treated as a 4-byte NOPW with a left-over zero byte which messes with all the following opcodes and causes issues like you observed.
However, gcc is probably not fully to blame, as it cannot know that it's generating 16-bit code. That piece is hidden in a asm(".code16gcc") statement.
Quoted 32 lines
I can't for the life of me figure out how to get
Makefilevariables to do what I want, so for now I'm going to drop the x86 change so as not to block the rest of the stuff I've got applied.I'll keep the rest applied, so just resubmit the x86 patch against kvm-x86/next.
FWIW, conceptually I think we want something like this:
x86/Makefile.common +2
-0@@ -44,6 +44,7 @@ COMMON_CFLAGS += -O1 KEEP_FRAME_POINTER := y ifneq ($(KEEP_FRAME_POINTER),) +ifneq ($(no_profiling),y) # Fake profiling to force the compiler to emit a frame pointer setup also in # leaf function (-mno-omit-leaf-frame-pointer doesn't work, unfortunately). #@@ -53,6 +54,7 @@ ifneq ($(KEEP_FRAME_POINTER),) # during compilation makes this do "The Right Thing." LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "") endif +endif FLATLIBS = lib/libcflat.a
x86/Makefile.common $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode. · line 126
Quoted 5 lines
x86/Makefile.common +1
-0@@ -120,6 +122,7 @@ $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode. $(LD) -m elf_i386 -nostdlib -o $@ \ -T $(SRCDIR)/$(TEST_DIR)/realmode.lds $(filter %.o, $^) +$(TEST_DIR)/realmode.o: no_profiling = y $(TEST_DIR)/realmode.o: bits = $(realmode_bits)
Kind of, yes. Just with the $(no_profiling) part embedded into LATE_CFLAGS. However, as that gets forcibly evaluated early (on purpose!), that can't work.
I tried to play with $(filter-out ...,$(CFLAGS)) but that didn't work out either. What does work, however, is to disable -mnop-mcount and make it a call again. This just needs a stub mcount() and the realmode test is working again.
I see that kvm-x86/next as of now (e403d30a8210 ("x86/emulator: Treat DR6_BUS_LOCK as writable if CPU has BUS_LOCK_DETECT")) still has f01ea38a385a ("x86: Better backtraces for leaf functions"), so I just created a fixup commit (attached). Feel free to squash it into f01ea38a385a or yell at if you want something else.
Thanks,
Mathias
From d2ac15881b794a4e3932c49a97278d0426f0aa67 Mon Sep 17 00:00:00 2001
From: Mathias Krause <[email protected]>
Date: Tue, 18 Nov 2025 02:14:27 +0100
Subject: [kvm-unit-tests PATCH] x86: Prevent realmode test code
instrumentation with nop-mcount
Commit f01ea38a385a ("x86: Better backtraces for leaf functions") made use of '-pg -mnop-mcount' to provide a lightweight way to force leaf functions to emit a proper prologue for the backtracing code. However, -mnop-mcount doesn't play well with 16-bit code generation for C code. gcc happily emits a 5-byte NOP that transmutes to a 4-byte NOP followed by a zero byte when executed in real mode, wrecking all code that follows.
Fix that by selectively disabling '-mnop-mcount' for realmode.c, making it call mcount(), which is provided as a stub function.
--- x86/Makefile.common | 5 ++++- x86/realmode.c | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-)
x86/Makefile.common +4 -1
@@ -51,7 +51,9 @@ ifneq ($(KEEP_FRAME_POINTER),) # We need to defer the cc-option test until -fno-pic or -no-pie have been # added to CFLAGS as -mnop-mcount needs it. The lazy evaluation of CFLAGS # during compilation makes this do "The Right Thing." -LATE_CFLAGS += $(call cc-option, -pg -mnop-mcount, "") +NOP_PGFLAGS := -pg -mnop-mcount +LATE_CFLAGS += $(call cc-option, $(NOP_PGFLAGS), "") +NO_NOP_MCOUNT = $(if $(filter $(NOP_PGFLAGS),$(LATE_CFLAGS)),-mno-nop-mcount) endif FLATLIBS = lib/libcflat.a
@@ -121,6 +123,7 @@ $(TEST_DIR)/realmode.elf: $(TEST_DIR)/realmode.o $(SRCDIR)/$(TEST_DIR)/realmode. -T $(SRCDIR)/$(TEST_DIR)/realmode.lds $(filter %.o, $^) $(TEST_DIR)/realmode.o: bits = $(realmode_bits) +$(TEST_DIR)/realmode.o: CFLAGS += $(NO_NOP_MCOUNT) $(TEST_DIR)/access_test.$(bin): $(TEST_DIR)/access.o
x86/realmode.c +3 -0
@@ -23,6 +23,9 @@ void test_function(void); asm("test_function: \n\t" "mov $0x1234, %eax \n\t" +"ret\n\t" +/* mcount() stub */ +"mcount:\n\t" "ret" );
Signature · 1 line
2.47.3
Quoted 29 lines · Mathias Krause
Quoted 28 lines · Sean Christopherson
Quoted 26 lines · Mathias Krause
Quoted 4 lines · Sean Christopherson
Quoted 3 lines · Mathias Krause
This is v2 of [1], trying to enhance backtraces involving leaf functions.
Quoted 8 lines
Quoted 6 lines
This version fixes backtraces on ARM and ARM64 as well, as ARM currently fails hard for leaf functions lacking a proper stack frame setup, making it dereference invalid pointers. ARM64 just skips frames, much like x86 does.
[...]
Applied to kvm-x86 next, thanks!
Thanks a lot, Sean!
Quoted 2 lines
P.S. This also prompted me to getpretty_print_stacks.pyworking in my
environment, so double thanks!Haha, you're welcome! :D
Quoted 5 lines
[1/4]
Makefile: Provide a concept of late CFLAGShttps://github.com/kvm-x86/kvm-unit-tests/commit/816fe2d45aed [2/4] x86: Better backtraces for leaf functions https://github.com/kvm-x86/kvm-unit-tests/commit/f01ea38a385a
Quoted 4 lines
Quoted 4 lines
Spoke too soon :-(
The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].
Bleh, I just noticed, f01ea38a385a ("x86: Better backtraces for leaf functions") broke vmx_sipi_signal_test too :(
Looking into it!
Mathias
Quoted 11 lines · Mathias Krause
Quoted 5 lines · Mathias Krause
Quoted 4 lines · Sean Christopherson
Spoke too soon :-(
The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].Bleh, I just noticed,
f01ea38a385a("x86: Better backtraces for leaf functions") brokevmx_sipi_signal_testtoo :(Looking into it!
Can't see it. The issue is, the call to hypercall(1) does return in x86/vmx.c:guest_entry() with f01ea38a385a applied but not w/o it. But that makes no sense to me whatsoever. :/
Below debug diff shows the issue:
x86/vmx.c +1 -0
@@ -806,4 +806,5 @@ asm(" mov $1, %edi\n\t" " call hypercall\n\t" +" ud2" );
It also explains why the test fails: it's executing random code that follows, in my case, __this_cpu_has().
Catching some sleep!
Mathias
Quoted 8 lines · Mathias Krause
Quoted 7 lines · Mathias Krause
Quoted 5 lines · Mathias Krause
Quoted 4 lines · Sean Christopherson
Spoke too soon :-(
The x86 change breaks the realmode test. I didn't try hard to debug, as that
test is brittle, e.g. see https://lore.kernel.org/all/[email protected].
Quoted 8 lines
Quoted 4 lines
Bleh, I just noticed,
f01ea38a385a("x86: Better backtraces for leaf functions") brokevmx_sipi_signal_testtoo :(Looking into it!
Can't see it. The issue is, the call to hypercall(1) does return in x86/vmx.c:guest_entry() with
f01ea38a385aapplied but not w/o it. But that makes no sense to me whatsoever. :/
After some sleep it makes lot more sense. Well, not really, but it's a race that f01ea38a385a, apparently, makes easier to hit.
x86/vmx.c asm( · line 809
Quoted 12 lines
Below debug diff shows the issue:
x86/vmx.c +1
-0@@ -806,4 +806,5 @@ asm(" mov $1, %edi\n\t" " call hypercall\n\t" +" ud2" );
That hypercall call shouldn't return, as it's HYPERCALL_VMEXIT which makes vmx_run() return and not re-enter the guest. But, apparently, it does so and it does so because the guest/VMM shared variable 'hypercall_field' to communicate to the VMM what to do can also be modified concurrently by the other CPU/guest #2 after it was written by CPU/guest #1 but before read by the VMM. Bummer!
I'll send a fix in a few.
Thanks,
Mathias
Quoted 2 lines · Mathias Krause
On 18.11.25 05:04, Mathias Krause wrote:
[...]> After some sleep it makes lot more sense. Well, not really, but
it's a
Quoted 5 lines
race that
f01ea38a385a, apparently, makes easier to hit.Quoted 3 lines
Below debug diff shows the issue:
Quoted 15 lines
x86/vmx.c asm( · line 809
Quoted 9 lines
x86/vmx.c +1
-0@@ -806,4 +806,5 @@ asm(" mov $1, %edi\n\t" " call hypercall\n\t" +" ud2" );That hypercall call shouldn't return, as it's
HYPERCALL_VMEXITwhich makesvmx_run()return and not re-enter the guest. But, apparently, it does so and it does so because the guest/VMM shared variable 'hypercall_field' to communicate to the VMM what to do can also be modified concurrently by the other CPU/guest #2 after it was written by CPU/guest #1 but before read by the VMM. Bummer!
Bleh, I messed up my test setup and it's not it. Back to square one.
Quoted 6 lines · Mathias Krause
On 18.11.25 02:33, Mathias Krause wrote:
[...]
Bleh, I just noticed,f01ea38a385a("x86: Better backtraces for leaf functions") brokevmx_sipi_signal_testtoo :(Looking into it!
Finally found it. It's register corruption within both host and guest.
It's not related to f01ea38a385a at all but, apparently, it actually exposes it, likely because of the enforced stack frame setup, making the code rely on (a corrupted) RBP instead of the properly restored (because VMCS managed) RSP.
The core issue is, 'regs' being a singleton, used by multiple CPUs, so all SMP VMX tests concurrently making use of vmx_enter_guest() are potentially affected.
When the first vCPU calls vmx_enter_guest() to launch a guest, it'll use 'regs' to load the guest registers but also store its host register state. Now, if while that vCPU is running, another vCPU gets launched via vmx_enter_guest(), it'll load the previous vCPU's host register values as guest register state and store its host registers in 'regs'.
Depending on which vCPU returns first, it'll either load the other vCPU's host registers effectively "switching threads" or, if it's the vCPU that called vmx_enter_guest() last, it'll resume just fine. Either way, the next vCPU returning will run with the guest register values.
The latter is what happens with vmx_sipi_signal_test, causing the crash.
I read a lot of vmx.c and vmx_test.c in the last few days and it's really not meant to be used concurrently by multiple guests. vmx_test.c has quite some hacks to work around obvious limitations (allocating dedicated stacks for APs) but state variables like 'launched', 'in_guest', 'guest_finished', 'hypercall_field' and 'regs' are shared but really meant to be used only by a single thread.
I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct vmx_test already has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM in vmx_enter_guest() to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)
To see the register corruption, one could start the vmx_sipi_signal_test test with -s -S, attach gdb to it and add a watch for regs.rax. Stepping through the test will clearly show how 'regs' get overwritten wrongly.
In one shell:
$ ./x86-run x86/vmx.flat -append vmx_sipi_signal_test \ -cpu max,+vmx -smp 2 -s -S
In another:
$ gdb -q -ex 'target remote :1234' x86/vmx.elf Reading symbols from x86/vmx.elf... Remote debugging using :1234 0x000000000000fff0 in ?? () (gdb) watch regs.rax Hardware watchpoint 1: regs.rax (gdb) c Continuing.
Thread 1 hit Hardware watchpoint 1: regs.rax
Old value = 0
New value = 75896400x00000000004006b5 in vmx_enter_guest (result=result@entry=0x73cf08) at x86/vmx.c:1754
1754 asm volatile (
(gdb)
Continuing.
[Switching to Thread 1.2]
Thread 2 hit Hardware watchpoint 1: regs.rax
Old value = 7589640
New value = 75773920x00000000004006b5 in vmx_enter_guest (result=result@entry=0x739f30) at x86/vmx.c:1754
1754 asm volatile (
(gdb)
Continuing.
Interrupting the test like this even makes it hang for me but above already shows that vCPU 1 makes use of 'regs' to load and store register state, followed by vCPU 2, doing the same, destroying vCPU 1's stored host register state. Bummer!
Now, the question is, how to properly fix that. (No, not my hack! :P)
Thanks,
Mathias
From 3fb07081769ae6680ac732168aab23cf45eaedc6 Mon Sep 17 00:00:00 2001
From: Mathias Krause <[email protected]>
Date: Fri, 21 Nov 2025 17:34:19 +0100
Subject: [kvm-unit-tests PATCH] x86: hacky attempt to make 'regs' per-cpu
The current concurrent use of globals like 'regs' causes register corruption when multiple CPUs are executing vmx_enter_guest() in parallel.
--- x86/vmx.h | 121 +++++++++++++++++++++++++++++++++-------------- x86/vmx.c | 70 ++++++++++++++++++--------- x86/vmx_tests.c | 122 ++++++++++++++++++++++++------------------------ 3 files changed, 196 insertions(+), 117 deletions(-)
x86/vmx.h +85 -36
@@ -120,7 +120,7 @@ struct vmx_test {const char *name;int (*init)(struct vmcs *vmcs);void (*guest_main)(void ); -int (*exit_handler)(union exit_reason exit_reason); +int (*exit_handler)(struct regs *regs,union exit_reason exit_reason);void (*syscall_handler)(u64 syscall_no);struct regs guest_regs;int (*entry_failure_handler)(struct vmentry_result *result);
@@ -588,43 +588,92 @@ enum vm_entry_failure_code { ENTRY_FAIL_VMCS_LINK_PTR = 4, }; -#define SAVE_GPR \ - "xchg %rax, regs\n\t" \ - "xchg %rcx, regs+0x8\n\t" \ - "xchg %rdx, regs+0x10\n\t" \ - "xchg %rbx, regs+0x18\n\t" \ - "xchg %rbp, regs+0x28\n\t" \ - "xchg %rsi, regs+0x30\n\t" \ - "xchg %rdi, regs+0x38\n\t" \ - "xchg %r8, regs+0x40\n\t" \ - "xchg %r9, regs+0x48\n\t" \ - "xchg %r10, regs+0x50\n\t" \ - "xchg %r11, regs+0x58\n\t" \ - "xchg %r12, regs+0x60\n\t" \ - "xchg %r13, regs+0x68\n\t" \ - "xchg %r14, regs+0x70\n\t" \ - "xchg %r15, regs+0x78\n\t" +#define SAVE_GPR (regs) \ + "xchg %rax, " regs "\n\t" \ + "xchg %rcx, " regs "+0x8\n\t" \ + "xchg %rdx, " regs "+0x10\n\t" \ + "xchg %rbx, " regs "+0x18\n\t" \ + "xchg %rbp, " regs "+0x28\n\t" \ + "xchg %rsi, " regs "+0x30\n\t" \ + "xchg %rdi, " regs "+0x38\n\t" \ + "xchg %r8, " regs "+0x40\n\t" \ + "xchg %r9, " regs "+0x48\n\t" \ + "xchg %r10, " regs "+0x50\n\t" \ + "xchg %r11, " regs "+0x58\n\t" \ + "xchg %r12, " regs "+0x60\n\t" \ + "xchg %r13, " regs "+0x68\n\t" \ + "xchg %r14, " regs "+0x70\n\t" \ + "xchg %r15, " regs "+0x78\n\t"#define LOAD_GPR SAVE_GPR -#define SAVE_GPR_C \ - "xchg %%rax, regs\n\t" \ - "xchg %%rcx, regs+0x8\n\t" \ - "xchg %%rdx, regs+0x10\n\t" \ - "xchg %%rbx, regs+0x18\n\t" \ - "xchg %%rbp, regs+0x28\n\t" \ - "xchg %%rsi, regs+0x30\n\t" \ - "xchg %%rdi, regs+0x38\n\t" \ - "xchg %%r8, regs+0x40\n\t" \ - "xchg %%r9, regs+0x48\n\t" \ - "xchg %%r10, regs+0x50\n\t" \ - "xchg %%r11, regs+0x58\n\t" \ - "xchg %%r12, regs+0x60\n\t" \ - "xchg %%r13, regs+0x68\n\t" \ - "xchg %%r14, regs+0x70\n\t" \ - "xchg %%r15, regs+0x78\n\t" +#define PUSH_GPR_C \ + "push %%rax\n\t" \ + "push %%rcx\n\t" \ + "push %%rdx\n\t" \ + "push %%rbx\n\t" \ + "push %%rbp\n\t" \ + "push %%rsi\n\t" \ + "push %%rdi\n\t" \ + "push %%r8\n\t" \ + "push %%r9\n\t" \ + "push %%r10\n\t" \ + "push %%r11\n\t" \ + "push %%r12\n\t" \ + "push %%r13\n\t" \ + "push %%r14\n\t" \ + "push %%r15\n\t" -#define LOAD_GPR_C SAVE_GPR_C +#define LOAD_GPR_C (regs) \ + "mov " regs ", %%rax\n\t" \ + "mov " regs "+0x8, %%rcx\n\t" \ + "mov " regs "+0x10, %%rdx\n\t" \ + "mov " regs "+0x18, %%rbx\n\t" \ + "mov " regs "+0x28, %%rbp\n\t" \ + "mov " regs "+0x30, %%rsi\n\t" \ + "mov " regs "+0x38, %%rdi\n\t" \ + "mov " regs "+0x40, %%r8\n\t" \ + "mov " regs "+0x48, %%r9\n\t" \ + "mov " regs "+0x50, %%r10\n\t" \ + "mov " regs "+0x58, %%r11\n\t" \ + "mov " regs "+0x60, %%r12\n\t" \ + "mov " regs "+0x68, %%r13\n\t" \ + "mov " regs "+0x70, %%r14\n\t" \ + "mov " regs "+0x78, %%r15\n\t" + +#define SAFE_GPR_C (regs) \ + "mov %%rax, " regs "\n\t" \ + "mov %%rcx, " regs "+0x8\n\t" \ + "mov %%rdx, " regs "+0x10\n\t" \ + "mov %%rbx, " regs "+0x18\n\t" \ + "mov %%rbp, " regs "+0x28\n\t" \ + "mov %%rsi, " regs "+0x30\n\t" \ + "mov %%rdi, " regs "+0x38\n\t" \ + "mov %%r8, " regs "+0x40\n\t" \ + "mov %%r9, " regs "+0x48\n\t" \ + "mov %%r10, " regs "+0x50\n\t" \ + "mov %%r11, " regs "+0x58\n\t" \ + "mov %%r12, " regs "+0x60\n\t" \ + "mov %%r13, " regs "+0x68\n\t" \ + "mov %%r14, " regs "+0x70\n\t" \ + "mov %%r15, " regs "+0x78\n\t" + +#define POP_GPR_C \ + "pop %%r15\n\t" \ + "pop %%r14\n\t" \ + "pop %%r13\n\t" \ + "pop %%r12\n\t" \ + "pop %%r11\n\t" \ + "pop %%r10\n\t" \ + "pop %%r9\n\t" \ + "pop %%r8\n\t" \ + "pop %%rdi\n\t" \ + "pop %%rsi\n\t" \ + "pop %%rbp\n\t" \ + "pop %%rbx\n\t" \ + "pop %%rdx\n\t" \ + "pop %%rcx\n\t" \ + "pop %%rax\n\t"#define VMX_IO_SIZE_MASK 0x7#define _VMX_IO_BYTE 0
@@ -755,7 +804,7 @@ enum vm_entry_failure_code {#define VMCS_FIELD_RESERVED_SHIFT (15)#define VMCS_FIELD_BIT_SIZE (BITS_PER_LONG) -extern struct regs regs; +struct regs *guest_regs (void );extern union vmx_basic_msr basic_msr;extern union vmx_ctrl_msr ctrl_pin_rev;
@@ -1017,7 +1066,7 @@ void init_vmx(u64 *vmxon_region);int init_vmcs (struct vmcs **vmcs);const char *exit_reason_description (u64 reason); -void print_vmexit_info (union exit_reason exit_reason); +void print_vmexit_info (struct regs *regs,union exit_reason exit_reason);void print_vmentry_failure_info (struct vmentry_result *result); void install_ept_entry(unsigned long *pml4, int pte_level, unsigned long guest_addr, unsigned long pte,
x86/vmx.c +49 -21
171 lines not on this page · raw message
x86/vmx_tests.c +62 -60
411 lines not on this page · raw message
Signature · 1 line
2.47.3
Sean Christopherson
reviewerQuoted 10 lines · Mathias Krause
Quoted 6 lines · Mathias Krause
On 18.11.25 02:33, Mathias Krause wrote:
[...]
Bleh, I just noticed,f01ea38a385a("x86: Better backtraces for leaf functions") brokevmx_sipi_signal_testtoo :(Looking into it!
Finally found it. It's register corruption within both host and guest.
Quoted 8 lines
It's not related to
f01ea38a385aat all but, apparently, it actually exposes it, likely because of the enforced stack frame setup, making the code rely on (a corrupted) RBP instead of the properly restored (because VMCS managed) RSP.The core issue is, 'regs' being a singleton, used by multiple CPUs, so all SMP VMX tests concurrently making use of
vmx_enter_guest()are potentially affected.
*sigh*
I'm not going to type out the first dozen words that escaped my mouth when reading this. I happen to like my job :-)
Quoted 11 lines
When the first vCPU calls
vmx_enter_guest()to launch a guest, it'll use 'regs' to load the guest registers but also store its host register state. Now, if while that vCPU is running, another vCPU gets launched viavmx_enter_guest(), it'll load the previous vCPU's host register values as guest register state and store its host registers in 'regs'.Depending on which vCPU returns first, it'll either load the other vCPU's host registers effectively "switching threads" or, if it's the vCPU that called
vmx_enter_guest()last, it'll resume just fine. Either way, the next vCPU returning will run with the guest register values.
Quoted 8 lines
The latter is what happens with
vmx_sipi_signal_test, causing the crash.I read a lot of
vmx.candvmx_test.cin the last few days and it's really not meant to be used concurrently by multiple guests.vmx_test.chas quite some hacks to work around obvious limitations (allocating dedicated stacks for APs) but state variables like 'launched', 'in_guest', 'guest_finished', 'hypercall_field' and 'regs' are shared but really meant to be used only by a single thread.
You're much more generous than me in your description.
Quoted 8 lines
I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct
vmx_testalready has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM invmx_enter_guest()to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)
So, much to my chagrin, I started coding away before reading your entire mail (I got through the paragraph about 'regs' getting clobbered, and then came back to the rest later; that was a mistake).
I had the exact same idea about making regs per-CPU, but without the quotes. Of course, because I didn't read your entire mail, converting only regs to be per-CPU didn't help. It actually made things worse (TIL the INIT test shares a VMCS between CPUs... WTF).
After making launch and in_guest per-CPU, and using RAX to communicate hypercalls, I've got everything passing (module one unsolvable SIPI wart; see below). I need to write changelogs and squash a few fixups, but overall it ended up being a nice cleanuped (de-duplicating the VMX vs. SVM GPR handling drops a lot of code). I'm tempted to delete a blank lines just to get to net negative LoC :-D
lib/x86/smp.h | 32 ++++++++++++++++++++++++++++++++ lib/x86/virt.h | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ x86/Makefile.common | 14 ++++++++++++++ x86/realmode.c | 3 +++ x86/svm.c | 19 ++++++++----------- x86/svm.h | 61 +++++++++---------------------------------------------------- x86/svm_tests.c | 5 +++-- x86/vmx.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------------- x86/vmx.h | 72 +++--------------------------------------------------------------------- x86/vmx_tests.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------ 10 files changed, 247 insertions(+), 246 deletions(-)
Quoted 3 lines
To see the register corruption, one could start the
vmx_sipi_signal_testtest with -s -S, attach gdb to it and add a watch for regs.rax. Stepping through the test will clearly show how 'regs' get overwritten wrongly.
The test itself is also flawed. On top of this race that you spotted:
@@ -9985,11 +9985,11 @@ static void vmx_sipi_signal_test(void)
/* update CR3 on AP */
on_cpu(1, update_cr3, (void *)read_cr3()); + vmx_set_test_stage(0);
+
/* start AP */
on_cpu_async(1, sipi_test_ap_thread, NULL); - vmx_set_test_stage(0);
-
/* BSP enter guest */
enter_guest(); }
This snippet is also broken:
vmx_set_test_stage(1); /* AP enter guest */ enter_guest();
because the BSP can think the AP has entered WFS before it has even attempted VMLAUNCH. It's "fine" so long as there are host CPUs available, but the test fails 100% for me if I run KUT in a VM with e.g. j<number of CPUs>, and on an Ivybridge server CPU, it fails pretty consistently. No idea why, maybe a slower nested VM-Enter path? E.g. the test passes on EMR even if I run with j<2x CPUs>.
Unfortunately, I can't think of any way to fix that problem. To recognize the SIPI, the AP needs to do VM-Enter with a WFS activity state, and I'm struggling to think of a way to atomically write software-visible state at the time of VM-Enter. E.g. in theory, the test could peek at the LAUNCHED field in the VMCS, but that would require reverse engineering the VMCS layout for every CPU. If KVM emulated any VM-wide MSRs, maybe we could throw one in the MSR load list? But all the ones I can think of, e.g. Hyper-V's partition-wide MSRs, aren't guaranteed to be available.
Anywho, now that I know what to look for, I can ignore false failures easily enough. If someone cares enough to come up with a clever fix, then yay. Otherwise, I'll just deal with the intermittent failures (or maybe nuke and pave).
As for this patch, I'll write changelogs and get a series posted with the backtrace and realmode patch at the end.
Hi Sean,
Quoted 10 lines · Sean Christopherson
Quoted 4 lines · Mathias Krause
On 18.11.25 02:47, Mathias Krause wrote:
Finally found it. It's register corruption within both host and guest. [...]
*sigh*
I'm not going to type out the first dozen words that escaped my mouth when reading this. I happen to like my job :-)
Me too, that's why I had to switch tasks to gain back some sanity ;)
Quoted 10 lines
[...]
Quoted 8 lines
I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct
vmx_testalready has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM invmx_enter_guest()to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)
Quoted 3 lines
So, much to my chagrin, I started coding away before reading your entire mail (I got through the paragraph about 'regs' getting clobbered, and then came back to the rest later; that was a mistake).
A common mistake of mine: writing lengthy explanations that don't get read. Maybe I should start included some tl;dr section for such cases ;)
Quoted 1 line
I had the exact same idea about making regs per-CPU, but without the quotes.
Actually, I wanted something quick to test, so I went the easy route and used a global array. However, my plan for a proper fix was to do the real per-cpu thing, as that would still fit the "offsetable absolute memory reference" needs for inline asm, as GS/FS-prefixed addresses would be just fine.
Quoted 3 lines
Of course, because I didn't read your entire mail, converting only regs to be per-CPU didn't help. It actually made things worse (TIL the INIT test shares a VMCS between CPUs... WTF).
Hah! This rabbit hole has way to many branches!
Quoted 2 lines
After making launch and
in_guestper-CPU, and using RAX to communicate hypercalls,
Nice! The use of a global 'hypercall_field' for communicating the intended hypercall was just puzzling to me. Using registers is such a more natural way of doing it.
Quoted 5 lines
I've got everything passing (module one unsolvable SIPI wart; see below). I need to write changelogs and squash a few fixups, but overall it ended up being a nice cleanuped (de-duplicating the VMX vs. SVM GPR handling drops a lot of code). I'm tempted to delete a blank lines just to get to net negative LoC :-D
Quoted 11 lines
lib/x86/smp.h | 32 ++++++++++++++++++++++++++++++++ lib/x86/virt.h | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ x86/Makefile.common | 14 ++++++++++++++ x86/realmode.c | 3 +++ x86/svm.c | 19 ++++++++----------- x86/svm.h | 61 +++++++++---------------------------------------------------- x86/svm_tests.c | 5 +++-- x86/vmx.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------------- x86/vmx.h | 72 +++--------------------------------------------------------------------- x86/vmx_tests.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------ 10 files changed, 247 insertions(+), 246 deletions(-)
Not too bad!
Quoted 29 lines
Quoted 4 lines
To see the register corruption, one could start the
vmx_sipi_signal_testtest with -s -S, attach gdb to it and add a watch for regs.rax. Stepping through the test will clearly show how 'regs' get overwritten wrongly.The test itself is also flawed. On top of this race that you spotted:
@@ -9985,11 +9985,11 @@ static void
vmx_sipi_signal_test(void)/* update CR3 on AP */ on_cpu(1, update_cr3, (void *)read_cr3());+
vmx_set_test_stage(0);
+/* start AP */ on_cpu_async(1, sipi_test_ap_thread, NULL);-
vmx_set_test_stage(0);
-/* BSP enter guest */ enter_guest();}
This snippet is also broken:
vmx_set_test_stage(1); /* AP enter guest */ enter_guest();
Quoted 5 lines
because the BSP can think the AP has entered WFS before it has even attempted VMLAUNCH. It's "fine" so long as there are host CPUs available, but the test fails 100% for me if I run KUT in a VM with e.g. j<number of CPUs>, and on an Ivybridge server CPU, it fails pretty consistently. No idea why, maybe a slower nested VM-Enter path? E.g. the test passes on EMR even if I run with j<2x CPUs>.
Yeah, I noticed the failing synchronization between the vCPUs too. I even tried to fix them by introducing a few more states (simply more steps). However, then I ended up in deadlocks, as the AP re-enters vmx_sipi_test_guest() multiple times (well, twice). It was likely related to the register corruption, I only noticed later. So I believed those magic sleeps (delay(SIPI_SIGNAL_TEST_DELAY)) are for the lack of synchronization and should account for the time span of the AP setting a new step value but not yet having switched to the guest.
It might be worth revisiting my attempts of explicitly adding new step states, now with the register corrupting being handled...
Quoted 8 lines
Unfortunately, I can't think of any way to fix that problem. To recognize the SIPI, the AP needs to do VM-Enter with a WFS activity state, and I'm struggling to think of a way to atomically write software-visible state at the time of VM-Enter. E.g. in theory, the test could peek at the LAUNCHED field in the VMCS, but that would require reverse engineering the VMCS layout for every CPU. If KVM emulated any VM-wide MSRs, maybe we could throw one in the MSR load list? But all the ones I can think of, e.g. Hyper-V's partition-wide MSRs, aren't guaranteed to be available.
Ahh, that explains the deadlock I observed. So adding more states won't help. Yeah, it needs some trickery... or those magic sleeps :D
Quoted 8 lines
Anywho, now that I know what to look for, I can ignore false failures easily enough. If someone cares enough to come up with a clever fix, then yay. Otherwise, I'll just deal with the intermittent failures (or maybe nuke and pave).
As for this patch, I'll write changelogs and get a series posted with the backtrace and realmode patch at the end.
Thanks a lot for picking up the work! For me, KUT is far from my daily work, so spending time on it cuts on the time budget what I'm actually supposed to do. After having spent nearly a week in chasing that bug made me realize, the work to fix it properly would be, at least for me, yet another week and I didn't had the time for it back then.
Of course, if changes of mine cause regressions, I feel strongly obligated to fix these -- even if it isn't my code's fault after all. However, in this case, I just didn't found the time to do so yet. So thanks again for working on this!
I'll be off for the season from tomorrow on, so may not provide instant feedback for the series, in case you post it in the next couple of days. But I'm looking forward reviewing it! :)
One more thing. I was able to land the gcc bug fix[1], so the realmode patch could be reworded to mention that only "pre gcc-16" is broken but the patch is still needed as gcc 16 isn't even released yet and far from being the minimal supported version for KUT.
Thanks,
Mathias
[1] https://gcc.gnu.org/git/?p=gcc.git;a=commitdiff;h=114a19fae9bd
Sean Christopherson
reviewerVICTORY IS MINE!!!!!!!
Quoted 24 lines · Mathias Krause
Quoted 10 lines · Sean Christopherson
Quoted 4 lines · Mathias Krause
On 18.11.25 02:47, Mathias Krause wrote:
Finally found it. It's register corruption within both host and guest. [...]
*sigh*
I'm not going to type out the first dozen words that escaped my mouth when reading this. I happen to like my job :-)
Me too, that's why I had to switch tasks to gain back some sanity ;)
Quoted 10 lines
[...]
Quoted 8 lines
I hacked up something to verify my theory and made 'regs' "per-cpu". It needs quite some code churn and I'm not all that happy with it. IMHO, 'regs' and lots of the other VMX management state should be part of some vcpu struct or something. In fact, struct
vmx_testalready has a 'guest_regs' but using it won't work, as we need offsetable absolute memory references for the inline ASM invmx_enter_guest()to work as it cannot make use of register-based memory references at all. (My hack uses a global 'scratch_regs' with mutual exclusion on its usage.)
Quoted 6 lines
Quoted 3 lines
So, much to my chagrin, I started coding away before reading your entire mail (I got through the paragraph about 'regs' getting clobbered, and then came back to the rest later; that was a mistake).
A common mistake of mine: writing lengthy explanations that don't get read. Maybe I should start included some tl;dr section for such cases ;)
FWIW, I like the lengthy, detailed explanations. I just got excited about having a clever idea for addressing the regs.
Quoted 50 lines
Quoted 24 lines
The test itself is also flawed. On top of this race that you spotted:
@@ -9985,11 +9985,11 @@ static void
vmx_sipi_signal_test(void)/* update CR3 on AP */ on_cpu(1, update_cr3, (void *)read_cr3());+
vmx_set_test_stage(0);
+/* start AP */ on_cpu_async(1, sipi_test_ap_thread, NULL);-
vmx_set_test_stage(0);
-/* BSP enter guest */ enter_guest();}
This snippet is also broken:
vmx_set_test_stage(1); /* AP enter guest */ enter_guest();Quoted 5 lines
because the BSP can think the AP has entered WFS before it has even attempted VMLAUNCH. It's "fine" so long as there are host CPUs available, but the test fails 100% for me if I run KUT in a VM with e.g. j<number of CPUs>, and on an Ivybridge server CPU, it fails pretty consistently. No idea why, maybe a slower nested VM-Enter path? E.g. the test passes on EMR even if I run with j<2x CPUs>.
Yeah, I noticed the failing synchronization between the vCPUs too. I even tried to fix them by introducing a few more states (simply more steps). However, then I ended up in deadlocks, as the AP re-enters
vmx_sipi_test_guest()multiple times (well, twice). It was likely related to the register corruption, I only noticed later. So I believed those magic sleeps (delay(SIPI_SIGNAL_TEST_DELAY)) are for the lack of synchronization and should account for the time span of the AP setting a new step value but not yet having switched to the guest.
It might be worth revisiting my attempts of explicitly adding new step states, now with the register corrupting being handled...Quoted 8 lines
Unfortunately, I can't think of any way to fix that problem. To recognize the SIPI, the AP needs to do VM-Enter with a WFS activity state, and I'm struggling to think of a way to atomically write software-visible state at the time of VM-Enter. E.g. in theory, the test could peek at the LAUNCHED field in the VMCS, but that would require reverse engineering the VMCS layout for every CPU. If KVM emulated any VM-wide MSRs, maybe we could throw one in the MSR load list? But all the ones I can think of, e.g. Hyper-V's partition-wide MSRs, aren't guaranteed to be available.
Quoted 2 lines
Ahh, that explains the deadlock I observed. So adding more states won't help. Yeah, it needs some trickery... or those magic sleeps :D
Trickery obtained! MSR_KVM_WALL_CLOCK_NEW to the rescue. Even if with a delay on the AP but not the BSP:
x86/vmx_tests.c +2 -1
@@ -9869,7 +9869,7 @@ static void vmx_sipi_test_guest(void)/* wait AP enter guest with activity=WAIT_SIPI */ while (vmx_get_test_stage () !=1 ) ; -delay (SIPI_SIGNAL_TEST_DELAY ); +// delay(SIPI_SIGNAL_TEST_DELAY); /* First SIPI signal */ apic_icr_write (APIC_DEST_PHYSICAL |APIC_DM_STARTUP |APIC_INT_ASSERT , id_map[1 ]);
@@ -9938,6 +9938,7 @@ static void sipi_test_ap_thread(void *data)vmcs_write (GUEST_ACTV_STATE ,ACTV_WAIT_SIPI );vmx_set_test_stage (1 ); +delay (SIPI_SIGNAL_TEST_DELAY );/* AP enter guest */ enter_guest ();
Writing MSR_KVM_WALL_CLOCK_NEW via the AP's VM-Entry load list passes with.
x86/vmx_tests.c +29 -3
@@ -6,6 +6,7 @@#include <asm/debugreg.h> +#include "kvmclock.h" #include "vmx.h" #include "msr.h" #include "processor.h"
@@ -1967,7 +1968,7 @@ struct vmx_msr_entry {u32 index;u32 reserved;u64 value; -} __attribute__((packed)); +} __attribute__((packed)) __attribute__((aligned(16)));#define MSR_MAGIC 0x31415926struct vmx_msr_entry *exit_msr_store, *entry_msr_load, *exit_msr_load;
@@ -9861,6 +9862,8 @@ static void vmx_init_signal_test(void)*/ } +static bool use_kvm_wall_clock; +#define SIPI_SIGNAL_TEST_DELAY 100000000ULLstatic void vmx_sipi_test_guest (void )
@@ -9869,7 +9872,7 @@ static void vmx_sipi_test_guest(void)/* wait AP enter guest with activity=WAIT_SIPI */ while (vmx_get_test_stage () !=1 ) ; -delay (SIPI_SIGNAL_TEST_DELAY ); +// delay(SIPI_SIGNAL_TEST_DELAY); /* First SIPI signal */ apic_icr_write (APIC_DEST_PHYSICAL |APIC_DM_STARTUP |APIC_INT_ASSERT , id_map[1 ]);
@@ -9903,6 +9906,12 @@ static void vmx_sipi_test_guest(void) } } +static const struct vmx_msr_entry msr_load_wall_clock = { + .index =MSR_KVM_WALL_CLOCK_NEW , + .reserved =0 , + .value =1 , +}; +static void sipi_test_ap_thread (void *data) {struct guest_regs *regs =this_cpu_guest_regs ();
@@ -9937,7 +9946,15 @@ static void sipi_test_ap_thread(void *data)/* Set guest activity state to wait-for-SIPI state */ vmcs_write (GUEST_ACTV_STATE ,ACTV_WAIT_SIPI ); -vmx_set_test_stage (1 ); +if (use_kvm_wall_clock) { +wrmsr (MSR_KVM_WALL_CLOCK_NEW ,0 ); +vmcs_write (ENT_MSR_LD_CNT ,1 ); +vmcs_write (ENTER_MSR_LD_ADDR ,virt_to_phys (&msr_load_wall_clock)); + }else { +vmx_set_test_stage (1 ); + } + +delay (SIPI_SIGNAL_TEST_DELAY );/* AP enter guest */ enter_guest ();
@@ -9980,6 +9997,8 @@ static void vmx_sipi_signal_test(void)u64 cpu_ctrl_0 =CPU_SECONDARY ;u64 cpu_ctrl_1 =0 ; + use_kvm_wall_clock =this_cpu_has_kvm () &&this_cpu_has (KVM_FEATURE_CLOCKSOURCE2 ); +/* passthrough lapic to L2 */ disable_intercept_for_x2apic_msrs ();vmcs_write (PIN_CONTROLS ,vmcs_read (PIN_CONTROLS ) & ~PIN_EXTINT );
@@ -9996,6 +10015,13 @@ static void vmx_sipi_signal_test(void)/* start AP */ on_cpu_async (1 , sipi_test_ap_thread,NULL ); +if (use_kvm_wall_clock) { +while (rdmsr (MSR_KVM_WALL_CLOCK_NEW ) !=1 ) +cpu_relax (); + +vmx_set_test_stage (1 ); + } +/* BSP enter guest */ enter_guest (); }
Quoted 1 line · Sean Christopherson
VICTORY IS MINE!!!!!!!
🏆
Quoted 86 lines
[...]
WritingMSR_KVM_WALL_CLOCK_NEWvia the AP's VM-Entry load list passes with.x86/vmx_tests.c +22
-3@@ -6,6 +6,7 @@#include <asm/debugreg.h> +#include "kvmclock.h" #include "vmx.h" #include "msr.h" #include "processor.h" @@ -1967,7 +1968,7 @@ struct vmx_msr_entry {u32 index;u32 reserved;u64 value; -} __attribute__((packed)); +} __attribute__((packed)) __attribute__((aligned(16)));#define MSR_MAGIC 0x31415926struct vmx_msr_entry *exit_msr_store, *entry_msr_load, *exit_msr_load;@@ -9861,6 +9862,8 @@ static void vmx_init_signal_test(void)*/ } +static bool use_kvm_wall_clock; +#define SIPI_SIGNAL_TEST_DELAY 100000000ULLstatic void vmx_sipi_test_guest (void )@@ -9869,7 +9872,7 @@ static void vmx_sipi_test_guest(void)/* wait AP enter guest with activity=WAIT_SIPI */ while (vmx_get_test_stage () !=1 ) ; -delay (SIPI_SIGNAL_TEST_DELAY ); +// delay(SIPI_SIGNAL_TEST_DELAY); /* First SIPI signal */ apic_icr_write (APIC_DEST_PHYSICAL |APIC_DM_STARTUP |APIC_INT_ASSERT , id_map[1 ]);@@ -9903,6 +9906,12 @@ static void vmx_sipi_test_guest(void) } } +static const struct vmx_msr_entry msr_load_wall_clock = { + .index =MSR_KVM_WALL_CLOCK_NEW , + .reserved =0 , + .value =1 , +}; +static void sipi_test_ap_thread (void *data) {struct guest_regs *regs =this_cpu_guest_regs ();@@ -9937,7 +9946,15 @@ static void sipi_test_ap_thread(void *data)/* Set guest activity state to wait-for-SIPI state */ vmcs_write (GUEST_ACTV_STATE ,ACTV_WAIT_SIPI ); -vmx_set_test_stage (1 ); +if (use_kvm_wall_clock) { +wrmsr (MSR_KVM_WALL_CLOCK_NEW ,0 ); +vmcs_write (ENT_MSR_LD_CNT ,1 ); +vmcs_write (ENTER_MSR_LD_ADDR ,virt_to_phys (&msr_load_wall_clock)); + }else { +vmx_set_test_stage (1 ); + } + +delay (SIPI_SIGNAL_TEST_DELAY );/* AP enter guest */ enter_guest ();@@ -9980,6 +9997,8 @@ static void vmx_sipi_signal_test(void)u64 cpu_ctrl_0 =CPU_SECONDARY ;u64 cpu_ctrl_1 =0 ; + use_kvm_wall_clock =this_cpu_has_kvm () &&this_cpu_has (KVM_FEATURE_CLOCKSOURCE2 ); +/* passthrough lapic to L2 */ disable_intercept_for_x2apic_msrs ();vmcs_write (PIN_CONTROLS ,vmcs_read (PIN_CONTROLS ) & ~PIN_EXTINT );@@ -9996,6 +10015,13 @@ static void vmx_sipi_signal_test(void)/* start AP */ on_cpu_async (1 , sipi_test_ap_thread,NULL );
x86/vmx_tests.c static void vmx_sipi_signal_test(void) · line 10027
Quoted 10 lines
x86/vmx_tests.c +7
-0+if (use_kvm_wall_clock) { +while (rdmsr (MSR_KVM_WALL_CLOCK_NEW ) !=1 ) +cpu_relax (); + +vmx_set_test_stage (1 ); + } +/* BSP enter guest */ enter_guest (); }
What a hack! I like it :D
Happy holidays!
Thanks,
Mathias