Debug info for nVHE hyp panics
Subsystems (3)
v2 cover Debug info for nVHE hyp panics
Panics from arm64's nVHE hyp mode are hard to interpret. This series adds some more debug info to help with diagnosis.
Using BUG() in nVHE hyp gives a meaningful address to locate invariants that fail to hold. The host can also look up the bug to provide the file and line, if the debug configs are enabled. Otherwise a kimg address is much more useful than a hyp VA since it can be looked up in vmlinux.
The lib/bug.c changes apply on 5.11.
This arm64 KVM changes apply on top of the previous panic fix at
https://lore.kernel.org/r/[email protected]/
From v1 (https://lore.kernel.org/r/[email protected]/):
- keeping struct bug details in bug.c
- using SPSR to distinguish hyp from host
- inlined __hyp_panic_string
Andrew Scull (4):
bug: Remove redundant condition check in report_bug
bug: Factor out a getter for a bug's file line
KVM: arm64: Use BUG and BUG_ON in nVHE hyp
KVM: arm64: Log source when panicking from nVHE hyp
arch/arm64/include/asm/kvm_hyp.h | 1 - arch/arm64/include/asm/kvm_mmu.h | 2 + arch/arm64/kernel/image-vars.h | 3 +- arch/arm64/kvm/handle_exit.c | 31 ++++++++++++++ arch/arm64/kvm/hyp/include/hyp/switch.h | 2 - arch/arm64/kvm/hyp/nvhe/host.S | 17 ++++---- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 6 +-- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 2 - arch/arm64/kvm/hyp/vhe/switch.c | 4 +- include/linux/bug.h | 3 ++ lib/bug.c | 54 +++++++++++++------------ 12 files changed, 77 insertions(+), 50 deletions(-)
Signature · 6 lines
2.30.0.617.g56c4b15f3c-goog _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
PATCH v2 1/4 bug: Remove redundant condition check in report_bug
report_bug() will return early if it cannot find a bug corresponding to the provided address. The subsequent test for the bug will always be true so remove it.
--- lib/bug.c | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-)
lib/bug.c +15 -18
@@ -158,30 +158,27 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) file =NULL ; line =0 ; - warning =0 ; -if (bug) {#ifdef CONFIG_DEBUG_BUGVERBOSE #ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - file = bug->file; + file = bug->file;#else - file = (const char *)bug + bug->file_disp; + file = (const char *)bug + bug->file_disp;#endif - line = bug->line; + line = bug->line;#endif - warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; - once = (bug->flags &BUGFLAG_ONCE ) !=0 ; - done = (bug->flags &BUGFLAG_DONE ) !=0 ; - -if (warning && once) { -if (done) -return BUG_TRAP_TYPE_WARN ; - -/* -* Since this is the only store, concurrency is not an issue. -*/ - bug->flags |=BUGFLAG_DONE ; - } + warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; + once = (bug->flags &BUGFLAG_ONCE ) !=0 ; + done = (bug->flags &BUGFLAG_DONE ) !=0 ; + +if (warning && once) { +if (done) +return BUG_TRAP_TYPE_WARN ; + +/* +* Since this is the only store, concurrency is not an issue. +*/ + bug->flags |=BUGFLAG_DONE ; }/*
Signature · 6 lines
2.30.0.617.g56c4b15f3c-goog _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v2 1/4] bug: Remove redundant condition check in report_bug
Quoted 3 lines · On Tue, 23 Feb 2021 15:57:56 +0000 Andrew Scull <[email protected]> wrote:
report_bug()will return early if it cannot find a bug corresponding to the provided address. The subsequent test for the bug will always be true so remove it.
-- Steve
Quoted 48 lines
Signed-off-by Andrew Scull <[email protected]>Cc Peter Zijlstra <[email protected]>Cc "Steven Rostedt (VMware)" <[email protected]>--- lib/bug.c | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-)lib/bug.c +8
-18@@ -158,30 +158,27 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) file =NULL ; line =0 ; - warning =0 ; -if (bug) {#ifdef CONFIG_DEBUG_BUGVERBOSE #ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - file = bug->file; + file = bug->file;#else - file = (const char *)bug + bug->file_disp; + file = (const char *)bug + bug->file_disp;#endif - line = bug->line; + line = bug->line;#endif - warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; - once = (bug->flags &BUGFLAG_ONCE ) !=0 ; - done = (bug->flags &BUGFLAG_DONE ) !=0 ; - -if (warning && once) { -if (done) -return BUG_TRAP_TYPE_WARN ; - -/* -* Since this is the only store, concurrency is not an issue. -*/ - bug->flags |=BUGFLAG_DONE ; - } + warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; + once = (bug->flags &BUGFLAG_ONCE ) !=0 ; + done = (bug->flags &BUGFLAG_DONE ) !=0 ; + +if (warning && once) {
lib/bug.c enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) · line 184
Quoted 10 lines
lib/bug.c +7
-0+if (done) +return BUG_TRAP_TYPE_WARN ; + +/* +* Since this is the only store, concurrency is not an issue. +*/ + bug->flags |=BUGFLAG_DONE ; }/*
_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
PATCH v2 2/4 bug: Factor out a getter for a bug's file line
In reply to Andrew ScullThere is some non-trivial config-based logic to get the file name and line number associated with a bug. Factor this out to a getter that can be resused.
--- include/linux/bug.h | 3 +++ lib/bug.c | 27 +++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-)
include/linux/bug.h +3 -0
@@ -36,6 +36,9 @@ static inline int is_warning_bug(const struct bug_entry *bug)return bug->flags &BUGFLAG_WARNING ; } +void bug_get_file_line (struct bug_entry *bug,const char **file, +unsigned int *line); +struct bug_entry *find_bug (unsigned long bugaddr);enum bug_trap_type report_bug (unsigned long bug_addr,struct pt_regs *regs);
lib/bug.c +17 -10
@@ -130,6 +130,22 @@ static inline struct bug_entry *module_find_bug(unsigned long bugaddr) }#endif +void bug_get_file_line (struct bug_entry *bug,const char **file, +unsigned int *line) +{ + *file =NULL ; + *line =0 ; + +#ifdef CONFIG_DEBUG_BUGVERBOSE +#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS + *file = bug->file; +#else + *file = (const char *)bug + bug->file_disp; +#endif + *line = bug->line; +#endif +} +struct bug_entry *find_bug (unsigned long bugaddr) {struct bug_entry *bug;
@@ -156,17 +172,8 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)disable_trace_on_warning (); - file =NULL ; - line =0 ; +bug_get_file_line (bug, &file, &line); -#ifdef CONFIG_DEBUG_BUGVERBOSE -#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - file = bug->file; -#else - file = (const char *)bug + bug->file_disp; -#endif - line = bug->line; -#endif warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; once = (bug->flags &BUGFLAG_ONCE ) !=0 ; done = (bug->flags &BUGFLAG_DONE ) !=0 ;
Signature · 6 lines
2.30.0.617.g56c4b15f3c-goog _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v2 2/4] bug: Factor out a getter for a bug's file line
Quoted 37 lines · On Tue, 23 Feb 2021 15:57:57 +0000 Andrew Scull <[email protected]> wrote:
There is some non-trivial config-based logic to get the file name and line number associated with a bug. Factor this out to a getter that can be resused.
Signed-off-by Andrew Scull <[email protected]>Cc Peter Zijlstra <[email protected]>Cc "Steven Rostedt (VMware)" <[email protected]>--- include/linux/bug.h | 3 +++ lib/bug.c | 27 +++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-)include/linux/bug.h +3
-0@@ -36,6 +36,9 @@ static inline int is_warning_bug(const struct bug_entry *bug)return bug->flags &BUGFLAG_WARNING ; } +void bug_get_file_line (struct bug_entry *bug,const char **file, +unsigned int *line); +struct bug_entry *find_bug (unsigned long bugaddr);enum bug_trap_type report_bug (unsigned long bug_addr,struct pt_regs *regs);lib/bug.c +3
-0@@ -130,6 +130,22 @@ static inline struct bug_entry *module_find_bug(unsigned long bugaddr) }#endif +void bug_get_file_line (struct bug_entry *bug,const char **file, +unsigned int *line) +{
lib/bug.c static inline struct bug_entry *module_find_bug(unsigned long bugaddr) · line 145
Quoted 10 lines
lib/bug.c +10
-0+ *file =NULL ; + *line =0 ; + +#ifdef CONFIG_DEBUG_BUGVERBOSE +#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS + *file = bug->file; +#else + *file = (const char *)bug + bug->file_disp; +#endif + *line = bug->line;
Not that it should be part of this patch, as this patch is moving the code and shouldn't modify it, but as a micro optimization, we could remove the initialization from the beginning and place it here:
#else *file = NULL; *line = 0;
But again, this patch is fine.
-- Steve
Quoted 15 lines
lib/bug.c +4
-3+#endif +} +struct bug_entry *find_bug (unsigned long bugaddr) {struct bug_entry *bug;@@ -156,17 +172,8 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)disable_trace_on_warning (); - file =NULL ; - line =0 ; +bug_get_file_line (bug, &file, &line); -#ifdef CONFIG_DEBUG_BUGVERBOSE
lib/bug.c enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) · line 179
Quoted 10 lines
lib/bug.c +0
-7-#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - file = bug->file; -#else - file = (const char *)bug + bug->file_disp; -#endif - line = bug->line; -#endif warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; once = (bug->flags &BUGFLAG_ONCE ) !=0 ; done = (bug->flags &BUGFLAG_DONE ) !=0 ;
_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
PATCH v2 3/4 KVM: arm64: Use BUG and BUG_ON in nVHE hyp
In reply to Andrew Scullhyp_panic() reports the address of the panic by using ELR_EL2, but this isn't a useful address when hyp_panic() is called directly. Replace such direct calls with BUG() and BUG_ON() which use BRK to trigger and exception that then goes to hyp_panic() with the correct address. Also remove the hyp_panic() declaration from the header file to avoid accidental misuse.
--- arch/arm64/include/asm/kvm_hyp.h | 1 - arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 6 ++---- 3 files changed, 3 insertions(+), 6 deletions(-)
arch/arm64/include/asm/kvm_hyp.h +0 -1
@@ -95,7 +95,6 @@ u64 __guest_enter(struct kvm_vcpu *vcpu);bool kvm_host_psci_handler (struct kvm_cpu_context *host_ctxt); -void __noreturn hyp_panic (void );#ifdef __KVM_NVHE_HYPERVISOR__void __noreturn __hyp_do_panic (struct kvm_cpu_context *host_ctxt,u64 spsr,u64 elr,u64 par);
arch/arm64/kvm/hyp/nvhe/hyp-main.c +1 -1
@@ -181,6 +181,6 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)handle_host_smc (host_ctxt);break ;default : -hyp_panic (); +BUG (); } }
arch/arm64/kvm/hyp/nvhe/hyp-smp.c +2 -4
@@ -18,8 +18,7 @@ u64 __ro_after_init hyp_cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALIDu64 cpu_logical_map (unsigned int cpu) { -if (cpu >=ARRAY_SIZE (hyp_cpu_logical_map)) -hyp_panic (); +BUG_ON (cpu >=ARRAY_SIZE (hyp_cpu_logical_map));return hyp_cpu_logical_map[cpu]; }
@@ -30,8 +29,7 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu)unsigned long this_cpu_base;unsigned long elf_base; -if (cpu >=ARRAY_SIZE (kvm_arm_hyp_percpu_base)) -hyp_panic (); +BUG_ON (cpu >=ARRAY_SIZE (kvm_arm_hyp_percpu_base)); cpu_base_array = (unsigned long *)hyp_symbol_addr (kvm_arm_hyp_percpu_base); this_cpu_base =kern_hyp_va (cpu_base_array[cpu]);
Signature · 6 lines
2.30.0.617.g56c4b15f3c-goog _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
PATCH v2 4/4 KVM: arm64: Log source when panicking from nVHE hyp
In reply to Andrew ScullTo aid with debugging, add details of the source of a panic. This is done by having nVHE hyp exit to nvhe_hyp_panic_handler() rather than directly to panic(). The handler will then add the extra details for debugging before panicking the kernel.
If the panic was due to a BUG(), look up the metadata to log the file and line, if available, otherwise log the kimg address that can be looked up in vmlinux.
__hyp_panic_string is now inlined since it no longer needs to be references as a symbol and message is free to diverge between VHE and nVHE.
--- arch/arm64/include/asm/kvm_mmu.h | 2 ++ arch/arm64/kernel/image-vars.h | 3 +-- arch/arm64/kvm/handle_exit.c | 31 +++++++++++++++++++++++++ arch/arm64/kvm/hyp/include/hyp/switch.h | 2 -- arch/arm64/kvm/hyp/nvhe/host.S | 17 ++++++-------- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 2 -- arch/arm64/kvm/hyp/vhe/switch.c | 4 +--- 7 files changed, 42 insertions(+), 19 deletions(-)
arch/arm64/include/asm/kvm_mmu.h +2 -0
@@ -130,6 +130,8 @@ void kvm_update_va_mask(struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst);void kvm_compute_layout (void ); +#define __hyp_pa (x) ((phys_addr_t)((x)) + hyp_physvirt_offset) +static __always_inline unsigned long __kern_hyp_va (unsigned long v) { asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n"
arch/arm64/kernel/image-vars.h +1 -2
@@ -71,8 +71,7 @@ KVM_NVHE_ALIAS(kvm_get_kimage_voffset);KVM_NVHE_ALIAS (kvm_vgic_global_state);/* Kernel symbols used to call panic() from nVHE hyp code (via ERET). */ -KVM_NVHE_ALIAS (__hyp_panic_string); -KVM_NVHE_ALIAS (panic); +KVM_NVHE_ALIAS (nvhe_hyp_panic_handler);/* Vectors installed by hyp-init on reset HVC. */ KVM_NVHE_ALIAS (__hyp_stub_vectors);
arch/arm64/kvm/handle_exit.c +31 -0
@@ -291,3 +291,34 @@ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index)if (exception_index ==ARM_EXCEPTION_EL1_SERROR )kvm_handle_guest_serror (vcpu,kvm_vcpu_get_esr (vcpu)); } + +void __noreturn __cold nvhe_hyp_panic_handler (u64 esr,u64 spsr,u64 elr, +u64 par,uintptr_t vcpu, +u64 far,u64 hpfar) { +u64 elr_in_kimg =__phys_to_kimg (__hyp_pa (elr)); +u64 mode = spsr &PSR_MODE_MASK ; + +if (mode != PSR_MODE_EL2t && mode != PSR_MODE_EL2h) { +kvm_err ("Invalid host exception to nVHE hyp!\n" ); + }else if (ESR_ELx_EC (esr) == ESR_ELx_EC_BRK64 && + (esr & ESR_ELx_BRK64_ISS_COMMENT_MASK) ==BUG_BRK_IMM ) { +struct bug_entry *bug =find_bug (elr_in_kimg); +const char *file =NULL ; +unsigned int line =0 ; + +/* All hyp bugs, including warnings, are treated as fatal. */ +if (bug) +bug_get_file_line (bug, &file, &line); + +if (file) { +kvm_err ("nVHE hyp BUG at: %s:%u!\n" , file, line); + }else { +kvm_err ("nVHE hyp BUG at: %016llx!\n" , elr_in_kimg); + } + }else { +kvm_err ("nVHE hyp panic at: %016llx!\n" , elr_in_kimg); + } + +panic ("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%016lx\n" , + spsr, elr, esr, far, hpfar, par, vcpu); +}
arch/arm64/kvm/hyp/include/hyp/switch.h +0 -2
@@ -30,8 +30,6 @@#include <asm/processor.h> #include <asm/thread_info.h> -extern const char __hyp_panic_string[]; -extern struct exception_table_entry __start___kvm_ex_table;extern struct exception_table_entry __stop___kvm_ex_table;
arch/arm64/kvm/hyp/nvhe/host.S +7 -10
@@ -77,21 +77,18 @@ SYM_FUNC_END(__host_enter) SYM_FUNC_START(__hyp_do_panic) mov x29, x0 - /* Load the format string into x0 and arguments into x1-7 */ - ldr x0, =__hyp_panic_string - - mov x6, x3 - get_vcpu_ptr x7, x3 - - mrs x3, esr_el2 - mrs x4, far_el2 - mrs x5, hpfar_el2 + /* Load the panic arguments into x0-7 */ + mrs x0, esr_el2 + get_vcpu_ptr x4, x5 + mrs x5, far_el2 + mrs x6, hpfar_el2 + mov x7, x0 // Unused argument /* Prepare and exit to the host's panic funciton. */ mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ PSR_MODE_EL1h) msr spsr_el2, lr - ldr lr, =panic + ldr lr, =nvhe_hyp_panic_handler msr elr_el2, lr /* Enter the host, restoring the host context if it was provided. */
arch/arm64/kvm/hyp/nvhe/psci-relay.c +0 -2
@@ -22,8 +22,6 @@ void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);struct kvm_host_psci_config __ro_after_init kvm_host_psci_config;s64 __ro_after_init hyp_physvirt_offset; -#define __hyp_pa (x) ((phys_addr_t)((x)) + hyp_physvirt_offset) -#define INVALID_CPU_ID UINT_MAXstruct psci_boot_args {
arch/arm64/kvm/hyp/vhe/switch.c +1 -3
@@ -27,8 +27,6 @@#include <asm/processor.h> #include <asm/thread_info.h> -const char __hyp_panic_string[] ="HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n" ; -/* VHE specific context */ DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
@@ -207,7 +205,7 @@ static void __hyp_call_panic(u64 spsr, u64 elr, u64 par)__deactivate_traps (vcpu);sysreg_restore_host_state_vhe (host_ctxt); -panic (__hyp_panic_string, +panic ("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n" , spsr, elr,read_sysreg_el2 (SYS_ESR ),read_sysreg_el2 (SYS_FAR ),read_sysreg (hpfar_el2), par, vcpu);
Signature · 6 lines
2.30.0.617.g56c4b15f3c-goog _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm