perf: Use LBR for machine/oops debugging
Subsystems (3)
PATCH 1/2 perf, x86: fix __initconst vs const
In reply to Peter ZijlstraAll variables that have __initconst should also be const.
--- arch/x86/kernel/cpu/perf_event_amd.c | 4 ++-- arch/x86/kernel/cpu/perf_event_intel.c | 12 ++++++------ arch/x86/kernel/cpu/perf_event_p4.c | 4 ++-- arch/x86/kernel/cpu/perf_event_p6.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-)
arch/x86/kernel/cpu/perf_event_amd.c +2 -2
@@ -2,7 +2,7 @@static DEFINE_RAW_SPINLOCK (amd_nb_lock); -static __initconst u64 amd_hw_cache_event_ids +static __initconst const u64 amd_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ] [PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -371,7 +371,7 @@ static void amd_pmu_cpu_dead(int cpu)raw_spin_unlock (&amd_nb_lock); } -static __initconst struct x86_pmu amd_pmu = { +static __initconst const struct x86_pmu amd_pmu = { .name ="AMD" , .handle_irq = x86_pmu_handle_irq, .disable_all = x86_pmu_disable_all,
arch/x86/kernel/cpu/perf_event_intel.c +6 -6
@@ -88,7 +88,7 @@ static u64 intel_pmu_event_map(int hw_evreturn intel_perfmon_event_map[hw_event]; } -static __initconst u64 westmere_hw_cache_event_ids +static __initconst const u64 westmere_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ] [PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -179,7 +179,7 @@ static __initconst u64 westmere_hw_cache }, }; -static __initconst u64 nehalem_hw_cache_event_ids +static __initconst const u64 nehalem_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ] [PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -270,7 +270,7 @@ static __initconst u64 nehalem_hw_cache_ }, }; -static __initconst u64 core2_hw_cache_event_ids +static __initconst const u64 core2_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ] [PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -361,7 +361,7 @@ static __initconst u64 core2_hw_cache_ev }, }; -static __initconst u64 atom_hw_cache_event_ids +static __initconst const u64 atom_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ] [PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -776,7 +776,7 @@ intel_get_event_constraints(struct cpu_hreturn x86_get_event_constraints (cpuc, event); } -static __initconst struct x86_pmu core_pmu = { +static __initconst const struct x86_pmu core_pmu = { .name ="core" , .handle_irq = x86_pmu_handle_irq, .disable_all = x86_pmu_disable_all,
@@ -815,7 +815,7 @@ static void intel_pmu_cpu_dying(int cpu)fini_debug_store_on_cpu (cpu); } -static __initconst struct x86_pmu intel_pmu = { +static __initconst const struct x86_pmu intel_pmu = { .name ="Intel" , .handle_irq = intel_pmu_handle_irq, .disable_all = intel_pmu_disable_all,
arch/x86/kernel/cpu/perf_event_p4.c +2 -2
@@ -287,7 +287,7 @@ static struct p4_event_bind p4_event_binp4_config_pack_cccr (cache_event | \ P4_CCCR_ESEL(P4_OPCODE_ESEL(P4_OPCODE(event)))) -static __initconst u64 p4_hw_cache_event_ids +static __initconst const u64 p4_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ] [PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -775,7 +775,7 @@ done:return num ? -ENOSPC :0 ; } -static __initconst struct x86_pmu p4_pmu = { +static __initconst const struct x86_pmu p4_pmu = { .name ="Netburst P4/Xeon" , .handle_irq = p4_pmu_handle_irq, .disable_all = p4_pmu_disable_all,
arch/x86/kernel/cpu/perf_event_p6.c +1 -1
@@ -102,7 +102,7 @@ static void p6_pmu_enable_event(struct p (void )checking_wrmsrl (hwc->config_base + hwc->idx, val); } -static __initconst struct x86_pmu p6_pmu = { +static __initconst const struct x86_pmu p6_pmu = { .name ="p6" , .handle_irq = x86_pmu_handle_irq, .disable_all = p6_pmu_disable_all,
cover perf: Use LBR for machine/oops debugging
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().
RFC PATCH 2/2 perf, x86: Utilize the LBRs for machine/oops debugging
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().
--- arch/x86/include/asm/perf_event.h | 7 ++ arch/x86/kernel/cpu/perf_event_intel.c | 5 - arch/x86/kernel/cpu/perf_event_intel_lbr.c | 86 +++++++++++++++++++++++++++-- arch/x86/kernel/dumpstack.c | 5 + 4 files changed, 95 insertions(+), 8 deletions(-)
arch/x86/include/asm/perf_event.h +6 -1
@@ -155,9 +155,14 @@ extern void perf_events_lapic_init(void)#define perf_instruction_pointer (regs) ((regs)->ip) +void dump_lbr_state (void ); +void lbr_off (void ); +#else static inline void init_hw_perf_events (void ) { } -static inline void perf_events_lapic_init (void ) { } +static inline void perf_events_lapic_init (void ) { } +static inline void dump_lbr_state (void ) { } +static inline void lbr_off (void ) { }#endif #endif /* _ASM_X86_PERF_EVENT_H */
arch/x86/kernel/cpu/perf_event_intel.c +1 -4
@@ -804,10 +804,7 @@ static __initconst const struct x86_pmustatic void intel_pmu_cpu_starting (int cpu) {init_debug_store_on_cpu (cpu); -/* -* Deal with CPUs that don't clear their LBRs on power-up. -*/ -intel_pmu_lbr_reset (); +intel_pmu_lbr_starting (); }static void intel_pmu_cpu_dying (int cpu)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +83 -3
114 lines not on this page · raw message
arch/x86/kernel/dumpstack.c +5 -0
@@ -17,6 +17,7 @@#include <linux/sysfs.h> #include <asm/stacktrace.h> +#include <asm/perf_event.h> #include "dumpstack.h"
@@ -224,6 +225,8 @@ unsigned __kprobes long oops_begin(void)int cpu;unsigned long flags; +lbr_off (); +oops_enter ();/* racy, but better than risking deadlock. */
@@ -306,6 +309,7 @@ int __kprobes __die(const char *str, strprintk_address (regs->ip,1 );printk (" RSP <%016lx>\n" , regs->sp);#endif +dump_lbr_state ();return 0 ; }
@@ -343,6 +347,7 @@ die_nmi(char *str, struct pt_regs *regs,printk (" on CPU%d, ip %08lx, registers:\n" ,smp_processor_id (), regs->ip);show_registers (regs); +dump_lbr_state ();oops_end (flags, regs,0 );if (do_panic || panic_on_oops)panic ("Non maskable interrupt" );
Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging
Quoted 7 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().
When this is enabled, it will prevent changing the LBR configuration to record only selected branches. Unless you are willing to accept filtered content in the kernel dump.
Quoted 207 lines
Requested-by Linus Torvalds <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>---
arch/x86/include/asm/perf_event.h | 7 ++arch/x86/kernel/cpu/perf_event_intel.c| 5 -arch/x86/kernel/cpu/perf_event_intel_lbr.c| 86 +++++++++++++++++++++++++++--arch/x86/kernel/dumpstack.c| 5 +4 files changed, 95 insertions(+), 8 deletions(-)Index:
linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================arch/x86/include/asm/perf_event.h +6
-1@@ -155,9 +155,14 @@ extern void perf_events_lapic_init(void) #define perf_instruction_pointer(regs) ((regs)->ip) +void dump_lbr_state (void ); +void lbr_off (void ); + #else static inline void init_hw_perf_events(void) { } -static inline void perf_events_lapic_init(void) { } +static inline void perf_events_lapic_init(void) { } +static inline void dump_lbr_state(void) { } +static inline void lbr_off(void) { } #endif #endif /* _ASM_X86_PERF_EVENT_H */Index:
linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================arch/x86/kernel/cpu/perf_event_intel.c +1
-4@@ -804,10 +804,7 @@ static __initconst const struct x86_pmu static void intel_pmu_cpu_starting(int cpu) { init_debug_store_on_cpu(cpu); - /* - * Deal with CPUs that don't clear their LBRs on power-up. - */ - intel_pmu_lbr_reset(); + intel_pmu_lbr_starting(); } static void intel_pmu_cpu_dying(int cpu)Index:
linux-2.6/arch/x86/kernel/cpu/perf_event_intel_lbr.c
===================================================================arch/x86/kernel/cpu/perf_event_intel_lbr.c +20
-3@@ -1,12 +1,32 @@ #ifdef CONFIG_CPU_SUP_INTEL enum { + LBR_DEBUG_OFF = 0, + LBR_DEBUG_ON = 1, +}; + +static int lbr_debug_state__read_mostly ; + +static int __init setup_lbr_debug (char *str) +{ + if (!strcmp(str, "on")) + lbr_debug_state = LBR_DEBUG_ON; + else + return 0; + return 1; +} + +__setup ("lbr_debug=" , setup_lbr_debug); + +enum { LBR_FORMAT_32 = 0x00, LBR_FORMAT_LIP = 0x01, LBR_FORMAT_EIP = 0x02, LBR_FORMAT_EIP_FLAGS = 0x03, }; +static DEFINE_PER_CPU(int, lbr_print_done); + /** We only support LBR implementations that have FREEZE_LBRS_ON_PMI * otherwise it becomes near impossible to get a reliable stack. @@ -50,9 +70,6 @@ static void intel_pmu_lbr_reset_64(void) static void intel_pmu_lbr_reset(void) { - if (!x86_pmu.lbr_nr) - return; - if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) intel_pmu_lbr_reset_32();else
arch/x86/kernel/cpu/perf_event_intel_lbr.c +63
-0@@ -182,6 +199,8 @@ static void intel_pmu_lbr_read(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + cpuc->lbr_stack.nr = 0; + if (!cpuc->lbr_users) return;@@ -215,4 +234,65 @@ static void intel_pmu_lbr_init_atom(void x86_pmu.lbr_to = 0x60; } +static void __dump_lbr_state (void ) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + int i; + + intel_pmu_lbr_read(); + for (i = 0; i < cpuc->lbr_stack.nr; i++) { + printk(KERN_DEBUG "CPU%d LBR%d: %pF -> %pF\n", + smp_processor_id(), i, + (void *)cpuc->lbr_entries[i].from, + (void *)cpuc->lbr_entries[i].to); + } +} + +static void intel_pmu_lbr_starting (void ) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + + if (!x86_pmu.lbr_nr) + return; + + cpuc->lbr_users = 1; + if (lbr_debug_state && !__get_cpu_var(lbr_print_done)) { + __get_cpu_var(lbr_print_done) = 1; + __dump_lbr_state(); + } + + intel_pmu_lbr_reset(); + + if (lbr_debug_state) + __intel_pmu_lbr_enable(); + else + cpuc->lbr_users = 0; +} + +void dump_lbr_state (void ) +{ + if (!lbr_debug_state) + return; + + __dump_lbr_state(); +} + +void lbr_off (void ) +{ + if (!lbr_debug_state) + return; + + __intel_pmu_lbr_disable(); +} + +#else + +void dump_lbr_state (void ) +{ +} + +void lbr_off (void ) +{ +} + #endif /* CONFIG_CPU_SUP_INTEL */Index:
linux-2.6/arch/x86/kernel/dumpstack.c
===================================================================arch/x86/kernel/dumpstack.c +5
-0@@ -17,6 +17,7 @@ #include <linux/sysfs.h> #include <asm/stacktrace.h> +#include <asm/perf_event.h> #include "dumpstack.h"@@ -224,6 +225,8 @@ unsigned __kprobes long oops_begin(void) int cpu; unsigned long flags; + lbr_off(); + oops_enter(); /* racy, but better than risking deadlock. */@@ -306,6 +309,7 @@ int __kprobes __die(const char *str, str printk_address(regs->ip, 1); printk(" RSP <%016lx>\n", regs->sp); #endif + dump_lbr_state(); return 0; }@@ -343,6 +347,7 @@ die_nmi(char *str, struct pt_regs *regs, printk(" on CPU%d, ip %08lx, registers:\n", smp_processor_id(), regs->ip); show_registers(regs); + dump_lbr_state(); oops_end(flags, regs, 0); if (do_panic || panic_on_oops) panic("Non maskable interrupt");
Quoted 6 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().
die is too late. they will only contain the oops code then.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Quoted 6 lines · Andi Kleen
Quoted 4 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Quoted 6 lines
Quoted 2 lines
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().die is too late. they will only contain the oops code then.
Most likely yes, especially with small LBR, such as on Intel Core.
Quoted 5 lines
-Andi
--
[email protected] -- Speaking for myself only.
Peter Zijlstra
maintainerRe: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging
In reply to Stephane EranianQuoted 5 lines · Stephane Eranian
Quoted 4 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Quoted 7 lines
Quoted 3 lines
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().When this is enabled, it will prevent changing the LBR configuration to record only selected branches. Unless you are willing to accept filtered content in the kernel dump.
Sure, but since we don't support that silly config reg anyway that's pretty much not an issue ;-)
Peter Zijlstra
maintainerQuoted 11 lines · Andi Kleen
Quoted 6 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().die is too late. they will only contain the oops code then.
We do an lbr_off() in oops_begin(), or is there a better/earlier place we can do that?
Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
Quoted 9 lines · Stephane Eranian
Quoted 7 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().
Quoted 7 lines
Quoted 3 lines
When this is enabled, it will prevent changing the LBR configuration to record only selected branches. Unless you are willing to accept filtered content in the kernel dump.
Sure, but since we don't support that silly config reg anyway that's pretty much not an issue ;-)
I will provide a patch to make it available. This is needed for certain measurements.
Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging
In reply to Peter ZijlstraQuoted 6 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().
Yummie!
Have you got some sample lbr_debug=1 output as well by any chance, with a crash provoked somewhere? How good is the output in practice? (i.e. how many artificial entries do we have at the end of the buffer, filled with crash related addresses?)
Also, i think we should use something more descriptive than lbr_debug=y. Perhaps crash_trace=1 or so?
Plus, it would be nice to have a sysctl entry for this as well - so that production systems can enable this if they want to enrich the output of some difficult-to-analyze kernel crash, without yet another reboot.
Ingo
Peter Zijlstra
maintainerRe: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging
Quoted 9 lines · Ingo Molnar
Quoted 6 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().
Quoted 6 lines
Yummie!
Have you got some sample lbr_debug=1 output as well by any chance, with a crash provoked somewhere? How good is the output in practice? (i.e. how many artificial entries do we have at the end of the buffer, filled with crash related addresses?)
lbr_debug=on actually, =1 doesn't parse. I had some output, but I was still looking at finding out why my %pF formats for CPU0 didn't have symbols in, probably a init sequence issue.
Quoted 2 lines
Also, i think we should use something more descriptive than lbr_debug=y. Perhaps crash_trace=1 or so?
Well, I would like to keep LBR in the name, since that is the mechanism used, we could maybe do lbr_trace or somesuch?
Quoted 3 lines
Plus, it would be nice to have a sysctl entry for this as well - so that production systems can enable this if they want to enrich the output of some difficult-to-analyze kernel crash, without yet another reboot.
Right, could do, but once it crashed it clearly to late to enable anything ;-)
Quoted 7 lines · Peter Zijlstra
Quoted 6 lines · Andi Kleen
Quoted 4 lines · Peter Zijlstra
The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and
die().die is too late. they will only contain the oops code then.
We do an
lbr_off()inoops_begin(), or is there a better/earlier place we can do that?
I had an old patch in the P4 era (slightly different but larger LBRs) which saved them all early in the exception handlers and then dumped them from the buffer. That's early enough that you only miss one or two.
The problem is that it's somewhat more expensive, the MSR reads are not cheap and they will slow down all your page faults.
I checked, but I can't find the old patch anymore. Could be probably redone.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging
In reply to Peter ZijlstraQuoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
Plus, it would be nice to have a sysctl entry for this as well - so that production systems can enable this if they want to enrich the output of some difficult-to-analyze kernel crash, without yet another reboot.
Right, could do, but once it crashed it clearly to late to enable anything ;-)
No. What i mean is that with your patch, a debugging session would go like this:
< kernel crashes > # reboot #1
< admin logs in and scratches head >
< admin consults kernel hackers and enables lbr_debug=1 in /etc/grub.conf >
< admin reboots > # reboot #2
< kernel crashes again > # reboot #3
With the sysctl we'd have one reboot less:
< kernel crashes > # reboot #1
< admin logs in and scratches head >
< admin consults kernel hackers and tweaks /proc/sys/kernel/x86/lbr_debug >
< kernel crashes again > # reboot #2
Thanks,
Ingo