Skip to contentSkip to conversation

perf: Use LBR for machine/oops debugging

PATCH In review Peter Zijlstra 5 participants 13 messages Mar 29, 2010 – Apr 2, 2010

Subsystems (3)
lore
Share & tools
lore
#1 lore raw

PATCH 1/2 perf, x86: fix __initconst vs const

In reply to Peter Zijlstra

All variables that have __initconst should also be const.

---
 arch/x86/kernel/cpu/perf_event_amd.c   |    4 ++--
 arch/x86/kernel/cpu/perf_event_intel.c |   12 ++++++------
 arch/x86/kernel/cpu/perf_event_p4.c    |    4 ++--
 arch/x86/kernel/cpu/perf_event_p6.c    |    2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)
arch/x86/kernel/cpu/perf_event_amd.c +2 -2
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_amd.c
===================================================================
@@ -2,7 +2,7 @@
 
 static DEFINE_RAW_SPINLOCK(amd_nb_lock);
 
-static __initconst u64 amd_hw_cache_event_ids
+static __initconst const u64 amd_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -371,7 +371,7 @@ static void amd_pmu_cpu_dead(int cpu)
 	raw_spin_unlock(&amd_nb_lock);
 }
 
-static __initconst struct x86_pmu amd_pmu = {
+static __initconst const struct x86_pmu amd_pmu = {
 	.name			= "AMD",
 	.handle_irq		= x86_pmu_handle_irq,
 	.disable_all		= x86_pmu_disable_all,
arch/x86/kernel/cpu/perf_event_intel.c +6 -6
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================
@@ -88,7 +88,7 @@ static u64 intel_pmu_event_map(int hw_ev
 	return intel_perfmon_event_map[hw_event];
 }
 
-static __initconst u64 westmere_hw_cache_event_ids
+static __initconst const u64 westmere_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -179,7 +179,7 @@ static __initconst u64 westmere_hw_cache
  },
 };
 
-static __initconst u64 nehalem_hw_cache_event_ids
+static __initconst const u64 nehalem_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -270,7 +270,7 @@ static __initconst u64 nehalem_hw_cache_
  },
 };
 
-static __initconst u64 core2_hw_cache_event_ids
+static __initconst const u64 core2_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -361,7 +361,7 @@ static __initconst u64 core2_hw_cache_ev
  },
 };
 
-static __initconst u64 atom_hw_cache_event_ids
+static __initconst const u64 atom_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -776,7 +776,7 @@ intel_get_event_constraints(struct cpu_h
 	return x86_get_event_constraints(cpuc, event);
 }
 
-static __initconst struct x86_pmu core_pmu = {
+static __initconst const struct x86_pmu core_pmu = {
 	.name			= "core",
 	.handle_irq		= x86_pmu_handle_irq,
 	.disable_all		= x86_pmu_disable_all,
@@ -815,7 +815,7 @@ static void intel_pmu_cpu_dying(int cpu)
 	fini_debug_store_on_cpu(cpu);
 }
 
-static __initconst struct x86_pmu intel_pmu = {
+static __initconst const struct x86_pmu intel_pmu = {
 	.name			= "Intel",
 	.handle_irq		= intel_pmu_handle_irq,
 	.disable_all		= intel_pmu_disable_all,
arch/x86/kernel/cpu/perf_event_p4.c +2 -2
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_p4.c
===================================================================
@@ -287,7 +287,7 @@ static struct p4_event_bind p4_event_bin
 	p4_config_pack_cccr(cache_event					| \
 			    P4_CCCR_ESEL(P4_OPCODE_ESEL(P4_OPCODE(event))))
 
-static __initconst u64 p4_hw_cache_event_ids
+static __initconst const u64 p4_hw_cache_event_ids
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
@@ -775,7 +775,7 @@ done:
 	return num ? -ENOSPC : 0;
 }
 
-static __initconst struct x86_pmu p4_pmu = {
+static __initconst const struct x86_pmu p4_pmu = {
 	.name			= "Netburst P4/Xeon",
 	.handle_irq		= p4_pmu_handle_irq,
 	.disable_all		= p4_pmu_disable_all,
arch/x86/kernel/cpu/perf_event_p6.c +1 -1
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_p6.c
===================================================================
@@ -102,7 +102,7 @@ static void p6_pmu_enable_event(struct p
 	(void)checking_wrmsrl(hwc->config_base + hwc->idx, val);
 }
 
-static __initconst struct x86_pmu p6_pmu = {
+static __initconst const struct x86_pmu p6_pmu = {
 	.name			= "p6",
 	.handle_irq		= x86_pmu_handle_irq,
 	.disable_all		= p6_pmu_disable_all,
#0 lore raw

cover perf: Use LBR for machine/oops debugging

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

#2 lore raw

RFC PATCH 2/2 perf, x86: Utilize the LBRs for machine/oops debugging

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

---
 arch/x86/include/asm/perf_event.h          |    7 ++
 arch/x86/kernel/cpu/perf_event_intel.c     |    5 -
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   86 +++++++++++++++++++++++++++--
 arch/x86/kernel/dumpstack.c                |    5 +
 4 files changed, 95 insertions(+), 8 deletions(-)
arch/x86/include/asm/perf_event.h +6 -1
Index: linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================
@@ -155,9 +155,14 @@ extern void perf_events_lapic_init(void)
 
 #define perf_instruction_pointer(regs)	((regs)->ip)
 
+void dump_lbr_state(void);
+void lbr_off(void);
+
 #else
 static inline void init_hw_perf_events(void)		{ }
-static inline void perf_events_lapic_init(void)	{ }
+static inline void perf_events_lapic_init(void)		{ }
+static inline void dump_lbr_state(void)			{ }
+static inline void lbr_off(void)			{ }
 #endif
 
 #endif /* _ASM_X86_PERF_EVENT_H */
arch/x86/kernel/cpu/perf_event_intel.c +1 -4
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================
@@ -804,10 +804,7 @@ static __initconst const struct x86_pmu 
 static void intel_pmu_cpu_starting(int cpu)
 {
 	init_debug_store_on_cpu(cpu);
-	/*
-	 * Deal with CPUs that don't clear their LBRs on power-up.
-	 */
-	intel_pmu_lbr_reset();
+	intel_pmu_lbr_starting();
 }
 
 static void intel_pmu_cpu_dying(int cpu)
arch/x86/kernel/cpu/perf_event_intel_lbr.c +83 -3

114 lines not on this page · raw message

arch/x86/kernel/dumpstack.c +5 -0
Index: linux-2.6/arch/x86/kernel/dumpstack.c
===================================================================
@@ -17,6 +17,7 @@
 #include <linux/sysfs.h>
 
 #include <asm/stacktrace.h>
+#include <asm/perf_event.h>
 
 #include "dumpstack.h"
 
@@ -224,6 +225,8 @@ unsigned __kprobes long oops_begin(void)
 	int cpu;
 	unsigned long flags;
 
+	lbr_off();
+
 	oops_enter();
 
 	/* racy, but better than risking deadlock. */
@@ -306,6 +309,7 @@ int __kprobes __die(const char *str, str
 	printk_address(regs->ip, 1);
 	printk(" RSP <%016lx>\n", regs->sp);
 #endif
+	dump_lbr_state();
 	return 0;
 }
 
@@ -343,6 +347,7 @@ die_nmi(char *str, struct pt_regs *regs,
 	printk(" on CPU%d, ip %08lx, registers:\n",
 		smp_processor_id(), regs->ip);
 	show_registers(regs);
+	dump_lbr_state();
 	oops_end(flags, regs, 0);
 	if (do_panic || panic_on_oops)
 		panic("Non maskable interrupt");
#3 lore raw

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

Quoted 7 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

When this is enabled, it will prevent changing the LBR configuration to record only selected branches. Unless you are willing to accept filtered content in the kernel dump.

Quoted 207 lines

---
 arch/x86/include/asm/perf_event.h          |    7 ++ arch/x86/kernel/cpu/perf_event_intel.c     |    5 - arch/x86/kernel/cpu/perf_event_intel_lbr.c |   86 +++++++++++++++++++++++++++-- arch/x86/kernel/dumpstack.c                |    5 +

 4 files changed, 95 insertions(+), 8 deletions(-)

Index: linux-2.6/arch/x86/include/asm/perf_event.h
===================================================================

arch/x86/include/asm/perf_event.h +6 -1
@@ -155,9 +155,14 @@ extern void perf_events_lapic_init(void)
 
  #define perf_instruction_pointer(regs) ((regs)->ip)
 
+void dump_lbr_state(void);
+void lbr_off(void);
+
  #else
  static inline void init_hw_perf_events(void)           { }
-static inline void perf_events_lapic_init(void)        { }
+static inline void perf_events_lapic_init(void)                { }
+static inline void dump_lbr_state(void)                        { }
+static inline void lbr_off(void)                       { }
  #endif
 
  #endif /* _ASM_X86_PERF_EVENT_H */

Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================

arch/x86/kernel/cpu/perf_event_intel.c +1 -4
@@ -804,10 +804,7 @@ static __initconst const struct x86_pmu
  static void intel_pmu_cpu_starting(int cpu)
  {
        init_debug_store_on_cpu(cpu);
-       /*
-        * Deal with CPUs that don't clear their LBRs on power-up.
-        */
-       intel_pmu_lbr_reset();
+       intel_pmu_lbr_starting();
  }
 
  static void intel_pmu_cpu_dying(int cpu)

Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel_lbr.c
===================================================================

arch/x86/kernel/cpu/perf_event_intel_lbr.c +20 -3
@@ -1,12 +1,32 @@
  #ifdef CONFIG_CPU_SUP_INTEL
 
  enum {
+       LBR_DEBUG_OFF           = 0,
+       LBR_DEBUG_ON            = 1,
+};
+
+static int lbr_debug_state __read_mostly;
+
+static int __init setup_lbr_debug(char *str)
+{
+       if (!strcmp(str, "on"))
+               lbr_debug_state = LBR_DEBUG_ON;
+       else
+               return 0;
+       return 1;
+}
+
+__setup("lbr_debug=", setup_lbr_debug);
+
+enum {
        LBR_FORMAT_32           = 0x00,
        LBR_FORMAT_LIP          = 0x01,
        LBR_FORMAT_EIP          = 0x02,
        LBR_FORMAT_EIP_FLAGS    = 0x03,
  };
 
+static DEFINE_PER_CPU(int, lbr_print_done);
+
  /*
  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  * otherwise it becomes near impossible to get a reliable stack.
@@ -50,9 +70,6 @@ static void intel_pmu_lbr_reset_64(void)
 
  static void intel_pmu_lbr_reset(void)
  {
-       if (!x86_pmu.lbr_nr)
-               return;
-
        if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
                intel_pmu_lbr_reset_32();

       else

arch/x86/kernel/cpu/perf_event_intel_lbr.c +63 -0
@@ -182,6 +199,8 @@ static void intel_pmu_lbr_read(void)
  {
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
+       cpuc->lbr_stack.nr = 0;
+
        if (!cpuc->lbr_users)
                return;
 
@@ -215,4 +234,65 @@ static void intel_pmu_lbr_init_atom(void
        x86_pmu.lbr_to     = 0x60;
  }
 
+static void __dump_lbr_state(void)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       int i;
+
+       intel_pmu_lbr_read();
+       for (i = 0; i < cpuc->lbr_stack.nr; i++) {
+               printk(KERN_DEBUG "CPU%d LBR%d: %pF -> %pF\n",
+                               smp_processor_id(), i,
+                               (void *)cpuc->lbr_entries[i].from,
+                               (void *)cpuc->lbr_entries[i].to);
+       }
+}
+
+static void intel_pmu_lbr_starting(void)
+{
+       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+       if (!x86_pmu.lbr_nr)
+               return;
+
+       cpuc->lbr_users = 1;
+       if (lbr_debug_state && !__get_cpu_var(lbr_print_done)) {
+               __get_cpu_var(lbr_print_done) = 1;
+               __dump_lbr_state();
+       }
+
+       intel_pmu_lbr_reset();
+
+       if (lbr_debug_state)
+               __intel_pmu_lbr_enable();
+       else
+               cpuc->lbr_users = 0;
+}
+
+void dump_lbr_state(void)
+{
+       if (!lbr_debug_state)
+               return;
+
+       __dump_lbr_state();
+}
+
+void lbr_off(void)
+{
+       if (!lbr_debug_state)
+               return;
+
+       __intel_pmu_lbr_disable();
+}
+
+#else
+
+void dump_lbr_state(void)
+{
+}
+
+void lbr_off(void)
+{
+}
+
  #endif /* CONFIG_CPU_SUP_INTEL */

Index: linux-2.6/arch/x86/kernel/dumpstack.c
===================================================================

arch/x86/kernel/dumpstack.c +5 -0
@@ -17,6 +17,7 @@
  #include <linux/sysfs.h>
 
  #include <asm/stacktrace.h>
+#include <asm/perf_event.h>
 
  #include "dumpstack.h"
 
@@ -224,6 +225,8 @@ unsigned __kprobes long oops_begin(void)
        int cpu;
        unsigned long flags;
 
+       lbr_off();
+
        oops_enter();
 
        /* racy, but better than risking deadlock. */
@@ -306,6 +309,7 @@ int __kprobes __die(const char *str, str
        printk_address(regs->ip, 1);
        printk(" RSP <%016lx>\n", regs->sp);
  #endif
+       dump_lbr_state();
        return 0;
  }
 
@@ -343,6 +347,7 @@ die_nmi(char *str, struct pt_regs *regs,
        printk(" on CPU%d, ip %08lx, registers:\n",
                smp_processor_id(), regs->ip);
        show_registers(regs);
+       dump_lbr_state();
        oops_end(flags, regs, 0);
        if (do_panic || panic_on_oops)
                panic("Non maskable interrupt");
#9 lore raw
In reply to Peter Zijlstra
Quoted 6 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

die is too late. they will only contain the oops code then.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#10 lore raw
Quoted 6 lines · Andi Kleen
Quoted 4 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Quoted 6 lines
Quoted 2 lines

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

die is too late. they will only contain the oops code then.

Most likely yes, especially with small LBR, such as on Intel Core.

Quoted 5 lines

-Andi

--
[email protected] -- Speaking for myself only.

maintainer
#4 lore raw

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

In reply to Stephane Eranian
Quoted 5 lines · Stephane Eranian
Quoted 4 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Quoted 7 lines
Quoted 3 lines

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

When this is enabled, it will prevent changing the LBR configuration to record only selected branches. Unless you are willing to accept filtered content in the kernel dump.

Sure, but since we don't support that silly config reg anyway that's pretty much not an issue ;-)

maintainer
#11 lore raw
In reply to Andi Kleen
Quoted 11 lines · Andi Kleen
Quoted 6 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

die is too late. they will only contain the oops code then.

We do an lbr_off() in oops_begin(), or is there a better/earlier place we can do that?

#5 lore raw

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 9 lines · Stephane Eranian
Quoted 7 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

Quoted 7 lines
Quoted 3 lines

When this is enabled, it will prevent changing the LBR configuration to record only selected branches. Unless you are willing to accept filtered content in the kernel dump.

Sure, but since we don't support that silly config reg anyway that's pretty much not an issue ;-)

I will provide a patch to make it available. This is needed for certain measurements.

#6 lore raw

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

In reply to Peter Zijlstra
Quoted 6 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

Yummie!

Have you got some sample lbr_debug=1 output as well by any chance, with a crash provoked somewhere? How good is the output in practice? (i.e. how many artificial entries do we have at the end of the buffer, filled with crash related addresses?)

Also, i think we should use something more descriptive than lbr_debug=y. Perhaps crash_trace=1 or so?

Plus, it would be nice to have a sysctl entry for this as well - so that production systems can enable this if they want to enrich the output of some difficult-to-analyze kernel crash, without yet another reboot.

Ingo

maintainer
#7 lore raw

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

Quoted 9 lines · Ingo Molnar
Quoted 6 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

Quoted 6 lines

Yummie!

Have you got some sample lbr_debug=1 output as well by any chance, with a crash provoked somewhere? How good is the output in practice? (i.e. how many artificial entries do we have at the end of the buffer, filled with crash related addresses?)

lbr_debug=on actually, =1 doesn't parse. I had some output, but I was still looking at finding out why my %pF formats for CPU0 didn't have symbols in, probably a init sequence issue.

Quoted 2 lines

Also, i think we should use something more descriptive than lbr_debug=y. Perhaps crash_trace=1 or so?

Well, I would like to keep LBR in the name, since that is the mechanism used, we could maybe do lbr_trace or somesuch?

Quoted 3 lines

Plus, it would be nice to have a sysctl entry for this as well - so that production systems can enable this if they want to enrich the output of some difficult-to-analyze kernel crash, without yet another reboot.

Right, could do, but once it crashed it clearly to late to enable anything ;-)

#12 lore raw
In reply to Peter Zijlstra
Quoted 7 lines · Peter Zijlstra
Quoted 6 lines · Andi Kleen
Quoted 4 lines · Peter Zijlstra

The LBRs are relatively cheap to keep enabled and provide some history to OOPSen, also some CPUs are reported to keep them over soft-reset, which allows us to use them to debug things like tripple faults.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Therefore introduce a boot option: lbr_debug=on, which always enable the LBRs and will print the LBRs on CPU init and die().

die is too late. they will only contain the oops code then.

We do an lbr_off() in oops_begin(), or is there a better/earlier place we can do that?

I had an old patch in the P4 era (slightly different but larger LBRs) which saved them all early in the exception handlers and then dumped them from the buffer. That's early enough that you only miss one or two.

The problem is that it's somewhat more expensive, the MSR reads are not cheap and they will slow down all your page faults.

I checked, but I can't find the old patch anymore. Could be probably redone.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#8 lore raw

Re: [RFC,PATCH 2/2] perf, x86: Utilize the LBRs for machine/oops debugging

In reply to Peter Zijlstra
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar

Plus, it would be nice to have a sysctl entry for this as well - so that production systems can enable this if they want to enrich the output of some difficult-to-analyze kernel crash, without yet another reboot.

Right, could do, but once it crashed it clearly to late to enable anything ;-)

No. What i mean is that with your patch, a debugging session would go like this:

< kernel crashes > # reboot #1

< admin logs in and scratches head >
< admin consults kernel hackers and enables lbr_debug=1 in /etc/grub.conf >

< admin reboots > # reboot #2

< kernel crashes again > # reboot #3

With the sysctl we'd have one reboot less:

< kernel crashes > # reboot #1

< admin logs in and scratches head >
< admin consults kernel hackers and tweaks /proc/sys/kernel/x86/lbr_debug >

< kernel crashes again > # reboot #2

Thanks,

Ingo