Skip to contentSkip to conversation

ftrace based hard lockup detector

PATCH New Frederic Weisbecker 1 participant 1 message Jan 19, 2009

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 ftrace based hard lockup detector

[Sorry, I forgot the function tracer dependency]
--
Like the NMI watchdog, this feature try to detect hard lockups by lurking at the non-progress of the timer interrupts.

You can enable it at boot time by passing the ftrace_hardlockup parameter. I plan to add a debugfs file to enable/disable at runtime.

When a hardlockup is detected, it will print a backtrace. Perhaps it would be good to print the locks held from lockdep too?

It only support x86 for the moment, because a kind

---
 Documentation/kernel-parameters.txt |    4 +
 arch/x86/Kconfig.debug              |   15 ++++-
 arch/x86/include/asm/nmi.h          |   15 +++++
 arch/x86/kernel/Makefile            |    2 +
 arch/x86/kernel/ftrace_hardlockup.c |  115 +++++++++++++++++++++++++++++++++++
 arch/x86/kernel/nmi.c               |   14 ----
 include/linux/seqlock.h             |    6 ++
 7 files changed, 156 insertions(+), 15 deletions(-)
 create mode 100644 arch/x86/kernel/ftrace_hardlockup.c
Documentation/kernel-parameters.txt +4 -0
index ac613a6..9881426 100644
@@ -774,6 +774,10 @@ and is between 256 and 4096 characters. It is defined in the file
 	ftrace_dump_on_oops
 			[ftrace] will dump the trace buffers on oops.
 
+	ftrace_hardlockup
+			[ftrace] will hook most of the kernel functions and
+			try to detect hard lockups.
+
 	gamecon.map[2|3]=
 			[HW,JOY] Multisystem joystick and NES/SNES/PSX pad
 			support via parallel port (up to 5 devices per port)
arch/x86/Kconfig.debug +14 -1
index 097d79a..33ae6c5 100644
@@ -367,5 +367,18 @@ config OPTIMIZE_INLINING
 
 	  If unsure, say N.
 
-endmenu
+config FTRACE_HARLOCKUPS_DETECT
+	bool "Ftrace based hard-lockup detection"
+	select FUNCTION_TRACER
+	help
+	  This options makes ftrace hooking most of the kernel functions, trying to
+	  detect a hard lockup and then print a backtrace if so. Its check is
+	  similar to the one performed by the nmi watchdog which lurks in the
+	  non-progress of the timer interrupts. By default it is not enabled.
+	  If you want to launch it, add the ftrace_hardlockup kernel parameter.
 
+	  Enabling this option will bring almost no overhead, assuming you
+	  enabled CONFIG_DYNAMIC_FTRACE. But once you activate it through the
+	  ftrace_hardlockup parameter, your system will be a bit slower.
+
+endmenu
arch/x86/include/asm/nmi.h +15 -0
index c45a0a5..9177b60 100644
@@ -48,6 +48,21 @@ extern int unknown_nmi_panic;
 void __trigger_all_cpu_backtrace(void);
 #define trigger_all_cpu_backtrace() __trigger_all_cpu_backtrace()
 
+/*
+ * Take the local apic timer and PIT/HPET into account. We don't
+ * know which one is active, when we have highres/dyntick on
+ */
+static inline unsigned int get_timer_irqs(int cpu)
+{
+#ifdef CONFIG_X86_64
+	return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
+#else
+	return per_cpu(irq_stat, cpu).apic_timer_irqs +
+		per_cpu(irq_stat, cpu).irq0_irqs;
+#endif
+}
+
+
 static inline void localise_nmi_watchdog(void)
 {
 	if (nmi_watchdog == NMI_IO_APIC)
arch/x86/kernel/Makefile +2 -0
index eb07453..c488d76 100644
@@ -13,6 +13,7 @@ CFLAGS_REMOVE_rtc.o = -pg
 CFLAGS_REMOVE_paravirt-spinlocks.o = -pg
 CFLAGS_REMOVE_ftrace.o = -pg
 CFLAGS_REMOVE_early_printk.o = -pg
+CFLAGS_REMOVE_ftrace_hardlockup.o = -pg
 endif
 
 #
@@ -68,6 +69,7 @@ obj-$(CONFIG_X86_IO_APIC)	+= io_apic.o
 obj-$(CONFIG_X86_REBOOTFIXUPS)	+= reboot_fixups_32.o
 obj-$(CONFIG_DYNAMIC_FTRACE)	+= ftrace.o
 obj-$(CONFIG_FUNCTION_GRAPH_TRACER)	+= ftrace.o
+obj-$(CONFIG_FTRACE_HARLOCKUPS_DETECT) += ftrace_hardlockup.o
 obj-$(CONFIG_KEXEC)		+= machine_kexec_$(BITS).o
 obj-$(CONFIG_KEXEC)		+= relocate_kernel_$(BITS).o crash.o
 obj-$(CONFIG_CRASH_DUMP)	+= crash_dump_$(BITS).o
arch/x86/kernel/ftrace_hardlockup.c new file +115 -0

115 lines not on this page · raw message

arch/x86/kernel/nmi.c +0 -14
index 7228979..86225f8 100644
@@ -76,20 +76,6 @@ static inline int mce_in_progress(void)
 	return 0;
 }
 
-/*
- * Take the local apic timer and PIT/HPET into account. We don't
- * know which one is active, when we have highres/dyntick on
- */
-static inline unsigned int get_timer_irqs(int cpu)
-{
-#ifdef CONFIG_X86_64
-	return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
-#else
-	return per_cpu(irq_stat, cpu).apic_timer_irqs +
-		per_cpu(irq_stat, cpu).irq0_irqs;
-#endif
-}
-
 #ifdef CONFIG_SMP
 /*
  * The performance counters used by NMI_LOCAL_APIC don't trigger when
include/linux/seqlock.h +6 -0
index 632205c..545cb85 100644
@@ -82,6 +82,12 @@ static inline int write_tryseqlock(seqlock_t *sl)
 	return ret;
 }
 
+static inline int write_is_seqlocked(seqlock_t *sl)
+{
+	return spin_is_locked(&sl->lock);
+}
+
+
 /* Start of read calculation -- fetch last complete writer token */
 static __always_inline unsigned read_seqbegin(const seqlock_t *sl)
 {
Signature · 1 line
1.6.0.4