Skip to contentSkip to conversation

x86: Enable NMI on all cpus on UV

PATCH Applied Russ Anderson 2 participants 8 messages Feb 17, 2010 – Feb 27, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86: Enable NMI on all cpus on UV

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

---

By default on x86 all the cpus except the boot cpu have NMI masked off. This patch enables NMI on all cpus in UV system and adds an NMI handler to dump_stack on each cpu. This way if a system hangs we can NMI the machine and get a backtrace from all the cpus.

 arch/x86/include/asm/uv/uv.h       |    1 
 arch/x86/kernel/apic/x2apic_uv_x.c |   49 +++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/smpboot.c          |    2 +
 3 files changed, 52 insertions(+)
arch/x86/kernel/apic/x2apic_uv_x.c +49 -0
Index: linux/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================
@@ -20,6 +20,7 @@
 #include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/io.h>
+#include <linux/kdebug.h>
 
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
@@ -39,6 +40,53 @@ static u64 gru_start_paddr, gru_end_padd
 int uv_min_hub_revision_id;
 EXPORT_SYMBOL_GPL(uv_min_hub_revision_id);
 
+int uv_handle_nmi(struct notifier_block *self,
+		  unsigned long reason, void *data)
+{
+	unsigned long flags;
+	static DEFINE_SPINLOCK(uv_nmi_lock);
+
+	if (reason != DIE_NMI_IPI)
+		return NOTIFY_OK;
+	/*
+	 * Use a lock so only one cpu prints at a time
+	 * to prevent intermixed output.
+	 */
+	spin_lock_irqsave(&uv_nmi_lock, flags);
+	printk(KERN_INFO "NMI stack dump cpu %u:\n",
+				smp_processor_id());
+	dump_stack();
+	spin_unlock_irqrestore(&uv_nmi_lock, flags);
+
+	return NOTIFY_STOP;
+}
+
+static struct notifier_block uv_dump_stack_nmi_nb = {
+  .notifier_call = uv_handle_nmi,
+  .next = NULL,
+  .priority = 0
+};
+
+void uv_register_nmi_notifier(void)
+{
+	if (register_die_notifier(&uv_dump_stack_nmi_nb))
+		printk(KERN_WARNING "UV NMI handler failed to register\n");
+}
+
+/*
+ * Called on each cpu to unmask NMI.
+ */
+void __cpuinit uv_nmi_init(void)
+{
+	unsigned int value;
+
+	/*
+	 * Unmask NMI on all cpus
+	 */
+	value = apic_read(APIC_LVT1) | APIC_DM_NMI;
+	value &= ~APIC_LVT_MASKED;
+	apic_write(APIC_LVT1, value);
+}
 
 static int is_GRU_range(u64 start, u64 end)
 {
@@ -718,5 +766,6 @@ void __init uv_system_init(void)
 
 	uv_cpu_init();
 	uv_scir_register_cpu_notifier();
+	uv_register_nmi_notifier();
 	proc_mkdir("sgi_uv", NULL);
 }
arch/x86/include/asm/uv/uv.h +1 -0
Index: linux/arch/x86/include/asm/uv/uv.h
===================================================================
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,
arch/x86/kernel/smpboot.c +2 -0
Index: linux/arch/x86/kernel/smpboot.c
===================================================================
@@ -320,6 +320,8 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	if (is_uv_system())
+		uv_nmi_init();
 
 	/* enable local interrupts */
 	local_irq_enable();
Signature · 2 lines
Russ Anderson, OS RAS/Partitioning Project Lead  
SGI - Silicon Graphics Inc          [email protected]
#1 lore raw
Quoted 109 lines · Russ Anderson

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

---

By default on x86 all the cpus except the boot cpu have NMI masked off. This patch enables NMI on all cpus in UV system and adds an NMI handler to dump_stack on each cpu. This way if a system hangs we can NMI the machine and get a backtrace from all the cpus.

 arch/x86/include/asm/uv/uv.h       |    1 
 arch/x86/kernel/apic/x2apic_uv_x.c |   49 +++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/smpboot.c          |    2 +
 3 files changed, 52 insertions(+)

Index: linux/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================

arch/x86/kernel/apic/x2apic_uv_x.c +49 -0
@@ -20,6 +20,7 @@
 #include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/io.h>
+#include <linux/kdebug.h>
 
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
@@ -39,6 +40,53 @@ static u64 gru_start_paddr, gru_end_padd
 int uv_min_hub_revision_id;
 EXPORT_SYMBOL_GPL(uv_min_hub_revision_id);
 
+int uv_handle_nmi(struct notifier_block *self,
+		  unsigned long reason, void *data)
+{
+	unsigned long flags;
+	static DEFINE_SPINLOCK(uv_nmi_lock);
+
+	if (reason != DIE_NMI_IPI)
+		return NOTIFY_OK;
+	/*
+	 * Use a lock so only one cpu prints at a time
+	 * to prevent intermixed output.
+	 */
+	spin_lock_irqsave(&uv_nmi_lock, flags);
+	printk(KERN_INFO "NMI stack dump cpu %u:\n",
+				smp_processor_id());
+	dump_stack();
+	spin_unlock_irqrestore(&uv_nmi_lock, flags);
+
+	return NOTIFY_STOP;
+}
+
+static struct notifier_block uv_dump_stack_nmi_nb = {
+  .notifier_call = uv_handle_nmi,
+  .next = NULL,
+  .priority = 0
+};
+
+void uv_register_nmi_notifier(void)
+{
+	if (register_die_notifier(&uv_dump_stack_nmi_nb))
+		printk(KERN_WARNING "UV NMI handler failed to register\n");
+}
+
+/*
+ * Called on each cpu to unmask NMI.
+ */
+void __cpuinit uv_nmi_init(void)
+{
+	unsigned int value;
+
+	/*
+	 * Unmask NMI on all cpus
+	 */
+	value = apic_read(APIC_LVT1) | APIC_DM_NMI;
+	value &= ~APIC_LVT_MASKED;
+	apic_write(APIC_LVT1, value);
+}
 
 static int is_GRU_range(u64 start, u64 end)
 {
@@ -718,5 +766,6 @@ void __init uv_system_init(void)
 
 	uv_cpu_init();
 	uv_scir_register_cpu_notifier();
+	uv_register_nmi_notifier();
 	proc_mkdir("sgi_uv", NULL);
 }

Index: linux/arch/x86/include/asm/uv/uv.h
===================================================================

arch/x86/include/asm/uv/uv.h +1 -0
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,

Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c notrace static void __cpuinit start_seco · line 324

Quoted 5 lines
arch/x86/kernel/smpboot.c +2 -0
@@ -320,6 +320,8 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	if (is_uv_system())
+		uv_nmi_init();

Instead of cramming it into the init sequence open-coded, shouldnt this be done via the x86_platform driver mechanism?

Thanks,

Ingo

#2 lore raw
Quoted 16 lines · Ingo Molnar

arch/x86/kernel/smpboot.c notrace static void __cpuinit start_seco · line 324

Quoted 10 lines · Russ Anderson

Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c +2 -0
@@ -320,6 +320,8 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	if (is_uv_system())
+		uv_nmi_init();

Instead of cramming it into the init sequence open-coded, shouldnt this be done via the x86_platform driver mechanism?

OK, I'm working on it.

Quoted 3 lines

Thanks,

Ingo

Thanks,

Signature · 2 lines
Russ Anderson, OS RAS/Partitioning Project Lead  
SGI - Silicon Graphics Inc          [email protected]
#3 lore raw
In reply to Ingo Molnar
Quoted 7 lines · Ingo Molnar
Quoted 4 lines · Russ Anderson

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

[...]

Quoted 6 lines
Quoted 6 lines

struct mm_struct *mm,
Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c +0 -0
@@ -320,6 +320,8 @@ notrace static void __cpuinit start_seco
Quoted 8 lines

arch/x86/kernel/smpboot.c notrace static void __cpuinit start_seco · line 324

Quoted 5 lines
arch/x86/kernel/smpboot.c +2 -0
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	if (is_uv_system())
+		uv_nmi_init();

Instead of cramming it into the init sequence open-coded, shouldnt this be done via the x86_platform driver mechanism?

Attached is the updated patch with the x86_platform driver mechanism used for uv_nmi_init.

Quoted 3 lines

Thanks,

Ingo

Signature · 2 lines
Russ Anderson, OS RAS/Partitioning Project Lead  
SGI - Silicon Graphics Inc          [email protected]

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

---

By default on x86 all the cpus except the boot cpu have NMI masked off. This patch enables NMI on all cpus in UV system and adds an NMI handler to dump_stack on each cpu. This way if a system hangs we can NMI the machine and get a backtrace from all the cpus.

Version 2: Use x86_platform driver mechanism for nmi init, per Ingo's suggestion.

 arch/x86/include/asm/nmi.h         |    1 
 arch/x86/include/asm/uv/uv.h       |    1 
 arch/x86/include/asm/x86_init.h    |    2 +
 arch/x86/kernel/apic/x2apic_uv_x.c |   50 +++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/smpboot.c          |    6 ++++
 arch/x86/kernel/x86_init.c         |    2 +
 6 files changed, 62 insertions(+)
arch/x86/kernel/apic/x2apic_uv_x.c +50 -0
Index: linux/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================
@@ -20,6 +20,7 @@
 #include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/io.h>
+#include <linux/kdebug.h>
 
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
@@ -71,6 +72,7 @@ static int __init uv_acpi_madt_oem_check
 	if (!strcmp(oem_id, "SGI")) {
 		nodeid = early_get_nodeid();
 		x86_platform.is_untracked_pat_range =  uv_is_untracked_pat_range;
+		x86_platform.nmi_init =  uv_nmi_init;
 		if (!strcmp(oem_table_id, "UVL"))
 			uv_system_type = UV_LEGACY_APIC;
 		else if (!strcmp(oem_table_id, "UVX"))
@@ -569,6 +571,53 @@ void __cpuinit uv_cpu_init(void)
 		set_x2apic_extra_bits(uv_hub_info->pnode);
 }
 
+/*
+ * When NMI is received, print a stack trace.
+ */
+int uv_handle_nmi(struct notifier_block *self,
+		  unsigned long reason, void *data)
+{
+	unsigned long flags;
+	static DEFINE_SPINLOCK(uv_nmi_lock);
+
+	if (reason != DIE_NMI_IPI)
+		return NOTIFY_OK;
+	/*
+	 * Use a lock so only one cpu prints at a time
+	 * to prevent intermixed output.
+	 */
+	spin_lock_irqsave(&uv_nmi_lock, flags);
+	printk(KERN_INFO "NMI stack dump cpu %u:\n",
+				smp_processor_id());
+	dump_stack();
+	spin_unlock_irqrestore(&uv_nmi_lock, flags);
+
+	return NOTIFY_STOP;
+}
+
+static struct notifier_block uv_dump_stack_nmi_nb = {
+  .notifier_call = uv_handle_nmi,
+  .next = NULL,
+  .priority = 0
+};
+
+void uv_register_nmi_notifier(void)
+{
+	if (register_die_notifier(&uv_dump_stack_nmi_nb))
+		printk(KERN_WARNING "UV NMI handler failed to register\n");
+}
+
+void uv_nmi_init(void)
+{
+	unsigned int value;
+
+	/*
+	 * Unmask NMI on all cpus
+	 */
+	value = apic_read(APIC_LVT1) | APIC_DM_NMI;
+	value &= ~APIC_LVT_MASKED;
+	apic_write(APIC_LVT1, value);
+}
 
 void __init uv_system_init(void)
 {
@@ -690,5 +739,6 @@ void __init uv_system_init(void)
 
 	uv_cpu_init();
 	uv_scir_register_cpu_notifier();
+	uv_register_nmi_notifier();
 	proc_mkdir("sgi_uv", NULL);
 }
arch/x86/include/asm/uv/uv.h +1 -0
Index: linux/arch/x86/include/asm/uv/uv.h
===================================================================
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,
arch/x86/kernel/smpboot.c +6 -0
Index: linux/arch/x86/kernel/smpboot.c
===================================================================
@@ -263,6 +263,11 @@ static void __cpuinit smp_callin(void)
 	cpumask_set_cpu(cpuid, cpu_callin_mask);
 }
 
+void default_nmi_init(void)
+{
+	return;
+}
+
 /*
  * Activate a secondary processor.
  */
@@ -320,6 +325,7 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	x86_platform.nmi_init();
 
 	/* enable local interrupts */
 	local_irq_enable();
arch/x86/include/asm/x86_init.h +2 -0
Index: linux/arch/x86/include/asm/x86_init.h
===================================================================
@@ -126,6 +126,7 @@ struct x86_cpuinit_ops {
  * @get_wallclock:		get time from HW clock like RTC etc.
  * @set_wallclock:		set time back to HW clock
  * @is_untracked_pat_range	exclude from PAT logic
+ * @nmi_init			enable NMI on cpus
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -133,6 +134,7 @@ struct x86_platform_ops {
 	int (*set_wallclock)(unsigned long nowtime);
 	void (*iommu_shutdown)(void);
 	bool (*is_untracked_pat_range)(u64 start, u64 end);
+	void (*nmi_init)(void);
 };
 
 extern struct x86_init_ops x86_init;
arch/x86/kernel/x86_init.c +2 -0
Index: linux/arch/x86/kernel/x86_init.c
===================================================================
@@ -13,6 +13,7 @@
 #include <asm/e820.h>
 #include <asm/time.h>
 #include <asm/irq.h>
+#include <asm/nmi.h>
 #include <asm/pat.h>
 #include <asm/tsc.h>
 #include <asm/iommu.h>
@@ -82,4 +83,5 @@ struct x86_platform_ops x86_platform = {
 	.set_wallclock			= mach_set_rtc_mmss,
 	.iommu_shutdown			= iommu_shutdown_noop,
 	.is_untracked_pat_range		= is_ISA_range,
+	.nmi_init			= default_nmi_init
 };
arch/x86/include/asm/nmi.h +1 -0
Index: linux/arch/x86/include/asm/nmi.h
===================================================================
@@ -24,6 +24,7 @@ extern int reserve_perfctr_nmi(unsigned 
 extern void release_perfctr_nmi(unsigned int);
 extern int reserve_evntsel_nmi(unsigned int);
 extern void release_evntsel_nmi(unsigned int);
+extern void default_nmi_init(void);
 
 extern void setup_apic_nmi_watchdog(void *);
 extern void stop_apic_nmi_watchdog(void *);
#4 lore raw
Quoted 24 lines · Russ Anderson
Quoted 7 lines · Ingo Molnar
Quoted 4 lines · Russ Anderson

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

[...]

Quoted 6 lines
Quoted 6 lines

struct mm_struct *mm,
Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c +0 -0
@@ -320,6 +320,8 @@ notrace static void __cpuinit start_seco
Quoted 8 lines

arch/x86/kernel/smpboot.c notrace static void __cpuinit start_seco

Quoted 5 lines
arch/x86/kernel/smpboot.c +2 -0
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	if (is_uv_system())
+		uv_nmi_init();

Instead of cramming it into the init sequence open-coded, shouldnt this be done via the x86_platform driver mechanism?

Quoted 2 lines

Attached is the updated patch with the x86_platform driver mechanism used for uv_nmi_init.

ok, this looks far cleaner. A few nits:

arch/x86/kernel/apic/x2apic_uv_x.c void __cpuinit uv_cpu_init(void) · line 578

Quoted 4 lines

+ * When NMI is received, print a stack trace.
+ */

arch/x86/kernel/apic/x2apic_uv_x.c +2 -0
+int uv_handle_nmi(struct notifier_block *self,
+		  unsigned long reason, void *data)

Please put prototypes on a single line if it's still below 100 cols or so.

arch/x86/kernel/apic/x2apic_uv_x.c void __cpuinit uv_cpu_init(void) · line 581

Quoted 3 lines
arch/x86/kernel/apic/x2apic_uv_x.c +3 -0
+{
+	unsigned long flags;
+	static DEFINE_SPINLOCK(uv_nmi_lock);

Please dont hide locks amongst local variables. (even if they are only used by that function)

arch/x86/kernel/apic/x2apic_uv_x.c void __cpuinit uv_cpu_init(void) · line 591

Quoted 10 lines
arch/x86/kernel/apic/x2apic_uv_x.c +10 -0
+
+	if (reason != DIE_NMI_IPI)
+		return NOTIFY_OK;
+	/*
+	 * Use a lock so only one cpu prints at a time
+	 * to prevent intermixed output.
+	 */
+	spin_lock_irqsave(&uv_nmi_lock, flags);
+	printk(KERN_INFO "NMI stack dump cpu %u:\n",
+				smp_processor_id());

Can be on a single line too.

Can use pr_info().

Should use a raw spinlock - this is NMI context.

arch/x86/kernel/apic/x2apic_uv_x.c void __cpuinit uv_cpu_init(void) · line 602

Quoted 11 lines
arch/x86/kernel/apic/x2apic_uv_x.c +11 -0
+	dump_stack();
+	spin_unlock_irqrestore(&uv_nmi_lock, flags);
+
+	return NOTIFY_STOP;
+}
+
+static struct notifier_block uv_dump_stack_nmi_nb = {
+  .notifier_call = uv_handle_nmi,
+  .next = NULL,
+  .priority = 0
+};

Please align structure initializations vertically.

Plus no need to open-code the setting of priority to 0 i guess.

Quoted 45 lines
arch/x86/kernel/apic/x2apic_uv_x.c +19 -0
+
+void uv_register_nmi_notifier(void)
+{
+	if (register_die_notifier(&uv_dump_stack_nmi_nb))
+		printk(KERN_WARNING "UV NMI handler failed to register\n");
+}
+
+void uv_nmi_init(void)
+{
+	unsigned int value;
+
+	/*
+	 * Unmask NMI on all cpus
+	 */
+	value = apic_read(APIC_LVT1) | APIC_DM_NMI;
+	value &= ~APIC_LVT_MASKED;
+	apic_write(APIC_LVT1, value);
+}
 
 void __init uv_system_init(void)
 {
@@ -690,5 +739,6 @@ void __init uv_system_init(void)
 
 	uv_cpu_init();
 	uv_scir_register_cpu_notifier();
+	uv_register_nmi_notifier();
 	proc_mkdir("sgi_uv", NULL);
 }

Index: linux/arch/x86/include/asm/uv/uv.h
===================================================================

arch/x86/include/asm/uv/uv.h +1 -0
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,

Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c static void __cpuinit smp_callin(void) · line 269

Quoted 7 lines
arch/x86/kernel/smpboot.c +4 -0
@@ -263,6 +263,11 @@ static void __cpuinit smp_callin(void)
 	cpumask_set_cpu(cpuid, cpu_callin_mask);
 }
 
+void default_nmi_init(void)
+{
+	return;
+}

That return is not needed.

Quoted 45 lines
arch/x86/kernel/smpboot.c +2 -0
+
 /*
  * Activate a secondary processor.
  */
@@ -320,6 +325,7 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	x86_platform.nmi_init();
 
 	/* enable local interrupts */
 	local_irq_enable();

Index: linux/arch/x86/include/asm/x86_init.h
===================================================================

arch/x86/include/asm/x86_init.h +2 -0
@@ -126,6 +126,7 @@ struct x86_cpuinit_ops {
  * @get_wallclock:		get time from HW clock like RTC etc.
  * @set_wallclock:		set time back to HW clock
  * @is_untracked_pat_range	exclude from PAT logic
+ * @nmi_init			enable NMI on cpus
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -133,6 +134,7 @@ struct x86_platform_ops {
 	int (*set_wallclock)(unsigned long nowtime);
 	void (*iommu_shutdown)(void);
 	bool (*is_untracked_pat_range)(u64 start, u64 end);
+	void (*nmi_init)(void);
 };
 
 extern struct x86_init_ops x86_init;

Index: linux/arch/x86/kernel/x86_init.c
===================================================================

arch/x86/kernel/x86_init.c +1 -0
@@ -13,6 +13,7 @@
 #include <asm/e820.h>
 #include <asm/time.h>
 #include <asm/irq.h>
+#include <asm/nmi.h>
 #include <asm/pat.h>
 #include <asm/tsc.h>
 #include <asm/iommu.h>

arch/x86/kernel/x86_init.c struct x86_platform_ops x86_platform = { · line 87

Quoted 5 lines
arch/x86/kernel/x86_init.c +1 -0
@@ -82,4 +83,5 @@ struct x86_platform_ops x86_platform = {
 	.set_wallclock			= mach_set_rtc_mmss,
 	.iommu_shutdown			= iommu_shutdown_noop,
 	.is_untracked_pat_range		= is_ISA_range,
+	.nmi_init			= default_nmi_init
 };

the default can be in this file too - then you can also make it 'static' and save some space and not touch smpboot.c.

arch/x86/include/asm/nmi.h extern int reserve_perfctr_nmi(unsigned · line 30

Quoted 12 lines

Index: linux/arch/x86/include/asm/nmi.h
===================================================================

arch/x86/include/asm/nmi.h +1 -0
@@ -24,6 +24,7 @@ extern int reserve_perfctr_nmi(unsigned 
 extern void release_perfctr_nmi(unsigned int);
 extern int reserve_evntsel_nmi(unsigned int);
 extern void release_evntsel_nmi(unsigned int);
+extern void default_nmi_init(void);
 
 extern void setup_apic_nmi_watchdog(void *);
 extern void stop_apic_nmi_watchdog(void *);

This will go away in that case as well.

Thanks,

Ingo

#5 lore raw

Attached is a patch that cleans up Ingo's nits.

Quoted 30 lines · Ingo Molnar
Quoted 24 lines · Russ Anderson
Quoted 7 lines · Ingo Molnar
Quoted 4 lines · Russ Anderson

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

[...]

Quoted 6 lines
Quoted 6 lines

struct mm_struct *mm,
Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c +0 -0
@@ -320,6 +320,8 @@ notrace static void __cpuinit start_seco
Quoted 8 lines

arch/x86/kernel/smpboot.c notrace static void __cpuinit start_seco · line 324

Quoted 5 lines
arch/x86/kernel/smpboot.c +2 -0
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	if (is_uv_system())
+		uv_nmi_init();

Instead of cramming it into the init sequence open-coded, shouldnt this be done via the x86_platform driver mechanism?

Quoted 2 lines

Attached is the updated patch with the x86_platform driver mechanism used for uv_nmi_init.

Quoted 8 lines

ok, this looks far cleaner. A few nits:

Review

Quoted 4 lines

+ * When NMI is received, print a stack trace.
+ */

(quoted diff) +2 -0
+int uv_handle_nmi(struct notifier_block *self,
+		  unsigned long reason, void *data)

Please put prototypes on a single line if it's still below 100 cols or so.

Done.

Quoted 6 lines

Review

Quoted 3 lines
(quoted diff) +3 -0
+{
+	unsigned long flags;
+	static DEFINE_SPINLOCK(uv_nmi_lock);

Please dont hide locks amongst local variables. (even if they are only used by that function)

Done.

Quoted 13 lines

Review

Quoted 10 lines
(quoted diff) +10 -0
+
+	if (reason != DIE_NMI_IPI)
+		return NOTIFY_OK;
+	/*
+	 * Use a lock so only one cpu prints at a time
+	 * to prevent intermixed output.
+	 */
+	spin_lock_irqsave(&uv_nmi_lock, flags);
+	printk(KERN_INFO "NMI stack dump cpu %u:\n",
+				smp_processor_id());

Can be on a single line too.

Quoted 3 lines

Can use pr_info().

Should use a raw spinlock - this is NMI context.

Done.

Quoted 14 lines

Review

Quoted 11 lines
(quoted diff) +11 -0
+	dump_stack();
+	spin_unlock_irqrestore(&uv_nmi_lock, flags);
+
+	return NOTIFY_STOP;
+}
+
+static struct notifier_block uv_dump_stack_nmi_nb = {
+  .notifier_call = uv_handle_nmi,
+  .next = NULL,
+  .priority = 0
+};

Please align structure initializations vertically.

Quoted 1 line

Plus no need to open-code the setting of priority to 0 i guess.

Done.

Quoted 48 lines
Quoted 48 lines
(quoted diff) +19 -0
+
+void uv_register_nmi_notifier(void)
+{
+	if (register_die_notifier(&uv_dump_stack_nmi_nb))
+		printk(KERN_WARNING "UV NMI handler failed to register\n");
+}
+
+void uv_nmi_init(void)
+{
+	unsigned int value;
+
+	/*
+	 * Unmask NMI on all cpus
+	 */
+	value = apic_read(APIC_LVT1) | APIC_DM_NMI;
+	value &= ~APIC_LVT_MASKED;
+	apic_write(APIC_LVT1, value);
+}
 
 void __init uv_system_init(void)
 {
@@ -690,5 +739,6 @@ void __init uv_system_init(void)
 
 	uv_cpu_init();
 	uv_scir_register_cpu_notifier();
+	uv_register_nmi_notifier();
 	proc_mkdir("sgi_uv", NULL);
 }

Index: linux/arch/x86/include/asm/uv/uv.h
===================================================================

arch/x86/include/asm/uv/uv.h +1 -0
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,

Index: linux/arch/x86/kernel/smpboot.c
===================================================================

arch/x86/kernel/smpboot.c +0 -0
@@ -263,6 +263,11 @@ static void __cpuinit smp_callin(void)
 	cpumask_set_cpu(cpuid, cpu_callin_mask);
 }
 
Quoted 6 lines

arch/x86/kernel/smpboot.c static void __cpuinit smp_callin(void) · line 269

Quoted 4 lines
arch/x86/kernel/smpboot.c +4 -0
+void default_nmi_init(void)
+{
+	return;
+}

That return is not needed.

Done and moved to x86_init.c.

Quoted 45 lines
Quoted 45 lines
arch/x86/kernel/smpboot.c +2 -0
+
 /*
  * Activate a secondary processor.
  */
@@ -320,6 +325,7 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	x86_platform.nmi_init();
 
 	/* enable local interrupts */
 	local_irq_enable();

Index: linux/arch/x86/include/asm/x86_init.h
===================================================================

arch/x86/include/asm/x86_init.h +2 -0
@@ -126,6 +126,7 @@ struct x86_cpuinit_ops {
  * @get_wallclock:		get time from HW clock like RTC etc.
  * @set_wallclock:		set time back to HW clock
  * @is_untracked_pat_range	exclude from PAT logic
+ * @nmi_init			enable NMI on cpus
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -133,6 +134,7 @@ struct x86_platform_ops {
 	int (*set_wallclock)(unsigned long nowtime);
 	void (*iommu_shutdown)(void);
 	bool (*is_untracked_pat_range)(u64 start, u64 end);
+	void (*nmi_init)(void);
 };
 
 extern struct x86_init_ops x86_init;

Index: linux/arch/x86/kernel/x86_init.c
===================================================================

arch/x86/kernel/x86_init.c +1 -0
@@ -13,6 +13,7 @@
 #include <asm/e820.h>
 #include <asm/time.h>
 #include <asm/irq.h>
+#include <asm/nmi.h>
 #include <asm/pat.h>
 #include <asm/tsc.h>
 #include <asm/iommu.h>
@@ -82,4 +83,5 @@ struct x86_platform_ops x86_platform = {
Quoted 8 lines

arch/x86/kernel/x86_init.c struct x86_platform_ops x86_platform = { · line 87

Quoted 5 lines
arch/x86/kernel/x86_init.c +1 -0
 	.set_wallclock			= mach_set_rtc_mmss,
 	.iommu_shutdown			= iommu_shutdown_noop,
 	.is_untracked_pat_range		= is_ISA_range,
+	.nmi_init			= default_nmi_init
 };

the default can be in this file too - then you can also make it 'static' and save some space and not touch smpboot.c.

Done.

Quoted 10 lines
Quoted 10 lines

Index: linux/arch/x86/include/asm/nmi.h
===================================================================

arch/x86/include/asm/nmi.h +1 -0
@@ -24,6 +24,7 @@ extern int reserve_perfctr_nmi(unsigned 
 extern void release_perfctr_nmi(unsigned int);
 extern int reserve_evntsel_nmi(unsigned int);
 extern void release_evntsel_nmi(unsigned int);
+extern void default_nmi_init(void);
 
Quoted 4 lines

arch/x86/include/asm/nmi.h extern int reserve_perfctr_nmi(unsigned · line 30

Quoted 2 lines
arch/x86/include/asm/nmi.h +0 -0
 extern void setup_apic_nmi_watchdog(void *);
 extern void stop_apic_nmi_watchdog(void *);

This will go away in that case as well.

Removed.

Quoted 3 lines

Thanks,

Ingo

Signature · 2 lines
Russ Anderson, OS RAS/Partitioning Project Lead  
SGI - Silicon Graphics Inc          [email protected]

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

---

By default on x86 all the cpus except the boot cpu have NMI masked off. This patch enables NMI on all cpus in UV system and adds an NMI handler to dump_stack on each cpu. This way if a system hangs we can NMI the machine and get a backtrace from all the cpus.

Version 2: Use x86_platform driver mechanism for nmi init, per Ingo's suggestion.

Version 3: Clean up Ingo's nits.

 arch/x86/include/asm/uv/uv.h       |    1 
 arch/x86/include/asm/x86_init.h    |    2 +
 arch/x86/kernel/apic/x2apic_uv_x.c |   44 +++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/smpboot.c          |    1 
 arch/x86/kernel/x86_init.c         |    3 ++
 5 files changed, 51 insertions(+)
arch/x86/kernel/apic/x2apic_uv_x.c +44 -0
Index: linux/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================
@@ -20,6 +20,7 @@
 #include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/io.h>
+#include <linux/kdebug.h>
 
 #include <asm/uv/uv_mmrs.h>
 #include <asm/uv/uv_hub.h>
@@ -38,6 +39,7 @@ static enum uv_system_type uv_system_typ
 static u64 gru_start_paddr, gru_end_paddr;
 int uv_min_hub_revision_id;
 EXPORT_SYMBOL_GPL(uv_min_hub_revision_id);
+static DEFINE_SPINLOCK(uv_nmi_lock);
 
 static inline bool is_GRU_range(u64 start, u64 end)
 {
@@ -71,6 +73,7 @@ static int __init uv_acpi_madt_oem_check
 	if (!strcmp(oem_id, "SGI")) {
 		nodeid = early_get_nodeid();
 		x86_platform.is_untracked_pat_range =  uv_is_untracked_pat_range;
+		x86_platform.nmi_init =  uv_nmi_init;
 		if (!strcmp(oem_table_id, "UVL"))
 			uv_system_type = UV_LEGACY_APIC;
 		else if (!strcmp(oem_table_id, "UVX"))
@@ -569,6 +572,46 @@ void __cpuinit uv_cpu_init(void)
 		set_x2apic_extra_bits(uv_hub_info->pnode);
 }
 
+/*
+ * When NMI is received, print a stack trace.
+ */
+int uv_handle_nmi(struct notifier_block *self, unsigned long reason, void *data)
+{
+	if (reason != DIE_NMI_IPI)
+		return NOTIFY_OK;
+	/*
+	 * Use a lock so only one cpu prints at a time
+	 * to prevent intermixed output.
+	 */
+	spin_lock(&uv_nmi_lock);
+	pr_info("NMI stack dump cpu %u:\n", smp_processor_id());
+	dump_stack();
+	spin_unlock(&uv_nmi_lock);
+
+	return NOTIFY_STOP;
+}
+
+static struct notifier_block uv_dump_stack_nmi_nb = {
+	.notifier_call	= uv_handle_nmi
+};
+
+void uv_register_nmi_notifier(void)
+{
+	if (register_die_notifier(&uv_dump_stack_nmi_nb))
+		printk(KERN_WARNING "UV NMI handler failed to register\n");
+}
+
+void uv_nmi_init(void)
+{
+	unsigned int value;
+
+	/*
+	 * Unmask NMI on all cpus
+	 */
+	value = apic_read(APIC_LVT1) | APIC_DM_NMI;
+	value &= ~APIC_LVT_MASKED;
+	apic_write(APIC_LVT1, value);
+}
 
 void __init uv_system_init(void)
 {
@@ -690,5 +733,6 @@ void __init uv_system_init(void)
 
 	uv_cpu_init();
 	uv_scir_register_cpu_notifier();
+	uv_register_nmi_notifier();
 	proc_mkdir("sgi_uv", NULL);
 }
arch/x86/include/asm/uv/uv.h +1 -0
Index: linux/arch/x86/include/asm/uv/uv.h
===================================================================
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,
arch/x86/kernel/smpboot.c +1 -0
Index: linux/arch/x86/kernel/smpboot.c
===================================================================
@@ -320,6 +320,7 @@ notrace static void __cpuinit start_seco
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	x86_platform.nmi_init();
 
 	/* enable local interrupts */
 	local_irq_enable();
arch/x86/include/asm/x86_init.h +2 -0
Index: linux/arch/x86/include/asm/x86_init.h
===================================================================
@@ -126,6 +126,7 @@ struct x86_cpuinit_ops {
  * @get_wallclock:		get time from HW clock like RTC etc.
  * @set_wallclock:		set time back to HW clock
  * @is_untracked_pat_range	exclude from PAT logic
+ * @nmi_init			enable NMI on cpus
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -133,6 +134,7 @@ struct x86_platform_ops {
 	int (*set_wallclock)(unsigned long nowtime);
 	void (*iommu_shutdown)(void);
 	bool (*is_untracked_pat_range)(u64 start, u64 end);
+	void (*nmi_init)(void);
 };
 
 extern struct x86_init_ops x86_init;
arch/x86/kernel/x86_init.c +3 -0
Index: linux/arch/x86/kernel/x86_init.c
===================================================================
@@ -76,10 +76,13 @@ struct x86_cpuinit_ops x86_cpuinit __cpu
 	.setup_percpu_clockev		= setup_secondary_APIC_clock,
 };
 
+static void default_nmi_init(void) { };
+
 struct x86_platform_ops x86_platform = {
 	.calibrate_tsc			= native_calibrate_tsc,
 	.get_wallclock			= mach_get_cmos_time,
 	.set_wallclock			= mach_set_rtc_mmss,
 	.iommu_shutdown			= iommu_shutdown_noop,
 	.is_untracked_pat_range		= is_ISA_range,
+	.nmi_init			= default_nmi_init
 };
Applied
#6 lore raw
Quoted 1 line · Russ Anderson

Attached is a patch that cleans up Ingo's nits.

Applied, thanks Russ!

arch/x86/kernel/apic/x2apic_uv_x.c static int __init uv_acpi_madt_oem_check · line 76

Quoted 3 lines
arch/x86/kernel/apic/x2apic_uv_x.c +1 -0
 		nodeid = early_get_nodeid();
 		x86_platform.is_untracked_pat_range =  uv_is_untracked_pat_range;
+		x86_platform.nmi_init =  uv_nmi_init;

I did a "s/ / /" here.

Ingo

#7 lore raw
In reply to Russ Anderson
Commit-ID:  78c06176466cbd1b3f0f67709d3023c40dbebcbd
Gitweb:     http://git.kernel.org/tip/78c06176466cbd1b3f0f67709d3023c40dbebcbd
Author:     Russ Anderson <[email protected]>
AuthorDate: Fri, 26 Feb 2010 10:49:12 -0600
Committer:  Ingo Molnar <[email protected]>
CommitDate: Sat, 27 Feb 2010 12:34:21 +0100

x86: Enable NMI on all cpus on UV

Enable NMI on all cpus in UV system and add an NMI handler to dump_stack on each cpu.

By default on x86 all the cpus except the boot cpu have NMI masked off. This patch enables NMI on all cpus in UV system and adds an NMI handler to dump_stack on each cpu. This way if a system hangs we can NMI the machine and get a backtrace from all the cpus.

Version 2: Use x86_platform driver mechanism for nmi init, per
Ingo's suggestion.

Version 3: Clean up Ingo's nits.

LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/uv/uv.h       |    1 +
 arch/x86/include/asm/x86_init.h    |    2 +
 arch/x86/kernel/apic/x2apic_uv_x.c |   44 ++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/smpboot.c          |    1 +
 arch/x86/kernel/x86_init.c         |    3 ++
 5 files changed, 51 insertions(+), 0 deletions(-)
arch/x86/include/asm/uv/uv.h +1 -0
index c0a01b5..3bb9491 100644
@@ -11,6 +11,7 @@ struct mm_struct;
 extern enum uv_system_type get_uv_system_type(void);
 extern int is_uv_system(void);
 extern void uv_cpu_init(void);
+extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,
arch/x86/include/asm/x86_init.h +2 -0
index ea0e8ea..60cc352 100644
@@ -126,6 +126,7 @@ struct x86_cpuinit_ops {
  * @get_wallclock:		get time from HW clock like RTC etc.
  * @set_wallclock:		set time back to HW clock
  * @is_untracked_pat_range	exclude from PAT logic
+ * @nmi_init			enable NMI on cpus
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -133,6 +134,7 @@ struct x86_platform_ops {
 	int (*set_wallclock)(unsigned long nowtime);
 	void (*iommu_shutdown)(void);
 	bool (*is_untracked_pat_range)(u64 start, u64 end);
+	void (*nmi_init)(void);
 };
 
 extern struct x86_init_ops x86_init;
arch/x86/kernel/apic/x2apic_uv_x.c +44 -0

74 lines not on this page · raw message

arch/x86/kernel/smpboot.c +1 -0
index 678d0b8..838a118 100644
@@ -320,6 +320,7 @@ notrace static void __cpuinit start_secondary(void *unused)
 	unlock_vector_lock();
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+	x86_platform.nmi_init();
 
 	/* enable local interrupts */
 	local_irq_enable();
arch/x86/kernel/x86_init.c +3 -0
index ccd179d..ee5746c 100644
@@ -76,10 +76,13 @@ struct x86_cpuinit_ops x86_cpuinit __cpuinitdata = {
 	.setup_percpu_clockev		= setup_secondary_APIC_clock,
 };
 
+static void default_nmi_init(void) { };
+
 struct x86_platform_ops x86_platform = {
 	.calibrate_tsc			= native_calibrate_tsc,
 	.get_wallclock			= mach_get_cmos_time,
 	.set_wallclock			= mach_set_rtc_mmss,
 	.iommu_shutdown			= iommu_shutdown_noop,
 	.is_untracked_pat_range		= is_ISA_range,
+	.nmi_init			= default_nmi_init
 };