Skip to contentSkip to conversation

x86: avoid calling arch_trigger_all_cpu_backtrace() at the same time on SMP

PATCH In review Don Zickus 3 participants 4 messages Nov 2, 2010 – Nov 10, 2010 x86

lore
Share & tools
lore

Don Zickus

author
#0 lore raw

PATCH 1/1 x86: avoid calling arch_trigger_all_cpu_backtrace() at the same time on SMP

From: Dongdong Deng <[email protected]>

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---
 arch/x86/kernel/apic/hw_nmi.c |   14 ++++++++++++++
 arch/x86/kernel/apic/nmi.c    |   14 ++++++++++++++
 2 files changed, 28 insertions(+), 0 deletions(-)
arch/x86/kernel/apic/hw_nmi.c +14 -0
index cefd694..3aea0a5 100644
@@ -29,6 +29,16 @@ u64 hw_nmi_get_sample_period(void)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -41,6 +51,10 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);
 }
 
 static int __kprobes
arch/x86/kernel/apic/nmi.c +14 -0
index a43f71c..5fa8a13 100644
@@ -552,6 +552,16 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -564,4 +574,8 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);
 }
Signature · 1 line
1.7.2.3
#1 lore raw
Quoted 44 lines · Don Zickus

From: Dongdong Deng <[email protected]>

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---
 arch/x86/kernel/apic/hw_nmi.c |   14 ++++++++++++++
 arch/x86/kernel/apic/nmi.c    |   14 ++++++++++++++
 2 files changed, 28 insertions(+), 0 deletions(-)
arch/x86/kernel/apic/hw_nmi.c +0 -0
index cefd694..3aea0a5 100644

arch/x86/kernel/apic/hw_nmi.c u64 hw_nmi_get_sample_period(void) · line 32

Quoted 4 lines
arch/x86/kernel/apic/hw_nmi.c +1 -0
@@ -29,6 +29,16 @@ u64 hw_nmi_get_sample_period(void)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;

Please dont put statics into the middle of local variables - put them into file scope in a visible way.

Quoted 45 lines
arch/x86/kernel/apic/hw_nmi.c +13 -0
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -41,6 +51,10 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);
 }
 
 static int __kprobes
arch/x86/kernel/apic/nmi.c +10 -0
index a43f71c..5fa8a13 100644
@@ -552,6 +552,16 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 

arch/x86/kernel/apic/nmi.c void arch_trigger_all_cpu_backtrace(void) · line 580

Quoted 7 lines
arch/x86/kernel/apic/nmi.c +4 -0
@@ -564,4 +574,8 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);

This spinlock is never actually used as a spinlock - it's a "in progress" flag. Why not use a flag and test_and_set_bit()?

Also, the irq disabling really needed, will this code ever be called with irqs on?

Thanks,

Ingo

#2 lore raw
Quoted 41 lines · Ingo Molnar
Quoted 39 lines · Don Zickus

From: Dongdong Deng <[email protected]>

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---
 arch/x86/kernel/apic/hw_nmi.c |   14 ++++++++++++++
 arch/x86/kernel/apic/nmi.c    |   14 ++++++++++++++
 2 files changed, 28 insertions(+), 0 deletions(-)
Quoted 11 lines

arch/x86/kernel/apic/hw_nmi.c u64 hw_nmi_get_sample_period(void) · line 32

Quoted 9 lines
arch/x86/kernel/apic/hw_nmi.c +1 -0
index cefd694..3aea0a5 100644
@@ -29,6 +29,16 @@ u64 hw_nmi_get_sample_period(void)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;

Please dont put statics into the middle of local variables - put them into file scope in a visible way.

Got it. will change it.

Quoted 45 lines
Quoted 45 lines
arch/x86/kernel/apic/hw_nmi.c +13 -0
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -41,6 +51,10 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);
 }
 
 static int __kprobes
arch/x86/kernel/apic/nmi.c +10 -0
index a43f71c..5fa8a13 100644
@@ -552,6 +552,16 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
Quoted 10 lines

arch/x86/kernel/apic/nmi.c void arch_trigger_all_cpu_backtrace(void) · line 580

Quoted 8 lines
arch/x86/kernel/apic/nmi.c +4 -0
@@ -564,4 +574,8 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);

This spinlock is never actually used as a spinlock - it's a "in progress" flag. Why not use a flag and test_and_set_bit()?

Yep, it's a "in progress" flag, I will change to use a flag to replace the spinlock.

Quoted 1 line

Also, the irq disabling really needed, will this code ever be called with irqs on?

This code could be called with irqs on.

If the arch_trigger_all_cpu_backtrace() was triggered by "spin_lock()"'s spin_lock_debug detector, it is possible that the irq is enabled, thus we have to save and disable it here.

Thanks,

Dongdong

Quoted 5 lines

Thanks,

Ingo

#3 lore raw
Quoted 109 lines · DDD
Quoted 41 lines · Ingo Molnar
Quoted 39 lines · Don Zickus

From: Dongdong Deng <[email protected]>

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---

arch/x86/kernel/apic/hw_nmi.c |   14 ++++++++++++++
arch/x86/kernel/apic/nmi.c    |   14 ++++++++++++++
2 files changed, 28 insertions(+), 0 deletions(-)
Quoted 12 lines

arch/x86/kernel/apic/hw_nmi.c u64 hw_nmi_get_sample_period(void) · line 32

Quoted 9 lines
arch/x86/kernel/apic/hw_nmi.c +1 -0
index cefd694..3aea0a5 100644
@@ -29,6 +29,16 @@ u64 hw_nmi_get_sample_period(void)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;

Please dont put statics into the middle of local variables - put them into file scope in a visible way.

Got it. will change it.

Quoted 43 lines
Quoted 43 lines
arch/x86/kernel/apic/hw_nmi.c +13 -0
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
@@ -41,6 +51,10 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);
 }

static int __kprobes

arch/x86/kernel/apic/nmi.c +10 -0
index a43f71c..5fa8a13 100644
@@ -552,6 +552,16 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
+	unsigned long flags;
+
+	local_irq_save(flags);
+	if (!arch_spin_trylock(&lock))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace()
+		 * in progress, don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
@@ -564,4 +574,8 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
Quoted 8 lines

arch/x86/kernel/apic/nmi.c void arch_trigger_all_cpu_backtrace(void) · line 580

Quoted 5 lines
arch/x86/kernel/apic/nmi.c +4 -0
 	}
+
+	arch_spin_unlock(&lock);
+out_restore_irq:
+	local_irq_restore(flags);

This spinlock is never actually used as a spinlock - it's a "in progress" flag. Why not use a flag and test_and_set_bit()?

Quoted 2 lines

Yep, it's a "in progress" flag, I will change to use a flag to replace the spinlock.

Thanks.

Quoted 7 lines
Quoted 1 line

Also, the irq disabling really needed, will this code ever be called with irqs on?

This code could be called with irqs on.

If the arch_trigger_all_cpu_backtrace() was triggered by "spin_lock()"'s spin_lock_debug detector, it is possible that the irq is enabled, thus we have to save and disable it here.

Ok - please document that fact in the code.

Thanks,

Ingo