Skip to contentSkip to conversation

x86: avoid calling arch_trigger_all_cpu_backtrace() at the same time on SMP

PATCH In review Dongdong Deng 4 participants 8 messages Nov 11, 2010 – Nov 12, 2010 x86

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 x86: avoid calling arch_trigger_all_cpu_backtrace() at the same time on SMP

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---
 arch/x86/kernel/apic/hw_nmi.c |   23 +++++++++++++++++++++++
 arch/x86/kernel/apic/nmi.c    |   23 +++++++++++++++++++++++
 2 files changed, 46 insertions(+), 0 deletions(-)
arch/x86/kernel/apic/hw_nmi.c +23 -0
index cefd694..bfdab3b 100644
@@ -26,9 +26,27 @@ u64 hw_nmi_get_sample_period(void)
 }
 
 #ifdef ARCH_HAS_NMI_WATCHDOG
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace() in progress
+		 * (backtrace_flag == 1), don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -41,6 +59,11 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	clear_bit(0, &backtrace_flag);
+
+out_restore_irq:
+	local_irq_restore(flags);
 }
 
 static int __kprobes
arch/x86/kernel/apic/nmi.c +23 -0
index c90041c..bd9cb79 100644
@@ -549,9 +549,27 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
 	return 0;
 }
 
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace() in progress
+		 * (backtrace_flag == 1), don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -564,4 +582,9 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	clear_bit(0, &backtrace_flag);
+
+out_restore_irq:
+	local_irq_restore(flags);
 }
Signature · 1 line
1.6.0.4
#1 lore raw

Hm, another thing i noticed is that there's two of these:

Quoted 22 lines

#ifdef ARCH_HAS_NMI_WATCHDOG

(quoted diff) +14 -0
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
 
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;

Review

Quoted 10 lines
(quoted diff) +10 -0
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))

A fair amount of code is being duplicated in two places - which is not nice. Lets try to create a shared facility instead?

Thanks,

Ingo

author
#2 lore raw
Quoted 28 lines · Ingo Molnar

Hm, another thing i noticed is that there's two of these:

Quoted 26 lines

#ifdef ARCH_HAS_NMI_WATCHDOG

arch/x86/kernel/apic/hw_nmi.c +18 -0
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
 
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
Quoted 8 lines

arch/x86/kernel/apic/hw_nmi.c u64 hw_nmi_get_sample_period(void) · line 44

Quoted 6 lines
arch/x86/kernel/apic/hw_nmi.c +6 -0
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))

A fair amount of code is being duplicated in two places - which is not nice. Lets try to create a shared facility instead?

Yep, It is a good idea, I will try to do that. :-)

Dongdong

Quoted 5 lines

Thanks,

Ingo

#3 lore raw
Quoted 40 lines · DDD
Quoted 30 lines · Ingo Molnar

Hm, another thing i noticed is that there's two of these:

Quoted 27 lines

 #ifdef ARCH_HAS_NMI_WATCHDOG

arch/x86/kernel/apic/hw_nmi.c +3 -0
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+

 void arch_trigger_all_cpu_backtrace(void)
 {
       int i;
+       unsigned long flags;

arch/x86/kernel/apic/hw_nmi.c +10 -0
+
+       /*
+        * Have to disable irq here, as the
+        * arch_trigger_all_cpu_backtrace() could be
+        * triggered by "spin_lock()" with irqs on.
+        */
+       local_irq_save(flags);
 
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+

 void arch_trigger_all_cpu_backtrace(void)
 {
       int i;
+       unsigned long flags;
+
+       /*
+        * Have to disable irq here, as the
+        * arch_trigger_all_cpu_backtrace() could be

Quoted 8 lines
Quoted 5 lines

+        * triggered by "spin_lock()" with irqs on.
+        */
+       local_irq_save(flags);
+
+       if (test_and_set_bit(0, &backtrace_flag))

A fair amount of code is being duplicated in two places - which is not nice. Lets try to create a shared facility instead?

Quoted 1 line

Yep, It is a good idea, I will try to do that. :-)

Hmm, it is hard to create a shared facility for
arch_trigger_all_cpu_backtrace().

The arch_trigger_all_cpu_backtrace() need a local variable(backtrace_mask) which was used by the other difference functions in nmi.c and hw_nmi.c.

If we want to create a shared facility for arch_trigger_all_cpu_backtrace(), we have to create an independent .c file for the single one function arch_trigger_all_cpu_backtrace(). I think it is worse than before...

Maybe Don could say some things here as he is the owner of hw_nmi.c. :-)

Thanks,
Dongdong

#4 lore raw
In reply to Ingo Molnar
Quoted 2 lines · Ingo Molnar

Hm, another thing i noticed is that there's two of these:

I guess it depends how we want to deprecate the old nmi watchdog. Dongdong was just patching the old and new code.

There are 3 ways we can go here.

- rip out the 'arch_trigger_all_cpu_backtrace' from the old nmi_watchdog, rework the Makefile and have both watchdogs use common code in hw_nmi.c

- patch both places

- rely on the fact that the old watchdog doesn't really work any more and only patch the new watchdog

I can put together a patch to do the first one, then have Dongdong put his patch on top. Selfishly I wouldn't mind just doing the third option. ;-)

Thoughts?

Cheers,
Don

Quoted 40 lines
Quoted 23 lines

#ifdef ARCH_HAS_NMI_WATCHDOG

(quoted diff) +14 -0
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
 
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;

Review

Quoted 10 lines
(quoted diff) +10 -0
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))

A fair amount of code is being duplicated in two places - which is not nice. Lets try to create a shared facility instead?

Thanks,

Ingo

#5 lore raw
Quoted 5 lines · Don Zickus
Quoted 2 lines · Ingo Molnar

Hm, another thing i noticed is that there's two of these:

I guess it depends how we want to deprecate the old nmi watchdog.

ah, indeed, i forgot about the old watchdog.

Lets zap it right now and see who complains? Mind sending a patch for that? (please also pick up Dongdong's patch)

Thanks,

Ingo

author
#6 lore raw
In reply to Dongdong Deng
Quoted 70 lines · Dongdong Deng

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---
 arch/x86/kernel/apic/hw_nmi.c |   23 +++++++++++++++++++++++
 arch/x86/kernel/apic/nmi.c    |   23 +++++++++++++++++++++++
 2 files changed, 46 insertions(+), 0 deletions(-)
arch/x86/kernel/apic/hw_nmi.c +18 -0
index cefd694..bfdab3b 100644
@@ -26,9 +26,27 @@ u64 hw_nmi_get_sample_period(void)
 }
 
 #ifdef ARCH_HAS_NMI_WATCHDOG
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace() in progress
+		 * (backtrace_flag == 1), don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 

arch/x86/kernel/apic/hw_nmi.c void arch_trigger_all_cpu_backtrace(void) · line 63

Quoted 5 lines
arch/x86/kernel/apic/hw_nmi.c +2 -0
@@ -41,6 +59,11 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	clear_bit(0, &backtrace_flag);

Hi Don,

The clear_bit() on x86 (arch/x86/include/asm/bitops.h) does not
contain a memory barrier, Could you add a memory barrier ops here when you pick up this patch?

+ clear_bit(0, &backtrace_flag);
+ smp_mb__after_clear_bit();

Thanks,
Dongdong

Quoted 48 lines
arch/x86/kernel/apic/hw_nmi.c +3 -0
+
+out_restore_irq:
+	local_irq_restore(flags);
 }
 
 static int __kprobes
arch/x86/kernel/apic/nmi.c +23 -0
index c90041c..bd9cb79 100644
@@ -549,9 +549,27 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)
 	return 0;
 }
 
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace() in progress
+		 * (backtrace_flag == 1), don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);
 
@@ -564,4 +582,9 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	clear_bit(0, &backtrace_flag);
+
+out_restore_irq:
+	local_irq_restore(flags);
 }
#7 lore raw
Quoted 82 lines · DDD
Quoted 67 lines · Dongdong Deng

The spin_lock_debug/rcu_cpu_stall detector uses trigger_all_cpu_backtrace() to dump cpu backtrace. Therefore it is possible that trigger_all_cpu_backtrace() could be called at the same time on different CPUs, which triggers and 'unknown reason NMI' warning. The following case illustrates the problem:

CPU1 CPU2 ... CPU N
trigger_all_cpu_backtrace()
set "backtrace_mask" to cpu mask
|
generate NMI interrupts generate NMI interrupts ...
\ | /
\ | /
The "backtrace_mask" will be cleaned by the first NMI interrupt
at nmi_watchdog_tick(), then the following NMI interrupts generated
by other cpus's arch_trigger_all_cpu_backtrace() will be took as unknown reason NMI interrupts.

This patch uses a lock to avoid the problem, and stop the arch_trigger_all_cpu_backtrace() calling to avoid dumping double cpu backtrace info when there is already a trigger_all_cpu_backtrace() in progress.

---

arch/x86/kernel/apic/hw_nmi.c |   23 +++++++++++++++++++++++
arch/x86/kernel/apic/nmi.c    |   23 +++++++++++++++++++++++
2 files changed, 46 insertions(+), 0 deletions(-)
arch/x86/kernel/apic/hw_nmi.c +18 -0
index cefd694..bfdab3b 100644
@@ -26,9 +26,27 @@ u64 hw_nmi_get_sample_period(void)
 }
 #ifdef ARCH_HAS_NMI_WATCHDOG
+/* "in progress" flag of arch_trigger_all_cpu_backtrace */
+static unsigned long backtrace_flag;
+
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
+	unsigned long flags;
+
+	/*
+	 * Have to disable irq here, as the
+	 * arch_trigger_all_cpu_backtrace() could be
+	 * triggered by "spin_lock()" with irqs on.
+	 */
+	local_irq_save(flags);
+
+	if (test_and_set_bit(0, &backtrace_flag))
+		/*
+		 * If there is already a trigger_all_cpu_backtrace() in progress
+		 * (backtrace_flag == 1), don't output double cpu dump infos.
+		 */
+		goto out_restore_irq;
 	cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask);

arch/x86/kernel/apic/hw_nmi.c void arch_trigger_all_cpu_backtrace(void) · line 63

Quoted 5 lines
arch/x86/kernel/apic/hw_nmi.c +2 -0
@@ -41,6 +59,11 @@ void arch_trigger_all_cpu_backtrace(void)
 			break;
 		mdelay(1);
 	}
+
+	clear_bit(0, &backtrace_flag);

Hi Don,

The clear_bit() on x86 (arch/x86/include/asm/bitops.h) does not
contain a memory barrier, Could you add a memory barrier ops here when you pick up this patch?

Review

Quoted 2 lines
(quoted diff) +2 -0
+	clear_bit(0, &backtrace_flag);
+	smp_mb__after_clear_bit();

Ok. Thanks.

Cheers,
Don