Skip to contentSkip to conversation

fix race between stop_two_cpus and stop_cpus

PATCH In review Rik van Riel 5 participants 13 messages Oct 31, 2013 – Nov 11, 2013

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fix race between stop_two_cpus and stop_cpus

There is a race between stop_two_cpus, and the global stop_cpus.

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
Prarit is running a test with this patch. By now the kernel would have crashed already, yet it is still going. I expect Prarit will add his
Tested-by: some time tomorrow morning.

 kernel/stop_machine.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)
kernel/stop_machine.c +42 -1
index 32a6c44..46cb4c2 100644
@@ -40,8 +40,10 @@ struct cpu_stopper {
 };
 
 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
+static DEFINE_PER_CPU(bool, stop_two_cpus_queueing);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
+static bool stop_cpus_queueing = false;
 
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
@@ -261,16 +263,37 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
 	cpu_stop_init_done(&done, 2);
 	set_state(&msdata, MULTI_STOP_PREPARE);
 
+ wait_for_global:
+	/* If a global stop_cpus is queuing up stoppers, wait. */
+	while (unlikely(stop_cpus_queueing))
+		cpu_relax();
+
+	/* This CPU is queuing up stoppers. */
+	preempt_disable();
+	this_cpu_write(stop_two_cpus_queueing, true);
+	smp_mb(); /* matches the smp_wmb in queue_stop_cpus_work */
+
+	/* Global stop_cpus got busy simultaneously. Wait and retry. */
+	if (unlikely(stop_cpus_queueing)) {
+		smp_mb(); /* matches the smp_wmb in queue_stop_cpus_work */
+		this_cpu_write(stop_two_cpus_queueing, false);
+		preempt_enable();
+		goto wait_for_global;
+	}
+
 	/*
 	 * Queuing needs to be done by the lowest numbered CPU, to ensure
 	 * that works are always queued in the same order on every CPU.
 	 * This prevents deadlocks.
 	 */
 	call_cpu = min(cpu1, cpu2);
-
 	smp_call_function_single(call_cpu, &irq_cpu_stop_queue_work,
 				 &call_args, 0);
 
+	smp_wmb(); /* matches the smp_mb in wait_on_stop_two_cpus */
+	this_cpu_write(stop_two_cpus_queueing, false);
+	preempt_enable();
+
 	wait_for_completion(&done.completion);
 	return done.executed ? done.ret : -ENOENT;
 }
@@ -295,6 +318,19 @@ void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
 	cpu_stop_queue_work(cpu, work_buf);
 }
 
+static void wait_on_stop_two_cpus(const struct cpumask *cpumask)
+{
+	int cpu;
+
+	/* Do not reorder reads before this point */
+	smp_mb(); /* matches the smp_wmb in stop_two_cpus */
+	
+	/* Wait until no stop_two_cpus stopper tasks are being queued */
+	for_each_cpu(cpu, cpumask)
+		while (per_cpu(stop_two_cpus_queueing, cpu) == true)
+			cpu_relax();
+}
+
 /* static data for stop_cpus */
 static DEFINE_MUTEX(stop_cpus_mutex);
 static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work);
@@ -320,8 +356,12 @@ static void queue_stop_cpus_work(const struct cpumask *cpumask,
 	 * to enter @fn which can lead to deadlock.
 	 */
 	preempt_disable();
+	stop_cpus_queueing = true;
+	wait_on_stop_two_cpus(cpumask);
 	for_each_cpu(cpu, cpumask)
 		cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
+	smp_wmb(); /* matches the smp_mb in stop_two_cpus */
+	stop_cpus_queueing = false;
 	preempt_enable();
 }
 
@@ -509,6 +549,7 @@ static int __init cpu_stop_init(void)
 
 		spin_lock_init(&stopper->lock);
 		INIT_LIST_HEAD(&stopper->works);
+		per_cpu(stop_two_cpus_queueing, cpu) = false;
 	}
 
 	BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
#1 lore raw
Quoted 2 lines · Rik van Riel

There is a race between stop_two_cpus, and the global stop_cpus.

What was the trigger for this? I want to see what was missing from my own testing. I'm going to go out on a limb and guess that CPU hotplug was also running in the background to specifically stress this sort of rare condition. Something like running a standard test with the monitors/watch-cpuoffline.sh from mmtests running in parallel.

Quoted 33 lines

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
Prarit is running a test with this patch. By now the kernel would have crashed already, yet it is still going. I expect Prarit will add his
Tested-by: some time tomorrow morning.

 kernel/stop_machine.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)
kernel/stop_machine.c +2 -0
index 32a6c44..46cb4c2 100644
@@ -40,8 +40,10 @@ struct cpu_stopper {
 };
 
 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
+static DEFINE_PER_CPU(bool, stop_two_cpus_queueing);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
+static bool stop_cpus_queueing = false;
 
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {

kernel/stop_machine.c int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · line 270

Quoted 8 lines
kernel/stop_machine.c +5 -0
@@ -261,16 +263,37 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
 	cpu_stop_init_done(&done, 2);
 	set_state(&msdata, MULTI_STOP_PREPARE);
 
+ wait_for_global:
+	/* If a global stop_cpus is queuing up stoppers, wait. */
+	while (unlikely(stop_cpus_queueing))
+		cpu_relax();
+

This partially serialises callers to migrate_swap() while it is checked if the pair of CPUs are being affected at the moment. It's two-stage locking. The global lock is short-lived while the per-cpu data is updated and the per-cpu values allow a degree of parallelisation on call_cpu which could not be done with a spinlock held anyway. Why not make protection of the initial update a normal spinlock? i.e.

spin_lock(&stop_cpus_queue_lock);
this_cpu_write(stop_two_cpus_queueing, true);
spin_unlock(&stop_cpus_queue_lock);

and get rid of the barriers and gogo wait_for_global loop entirely? I'm not seeing the hidden advantage. The this_cpu_write(stop_two_cpus_queueing, false) would also need to be within the lock as would the checks in queue_stop_cpus_work.

The locks look bad but it's not clear to me why the barriers and retries are better.

Signature · 2 lines
Mel Gorman
SUSE Labs
#2 lore raw
Quoted 9 lines · Mel Gorman
Quoted 2 lines · Rik van Riel

There is a race between stop_two_cpus, and the global stop_cpus.

What was the trigger for this? I want to see what was missing from my own testing. I'm going to go out on a limb and guess that CPU hotplug was also running in the background to specifically stress this sort of rare condition. Something like running a standard test with the monitors/watch-cpuoffline.sh from mmtests running in parallel.

AFAIK the trigger was a test that continuously loads and unloads kernel modules, while doing other stuff.

Quoted 36 lines
Quoted 36 lines

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
Prarit is running a test with this patch. By now the kernel would have crashed already, yet it is still going. I expect Prarit will add his
Tested-by: some time tomorrow morning.

 kernel/stop_machine.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)
kernel/stop_machine.c +2 -0
index 32a6c44..46cb4c2 100644
@@ -40,8 +40,10 @@ struct cpu_stopper {
 };
 
 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
+static DEFINE_PER_CPU(bool, stop_two_cpus_queueing);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
+static bool stop_cpus_queueing = false;
 
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
@@ -261,16 +263,37 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
 	cpu_stop_init_done(&done, 2);
 	set_state(&msdata, MULTI_STOP_PREPARE);
 
Quoted 8 lines

kernel/stop_machine.c int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · line 270

Quoted 5 lines
kernel/stop_machine.c +5 -0
+ wait_for_global:
+	/* If a global stop_cpus is queuing up stoppers, wait. */
+	while (unlikely(stop_cpus_queueing))
+		cpu_relax();
+

This partially serialises callers to migrate_swap() while it is checked if the pair of CPUs are being affected at the moment. It's two-stage

Not really. This only serializes migrate_swap if there is a global stop_cpus underway.

If there is no global stop_cpus, migrate_swap will continue the way it did before, without locking.

Quoted 8 lines

locking. The global lock is short-lived while the per-cpu data is updated and the per-cpu values allow a degree of parallelisation on call_cpu which could not be done with a spinlock held anyway. Why not make protection of the initial update a normal spinlock? i.e.

spin_lock(&stop_cpus_queue_lock);
this_cpu_write(stop_two_cpus_queueing, true);
spin_unlock(&stop_cpus_queue_lock);

Because that would result in all migrate_swap instances serializing with each other.

Signature · 1 line
All rights reversed
Tested-by
#3 lore raw
Quoted 5 lines · Rik van Riel
Quoted 4 lines · Mel Gorman
Quoted 2 lines · Rik van Riel

There is a race between stop_two_cpus, and the global stop_cpus.

Quoted 9 lines
Quoted 5 lines

What was the trigger for this? I want to see what was missing from my own testing. I'm going to go out on a limb and guess that CPU hotplug was also running in the background to specifically stress this sort of rare condition. Something like running a standard test with the monitors/watch-cpuoffline.sh from mmtests running in parallel.

AFAIK the trigger was a test that continuously loads and unloads kernel modules, while doing other stuff.

With this patch in place the module load/unload test ran for ~16 hours without failure. Without the patch the test usually fails in 5-10 minutes.

P.

#4 lore raw
In reply to Rik van Riel
Quoted 5 lines · Rik van Riel
Quoted 4 lines · Mel Gorman
Quoted 2 lines · Rik van Riel

There is a race between stop_two_cpus, and the global stop_cpus.

Quoted 9 lines
Quoted 5 lines

What was the trigger for this? I want to see what was missing from my own testing. I'm going to go out on a limb and guess that CPU hotplug was also running in the background to specifically stress this sort of rare condition. Something like running a standard test with the monitors/watch-cpuoffline.sh from mmtests running in parallel.

AFAIK the trigger was a test that continuously loads and unloads kernel modules, while doing other stuff.

ok, thanks.

Quoted 9 lines
Quoted 8 lines

kernel/stop_machine.c int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * · line 270

Quoted 5 lines

+ wait_for_global:

kernel/stop_machine.c +4 -0
+	/* If a global stop_cpus is queuing up stoppers, wait. */
+	while (unlikely(stop_cpus_queueing))
+		cpu_relax();
+

This partially serialises callers to migrate_swap() while it is checked if the pair of CPUs are being affected at the moment. It's two-stage

Quoted 3 lines

Not really. This only serializes migrate_swap if there is a global stop_cpus underway.

Ok, I see your point now but still wonder if this is too specialised for what we are trying to do. Could it have been done with a read-write semaphore with the global stop_cpus taking it for write and stop_two_cpus taking it for read?

Signature · 2 lines
Mel Gorman
SUSE Labs
#5 lore raw
Quoted 4 lines · Mel Gorman

Ok, I see your point now but still wonder if this is too specialised for what we are trying to do. Could it have been done with a read-write semaphore with the global stop_cpus taking it for write and stop_two_cpus taking it for read?

rwsem for read is still global state.. That said it should be fairly easy to use lglock for this.

Or write it with spinlocks like:

DEFINE_PER_CPU(spinlock_t, local_lock);
DEFINE_PER_CPU(int, have_global);
spinlock_t global_lock;

void local_lock(void)

{
	preempt_disable();
	spin_lock(this_cpu_ptr(&local_lock));
	if (spin_is_locked(&global_lock)) {
		spin_unlock(this_cpu_ptr(&local_lock));
		spin_lock(&global_lock);
		this_cpu_write(have_global, true);
		spin_lock(this_cpu_ptr(&local_lock));
	}
}

void local_unlock(void)

{
	spin_unlock(this_cpu_ptr(&local_lock));
	if (this_cpu_read(have_global)) {
		this_cpu_write(have_global, false);
		spin_unlock(&global_lock);
	}
}

void global_lock(void)

{
	int cpu;

	spin_lock(&global_lock);
	for_each_possible_cpu(cpu)
		spin_unlock_wait(&per_cpu(local_lock, cpu));
}

void global_unlock(void)

{
	spin_unlock(&global_lock);
}

Or possibly make the global_lock a mutex, plenty variants possible.

#6 lore raw
Quoted 8 lines · Peter Zijlstra
Quoted 4 lines · Mel Gorman

Ok, I see your point now but still wonder if this is too specialised for what we are trying to do. Could it have been done with a read-write semaphore with the global stop_cpus taking it for write and stop_two_cpus taking it for read?

rwsem for read is still global state.. That said it should be fairly easy to use lglock for this.

I'll rewrite the patch using lglocks, that should make things more readable.

Signature · 1 line
All rights reversed
#7 lore raw

PATCH 1/1 fix race between stop_two_cpus and stop_cpus

Subject: fix race between stop_two_cpus and stop_cpus

There is a race between stop_two_cpus, and the global stop_cpus.

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
v2: use lglock, as suggested by Peter & Mel, thanks to both of you for insisting on nicer code :)

 kernel/stop_machine.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
kernel/stop_machine.c +13 -2
index 32a6c44..c5c1af1 100644
@@ -20,6 +20,7 @@
 #include <linux/kallsyms.h>
 #include <linux/smpboot.h>
 #include <linux/atomic.h>
+#include <linux/lglock.h>
 
 /*
  * Structure to determine completion condition and record errors.  May
@@ -43,6 +44,14 @@ static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
 
+/*
+ * Avoids a race between stop_two_cpus and global stop_cpus, where
+ * the stoppers could get queued up in reverse order, leading to
+ * system deadlock. Using an lglock means stop_two_cpus remains
+ * relatively cheap.
+ */
+DEFINE_STATIC_LGLOCK(stop_cpus_lock);
+
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
 	memset(done, 0, sizeof(*done));
@@ -268,8 +277,10 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
 	 */
 	call_cpu = min(cpu1, cpu2);
 
+	lg_local_lock(&stop_cpus_lock);
 	smp_call_function_single(call_cpu, &irq_cpu_stop_queue_work,
 				 &call_args, 0);
+	lg_local_unlock(&stop_cpus_lock);
 
 	wait_for_completion(&done.completion);
 	return done.executed ? done.ret : -ENOENT;
@@ -319,10 +330,10 @@ static void queue_stop_cpus_work(const struct cpumask *cpumask,
 	 * preempted by a stopper which might wait for other stoppers
 	 * to enter @fn which can lead to deadlock.
 	 */
-	preempt_disable();
+	lg_global_lock(&stop_cpus_lock);
 	for_each_cpu(cpu, cpumask)
 		cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
-	preempt_enable();
+	lg_global_unlock(&stop_cpus_lock);
 }
 
 static int __stop_cpus(const struct cpumask *cpumask,
Acked-by on v1 1/1
#8 lore raw
Quoted 9 lines · Rik van Riel

Subject: fix race between stop_two_cpus and stop_cpus

There is a race between stop_two_cpus, and the global stop_cpus.

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

Quoted 8 lines

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
v2: use lglock, as suggested by Peter & Mel, thanks to both of you for insisting on nicer code :)

This is a tad more comprehensible :). Thanks!

Signature · 2 lines
Mel Gorman
SUSE Labs
#9 lore raw
Quoted 19 lines · Mel Gorman
Quoted 9 lines · Rik van Riel

Subject: fix race between stop_two_cpus and stop_cpus

There is a race between stop_two_cpus, and the global stop_cpus.

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

Quoted 8 lines

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
v2: use lglock, as suggested by Peter & Mel, thanks to both of you for insisting on nicer code :)

Quoted 3 lines

This is a tad more comprehensible :). Thanks!

I'll test for a few hours and get back to everyone by the end of the day.

P.

Quoted 1 line
Tested-by
#10 lore raw
In reply to Mel Gorman
Quoted 19 lines · Mel Gorman
Quoted 9 lines · Rik van Riel

Subject: fix race between stop_two_cpus and stop_cpus

There is a race between stop_two_cpus, and the global stop_cpus.

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

Quoted 8 lines

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
v2: use lglock, as suggested by Peter & Mel, thanks to both of you for insisting on nicer code :)

Quoted 4 lines

This is a tad more comprehensible :). Thanks!

This ran all morning without issues.

P.

#11 lore raw

[tip:sched/core] stop_machine: Fix race between stop_two_cpus() and stop_cpus()

In reply to Rik van Riel
stop_machine: Fix race between stop_two_cpus() and stop_cpus()
Commit-ID:  7053ea1a34fa8567cb5e3c39e04ace4c5d0fbeaa
Gitweb:     http://git.kernel.org/tip/7053ea1a34fa8567cb5e3c39e04ace4c5d0fbeaa
Author:     Rik van Riel <[email protected]>
AuthorDate: Fri, 1 Nov 2013 10:41:46 -0400
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 11 Nov 2013 12:43:38 +0100

stop_machine: Fix race between stop_two_cpus() and stop_cpus()

There is a race between stop_two_cpus, and the global stop_cpus.

It is possible for two CPUs to get their stopper functions queued "backwards" from one another, resulting in the stopper threads getting stuck, and the system hanging. This can happen because queuing up stoppers is not synchronized.

This patch adds synchronization between stop_cpus (a rare operation), and stop_two_cpus.

---
 kernel/stop_machine.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
kernel/stop_machine.c +13 -2
index c530bc5..84571e0 100644
@@ -20,6 +20,7 @@
 #include <linux/kallsyms.h>
 #include <linux/smpboot.h>
 #include <linux/atomic.h>
+#include <linux/lglock.h>
 
 /*
  * Structure to determine completion condition and record errors.  May
@@ -43,6 +44,14 @@ static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
 
+/*
+ * Avoids a race between stop_two_cpus and global stop_cpus, where
+ * the stoppers could get queued up in reverse order, leading to
+ * system deadlock. Using an lglock means stop_two_cpus remains
+ * relatively cheap.
+ */
+DEFINE_STATIC_LGLOCK(stop_cpus_lock);
+
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
 	memset(done, 0, sizeof(*done));
@@ -276,6 +285,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
 		return -ENOENT;
 	}
 
+	lg_local_lock(&stop_cpus_lock);
 	/*
 	 * Queuing needs to be done by the lowest numbered CPU, to ensure
 	 * that works are always queued in the same order on every CPU.
@@ -284,6 +294,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
 	smp_call_function_single(min(cpu1, cpu2),
 				 &irq_cpu_stop_queue_work,
 				 &call_args, 0);
+	lg_local_unlock(&stop_cpus_lock);
 	preempt_enable();
 
 	wait_for_completion(&done.completion);
@@ -335,10 +346,10 @@ static void queue_stop_cpus_work(const struct cpumask *cpumask,
 	 * preempted by a stopper which might wait for other stoppers
 	 * to enter @fn which can lead to deadlock.
 	 */
-	preempt_disable();
+	lg_global_lock(&stop_cpus_lock);
 	for_each_cpu(cpu, cpumask)
 		cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
-	preempt_enable();
+	lg_global_unlock(&stop_cpus_lock);
 }
 
 static int __stop_cpus(const struct cpumask *cpumask,
#12 lore raw
In reply to Mel Gorman
Quoted 10 lines · Mel Gorman
Quoted 2 lines · Rik van Riel

There is a race between stop_two_cpus, and the global stop_cpus.

What was the trigger for this? I want to see what was missing from my own testing. I'm going to go out on a limb and guess that CPU hotplug was also running in the background to specifically stress this sort of rare condition. Something like running a standard test with the monitors/watch-cpuoffline.sh from mmtests running in parallel.

I have a test that loads and unloads each module in /lib/modules/3.*/...

Each run typically takes a few minutes. After running 4-5 times, the system issues a soft lockup warning with a CPU in multi_cpu_stop(). Unfortunately, kdump isn't working on this particular system (due to another bug) so I modified the code with (sorry for the cut-and-paste):

kernel/watchdog.c +3 -1
index 05039e3..4a8c9f9 100644
@@ -323,8 +323,10 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtime
                else
                        dump_stack();
 
-               if (softlockup_panic)
+               if (softlockup_panic) {
+                       show_state();
                        panic("softlockup: hung tasks");
+               }
                __this_cpu_write(soft_watchdog_warn, true);
        } else
                __this_cpu_write(soft_watchdog_warn, false);

and then 'echo 1 > /proc/sys/kernel/softlockup_panic' to get a full trace of all tasks.

When I did this and ran the kernel module load unload test ...

[prarit@prarit tmp]$ cat /tmp/intel.log | grep RIP

[  678.081168] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.156180] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.230190] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.244186] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.259194] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.274192] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.288195] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.303197] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.318200] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.333203] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.349206] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.364208] RIP: 0010:[<ffffffff810d328b>]  [<ffffffff810d328b>]
multi_cpu_stop+0x7b/0xf0
[  678.379211] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.394212] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.409215] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.424217] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.438219] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.452221] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.466228] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.481228] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.496230] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.511234] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.526236] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.541238] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.556244] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.571243] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.586247] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]
multi_cpu_stop+0x86/0xf0
[  678.601248] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.616251] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  678.632254] RIP: 0010:[<ffffffff810d328b>]  [<ffffffff810d328b>]
multi_cpu_stop+0x7b/0xf0
[  678.647257] RIP: 0010:[<ffffffff810d3292>]  [<ffffffff810d3292>]
multi_cpu_stop+0x82/0xf0
[  687.570464] RIP: 0010:[<ffffffff810d3296>]  [<ffffffff810d3296>]

multi_cpu_stop+0x86/0xf0

and,

[prarit@prarit tmp]$ cat /tmp/intel.log | grep RIP | wc -l 32

which shows all 32 cpus are "correctly" in the cpu stop threads. After some investigation, Rik came up with his patch.

Hope this explains things,

P.