reimplement flush_workqueue()
RFC PATCH 1/1 reimplement flush_workqueue()
Remove ->remove_sequence, ->insert_sequence, and ->work_done from struct cpu_workqueue_struct. To implement flush_workqueue() we can queue a barrier work on each CPU and wait for its completition.
We don't need to worry about CPU going down while we are are sleeping on the completition. take_over_work() will move this work on another CPU, and the handler will wake up us eventually.
NOTE: I removed 'int cpu' parameter, flush_workqueue() locks/unlocks workqueue_mutex unconditionally. It may be restored, but I think it doesn't make much sense, we take the mutex for the very short time, and the code becomes simpler.
workqueue.c | 87 ++++++++++++++++++++---------------------------------------- 1 files changed, 29 insertions(+), 58 deletions(-)
kernel/workqueue.c renamed from kernel/workqueue.c~1_flush_q +29 -58
@@ -36,23 +36,13 @@/* * The per-CPU workqueue (if single thread, we always use the first * possible cpu). -* -* The sequence counters are for flush_scheduled_work(). It wants to wait -* until all currently-scheduled works are completed, but it doesn't -* want to be livelocked by new, incoming ones. So it waits until -* remove_sequence is >= the insert_sequence which pertained when -* flush_scheduled_work() was called. */ struct cpu_workqueue_struct {spinlock_t lock; -long remove_sequence;/* Least-recently added (next to run) */ -long insert_sequence;/* Next to add */ -struct list_head worklist;wait_queue_head_t more_work; -wait_queue_head_t work_done;struct workqueue_struct *wq;struct task_struct *thread;
@@ -138,8 +128,6 @@ static int __run_work(struct cpu_workquef (work);spin_lock_irqsave (&cwq->lock, flags); - cwq->remove_sequence++; -wake_up (&cwq->work_done); ret =1 ; }spin_unlock_irqrestore (&cwq->lock, flags);
@@ -187,7 +175,6 @@ static void __queue_work(struct cpu_workspin_lock_irqsave (&cwq->lock, flags);set_wq_data (work, cwq);list_add_tail (&work->entry, &cwq->worklist); - cwq->insert_sequence++;wake_up (&cwq->more_work);spin_unlock_irqrestore (&cwq->lock, flags); }
@@ -338,8 +325,6 @@ static void run_workqueue(struct cpu_wor }spin_lock_irqsave (&cwq->lock, flags); - cwq->remove_sequence++; -wake_up (&cwq->work_done); } cwq->run_depth--;spin_unlock_irqrestore (&cwq->lock, flags);
@@ -394,45 +379,39 @@ static int worker_thread(void *__cwq)return 0 ; } -/* -* If cpu == -1 it's a single-threaded workqueue and the caller does not hold -* workqueue_mutex -*/ -static void flush_cpu_workqueue (struct cpu_workqueue_struct *cwq,int cpu) +struct wq_barrier { +struct work_struct work; +struct completion done; +}; + +static void wq_barrier_func (struct work_struct *work) +{ + struct wq_barrier *barr = container_of(work, struct wq_barrier, work); +complete (&barr->done); +} + +static void flush_cpu_workqueue (struct cpu_workqueue_struct *cwq) {if (cwq->thread == current) {/* * Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ -if (cpu !=-1 ) -mutex_unlock (&workqueue_mutex); +mutex_unlock (&workqueue_mutex);run_workqueue (cwq); -if (cpu !=-1 ) -mutex_lock (&workqueue_mutex); +mutex_lock (&workqueue_mutex); }else { -DEFINE_WAIT (wait); -long sequence_needed; +struct wq_barrier barr = { + .work =__WORK_INITIALIZER (barr.work, wq_barrier_func), + .done =COMPLETION_INITIALIZER_ONSTACK (barr.done), + }; -spin_lock_irq (&cwq->lock); - sequence_needed = cwq->insert_sequence; +__set_bit (WORK_STRUCT_PENDING , &barr.work.management); +__queue_work (cwq, &barr.work); -while (sequence_needed - cwq->remove_sequence >0 ) { -prepare_to_wait (&cwq->work_done, &wait, -TASK_UNINTERRUPTIBLE ); -spin_unlock_irq (&cwq->lock); -if (cpu !=-1 ) -mutex_unlock (&workqueue_mutex); -schedule (); -if (cpu !=-1 ) { -mutex_lock (&workqueue_mutex); -if (!cpu_online (cpu)) -return ;/* oops, CPU unplugged */ - } -spin_lock_irq (&cwq->lock); - } -finish_wait (&cwq->work_done, &wait); -spin_unlock_irq (&cwq->lock); +mutex_unlock (&workqueue_mutex); +wait_for_completion (&barr.done); +mutex_lock (&workqueue_mutex); } }
@@ -443,30 +422,25 @@ static void flush_cpu_workqueue(struct c* Forces execution of the workqueue and blocks until its completion. * This is typically used in driver shutdown handlers. * -* This function will sample each workqueue's current insert_sequence number and -* will sleep until the head sequence is greater than or equal to that. This -* means that we sleep until all works which were queued on entry have been -* handled, but we are not livelocked by new incoming ones. +* We sleep until all works which were queued on entry have been handled, +* but we are not livelocked by new incoming ones. * * This function used to run the workqueues itself. Now we just wait for the * helper threads to do it. */ void fastcall flush_workqueue(struct workqueue_struct *wq) { -might_sleep (); - +mutex_lock (&workqueue_mutex);if (is_single_threaded (wq)) {/* Always use first cpu's area. */ -flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu), --1 ); +flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu)); }else {int cpu; -mutex_lock (&workqueue_mutex);for_each_online_cpu (cpu) -flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu), cpu); -mutex_unlock (&workqueue_mutex); +flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu)); } +mutex_unlock (&workqueue_mutex); }EXPORT_SYMBOL_GPL (flush_workqueue);
@@ -479,12 +453,9 @@ static struct task_struct *create_workquspin_lock_init (&cwq->lock); cwq->wq = wq; cwq->thread =NULL ; - cwq->insert_sequence =0 ; - cwq->remove_sequence =0 ; cwq->freezeable = freezeable;INIT_LIST_HEAD (&cwq->worklist);init_waitqueue_head (&cwq->more_work); -init_waitqueue_head (&cwq->work_done);if (is_single_threaded (wq)) p =kthread_create (worker_thread, cwq,"%s" , wq->name);
Quoted 4 lines · Oleg Nesterov
Remove ->remove_sequence, ->insert_sequence, and ->work_done from struct
cpu_workqueue_struct. To implementflush_workqueue()we can queue a barrier work on each CPU and wait for its completition.
Looks fine to me. It's after -rc1 so I won't apply it, but it looks like a nice cleanup.
Linus
Quoted 3 lines · On Mon, 18 Dec 2006 01:34:16 +0300 Oleg Nesterov <[email protected]> wrote:
Remove ->remove_sequence, ->insert_sequence, and ->work_done from struct
cpu_workqueue_struct. To implementflush_workqueue()we can queue a barrier work on each CPU and wait for its completition.
Seems sensible. I seem to recall considering doing it that way when I initially implemeted flush_workqueue(), but I don't recall why I didn't do this. hmm.
Quoted 9 lines
We don't need to worry about CPU going down while we are are sleeping on the completition.
take_over_work()will move this work on another CPU, and the handler will wake up us eventually.NOTE: I removed 'int cpu' parameter,
flush_workqueue()locks/unlocksworkqueue_mutexunconditionally. It may be restored, but I think it doesn't make much sense, we take the mutex for the very short time, and the code becomes simpler.
Taking workqueue_mutex() unconditionally in flush_workqueue() means that we'll deadlock if a single-threaded workqueue callback handler calls flush_workqueue().
It's an idiotic thing to do, but I think I spotted a site last week which does this. scsi? Not sure..
Quoted 10 lines · Andrew Morton
Quoted 5 lines · On Mon, 18 Dec 2006 01:34:16 +0300 Oleg Nesterov <[email protected]> wrote:
NOTE: I removed 'int cpu' parameter,
flush_workqueue()locks/unlocksworkqueue_mutexunconditionally. It may be restored, but I think it doesn't make much sense, we take the mutex for the very short time, and the code becomes simpler.
Quoted 3 lines
Taking
workqueue_mutex()unconditionally inflush_workqueue()means that we'll deadlock if a single-threaded workqueue callback handler callsflush_workqueue().
Well. But flush_workqueue() drops workqueue_mutex before going to sleep ?
flush_workqueue(single_threaded_wq); ... mutex_lock(&workqueue_mutex); ... mutex_unlock(&workqueue_mutex); wait_for_completition(); handler runs, calls flush_workqueue(), workqueue_mutex is free
Quoted 2 lines
It's an idiotic thing to do, but I think I spotted a site last week which does this. scsi? Not sure..
Ok, it is time to sleep. I'll look tomorrov and re-send if flush_cpu_workqueue() really needs "bool workqueue_mutex_is_locked" parameter.
Oleg.
Quoted 17 lines · On Tue, 19 Dec 2006 03:43:19 +0300 Oleg Nesterov <[email protected]> wrote:
Quoted 10 lines · Andrew Morton
Quoted 5 lines · On Mon, 18 Dec 2006 01:34:16 +0300 Oleg Nesterov <[email protected]> wrote:
NOTE: I removed 'int cpu' parameter,
flush_workqueue()locks/unlocksworkqueue_mutexunconditionally. It may be restored, but I think it doesn't make much sense, we take the mutex for the very short time, and the code becomes simpler.Quoted 3 lines
Taking
workqueue_mutex()unconditionally inflush_workqueue()means that we'll deadlock if a single-threaded workqueue callback handler callsflush_workqueue().Well. But
flush_workqueue()dropsworkqueue_mutexbefore going to sleep ?
Quoted 9 lines
flush_workqueue(single_threaded_wq); ... mutex_lock(&workqueue_mutex); ... mutex_unlock(&workqueue_mutex); wait_for_completition(); handler runs, calls flush_workqueue(), workqueue_mutex is free
Oh. OK. In that case we can switch to preempt_disable() for the cpu-hotplug holdoff. Sometime.
Quoted 5 lines
Quoted 2 lines
It's an idiotic thing to do, but I think I spotted a site last week which does this. scsi? Not sure..
Ok, it is time to sleep. I'll look tomorrov and re-send if
flush_cpu_workqueue()really needs "boolworkqueue_mutex_is_locked" parameter.
Hopefully not.
Quoted 5 lines · Oleg Nesterov
Quoted 3 lines
Taking
workqueue_mutex()unconditionally inflush_workqueue()means that we'll deadlock if a single-threaded workqueue callback handler callsflush_workqueue().Well. But
flush_workqueue()dropsworkqueue_mutexbefore going to sleep ?
... and acquires it again after woken from sleep. That can be a problem, which will lead to the problem described here:
http://lkml.org/lkml/2006/12/7/374
In brief:
keventd thread hotplug thread
-------------- --------------
run_workqueue()
|
work_fn()
|
flush_workqueue()
|
flush_cpu_workqueue
| cpu_down()
mutex_unlock(wq_mutex); |
(above opens window for hotplug) mutex_lock(wq_mutex);
| /* bring down cpu */
wait_for_completition(); notifier(CPU_DEAD, ..)
| workqueue_cpu_callback
| cleanup_workqueue_thread
| kthread_stop()
|
|
mutex_lock(wq_mutex); <- Can deadlockThe kthread_stop() will wait for keventd() thread to exit, but keventd() is blocked on mutex_lock(wq_mutex) leading to a deadlock.
Quoted 10 lines
flush_workqueue(single_threaded_wq); ... mutex_lock(&workqueue_mutex); ... mutex_unlock(&workqueue_mutex); wait_for_completition(); handler runs, calls flush_workqueue(), workqueue_mutex is free
Signature · 2 lines
Regards, vatsa
Quoted 38 lines · Srivatsa Vaddagiri
Quoted 5 lines · Oleg Nesterov
Quoted 3 lines
Taking
workqueue_mutex()unconditionally inflush_workqueue()means that we'll deadlock if a single-threaded workqueue callback handler callsflush_workqueue().Well. But
flush_workqueue()dropsworkqueue_mutexbefore going to sleep ?... and acquires it again after woken from sleep. That can be a problem, which will lead to the problem described here:
http://lkml.org/lkml/2006/12/7/374In brief:
keventd thread hotplug thread
-------------- --------------
run_workqueue()| work_fn() | flush_workqueue() | flush_cpu_workqueue | cpu_down() mutex_unlock(wq_mutex); | (above opens window for hotplug) mutex_lock(wq_mutex); | /* bring down cpu */ wait_for_completition(); notifier(CPU_DEAD, ..) | workqueue_cpu_callback | cleanup_workqueue_thread | kthread_stop() | | mutex_lock(wq_mutex); <- Can deadlock
Quoted 2 lines
The
kthread_stop()will wait forkeventd()thread to exit, butkeventd()is blocked onmutex_lock(wq_mutex)leading to a deadlock.
Thanks, I need to think about this.
However I am not sure I fully understand the problem.
First, this deadlock was not introduced by recent changes (including "single threaded flush_workqueue() takes workqueue_mutex too"), yes?
Also, it seems to me we have a much more simple scenario for deadlock.
events/0 runs run_workqueue(), work->func() sleeps or takes a preemtion. CPU 0 dies, keventd thread migrates to another CPU. CPU_DEAD calls kthread_stop() under workqueue_mutex and waits for until kevents thread exits. Now, if this work (or another work pending on cwq->worklist) takes workqueue_mutex (for example, does flush_workqueue) we have a deadlock.
No?
Oleg.
Quoted 6 lines · Oleg Nesterov
Thanks, I need to think about this.
However I am not sure I fully understand the problem.
First, this deadlock was not introduced by recent changes (including "single threaded
flush_workqueue()takesworkqueue_mutextoo"), yes?
AFAIK this deadlock originated from Andrew's patch here:
http://lkml.org/lkml/2006/12/7/231
(Yes, your patches didnt introduce this. I was just reiterating here my earlier point that workqueue code is broken of late wrt cpu hotplug).
Quoted 9 lines
Also, it seems to me we have a much more simple scenario for deadlock.
events/0 runs
run_workqueue(),work->func()sleeps or takes a preemtion. CPU 0 dies, keventd thread migrates to another CPU.CPU_DEADcallskthread_stop()underworkqueue_mutexand waits for until kevents thread exits. Now, if this work (or another work pending oncwq->worklist) takesworkqueue_mutex(for example, doesflush_workqueue) we have a deadlock.No?
Yes, the above scenario also will cause a deadlock.
I supposed one could avoid the deadlock by having a 'workqueue_mutex_held' flag and avoid taking the mutex set under some conditions, but IMHO a more neater solution is to provide a cpu-hotplug lock which works under all these corner cases. One such proposal was made here:
http://lkml.org/lkml/2006/10/26/65
Signature · 2 lines
Regards, vatsa
Quoted 4 lines · Srivatsa Vaddagiri
Quoted 2 lines · Oleg Nesterov
Thanks, I need to think about this.
Quoted 8 lines
Quoted 4 lines
However I am not sure I fully understand the problem.
First, this deadlock was not introduced by recent changes (including "single threaded
flush_workqueue()takesworkqueue_mutextoo"), yes?AFAIK this deadlock originated from Andrew's patch here:
http://lkml.org/lkml/2006/12/7/231
I don't think so. The core problem is not that we are doing unlock/sleep/lock with this patch. The thing is: work->func() can't take wq_mutex (and thus use flush_work/workqueue) because it is possible that CPU_DEAD holds this mutex and waits for us to complete(kthread_stop_info). I believe this bug is old.
Quoted 11 lines
(Yes, your patches didnt introduce this. I was just reiterating here my earlier point that workqueue code is broken of late wrt cpu hotplug).
Quoted 8 lines
Also, it seems to me we have a much more simple scenario for deadlock.
events/0 runs
run_workqueue(),work->func()sleeps or takes a preemtion. CPU 0 dies, keventd thread migrates to another CPU.CPU_DEADcallskthread_stop()underworkqueue_mutexand waits for until kevents thread exits. Now, if this work (or another work pending oncwq->worklist) takesworkqueue_mutex(for example, doesflush_workqueue) we have a deadlock.
Quoted 3 lines
Quoted 1 line
No?
Yes, the above scenario also will cause a deadlock.
Ok, thanks for acknowledgement.
Quoted 2 lines
I supposed one could avoid the deadlock by having a '
workqueue_mutex_held' flag and avoid taking the mutex set under some conditions,
I am thinking about the same right now. Probably we can do something like this:
int xxx_lock(void)
{
for (;;) {
if (mutex_trylock(wq_mutex))
return 1;
// the owner of wq_mutex sleeps, we can proceed
if (kthread_should_stop())
return 0;
}
}
void xxx_unlock(int yesno)
{
if (yesno)
mutex_unlock(wq_mutext);
}and then do
locked = xxx_lock(); ... xxx_unlock(locked);
in flush_xxx() instead of plain lock/unlock.
Yes, ugly. I'll try to do something else on weekend.
Quoted 5 lines
but IMHO a
more neater solution is to provide a cpu-hotplug lock which works under all these corner cases. One such proposal was made here:http://lkml.org/lkml/2006/10/26/65
I'll take a look later, thanks.
Oleg.
Quoted 8 lines · Oleg Nesterov
Quoted 3 lines
AFAIK this deadlock originated from Andrew's patch here:
http://lkml.org/lkml/2006/12/7/231I don't think so. The core problem is not that we are doing unlock/sleep/lock with this patch. The thing is:
work->func()can't takewq_mutex(and thus use flush_work/workqueue) because it is possible thatCPU_DEADholds this mutex and waits for us tocomplete(kthread_stop_info). I believe this bug is old.
Yes, this bug is quite old looks like. Thanks for correcting me.
Signature · 2 lines
Regards, vatsa
Quoted 25 lines · On Thu, 4 Jan 2007 17:29:36 +0300 Oleg Nesterov <[email protected]> wrote:
Quoted 25 lines
In brief:
keventd thread hotplug thread
-------------- --------------
run_workqueue()| work_fn() | flush_workqueue() | flush_cpu_workqueue | cpu_down() mutex_unlock(wq_mutex); | (above opens window for hotplug) mutex_lock(wq_mutex); | /* bring down cpu */ wait_for_completition(); notifier(CPU_DEAD, ..) | workqueue_cpu_callback | cleanup_workqueue_thread | kthread_stop() | | mutex_lock(wq_mutex); <- Can deadlock
Quoted 2 lines
Quoted 2 lines
The
kthread_stop()will wait forkeventd()thread to exit, butkeventd()is blocked onmutex_lock(wq_mutex)leading to a deadlock.
This?
kernel/workqueue.c renamed from kernel/workqueue.c~flush_workqueue-use-preempt_disable-to-hold-off-cpu-hotplug +10 -6
@@ -419,18 +419,22 @@ static void flush_cpu_workqueue(struct c* Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ -mutex_unlock (&workqueue_mutex); +preempt_enable (); +/* +* We can still touch *cwq here because we are keventd, and +* hot-unplug will be waiting us to exit. +*/ run_workqueue (cwq); -mutex_lock (&workqueue_mutex); +preempt_disable (); }else {struct wq_barrier barr;init_wq_barrier (&barr);__queue_work (cwq, &barr.work); -mutex_unlock (&workqueue_mutex); +preempt_enable ();/* Can no longer touch *cwq */ wait_for_completion (&barr.done); -mutex_lock (&workqueue_mutex); +preempt_disable (); } }
@@ -449,7 +453,7 @@ static void flush_cpu_workqueue(struct c*/ void fastcall flush_workqueue(struct workqueue_struct *wq) { -mutex_lock (&workqueue_mutex); +preempt_disable ();/* CPU hotplug */ if (is_single_threaded (wq)) {/* Always use first cpu's area. */ flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu));
@@ -459,7 +463,7 @@ void fastcall flush_workqueue(struct worfor_each_online_cpu (cpu)flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu)); } -mutex_unlock (&workqueue_mutex); +preempt_enable (); }EXPORT_SYMBOL_GPL (flush_workqueue);
_
Quoted 38 lines · Andrew Morton
Quoted 25 lines · On Thu, 4 Jan 2007 17:29:36 +0300 Oleg Nesterov <[email protected]> wrote:
Quoted 25 lines
In brief:
keventd thread hotplug thread
-------------- --------------
run_workqueue()| work_fn() | flush_workqueue() | flush_cpu_workqueue | cpu_down() mutex_unlock(wq_mutex); | (above opens window for hotplug) mutex_lock(wq_mutex); | /* bring down cpu */ wait_for_completition(); notifier(CPU_DEAD, ..) | workqueue_cpu_callback | cleanup_workqueue_thread | kthread_stop() | | mutex_lock(wq_mutex); <- Can deadlockQuoted 2 lines
Quoted 2 lines
The
kthread_stop()will wait forkeventd()thread to exit, butkeventd()is blocked onmutex_lock(wq_mutex)leading to a deadlock.This?
kernel/workqueue.c static void flush_cpu_workqueue(struct c · line 422
Quoted 5 lines
kernel/workqueue.c renamed from kernel/workqueue.c~flush_workqueue-use-preempt_disable-to-hold-off-cpu-hotplug +1
-1@@ -419,18 +419,22 @@ static void flush_cpu_workqueue(struct c* Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ -mutex_unlock (&workqueue_mutex); +preempt_enable ();
Ah, (looking at _cpu_down()->stop_machine()), so preempt_disable() not only "pins" the current CPU, it blocks cpu_down(), yes ???
I guess this should work then. I'll try to re-check this code on weekend.
Oleg.
Quoted 11 lines · On Thu, 4 Jan 2007 21:09:01 +0300 Oleg Nesterov <[email protected]> wrote:
kernel/workqueue.c static void flush_cpu_workqueue(struct c · line 422
Quoted 8 lines
kernel/workqueue.c renamed from kernel/workqueue.c~flush_workqueue-use-preempt_disable-to-hold-off-cpu-hotplug +1
-1@@ -419,18 +419,22 @@ static void flush_cpu_workqueue(struct c* Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ -mutex_unlock (&workqueue_mutex); +preempt_enable ();Ah, (looking at _cpu_down()->stop_machine()), so
preempt_disable()not only "pins" the current CPU, it blockscpu_down(), yes ???
yep.
But before we do much more of this we should have a wrapper. Umm
static inline void block_cpu_hotplug(void)
{
preempt_disable();
}and use that, so people can see why it's being used.
I spose I'll do that and convert this patch to use it.
Quoted 6 lines · Andrew Morton
But before we do much more of this we should have a wrapper. Umm
static inline void
block_cpu_hotplug(void){ preempt_disable(); }
Nack.
This will only block cpu down, not cpu_up and hence is a misnomer. I would be vary of ignoring cpu_up events totally in writing hotplug safe code.
Signature · 2 lines
Regards, vatsa
Quoted 9 lines · Srivatsa Vaddagiri
Quoted 6 lines · Andrew Morton
But before we do much more of this we should have a wrapper. Umm
static inline void
block_cpu_hotplug(void){ preempt_disable(); }
Quoted 4 lines
Nack.
This will only block cpu down, not
cpu_upand hence is a misnomer. I would be vary of ignoringcpu_upevents totally in writing hotplug safe code.
How about block_cpu_down() ?
These cpu-hotplug races delayed the last workqueue patch I have in my queue. flush_workqueue() misses an important optimization: we don't need to insert a barrier and have an extra wake_up + wait_for_completion when cwq has no pending works. But we need ->current_work (introduced in the next patch) to implement this correctly.
I'll re-send the patch below later, when we finish with the bug you pointed out, but it would be nice if you can take a look now.
Oleg.
kernel/workqueue.c renamed from kernel/workqueue.c~4_speedup +17 -8
@@ -405,12 +405,15 @@ static void wq_barrier_func(struct work_complete (&barr->done); } -static inline void init_wq_barrier (struct wq_barrier *barr) +static void insert_wq_barrier (struct cpu_workqueue_struct *cwq, +struct wq_barrier *barr,int tail) {INIT_WORK (&barr->work, wq_barrier_func);__set_bit (WORK_STRUCT_PENDING ,work_data_bits (&barr->work));init_completion (&barr->done); + +insert_work (cwq, &barr->work, tail); }static void flush_cpu_workqueue (struct cpu_workqueue_struct *cwq)
@@ -425,13 +428,20 @@ static void flush_cpu_workqueue(struct cmutex_lock (&workqueue_mutex); }else {struct wq_barrier barr; +int active =0 ; -init_wq_barrier (&barr); -__queue_work (cwq, &barr.work); +spin_lock_irq (&cwq->lock); +if (!list_empty (&cwq->worklist) || cwq->current_work !=NULL ) { +insert_wq_barrier (cwq, &barr,1 ); + active =1 ; + } +spin_unlock_irq (&cwq->lock); -mutex_unlock (&workqueue_mutex); -wait_for_completion (&barr.done); -mutex_lock (&workqueue_mutex); +if (active) { +mutex_unlock (&workqueue_mutex); +wait_for_completion (&barr.done); +mutex_lock (&workqueue_mutex); + } } }
@@ -478,8 +488,7 @@ static void wait_on_work(struct cpu_workspin_lock_irq (&cwq->lock);if (unlikely (cwq->current_work == work)) { -init_wq_barrier (&barr); -insert_work (cwq, &barr.work,0 ); +insert_wq_barrier (cwq, &barr,0 ); running =1 ; }spin_unlock_irq (&cwq->lock);
Quoted 1 line · Oleg Nesterov
How about
block_cpu_down()?
Maybe ..not sure
If we do introduce such a function, we may need to convert several preempt_disable() that are there already (with intent of blocking cpu_down) to block_cpu_down() ..
Quoted 8 lines
These cpu-hotplug races delayed the last workqueue patch I have in my queue.
flush_workqueue()misses an important optimization: we don't need to insert a barrier and have an extrawake_up+wait_for_completionwhen cwq has no pending works. But we need ->current_work (introduced in the next patch) to implement this correctly.I'll re-send the patch below later, when we finish with the bug you pointed out, but it would be nice if you can take a look now.
The patch seems fine to me, though I dont see any cpu hotplug related problem being exposed/solved in this patch.
Signature · 2 lines
Regards, vatsa
Quoted 1 line · Andrew Morton
This?
This can still lead to the problem spotted by Oleg here:
http://lkml.org/lkml/2006/12/30/37
and you would need a similar patch he posted there.
Quoted 15 lines
void fastcall
flush_workqueue(struct workqueue_struct *wq){kernel/workqueue.c renamed from kernel/workqueue.c~flush_workqueue-use-preempt_disable-to-hold-off-cpu-hotplug +2
-2-mutex_lock (&workqueue_mutex); +preempt_disable ();/* CPU hotplug */ if (is_single_threaded (wq)) {/* Always use first cpu's area. */ flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu));@@ -459,7 +463,7 @@ void fastcall flush_workqueue(struct worfor_each_online_cpu (cpu)flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu)); } -mutex_unlock (&workqueue_mutex); +preempt_enable (); }EXPORT_SYMBOL_GPL (flush_workqueue);
Signature · 2 lines
Regards, vatsa
Quoted 9 lines · Srivatsa Vaddagiri
Quoted 1 line · Andrew Morton
This?
This can still lead to the problem spotted by Oleg here:
http://lkml.org/lkml/2006/12/30/37and you would need a similar patch he posted there.
preempt_disable() can't prevent cpu_up, but flush_workqueue() doesn't care _unless_ cpu_down also happened meantime (and hence a fresh CPU may have pending work_structs which were moved from a dead CPU).
So you are right, we still need the patch above, but I think we don't have new problems with preempt_disable().
I might have missed your point though.
Oleg.
Quoted 3 lines · Oleg Nesterov
preempt_disable()can't preventcpu_up, butflush_workqueue()doesn't care _unless_cpu_downalso happened meantime (and hence a fresh CPU may have pendingwork_structswhich were moved from a dead CPU).
Yes, that was what I had in mind.
Quoted 2 lines
So you are right, we still need the patch above, but I think we don't have new problems with
preempt_disable().
Right, preempt_disable() hasn't added any new problem. Its just revealing the same problem as earlier, by opening up window for cpu hotplug events to happen in the middle of flush_workqueue().
Ideally I would have liked a lock_cpu_hotplug() equivalent which blocks all cpu hotplug events during the entire flush_workqueue(). In its absence, I guess we just need to deal with all these ugly races ..
In summary, I think we need to go ahead with the preemp_disable() patch in flush_workqueue() from Andrew and the race fix you posted here:
http://lkml.org/lkml/2006/12/30/37
Signature · 2 lines
Regards, vatsa
PATCH 1/1 fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Andrew Morton( Srivatsa, Gautham, could you please verify my thinking ? )
On top of fix-flush_workqueue-vs-cpu_dead-race.patch
hotplug_sequence is incremented under "case CPU_DEAD:". This was ok before flush_workqueue() was changed to use preempt_disable() instead of workqueue_mutex. However preempt_disable() can't garantee that there is no CPU_DEAD event in progress (it is possible that flush_workqueue() runs after STOPMACHINE_EXIT), so flush_workqueue() can miss CPU_DEAD event.
Increment hotplug_sequence earlier, under CPU_DOWN_PREPARE. We can't miss the event, the task running flush_workqueue() will be re-scheduled at least once before CPU actually disappears from cpu_online_map.
We may have a false positive but this is very unlikely and only means we will have one unneeded "goto again".
Note: this patch depends on handle-cpu_lock_acquire-and-cpu_lock_release-in-workqueue_cpu_callback but only "textually".
kernel/workqueue.c renamed from kernel/workqueue.c~1_down +4 -1
@@ -886,12 +886,15 @@ static int __devinit workqueue_cpu_callb }break ; +case CPU_DOWN_PREPARE : + hotplug_sequence++; +break ; +case CPU_DEAD :list_for_each_entry (wq, &workqueues, list)cleanup_workqueue_thread (wq, hotcpu);list_for_each_entry (wq, &workqueues, list)take_over_work (wq, hotcpu); - hotplug_sequence++;break ;case CPU_LOCK_RELEASE :
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 3 lines · Oleg Nesterov
Increment
hotplug_sequenceearlier, underCPU_DOWN_PREPARE. We can't miss the event, the task runningflush_workqueue()will be re-scheduled at least once before CPU actually disappears fromcpu_online_map.
Eww ..what happens if flush_workqueue() starts after CPU_DOWN_PREPARE?
CPU_DOWN_PREPARE(8)
hotplug_sequence++ = 10 flush_workqueue() sequence = 10 flush cpus 1 ....7
CPU_DEAD(8)
take_over_work(8->1) return not flushing dead cpu8 (=BUG)
??
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 7 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
Increment
hotplug_sequenceearlier, underCPU_DOWN_PREPARE. We can't miss the event, the task runningflush_workqueue()will be re-scheduled at least once before CPU actually disappears fromcpu_online_map.Eww ..what happens if
flush_workqueue()starts afterCPU_DOWN_PREPARE?
^^^^^
Stupid me. Thanks.
Quoted 11 lines
CPU_DOWN_PREPARE(8)hotplug_sequence++ = 10 flush_workqueue() sequence = 10 flush cpus 1 ....7
CPU_DEAD(8)take_over_work(8->1) return not flushing dead cpu8 (=BUG)
I'll try to do something else tomorrow. Do you see a simple soulution?
The current usage of workqueue_mutex (I mean stable kernel) is broken and deadlockable. We really need to change it.
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 3 lines · Oleg Nesterov
Stupid me. Thanks.
I'll try to do something else tomorrow. Do you see a simple soulution?
Sigh ..I dont see a simple solution, unless we have something like lock_cpu_hotplug() ..
Andrew,
This workqueue problem has exposed a classic example of how
tough/miserable it can be to write hotplug safe code w/o something like lock_cpu_hotplug() ..Are you still inclined towards banning it? :)
FYI, the lock_cpu_hotplug() rewrite proposed by Gautham at
http://lkml.org/lkml/2006/10/26/65 may still need refinement to avoid all the kind of deadlocks we have unearthed with workqueue example. I can review that design with Gautham if there is some interest to revive lock_cpu_hotplug() ..
Quoted 2 lines
The current usage of
workqueue_mutex(I mean stable kernel) is broken and deadlockable. We really need to change it.
Yep ..
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 8 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
Stupid me. Thanks.
I'll try to do something else tomorrow. Do you see a simple soulution?
Sigh ..I dont see a simple solution, unless we have something like
lock_cpu_hotplug()..
I suspect this can't help either.
The problem is that flush_workqueue() may be called while cpu hotplug event in progress and CPU_DEAD waits for kthread_stop(), so we have the same dead lock if work->func() does flush_workqueue(). This means that Andrew's change to use preempt_disable() is good and anyway needed.
I am starting to believe we need some more intrusive changes in workquue.c.
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 6 lines · Oleg Nesterov
I suspect this can't help either.
The problem is that
flush_workqueue()may be called while cpu hotplug event in progress andCPU_DEADwaits forkthread_stop(), so we have the same dead lock ifwork->func()doesflush_workqueue(). This means that Andrew's change to usepreempt_disable()is good and anyway needed.
Well ..a lock_cpu_hotplug() in run_workqueue() and support for recursive calls to lock_cpu_hotplug() by the same thread will avoid the problem you mention. This will need changes to task_struct to track the recursion depth. Alternately this can be supported w/o changes to task_struct by 'biasing' readers over writers as I believe Gautham's patches [1] do.
1. http://lkml.org/lkml/2006/10/26/65
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 4 lines · Srivatsa Vaddagiri
Quoted 2 lines · Oleg Nesterov
I suspect this can't help either.
Quoted 8 lines
Quoted 4 lines
The problem is that
flush_workqueue()may be called while cpu hotplug event in progress andCPU_DEADwaits forkthread_stop(), so we have the same dead lock ifwork->func()doesflush_workqueue(). This means that Andrew's change to usepreempt_disable()is good and anyway needed.Well ..a
lock_cpu_hotplug()inrun_workqueue()and support for recursive calls tolock_cpu_hotplug()by the same thread will avoid the problem you mention.
Srivatsa, I'm completely new to cpu-hotplug, so please correct me if I'm wrong (in fact I _hope_ I am wrong) but as I see it, the hotplug/workqueue interaction is broken by design, it can't be fixed by changing just locking.
Once again. CPU dies, CPU_DEAD calls kthread_stop() and sleeps until cwq->thread exits. To do so, this thread must at least complete the currently running work->func().
work->func() calls flush_workque(WQ), it does lock_cpu_hotplug() or _whatever_. Now the question, does it block?
if YES:
This is what the stable tree does - deadlock.
if NOT:
This is what we have with Andrew's s/mutex_lock/preempt_disable/ patch - race or deadlock, we have a choice.
Suppose that WQ has pending works on that dead CPU. Note that at this point this CPU does not present on cpu_online_map. This means that (without other changes) we have lost.
- flush_workque(WQ) can't return until CPU_DEAD transfers these works to some another CPU on the cpu_online_map.
- CPU_DEAD can't do take_over_work() untill flush_workque() returns.
Andrew, Ingo, this also means that freezer can't solve this particular problem either (if i am right).
Thoughts?
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 2 lines · Oleg Nesterov
Thoughts?
How about:
CPU_DEAD does nothing. After __cpu_disable() cwq->thread runs on all CPUs and becomes idle when it flushes cwq->worklist: nobody will add work_struct on that list. CPU_UP: if (!cwq->thread) create_workqueue_thread(); else set_cpus_allowed(newcpu); flush_workqueue: for_each_possible_cpu() // NOT online! if (cwq->thread) flush_cpu_workqueue()
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 6 lines · Oleg Nesterov
How about:
CPU_DEAD does nothing. After __cpu_disable() cwq->thread runs on all CPUs and becomes idle when it flushes cwq->worklist: nobody will add work_struct on that list.
Also, we can add cpu_workqueue_struct->should_exit_after_flush flag, but we have to be careful to serialize with destroy_workqueue/CPU_UP.
Quoted 10 lines
CPU_UP: if (!cwq->thread) create_workqueue_thread(); else set_cpus_allowed(newcpu); flush_workqueue: for_each_possible_cpu() // NOT online! if (cwq->thread) flush_cpu_workqueue()
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Oleg NesterovQuoted 8 lines · Oleg Nesterov
Quoted 2 lines · Oleg Nesterov
Thoughts?
How about:
CPU_DEAD does nothing. After __cpu_disable() cwq->thread runs on all CPUs and becomes idle when it flushes cwq->worklist: nobody
^^^
all except dead cpus that is.
Quoted 1 line
will add
work_structon that list.
If CPU_DEAD does nothing, then the dead cpu's workqueue list may be non-empty. How will it be flushed, given that no thread can run on the dead cpu?
We could consider CPU_DEAD moving over work atleast (and not killing worker threads also). In that case, cwq->thread can flush its work, however it now requires serialization among worker threads, since more than one worker thread can now be servicing the same CPU's workqueue list (this will beat the very purpose of maintaining per-cpu threads to avoid synchronization between them).
Finally, I am concerned about the (un)friendliness of this programming model, where programmers are restricted in not having a stable access to cpu_online_map at all -and- also requiring them to code in non-obvious terms. Granted that writing hotplug-safe code is non-trivial, but the absence of "safe access to online_map" will make it more complicated.
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 10 lines · Srivatsa Vaddagiri
If
CPU_DEADdoes nothing, then the dead cpu's workqueue list may be non-empty. How will it be flushed, given that no thread can run on the dead cpu?We could consider
CPU_DEADmoving over work atleast (and not killing worker threads also). In that case,cwq->threadcan flush its work, however it now requires serialization among worker threads, since more than one worker thread can now be servicing the same CPU's workqueue list (this will beat the very purpose of maintaining per-cpu threads to avoid synchronization between them).
I guess you could have cwq->thread flush only it's cpu's workqueue by running on another cpu, which will avoid the need to synchronize between worker threads. I am not 100% sure if that breaks workqueue model in any way (since we could have two worker threads running on the same CPU, but servicing different queues). Hopefully it doesnt.
However the concern expressed below remains ..
Quoted 5 lines
Finally, I am concerned about the (un)friendliness of this programming model, where programmers are restricted in not having a stable access to
cpu_online_mapat all -and- also requiring them to code in non-obvious terms. Granted that writing hotplug-safe code is non-trivial, but the absence of "safe access toonline_map" will make it more complicated.
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 5 lines · Srivatsa Vaddagiri
On Sun, Jan 07, 2007 at 10:13:44PM +0530, Srivatsa Vaddagiri wrote:
I guess you could have
cwq->threadflush only it's cpu's workqueue by running on another cpu,
yes, this is what I meant,
Quoted 4 lines
which will avoid the need to synchronize
between worker threads. I am not 100% sure if that breaks workqueue model in any way (since we could have two worker threads running on the same CPU, but servicing different queues). Hopefully it doesnt.
We are already doing this on CPU_DEAD->kthread_stop().
Quoted 7 lines
However the concern expressed below remains ..
Quoted 5 lines
Finally, I am concerned about the (un)friendliness of this programming model, where programmers are restricted in not having a stable access to
cpu_online_mapat all -and- also requiring them to code in non-obvious terms. Granted that writing hotplug-safe code is non-trivial, but the absence of "safe access toonline_map" will make it more complicated.
please see the previous message.
Srivatsa, I don't claim my idea is the best. Actually I still hope somebody else will suggest something better and simpler :)
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Srivatsa VaddagiriQuoted 6 lines · Srivatsa Vaddagiri
Quoted 4 lines · Oleg Nesterov
Quoted 2 lines · Oleg Nesterov
Thoughts?
Quoted 7 lines
Quoted 4 lines
How about:
CPU_DEAD does nothing. After __cpu_disable() cwq->thread runs on all CPUs and becomes idle when it flushes cwq->worklist: nobody^^^all except dead cpus that is.
yes, of course.
Quoted 6 lines
Quoted 2 lines
will add
work_structon that list.If
CPU_DEADdoes nothing, then the dead cpu's workqueue list may be non-empty. How will it be flushed, given that no thread can run on the dead cpu?
But cwq->thread is not bound to the dead CPU at this point, it was aleady migrated (like all other threads which had that CPU in ->cpus_allowed).
Quoted 5 lines
Finally, I am concerned about the (un)friendliness of this programming model, where programmers are restricted in not having a stable access to
cpu_online_mapat all -and- also requiring them to code in non-obvious terms. Granted that writing hotplug-safe code is non-trivial, but the absence of "safe access toonline_map" will make it more complicated.
I guess you misunderstood me, I meant CPU_DEAD does nothing only in workqueue.c:workqueue_cpu_callback().
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Oleg NesterovQuoted 7 lines · Oleg Nesterov
Srivatsa, I'm completely new to cpu-hotplug, so please correct me if I'm wrong (in fact I _hope_ I am wrong) but as I see it, the hotplug/workqueue interaction is broken by design, it can't be fixed by changing just locking.
Once again. CPU dies,
CPU_DEADcallskthread_stop()and sleeps untilcwq->threadexits. To do so, this thread must at least complete the currently runningwork->func().
If run_workqueue() takes a lock_cpu_hotplug() successfully, then we shouldnt even reach till this point, as it will block writers (cpu_down/up) until it completes.
run_workqueue() ---------------
try_again:
rc = lock_cpu_hotplug_interruptible(); if (rc && kthread_should_stop()) return; if (rc != 0) goto try_again; /* cpu_down/up shouldnt happen now untill we call unlock_cpu_hotplug */ while (!list_empty(..)) work->func(); unlock_cpu_hotplug();
If work->func() calls something (say flush_workqueue()) which requires a lock_cpu_hotplug() again, there are two ways to support it:
Method 1: Add a field, hotplug_lock_held, in task_struct
If current->hotplug_lock_held > 1, then lock_cpu_hotplug() merely increments it and returns success. Its counterpart, unlock_cpu_hotplug() will decrement the count.
Easiest to implement. However additional field is required in each task_struct, which may not be attractive for some.
Method 2 : Bias readers over writers:
This method will support recursive calls to lock_cpu_hotplug() by the same thread, w/o requiring a field in task_struct. To accomplish this, readers are biased over writers i.e reader1_lock(); <- success writer1_lock(); <- blocks on reader1 reader2_lock(); <- success
A fair lock would have blocked reader2_lock() until writer1_lock()/writer1_unlock() is complete, but since we are required to support recursion w/o maintaining a task_struct field, we let reader2_lock() succeed, even though it could be from a different thread.
Quoted 2 lines
Andrew, Ingo, this also means that freezer can't solve this particular problem either (if i am right).
freezer wont give stable access to cpu_online_map either, as could typically be required in functions like flush_workqueue.
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 19 lines · Srivatsa Vaddagiri
Quoted 7 lines · Oleg Nesterov
Srivatsa, I'm completely new to cpu-hotplug, so please correct me if I'm wrong (in fact I _hope_ I am wrong) but as I see it, the hotplug/workqueue interaction is broken by design, it can't be fixed by changing just locking.
Once again. CPU dies,
CPU_DEADcallskthread_stop()and sleeps untilcwq->threadexits. To do so, this thread must at least complete the currently runningwork->func().If
run_workqueue()takes alock_cpu_hotplug()successfully, then we shouldnt even reach till this point, as it will block writers (cpu_down/up) until it completes.run_workqueue() ---------------
try_again:
Quoted 11 lines
rc = lock_cpu_hotplug_interruptible(); if (rc && kthread_should_stop()) return; if (rc != 0) goto try_again; /* cpu_down/up shouldnt happen now untill we call unlock_cpu_hotplug */ while (!list_empty(..)) work->func();
This mean that every work->func() which may sleep delays cpu_down/up unpredictable, not good. What about work->func which sleeps then re-queues itself? I guess we can solve this, but this is what I said "other changes".
Also, lock_cpu_hotplug() should be per-cpu, otherwise we have livelock.
Not that I am against lock_cpu_hotplug (I can't judge), but its usage in run_workqueue looks like complication to me. I may be wrong. But the main problem we don't have it :)
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Srivatsa VaddagiriQuoted 3 lines · On Sat, 6 Jan 2007 22:08:51 +0530 Srivatsa Vaddagiri <[email protected]> wrote:
This workqueue problem has exposed a classic example of how
tough/miserable it can be to write hotplug safe code w/o something likelock_cpu_hotplug()..Are you still inclined towards banning it? :)
I don't ban stuff - I just advocate ;)
I would still prefer that we not try to invent a new magical lock, but yes, the current approach is looking troublesome.
Quoted 5 lines
FYI, the
lock_cpu_hotplug()rewrite proposed by Gautham at
http://lkml.org/lkml/2006/10/26/65 may still need refinement to avoid all the kind of deadlocks we have unearthed with workqueue example. I can review that design with Gautham if there is some interest to revivelock_cpu_hotplug()..
Has anyone thought seriously about using the process freezer in the cpu-down/cpu-up paths? That way we don't need to lock anything anywhere?
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 9 lines · Andrew Morton
Quoted 5 lines
FYI, the
lock_cpu_hotplug()rewrite proposed by Gautham at
http://lkml.org/lkml/2006/10/26/65 may still need refinement to avoid all the kind of deadlocks we have unearthed with workqueue example. I can review that design with Gautham if there is some interest to revivelock_cpu_hotplug()..Has anyone thought seriously about using the process freezer in the cpu-down/cpu-up paths? That way we don't need to lock anything anywhere?
yes, yes, yes - lets please do that! The process freezer is already used for suspend, for hibernate and recently for kprobes - so its performance and robustness is being relied on and verified from multiple angles. I can see no reason why it couldnt be made really fast even on large boxes, if the need arises. (but even the current one is fast enough for any human-driven CPU hotplug stuff)
Ingo
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Andrew MortonQuoted 2 lines · Andrew Morton
Has anyone thought seriously about using the process freezer in the cpu-down/cpu-up paths? That way we don't need to lock anything anywhere?
How would this provide a stable access to cpu_online_map in functions that need to block while accessing it (as flush_workqueue requires)?
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 6 lines · On Sun, 7 Jan 2007 16:30:13 +0530 Srivatsa Vaddagiri <[email protected]> wrote:
Quoted 2 lines · Andrew Morton
Has anyone thought seriously about using the process freezer in the cpu-down/cpu-up paths? That way we don't need to lock anything anywhere?
How would this provide a stable access to
cpu_online_mapin functions that need to block while accessing it (asflush_workqueuerequires)?
If a thread simply blocks, that will not permit a cpu plug/unplug to proceed.
The thread had to explicitly call try_to_freeze(). CPU plug/unplug will not occur (and cpu_online_map will not change) until every process in the machine has called try_to_freeze()).
So the problem which you're referring to will only occur if a workqueue callback function calls try_to_freeze(), which would be mad.
Plus flush_workqueue() is on the way out. We're slowly edging towards a working cancel_work() which will only block if the work which you're trying to cancel is presently running. With that, pretty much all the flush_workqueue() calls go away, and all these accidental rarely-occurring deadlocks go away too.
PATCH 1/1 flush_cpu_workqueue: don't flush an empty ->worklist
Now when we have ->current_work we can avoid adding a barrier and waiting for its completition when cwq's queue is empty.
Note: this change is also useful if we change flush_workqueue() to also check the dead CPUs.
kernel/workqueue.c renamed from kernel/workqueue.c~1_opt +17 -8
@@ -405,12 +405,15 @@ static void wq_barrier_func(struct work_complete (&barr->done); } -static inline void init_wq_barrier (struct wq_barrier *barr) +static void insert_wq_barrier (struct cpu_workqueue_struct *cwq, +struct wq_barrier *barr,int tail) {INIT_WORK (&barr->work, wq_barrier_func);__set_bit (WORK_STRUCT_PENDING ,work_data_bits (&barr->work));init_completion (&barr->done); + +insert_work (cwq, &barr->work, tail); }static void flush_cpu_workqueue (struct cpu_workqueue_struct *cwq)
@@ -429,13 +432,20 @@ static void flush_cpu_workqueue(struct cpreempt_disable (); }else {struct wq_barrier barr; +int active =0 ; -init_wq_barrier (&barr); -__queue_work (cwq, &barr.work); +spin_lock_irq (&cwq->lock); +if (!list_empty (&cwq->worklist) || cwq->current_work !=NULL ) { +insert_wq_barrier (cwq, &barr,1 ); + active =1 ; + } +spin_unlock_irq (&cwq->lock); -preempt_enable ();/* Can no longer touch *cwq */ -wait_for_completion (&barr.done); -preempt_disable (); +if (active) { +preempt_enable (); +wait_for_completion (&barr.done); +preempt_disable (); + } } }
@@ -482,8 +492,7 @@ static void wait_on_work(struct cpu_workspin_lock_irq (&cwq->lock);if (unlikely (cwq->current_work == work)) { -init_wq_barrier (&barr); -insert_work (cwq, &barr.work,0 ); +insert_wq_barrier (cwq, &barr,0 ); running =1 ; }spin_unlock_irq (&cwq->lock);
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Just a sad little note that I've utterly lost the plot on the workqueue changes.
schedule_on_each_cpu-use-preempt_disable.patch reimplement-flush_workqueue.patch
implement-flush_work.patch
implement-flush_work-sanity.patch implement-flush_work_keventd.patch flush_workqueue-use-preempt_disable-to-hold-off-cpu-hotplug.patch flush_cpu_workqueue-dont-flush-an-empty-worklist.patch aio-use-flush_work.patch
kblockd-use-flush_work.patch relayfs-use-flush_keventd_work.patch tg3-use-flush_keventd_work.patch e1000-use-flush_keventd_work.patch
libata-use-flush_work.patch
phy-use-flush_work.patch #bridge-avoid-using-noautorel-workqueues.patch
#
extend-notifier_call_chain-to-count-nr_calls-made.patch extend-notifier_call_chain-to-count-nr_calls-made-fixes.patch extend-notifier_call_chain-to-count-nr_calls-made-fixes-2.patch define-and-use-new-eventscpu_lock_acquire-and-cpu_lock_release.patch define-and-use-new-eventscpu_lock_acquire-and-cpu_lock_release-fix.patch eliminate-lock_cpu_hotplug-in-kernel-schedc.patch eliminate-lock_cpu_hotplug-in-kernel-schedc-fix.patch handle-cpu_lock_acquire-and-cpu_lock_release-in-workqueue_cpu_callback.patch fix-flush_workqueue-vs-cpu_dead-race.patch
I don't know which of these are good, bad or indifferent.
Furthermore I don't know which of these need to be tossed overboard if/when we get around to using the task freezer for CPU hotplug synchronisation. Hopefully, a lot of them. I don't really understand why we're continuing to struggle with the existing approach before that question is settled.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 4 lines · Andrew Morton
Furthermore I don't know which of these need to be tossed overboard if/when we get around to using the task freezer for CPU hotplug synchronisation. Hopefully, a lot of them. I don't really understand why we're continuing to struggle with the existing approach before that question is settled.
Good point!
Fundamentally, I think we need to answer this question:
"Do we provide *some* mechanism to block concurrent hotplug operations from happening? By hotplug operations I mean both changes to the bitmap and execution of all baclbacks in CPU_DEAD/ONLINE etc"
If NO, then IMHO we will be forever fixing races
If YES, then what is that mechanism? freeze_processes()? or a magical lock?
freeze_processes() cant be that mechanism, if my understanding of it is correct - see http://lkml.org/lkml/2007/1/8/149 and
http://marc.theaimsgroup.com/?l=linux-kernel&m=116817460726058.
I would be happy to be corrected if the above impression of freeze_processes() is corrected ..
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 14 lines · On Tue, 9 Jan 2007 10:34:17 +0530 Srivatsa Vaddagiri <[email protected]> wrote:
Quoted 4 lines · Andrew Morton
Furthermore I don't know which of these need to be tossed overboard if/when we get around to using the task freezer for CPU hotplug synchronisation. Hopefully, a lot of them. I don't really understand why we're continuing to struggle with the existing approach before that question is settled.
Good point!
Fundamentally, I think we need to answer this question:
"Do we provide *some* mechanism to block concurrent hotplug operations from happening? By hotplug operations I mean both changes to the bitmap and execution of all baclbacks in CPU_DEAD/ONLINE etc"
Quoted 7 lines
If NO, then IMHO we will be forever fixing races
If YES, then what is that mechanism?
freeze_processes()? or a magical lock?
freeze_processes()cant be that mechanism, if my understanding of it is correct - see http://lkml.org/lkml/2007/1/8/149
That's not correct. freeze_processes() will freeze *all* processes. All of them are forced to enter refrigerator(). With the mysterious exception of some I/O-related kernel threads, which might need some thought.
Quoted 2 lines
and
http://marc.theaimsgroup.com/?l=linux-kernel&m=116817460726058.
Am not sure how that's related.
Quoted 2 lines
I would be happy to be corrected if the above impression of
freeze_processes()is corrected ..
It could be that the freezer needs a bit of work for this application. Obviously we're not interested in the handling of disk I/O, so we'd really like to do a simple try_to_freeze_tasks(FREEZER_USER_SPACE|FREEZER_KERNEL_THREADS), but the code isn't set up to do that (it should be). The other non-swsusp callers probably want this change as well. But that's all a minor matter.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 10 lines · Andrew Morton
Quoted 2 lines
I would be happy to be corrected if the above impression of
freeze_processes()is corrected ..It could be that the freezer needs a bit of work for this application. Obviously we're not interested in the handling of disk I/O, so we'd really like to do a simple
try_to_freeze_tasks(FREEZER_USER_SPACE|FREEZER_KERNEL_THREADS), but the code isn't set up to do that (it should be). The other non-swsusp callers probably want this change as well. But that's all a minor matter.
yes. The freezer does the fundamentally right thing: it stops all tasks in user-space or waits for them to return back to user-space to stop them there, or if it's a pure kernel-space task it waits until that kernel-space task voluntarily stop.
Once the system is in such a state, and all processing has been stopped, all of the kernel's data structures are completely 'unused', and we can:
- patch the kernel freely (kprobes)
- save+stop the kernel (sw-suspend)
- remove a CPU (CPU hotplug and suspend)
- (We could also use this mechanism to live-migrate the kernel to another system btw., possibly useful for containers)
- (We could also use this mechanism to create a live snapshot of a running kernel, together with an LVM snapshot of filesystem state, for possible restore point later on.)
It is a very powerful mechanism that has really nice properties - we should work on this one shared infrastructure instead of adding zillions of per-subsystem CPU hotplug locks.
Ingo
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
In reply to Andrew MortonQuoted 1 line · Andrew Morton
That's not correct.
freeze_processes()will freeze *all* processes.
I am not arguing whether all processes will be frozen. However my question was on the freeze point. Let me ask the question with an example:
rtasd thread (arch/powerpc/platforms/pseries/rtasd.c) executes this simple loop:
static int rtasd(void *unused)
{
i = first_cpu(cpu_online_map);
while (1) {
set_cpus_allowed(current, cpumask_of_cpu(i)); /* can block */
/* we should now be running on cpu i */
do_something_on_a_cpu(i);
/* sleep for some time */
i = next_cpu(cpu, cpu_online_map);
}}
This thread makes absolutely -no- calls to try_to_freeze() in its lifetime.
1. Does this mean that the thread can't be frozen? (lets say that the thread's PF_NOFREEZE is not set)
AFAICS it can still be frozen by sending it a signal and have the signal delivery code call try_to_freeze() ..
2. If the thread can be frozen at any arbitrary point of its execution, then I dont see what prevents cpu_online_map from changing under the feet of rtasd thread,
In other words, we would have failed to provide the ability to *block* hotplug operations from happening concurrently.
Quoted 1 line
All of them are forced to enter
refrigerator().
^^^^^^
*forced*, yes ..that's the point of concern ..
Warm regards,
vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 2 lines · Srivatsa Vaddagiri
This thread makes absolutely -no- calls to
try_to_freeze()in its lifetime.
that's a plain bug. suspend will fail for example. A kernel thread either needs to mark itself PF_NOFREEZE (which means it can be ignored and zapped with impunity), or needs to call into try_to_freeze() with a sane frequency.
Ingo
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
In reply to Srivatsa VaddagiriQuoted 28 lines · On Tue, 9 Jan 2007 15:03:02 +0530 Srivatsa Vaddagiri <[email protected]> wrote:
Quoted 1 line · Andrew Morton
That's not correct.
freeze_processes()will freeze *all* processes.I am not arguing whether all processes will be frozen. However my question was on the freeze point. Let me ask the question with an example:
rtasd thread (
arch/powerpc/platforms/pseries/rtasd.c) executes this simple loop:static int
rtasd(void *unused)
{i = first_cpu(cpu_online_map); while (1) { set_cpus_allowed(current, cpumask_of_cpu(i)); /* can block */ /* we should now be running on cpu i */ do_something_on_a_cpu(i); /* sleep for some time */ i = next_cpu(cpu, cpu_online_map); }
Quoted 3 lines
}
This thread makes absolutely -no- calls to
try_to_freeze()in its lifetime.
Looks like a bug to me. powerpc does appear to try to support the freezer.
Quoted 2 lines
1. Does this mean that the thread can't be frozen? (lets say that the thread's
PF_NOFREEZEis not set)
yup. I'd expect the freeze_processes() call would fail if this thread is running.
Quoted 2 lines
AFAICS it can still be frozen by sending it a signal and have the signal delivery code call
try_to_freeze()..
kernel threads don't take signals in the same manner as userspace. A kernel thread needs to explicitly poll, via
if (signal_pending(current))
do_something()
rtasd doesn't do that, and using signals in-kernel is considered lame.
Quoted 3 lines
2. If the thread can be frozen at any arbitrary point of its execution, then I dont see what prevents
cpu_online_mapfrom changing under the feet of rtasd thread,
It cannot.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 9 lines · Andrew Morton
Quoted 1 line
This thread makes absolutely -no- calls to
try_to_freeze()in its lifetime.Looks like a bug to me. powerpc does appear to try to support the freezer.
Quoted 2 lines
1. Does this mean that the thread can't be frozen? (lets say that the thread's
PF_NOFREEZEis not set)yup. I'd expect the
freeze_processes()call would fail if this thread is running.
ok.
Quoted 9 lines
Quoted 3 lines
AFAICS it can still be frozen by sending it a signal and have the signal delivery code call
try_to_freeze()..kernel threads don't take signals in the same manner as userspace. A kernel thread needs to explicitly poll, via
if (
signal_pending(current))
do_something()
Thanks for the education! I feel much better about the use of process freezer now ..
Quoted 5 lines
Quoted 3 lines
2. If the thread can be frozen at any arbitrary point of its execution, then I dont see what prevents
cpu_online_mapfrom changing under the feet of rtasd thread,It cannot.
Excellent ..
I just hope the latency of freeze_processes() is tolerable ..
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 1 line · On Tue, 9 Jan 2007 15:39:26 +0530 Srivatsa Vaddagiri <[email protected]> wrote:
I just hope the latency of
freeze_processes()is tolerable ..
It'll be roughly proportional to the number of processes, I guess: if we have 100,000 processes (or threads) doing sleep(1000000) then yeah, it'll take some time to wake them all up and capture them in refrigerator().
But I suspect a suitable fix for any problems which arise there is to implement gang-offlining and onlining, rather than the present one-cpu-at-a-time. That'd be pretty simple to do: we already have sysfs interfaces which take a cpumask.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
In reply to Andrew MortonQuoted 4 lines · Andrew Morton
That's not correct.
freeze_processes()will freeze *all* processes. All of them are forced to enterrefrigerator(). With the mysterious exception of some I/O-related kernel threads, which might need some thought.
Yes, and we can remove a dead CPU safely, this part is ok.
Quoted 4 lines
Quoted 2 lines
and
http://marc.theaimsgroup.com/?l=linux-kernel&m=116817460726058.Am not sure how that's related.
but at some point we should thaw processes, including cwq->thread which should die. So we are doing things like take_over_work() and this is the source of races, because the dead CPU is not on cpu_online_map.
flush_workqueue() doesn't use any locks now. If we use freezer to implement cpu-hotplug nothing will change, we still have races.
It looks so obvious to me, so I'm starting to suspect I missed something very simple :(
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 2 lines · Oleg Nesterov
but at some point we should thaw processes, including
cwq->threadwhich should die.
I am presuming we will thaw processes after all CPU_DEAD handlers have run.
Quoted 5 lines
So we are doing things like
take_over_work()and this is the source of races, because the dead CPU is not oncpu_online_map.
flush_workqueue()doesn't use any locks now. If we use freezer to implement cpu-hotplug nothing will change, we still have races.
We have races -if- CPU_DEAD handling can run concurrently with a ongoing flush_workqueue. From my recent understanding of process freezer, this is not possible. In other words, flush_workqueue() can be its old implementation as below w/o any races:
some_thread: for_each_online_cpu(i) flush_cpu_workqueue(i);
As long as this loop is running, cpu_down/up will not proceed. This means, cpu_online_map is stable even if flush_cpu_workqueue blocks ..
Once this loop is complete and all threads have called try_to_freeze, cpu_down will proceed to change the bit map and run CPU_DEAD handlers of everyone. I am presuimg we will thaw processes only after all CPU_DEAD/ONLINE handlers have run (dont know if that is a problem). In that case do you still see races? Yes, this would require some changes in worker_thread to check for kthread_should_stop() after try_to_freeze returns ...
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 24 lines · Srivatsa Vaddagiri
Quoted 2 lines · Oleg Nesterov
but at some point we should thaw processes, including
cwq->threadwhich should die.I am presuming we will thaw processes after all
CPU_DEADhandlers have run.Quoted 5 lines
So we are doing things like
take_over_work()and this is the source of races, because the dead CPU is not oncpu_online_map.
flush_workqueue()doesn't use any locks now. If we use freezer to implement cpu-hotplug nothing will change, we still have races.We have races -if-
CPU_DEADhandling can run concurrently with a ongoingflush_workqueue. From my recent understanding of process freezer, this is not possible. In other words,flush_workqueue()can be its old implementation as below w/o any races:some_thread: for_each_online_cpu(i) flush_cpu_workqueue(i);
Quoted 7 lines
As long as this loop is running, cpu_down/up will not proceed. This means,
cpu_online_mapis stable even ifflush_cpu_workqueueblocks ..Once this loop is complete and all threads have called
try_to_freeze,cpu_downwill proceed to change the bit map and runCPU_DEADhandlers of everyone. I am presuimg we will thaw processes only after all CPU_DEAD/ONLINE handlers have run.
We can't do this. We should thaw cwq->thread (which was bound to the dead CPU) to complete CPU_DEAD event. So we still need some changes.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 2 lines · Oleg Nesterov
We can't do this. We should thaw
cwq->thread(which was bound to the dead CPU) to completeCPU_DEADevent. So we still need some changes.
I noticed that, but I presumed kthread_stop() will post a wakeup which will bring it out of frozen state. Looking at refrigerator(), I realize that is not possible.
So CPU_DEAD should do a thaw_process on the kthread before doing a kthread_stop?
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 11 lines · Srivatsa Vaddagiri
Quoted 2 lines · Oleg Nesterov
We can't do this. We should thaw
cwq->thread(which was bound to the dead CPU) to completeCPU_DEADevent. So we still need some changes.I noticed that, but I presumed
kthread_stop()will post a wakeup which will bring it out of frozen state. Looking atrefrigerator(), I realize that is not possible.So
CPU_DEADshould do athaw_processon the kthread before doing akthread_stop?
Probably we can do this, or that. In any case we need changes, that was my point.
And the best change I believe is to _remove_ CPU_DEAD handling from workqueue.c as I suggested before. This kthread_stop() is not a good idea per se, it calls wake_up_process(), but we should in fact use wake_up(&cwq->more_work). Yes, work->func() should be ready for the false wakeups, but still.
But for now I hope the last "draft" patch is enough. I'll continue on next weekend.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
How about the pseudo-code below?
workqueue_mutex is only used to protect "struct list_head workqueues", all workqueue operations can run in parallel with cpuhotplug callback path. take_over_work(), migrate_sequence, CPU_LOCK_ACQUIRE/RELEASE go away.
I'd like to make a couple of cleanups (and fix schedule_on_each_cpu) before sending the patch, but if somebody doesn't like this intrusive change, he can nack it right now.
Oleg.
struct cpu_workqueue_srtuct {
...
int should_stop;
...
};// also used by flush_work/flush_workqueue
static cpumask_t cpu_populated_map __read_mostly;
/*
* NOTE: the caller must not touch *cwq if this func returns true */
static inline int cwq_should_stop(struct cpu_workqueue_struct *cwq)
{
int should_stop = cwq->should_stop;
if (unlikely(should_stop)) {
spin_lock_irq(&cwq->lock);
should_stop = cwq->should_stop && list_empty(&cwq->worklist);
if (should_stop)
cwq->thread = NULL;
spin_unlock_irq(&cwq->lock);
}
return should_stop;
}static int worker_thread(void *cwq)
{
while (!cwq_should_stop(cwq)) {
...
run_workqueue();
...
}
}static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
{
struct task_struct *p;
spin_lock_irq(&cwq->lock);
cwq->should_stop = 0;
p = cwq->thread;
spin_unlock_irq(&cwq->lock);
if (!p) {
struct workqueue_struct *wq = cwq->wq;
const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d";
p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
/*
* Nobody can add the work_struct to this cwq,
* if (caller is __create_workqueue)
* nobody should see this wq
* else // caller is CPU_UP_PREPARE
* cpu is not on cpu_online_map
* so we can abort safely.
*/
if (IS_ERR(p))
return PTR_ERR(p);
if (!is_single_threaded(wq))
kthread_bind(p, cpu);
/*
* Cancels affinity if the caller is CPU_UP_PREPARE.
* Needs a cleanup, but OK.
*/
wake_up_process(p);
cwq->thread = p;
}
return 0;
}struct workqueue_struct *__create_workqueue(const char *name,
int singlethread, int freezeable)
{
struct workqueue_struct *wq;
struct cpu_workqueue_struct *cwq;
int err = 0, cpu;
wq = kzalloc(sizeof(*wq), GFP_KERNEL);
if (!wq)
return NULL;
wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
if (!wq->cpu_wq) {
kfree(wq);
return NULL;
}
wq->name = name;
wq->freezeable = freezeable;
if (singlethread) {
INIT_LIST_HEAD(&wq->list);
cwq = init_cpu_workqueue(wq, singlethread_cpu);
err = create_workqueue_thread(cwq, singlethread_cpu);
} else {
mutex_lock(&workqueue_mutex);
list_add(&wq->list, &workqueues);
for_each_possible_cpu(cpu) {
cwq = init_cpu_workqueue(wq, cpu);
if (err || !cpu_isset(cpu, cpu_populated_map))
continue;
err = create_workqueue_thread(cwq, cpu);
}
mutex_unlock(&workqueue_mutex);
}
if (err) {
destroy_workqueue(wq);
wq = NULL;
}
return wq;
}static void cleanup_workqueue_thread(struct workqueue_struct *wq, int cpu)
{
struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
struct wq_barrier barr;
int alive = 0;
spin_lock_irq(&cwq->lock);
if (cwq->thread != NULL) {
insert_wq_barrier(cwq, &barr, 1);
cwq->should_stop = 1;
alive = 1;
}
spin_unlock_irq(&cwq->lock);
if (alive) {
wait_for_completion(&barr.done);
while (unlikely(cwq->thread != NULL))
cpu_relax();
/*
* Wait until cwq->thread unlocks cwq->lock,
* it won't touch *cwq after that.
*/
smp_rmb();
spin_unlock_wait(&cwq->lock);
}
}void destroy_workqueue(struct workqueue_struct *wq)
{
if (is_single_threaded(wq))
cleanup_workqueue_thread(wq, singlethread_cpu);
else {
int cpu;
mutex_lock(&workqueue_mutex);
list_del(&wq->list);
mutex_unlock(&workqueue_mutex);
for_each_cpu_mask(cpu, cpu_populated_map)
cleanup_workqueue_thread(wq, cpu);
}
free_percpu(wq->cpu_wq);
kfree(wq);
}static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
unsigned long action, void *hcpu)
{
struct workqueue_struct *wq;
struct cpu_workqueue_struct *cwq;
unsigned int cpu = (unsigned long)hcpu;
int ret = NOTIFY_OK;
mutex_lock(&workqueue_mutex);
if (action == CPU_UP_PREPARE)
cpu_set(cpu, cpu_populated_map);
list_for_each_entry(wq, &workqueues, list) {
cwq = per_cpu_ptr(wq->cpu_wq, cpu);
switch (action) {
case CPU_UP_PREPARE:
if (create_workqueue_thread(cwq, cpu))
ret = NOTIFY_BAD;
break;
case CPU_ONLINE:
set_cpus_allowed(cwq->thread, cpumask_of_cpu(cpu));
break;
case CPU_UP_CANCELED:
case CPU_DEAD:
cwq->should_stop = 1;
wake_up(&cwq->more_work);
break;
}
if (ret != NOTIFY_OK) {
printk(KERN_ERR "workqueue for %i failed\n", cpu);
break;
}
}
mutex_unlock(&workqueue_mutex);
return ret;
}void init_workqueues(void)
{
...
cpu_populated_map = cpu_online_map;
...
}Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 1 line · Oleg Nesterov
How about the pseudo-code below?
Some quick comments:
- singlethread_cpu needs to be hotplug safe (broken currently)
- Any reason why cpu_populated_map is not modified on CPU_DEAD?
- I feel more comfortable if workqueue_cpu_callback were to take workqueue_mutex in LOCK_ACQ and release it in LOCK_RELEASE notifications. This will provide stable access to cpu_populated_map to functions like __create_workqueue.
Finally, I wonder if these changes will be unnecessary if we move to process freezer based hotplug locking ...
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 7 lines · Srivatsa Vaddagiri
Quoted 1 line · Oleg Nesterov
How about the pseudo-code below?
Some quick comments:
-
singlethread_cpuneeds to be hotplug safe (broken currently)
Why? Could you explain?
Quoted 1 line
- Any reason why
cpu_populated_mapis not modified onCPU_DEAD?
Because CPU_DEAD/CPU_UP_CANCELED doesn't wait for cwq->thread to exit. cpu_populated_map never shrinks, it only grows on CPU_UP_PREPARE.
We can change this, but it needs some more code, and I am not sure we need it. Note that a "false" bit in cpu_populated_map only means that flush_work/flush_workqueue/destroy_workqueu will do lock/unlock of cwq->lock, nothing more.
Quoted 3 lines
- I feel more comfortable if
workqueue_cpu_callbackwere to takeworkqueue_mutexinLOCK_ACQand release it inLOCK_RELEASEnotifications.
The whole purpose of this change to avoid this!
Gautham R Shenoy wrote:
>
> Oh! I was refering to the other set of workqueue deadlock woes. The > ones caused when subsystems (like cpufreq) try to create/destroy > workqueue from the cpuhotplug callback path.
>
> Creation/destruction of workqueue requires us to take workqueue_mutex, > which would have already been taken during CPU_LOCK_ACQUIRE.
With this change workqueue_mutex is only taken to protect workqueues list, why should we hold it for (say) CPU_UP_PREPARE->CPU_ONLINE path?
Quoted 2 lines
This will provide stable access to
cpu_populated_map
to functions like__create_workqueue.
I think this is not needed.
Quoted 2 lines
Finally, I wonder if these changes will be unnecessary if we move to process freezer based hotplug locking ...
This change ir not strictly necessary but imho make the code better and shrinks .text by 379 bytes.
But I believe that freezer will change nothing for workqueue. We still need take_over_work(), and hacks like migrate_sequence. And no, CPU_DEAD can't just thaw cwq->thread which was bound to the dead CPU to complete kthread_stop(), we should thaw all processes.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 2 lines · Oleg Nesterov
cpu_populated_mapnever shrinks, it only grows onCPU_UP_PREPARE.
__create_workqueue() should not use it. Needs a fix.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
In reply to Oleg NesterovQuoted 3 lines · Oleg Nesterov
Quoted 1 line
-
singlethread_cpuneeds to be hotplug safe (broken currently)Why? Could you explain?
What if 'singlethread_cpu' dies?
Quoted 9 lines
Quoted 1 line
- Any reason why
cpu_populated_mapis not modified onCPU_DEAD?Because CPU_DEAD/CPU_UP_CANCELED doesn't wait for
cwq->threadto exit.cpu_populated_mapnever shrinks, it only grows onCPU_UP_PREPARE.We can change this, but it needs some more code, and I am not sure we need it. Note that a "false" bit in
cpu_populated_maponly means that flush_work/flush_workqueue/destroy_workqueu will do lock/unlock ofcwq->lock, nothing more.
What abt __create_workqueue/schedule_on_each_cpu?
Quoted 5 lines
Quoted 3 lines
- I feel more comfortable if
workqueue_cpu_callbackwere to takeworkqueue_mutexinLOCK_ACQand release it inLOCK_RELEASEnotifications.The whole purpose of this change to avoid this!
I guess it depends on how __create_workqueue/schedule_on_each_cpu is modified (whether we take/release lock upon LOCK_ACQ/LOCK_RELEASE)
Quoted 10 lines
Quoted 2 lines
Finally, I wonder if these changes will be unnecessary if we move to process freezer based hotplug locking ...
This change ir not strictly necessary but imho make the code better and shrinks .text by 379 bytes.
But I believe that freezer will change nothing for workqueue. We still need
take_over_work(), and hacks likemigrate_sequence. And no,CPU_DEADcan't just thawcwq->threadwhich was bound to the dead CPU to completekthread_stop(), we should thaw all processes.
What abt stopping that thread in CPU_DOWN_PREPARE (before freezing processes)? I understand that it may add to the latency, but compared to the overall latency of process freezer, I suspect it may not be much.
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 7 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
Quoted 1 line
-
singlethread_cpuneeds to be hotplug safe (broken currently)Why? Could you explain?
What if '
singlethread_cpu' dies?
Still can't understand you. Probably you missed what singlethread_cpu is.
singlethread_cpu is just a "random" bit from cpu_possible_map. Single threaded workqueue is not bound to any cpu. We only need it to be sure that percpu_data.ptrs[singlethread_cpu] is populated by __percpu_alloc_mask().
Quoted 11 lines
Quoted 9 lines
Quoted 1 line
- Any reason why
cpu_populated_mapis not modified onCPU_DEAD?Because CPU_DEAD/CPU_UP_CANCELED doesn't wait for
cwq->threadto exit.cpu_populated_mapnever shrinks, it only grows onCPU_UP_PREPARE.We can change this, but it needs some more code, and I am not sure we need it. Note that a "false" bit in
cpu_populated_maponly means that flush_work/flush_workqueue/destroy_workqueu will do lock/unlock ofcwq->lock, nothing more.What abt __create_workqueue/schedule_on_each_cpu?
As I said already __create_workqueue() needs a fix, schedule_on_each_cpu() is already broken, and should be fixed as well.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
- I feel more comfortable if
workqueue_cpu_callbackwere to takeworkqueue_mutexinLOCK_ACQand release it inLOCK_RELEASEnotifications.The whole purpose of this change to avoid this!
I guess it depends on how __create_workqueue/schedule_on_each_cpu is modified (whether we take/release lock upon LOCK_ACQ/LOCK_RELEASE)
Sorry, can't understand this...
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Finally, I wonder if these changes will be unnecessary if we move to process freezer based hotplug locking ...
This change ir not strictly necessary but imho make the code better and shrinks .text by 379 bytes.
Quoted 8 lines
Quoted 4 lines
But I believe that freezer will change nothing for workqueue. We still need
take_over_work(), and hacks likemigrate_sequence. And no,CPU_DEADcan't just thawcwq->threadwhich was bound to the dead CPU to completekthread_stop(), we should thaw all processes.What abt stopping that thread in
CPU_DOWN_PREPARE(before freezing processes)? I understand that it may add to the latency, but compared to the overall latency of process freezer, I suspect it may not be much.
Srivatsa, why do you think this would be better?
It add to the complexity! What do you mean by "stopping that thread" ? Kill it? - this is wrong. Make it TASK_STOPPED? - very untrivial and I can't see how this helps.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 3 lines · Oleg Nesterov
Quoted 1 line
What if '
singlethread_cpu' dies?Still can't understand you. Probably you missed what
singlethread_cpuis.
oops yes ..I had mistakenly thought that create_workqueue_thread() will bind worker thread to singlethread_cpu for single_threaded workqueue. So it isn't a problem.
Quoted 4 lines
Quoted 1 line
What abt __create_workqueue/schedule_on_each_cpu?
As I said already
__create_workqueue()needs a fix,schedule_on_each_cpu()is already broken, and should be fixed as well.
__create_workqueue() creates worker threads for all online CPUs currently. Accessing the online_map could be racy unless we serialize the access with hotplug event (thr' a mutex like workqueue mutex held between LOCK_ACQ/LOCK_RELEASE messages or process freezer) OR take special measures as was done in flush_workqueue. How were you considering to deal with that raciness?
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
The whole purpose of this change to avoid this!
I guess it depends on how __create_workqueue/schedule_on_each_cpu is modified (whether we take/release lock upon LOCK_ACQ/LOCK_RELEASE)
Sorry, can't understand this...
I meant to say that depending on how we modify __create_workqueue/schedule_on_each_cpu to avoid racy-access to online_map, we can debate whether workqueue mutex needs to be held between LOCK_ACQ/LOCK_RELEASE messages in the callback.
Quoted 8 lines
Quoted 3 lines
What abt stopping that thread in
CPU_DOWN_PREPARE(before freezing processes)? I understand that it may add to the latency, but compared to the overall latency of process freezer, I suspect it may not be much.Srivatsa, why do you think this would be better?
It add to the complexity! What do you mean by "stopping that thread" ? Kill it? - this is wrong.
I meant issuing kthread_stop() in DOWN_PREPARE so that worker thread exits itself (much before CPU is actually brought down).
Do you see any problems with that?
Even if there are problems with it, how abt something like below:
workqueue_cpu_callback()
{
CPU_DEAD: /* threads are still frozen at this point */ take_over_work(); kthread_mark_stop(worker_thread); break; CPU_CLEAN_THREADS: /* all threads resumed by now */ kthread_stop(worker_thread); /* task_struct ref required? */ break;
}
kthread_mark_stop() will mark somewhere in task_struct that the thread should exit when it comes out of refrigerator.
worker_thread()
{
while (!kthread_should_stop()) {
if (cwq->freezeable)
try_to_freeze();
if (kthread_marked_stop(current))
break;
...
}
}The advantage I see above is that, when take_over_work() is running, we wont race with functions like flush_workqueue() (threads still frozen at that point) and hence we avoid hacks like migrate_sequence. This will also let functions like flush_workqueue() easily access cpu_online_map as below -without- any special locking/hacks (which I consider a great benefit for programmers).
flush_workqueue()
{
for_each_online_cpu(i)
flush_cpu_workqueue(i);
}Do you see any problems with this later approach?
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 6 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
As I said already
__create_workqueue()needs a fix,schedule_on_each_cpu()is already broken, and should be fixed as well.
Quoted 6 lines
__create_workqueue()creates worker threads for all online CPUs currently. Accessing theonline_mapcould be racy unless we serialize the access with hotplug event (thr' a mutex like workqueue mutex held between LOCK_ACQ/LOCK_RELEASE messages or process freezer) OR take special measures as was done inflush_workqueue. How were you considering to deal with that raciness?
This is easy to fix in multiple ways. For example (this is not the best approach), we can make cpu_creation_map. Like cpu_populate_map, but CPU is cleared on CPU_DEAD/CPU_UP_CANCELED.
I'll continue when I have a time.
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
What abt stopping that thread in
CPU_DOWN_PREPARE(before freezing processes)? I understand that it may add to the latency, but compared to the overall latency of process freezer, I suspect it may not be much.I meant issuing
kthread_stop()inDOWN_PREPAREso that worker thread exits itself (much before CPU is actually brought down).
Deadlock if work_struct re-queues itself.
Quoted 8 lines
Even if there are problems with it, how abt something like below:
workqueue_cpu_callback()
{CPU_DEAD: /* threads are still frozen at this point */ take_over_work();
No, we can't move a currently executing work. This will break flush_workxxx().
Quoted 6 lines
kthread_mark_stop(worker_thread); break; CPU_CLEAN_THREADS: /* all threads resumed by now */ kthread_stop(worker_thread); /* task_struct ref required? */
yes, required,
Quoted 4 lines
kthread_mark_stop()will mark somewhere intask_structthat the thread should exit when it comes out of refrigerator.
worker_thread()
Quoted 8 lines
{
while (!kthread_should_stop()) {
if (cwq->freezeable)
try_to_freeze();
if (kthread_marked_stop(current))
break;Racy. Because of kthread_stop() above we should clear cwq->thread somehow. But we can't do this: this workqueue may be already destroyed.
Quoted 3 lines
The advantage I see above is that, when
take_over_work()is running, we wont race with functions likeflush_workqueue()(threads still frozen at that point) and hence we avoid hacks likemigrate_sequence.
See above.
Please note that the code I posted does something like kthread_mark_stop(), but it operates on cwq, not on task_struct, this makes a big difference. And it doesn't need take_over_work() at all. And it doesn't need additional complications. Instead, it lessens both the source and compiled code.
Note that I am not against using freezer for cpu-hotplug. My point is that this can't help much for the current workqueue.c, and it will completely transparent with the change I suggest.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 4 lines · Oleg Nesterov
Quoted 2 lines
I meant issuing
kthread_stop()inDOWN_PREPAREso that worker thread exits itself (much before CPU is actually brought down).Deadlock if
work_structre-queues itself.
Are you referring to the problem you described here?
http://lkml.org/lkml/2007/1/8/173
If so, then it can easily be prevented by having run_workqueue() check for kthread_should_stop() in its while loop?
Quoted 10 lines
Quoted 8 lines
Even if there are problems with it, how abt something like below:
workqueue_cpu_callback()
{CPU_DEAD: /* threads are still frozen at this point */ take_over_work();No, we can't move a currently executing work. This will break
flush_workxxx().
What do you mean by "currently" executing work? worker thread executing some work on the cpu? That is not possible, because all threads are frozen at this point. There cant be any ongoing flush_workxxx() as well because of this, which should avoid breaking flush_workxxx() ..
Quoted 5 lines
Quoted 2 lines
if (kthread_marked_stop(current)) break;Racy. Because of
kthread_stop()above we should clearcwq->threadsomehow. But we can't do this: this workqueue may be already destroyed.
We will surely take workqueue_mutex in CPU_CLEAN_THREADS (when it traverses the workqueues list), which should avoid this race?
Quoted 2 lines
Please note that the code I posted does something like
kthread_mark_stop(), but it operates on cwq, not ontask_struct, this makes a big difference.
Ok sure ..putting the flag in cwq makes sense. Others also can follow similar trick for stopping threads (like ksoftirqd).
Quoted 2 lines
And it doesn't need
take_over_work()at all. And it doesn't need additional complications. Instead, it lessens both the source and compiled code.
I guess either way, changes are required.
1st method, what you are suggesting:
- Needs separate bitmap(s), cpu_populated_map and possible another for create_workqueue()?
- flush_workqueue() traverses thr a separate bitmap cpu_populated_map (separate from the online map) while create_workqueue() traverses the other bitmap
2nd method:
- Avoids the need for maintenance of separate bitmaps (uses existing cpu_online_map). All functions can safely use the online_map w/o any races. Personally this is why I like this approach.
- Needs changes in worker_thread to exit right after it comes out of refrigerator.
I havent made any changes as per 2nd method to see the resulting code size, so I cant comment on code sizes.
Another point is that once we create code as in 1st method, which maintains separate bitmaps, that will easily get replicated (over time) to other subsystems. Is that a good thing?
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 5 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
Quoted 2 lines
I meant issuing
kthread_stop()inDOWN_PREPAREso that worker thread exits itself (much before CPU is actually brought down).
Quoted 8 lines
Quoted 1 line
Deadlock if
work_structre-queues itself.Are you referring to the problem you described here?
http://lkml.org/lkml/2007/1/8/173If so, then it can easily be prevented by having
run_workqueue()check forkthread_should_stop()in its while loop?
flush_workqueue() also calls run_workqueue().
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
workqueue_cpu_callback()
{CPU_DEAD: /* threads are still frozen at this point */ take_over_work();
Quoted 6 lines
Quoted 1 line
No, we can't move a currently executing work. This will break
flush_workxxx().What do you mean by "currently" executing work? worker thread executing some work on the cpu? That is not possible, because all threads are frozen at this point. There cant be any ongoing
flush_workxxx()as well because of this, which should avoid breakingflush_workxxx()..
work->func() sleeps/freezed. We can't move the rest of pending jobs before it completes. This will break flush_workxxx. And no, this is not because we use barriers now.
Quoted 7 lines
1st method, what you are suggesting:
- Needs separate bitmap(s),
cpu_populated_mapand possible another forcreate_workqueue()?
-flush_workqueue()traverses thr a separate bitmapcpu_populated_map(separate from the online map) whilecreate_workqueue()traverses the other bitmap
Yes, we need the additional bitmap. This is optimization, we can just use cpu_possible_map. create_workqueue() can use cpu_online_map + "int new_cpu".
Yes, this is a complication. But still this is much simpler (IMHO) than we have now. And imho better.
Quoted 11 lines
2nd method:
- Avoids the need for maintenance of separate bitmaps (uses existing
cpu_online_map). All functions can safely use theonline_mapw/o any races. Personally this is why I like this approach.
- Needs changes inworker_threadto exit right after it comes out of refrigerator.I havent made any changes as per 2nd method to see the resulting code size, so I cant comment on code sizes.
Yes, yes, yes, let's see the code first! :) Then we can compare. Right now:
- cpu-hotplug doesn't use freezer yet
- all ideas about using it to improve workqueue.c were wrong
Quoted 3 lines
Another point is that once we create code as in 1st method, which maintains separate bitmaps, that will easily get replicated (over time) to other subsystems. Is that a good thing?
Honestly, I can't understand your point. Why it will get replicated? Because another subsystem will need cpu_populated_map too? We can remove "static" and move cpu_populated_map to kernel/cpu.c then.
Btw, I agree it is good to have a sleeping lock to protect cpu_online_map. But it should be separate from workqueue_mutex, and it is not needed for create/destroy/flush funcs.
Oleg.
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 6 lines · Oleg Nesterov
Quoted 4 lines
What do you mean by "currently" executing work? worker thread executing some work on the cpu? That is not possible, because all threads are frozen at this point. There cant be any ongoing
flush_workxxx()as well because of this, which should avoid breakingflush_workxxx()..
work->func()sleeps/freezed.
Didnt Andrew call that (work->func calling try_to_freeze) madness?
http://lkml.org/lkml/2007/01/07/166
Does that happen in practice?
Signature · 2 lines
Regards, vatsa
Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
Quoted 7 lines · Srivatsa Vaddagiri
Quoted 5 lines · Oleg Nesterov
Quoted 4 lines
What do you mean by "currently" executing work? worker thread executing some work on the cpu? That is not possible, because all threads are frozen at this point. There cant be any ongoing
flush_workxxx()as well because of this, which should avoid breakingflush_workxxx()..
Quoted 5 lines
Quoted 1 line
work->func()sleeps/freezed.Didnt Andrew call that (
work->funccallingtry_to_freeze) madness?http://lkml.org/lkml/2007/01/07/166
This means we should move try_to_freeze() to run_workqueue() or we should forbid auto-requeued works.
I don't have a time to do anything till weekend, but please see a "final" version below.
- Do you see any bugs?
- Do you agree it is better than we have now?
If/when we use freezer/lock_cpu_hotplug() we probably can simplfy things further.
Note: schedule_on_each_cpu() remains broken.
Oleg.
struct cpu_workqueue_srtuct {
...
int should_stop;
...
};static cpumask_t cpu_populated_map __read_mostly; //also used in flush_work... static int embryonic_cpu __read_mostly = -1;
/*
* NOTE: the caller must not touch *cwq if this func returns true */
static inline int cwq_should_stop(struct cpu_workqueue_struct *cwq)
{
int should_stop = cwq->should_stop;
if (unlikely(should_stop)) {
spin_lock_irq(&cwq->lock);
should_stop = cwq->should_stop && list_empty(&cwq->worklist);
if (should_stop)
cwq->thread = NULL;
spin_unlock_irq(&cwq->lock);
}
return should_stop;
}static int worker_thread(void *cwq)
{
...
while (!cwq_should_stop(cwq)) {
if (cwq->wq->freezeable)
try_to_freeze();
prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
if (!cwq->should_stop && list_empty(&cwq->worklist))
schedule();
finish_wait(&cwq->more_work, &wait);
run_workqueue(cwq);
}
return 0;
}static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
{
struct task_struct *p;
spin_lock_irq(&cwq->lock);
cwq->should_stop = 0;
p = cwq->thread;
spin_unlock_irq(&cwq->lock);
if (!p) {
struct workqueue_struct *wq = cwq->wq;
const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d";
p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
/*
* Nobody can add the work_struct to this cwq,
* if (caller is __create_workqueue)
* nobody should see this wq
* else // caller is CPU_UP_PREPARE
* cpu is not on cpu_online_map
* so we can abort safely.
*/
if (IS_ERR(p))
return PTR_ERR(p);
if (!is_single_threaded(wq))
kthread_bind(p, cpu);
/*
* Cancels affinity if the caller is CPU_UP_PREPARE.
* Needs a cleanup, but OK.
*/
wake_up_process(p);
cwq->thread = p;
}
return 0;
}struct workqueue_struct *__create_workqueue(const char *name,
int singlethread, int freezeable)
{
struct workqueue_struct *wq;
struct cpu_workqueue_struct *cwq;
int err = 0, cpu;
wq = kzalloc(sizeof(*wq), GFP_KERNEL);
if (!wq)
return NULL;
wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
if (!wq->cpu_wq) {
kfree(wq);
return NULL;
}
wq->name = name;
wq->freezeable = freezeable;
if (singlethread) {
INIT_LIST_HEAD(&wq->list);
cwq = init_cpu_workqueue(wq, singlethread_cpu);
err = create_workqueue_thread(cwq, singlethread_cpu);
} else {
mutex_lock(&workqueue_mutex);
list_add(&wq->list, &workqueues);
for_each_possible_cpu(cpu) {
cwq = init_cpu_workqueue(wq, cpu);
if (err || !(cpu_online(cpu) || cpu == embryonic_cpu))
continue;
err = create_workqueue_thread(cwq, cpu);
}
mutex_unlock(&workqueue_mutex);
}
if (err) {
destroy_workqueue(wq);
wq = NULL;
}
return wq;
}static void cleanup_workqueue_thread(struct workqueue_struct *wq, int cpu)
{
struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
struct wq_barrier barr;
int alive = 0;
spin_lock_irq(&cwq->lock);
if (cwq->thread != NULL) {
insert_wq_barrier(cwq, &barr, 1);
cwq->should_stop = 1;
alive = 1;
}
spin_unlock_irq(&cwq->lock);
if (alive) {
wait_for_completion(&barr.done);
while (unlikely(cwq->thread != NULL))
cpu_relax();
/*
* Wait until cwq->thread unlocks cwq->lock,
* it won't touch *cwq after that.
*/
smp_rmb();
spin_unlock_wait(&cwq->lock);
}
}void destroy_workqueue(struct workqueue_struct *wq)
{
if (is_single_threaded(wq))
cleanup_workqueue_thread(wq, singlethread_cpu);
else {
int cpu;
mutex_lock(&workqueue_mutex);
list_del(&wq->list);
mutex_unlock(&workqueue_mutex);
for_each_cpu_mask(cpu, cpu_populated_map)
cleanup_workqueue_thread(wq, cpu);
}
free_percpu(wq->cpu_wq);
kfree(wq);
}static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
unsigned long action, void *hcpu)
{
struct workqueue_struct *wq;
struct cpu_workqueue_struct *cwq;
unsigned int cpu = (unsigned long)hcpu;
int ret = NOTIFY_OK;
mutex_lock(&workqueue_mutex);
embryonic_cpu = -1;
if (action == CPU_UP_PREPARE) {
cpu_set(cpu, cpu_populated_map);
embryonic_cpu = cpu;
}
list_for_each_entry(wq, &workqueues, list) {
cwq = per_cpu_ptr(wq->cpu_wq, cpu);
switch (action) {
case CPU_UP_PREPARE:
if (create_workqueue_thread(cwq, cpu))
ret = NOTIFY_BAD;
break;
case CPU_ONLINE:
set_cpus_allowed(cwq->thread, cpumask_of_cpu(cpu));
break;
case CPU_UP_CANCELED:
case CPU_DEAD:
cwq->should_stop = 1;
wake_up(&cwq->more_work);
break;
}
if (ret != NOTIFY_OK) {
printk(KERN_ERR "workqueue for %i failed\n", cpu);
break;
}
}
mutex_unlock(&workqueue_mutex);
return ret;
}void init_workqueues(void)
{
...
cpu_populated_map = cpu_online_map;
...
}Re: [PATCH] flush_cpu_workqueue: don't flush an empty ->worklist
In reply to Oleg NesterovQuoted 3 lines · Oleg Nesterov
Btw, I agree it is good to have a sleeping lock to protect
cpu_online_map. But it should be separate fromworkqueue_mutex, and it is not needed for create/destroy/flush funcs.
Which is what lock_cpu_hotplug() attempted to provide but which has recd lot of flak in recent days. I guess if someone implements process freezer based locking of cpu_online_map, we will know better how suited later is for cpu hotplug.
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Andrew MortonQuoted 6 lines · Andrew Morton
Plus
flush_workqueue()is on the way out. We're slowly edging towards a workingcancel_work()which will only block if the work which you're trying to cancel is presently running. With that, pretty much all theflush_workqueue()calls go away, and all these accidental rarely-occurring deadlocks go away too.
So. If we can forget about the race we have - fine. Otherwise, how about the patch below? It is untested and needs a review. I can't suggest any simpler now.
Change flush_workqueue() to use for_each_possible_cpu(). This means that flush_cpu_workqueue() may hit CPU which is already dead. However in that case
if (!list_empty(&cwq->worklist) || cwq->current_work != NULL)
means that CPU_DEAD in progress, it will do kthread_stop() + take_over_work() so we can proceed and insert a barrier. We hold cwq->lock, so we are safe.
This patch replaces fix-flush_workqueue-vs-cpu_dead-race.patch which was broken by switching to preempt_disable (now we don't need locking at all). Note that migrate_sequence (was hotplug_sequence) is incremented under cwq->lock. Since flush_workqueue does lock/unlock of cwq->lock on all CPUs, it must see the new value if take_over_work() happened before we checked this cwq, and this is the case we should worry about: otherwise we added a barrier.
Srivatsa?
kernel/workqueue.c renamed from kernel/workqueue.c~2_race +26 -19
@@ -65,6 +65,7 @@ struct workqueue_struct {/* All the per-cpu workqueues on the system, for hotplug cpu to add/remove threads to each one as cpus come/go. */ +static long migrate_sequence__read_mostly ;static DEFINE_MUTEX (workqueue_mutex);static LIST_HEAD (workqueues);
@@ -422,13 +423,7 @@ static void flush_cpu_workqueue(struct c* Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ -preempt_enable (); -/* -* We can still touch *cwq here because we are keventd, and -* hot-unplug will be waiting us to exit. -*/ run_workqueue (cwq); -preempt_disable (); }else {struct wq_barrier barr;int active =0 ;
@@ -441,9 +436,7 @@ static void flush_cpu_workqueue(struct cspin_unlock_irq (&cwq->lock);if (active) { -preempt_enable ();wait_for_completion (&barr.done); -preempt_disable (); } } }
@@ -463,17 +456,21 @@ static void flush_cpu_workqueue(struct c*/ void fastcall flush_workqueue(struct workqueue_struct *wq) { -preempt_disable ();/* CPU hotplug */ if (is_single_threaded (wq)) {/* Always use first cpu's area. */ flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu)); }else { +long sequence;int cpu; +again: + sequence = migrate_sequence; -for_each_online_cpu (cpu) +for_each_possible_cpu (cpu)flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu)); + +if (unlikely (sequence != migrate_sequence)) + goto again; } -preempt_enable (); }EXPORT_SYMBOL_GPL (flush_workqueue);
@@ -545,18 +542,22 @@ out: }EXPORT_SYMBOL_GPL (flush_work); -static struct task_struct *create_workqueue_thread (struct workqueue_struct *wq, -int cpu,int freezeable) +static void init_cpu_workqueue (struct workqueue_struct *wq, +struct cpu_workqueue_struct *cwq,int freezeable) { -struct cpu_workqueue_struct *cwq =per_cpu_ptr (wq->cpu_wq, cpu); -struct task_struct *p; -spin_lock_init (&cwq->lock); cwq->wq = wq; cwq->thread =NULL ; cwq->freezeable = freezeable;INIT_LIST_HEAD (&cwq->worklist);init_waitqueue_head (&cwq->more_work); +} + +static struct task_struct *create_workqueue_thread (struct workqueue_struct *wq, +int cpu) +{ +struct cpu_workqueue_struct *cwq =per_cpu_ptr (wq->cpu_wq, cpu); +struct task_struct *p;if (is_single_threaded (wq)) p =kthread_create (worker_thread, cwq,"%s" , wq->name);
@@ -589,15 +590,20 @@ struct workqueue_struct *__create_workqumutex_lock (&workqueue_mutex);if (singlethread) {INIT_LIST_HEAD (&wq->list); - p =create_workqueue_thread (wq, singlethread_cpu, freezeable); +init_cpu_workqueue (wq,per_cpu_ptr (wq->cpu_wq, singlethread_cpu), + freezeable); + p =create_workqueue_thread (wq, singlethread_cpu);if (!p) destroy =1 ;else wake_up_process (p); }else {list_add (&wq->list, &workqueues); +for_each_possible_cpu (cpu) +init_cpu_workqueue (wq,per_cpu_ptr (wq->cpu_wq, cpu), + freezeable);for_each_online_cpu (cpu) { - p =create_workqueue_thread (wq, cpu, freezeable); + p =create_workqueue_thread (wq, cpu);if (p) {kthread_bind (p, cpu);wake_up_process (p);
@@ -833,6 +839,7 @@ static void take_over_work(struct workquspin_lock_irq (&cwq->lock);list_replace_init (&cwq->worklist, &list); + migrate_sequence++;while (!list_empty (&list)) {printk ("Taking work for %s\n" , wq->name);
@@ -859,7 +866,7 @@ static int __devinit workqueue_cpu_callbcase CPU_UP_PREPARE :/* Create a new workqueue thread for it. */ list_for_each_entry (wq, &workqueues, list) { -if (!create_workqueue_thread (wq, hotcpu,0 )) { +if (!create_workqueue_thread (wq, hotcpu)) {printk ("workqueue for %i failed\n" , hotcpu);return NOTIFY_BAD ; }
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 17 lines · Oleg Nesterov
Change
flush_workqueue()to usefor_each_possible_cpu(). This means thatflush_cpu_workqueue()may hit CPU which is already dead. However in that caseif (!list_empty(&cwq->worklist) || cwq->current_work != NULL)means that
CPU_DEADin progress, it will dokthread_stop()+take_over_work()so we can proceed and insert a barrier. We holdcwq->lock, so we are safe.This patch replaces fix-flush_workqueue-vs-cpu_dead-race.patch which was broken by switching to
preempt_disable(now we don't need locking at all). Note thatmigrate_sequence(washotplug_sequence) is incremented undercwq->lock. Sinceflush_workqueuedoes lock/unlock ofcwq->lockon all CPUs, it must see the new value iftake_over_work()happened before we checked this cwq, and this is the case we should worry about: otherwise we added a barrier.
Quoted 1 line
Srivatsa?
This is head-spinning :)
Spotted atleast these problems:
1. run_workqueue()->work.func()->flush_work()->mutex_lock(workqueue_mutex) deadlocks if we are blocked in cleanup_workqueue_thread()->kthread_stop() for the same worker thread to exit.
Looks possible in practice to me.
2.
CPU_DEAD->cleanup_workqueue_thread->(cwq->thread = NULL)->kthread_stop() ..
^^^^^^^^^^^^^^^^^^^^ |___ Problematic
Now while we are blocked here, if a work->func() calls flush_workqueue->flush_cpu_workqueue, we clearly cant identify that event thread is trying to flush its own queue (cwq->thread == current test fails) and hence we will deadlock.
A lock_cpu_hotplug(), or any other ability to block concurrent hotplug operations from happening, in run_workqueue would have avoided both the above races.
Alternatively, for the second race, I guess we can avoid setting cwq->thread = NULL in cleanup_workqueue_thread() till the thread has exited, but I am not sure if that opens up any other race. The first race seems harder to fix ..
I wonder if spin (spinroot.com) or some other formal model can make this job of spotting-races easier for us ..
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 19 lines · Srivatsa Vaddagiri
Quoted 17 lines · Oleg Nesterov
Change
flush_workqueue()to usefor_each_possible_cpu(). This means thatflush_cpu_workqueue()may hit CPU which is already dead. However in that caseif (!list_empty(&cwq->worklist) || cwq->current_work != NULL)means that
CPU_DEADin progress, it will dokthread_stop()+take_over_work()so we can proceed and insert a barrier. We holdcwq->lock, so we are safe.This patch replaces fix-flush_workqueue-vs-cpu_dead-race.patch which was broken by switching to
preempt_disable(now we don't need locking at all). Note thatmigrate_sequence(washotplug_sequence) is incremented undercwq->lock. Sinceflush_workqueuedoes lock/unlock ofcwq->lockon all CPUs, it must see the new value iftake_over_work()happened before we checked this cwq, and this is the case we should worry about: otherwise we added a barrier.
Quoted 3 lines
Quoted 1 line
Srivatsa?
This is head-spinning :)
Thank you for review!
Quoted 7 lines
Spotted atleast these problems:
1. run_workqueue()->work.func()->flush_work()->mutex_lock(workqueue_mutex) deadlocks if we are blocked in cleanup_workqueue_thread()->kthread_stop() for the same worker thread to exit.
Looks possible in practice to me.
Yes, this is the same (old) problem as we have/had with flush_workqueue(). We can convert flush_work() to use preempt_disable (this is not straightforward, but easy), or forbid to call flush_work() from work.func().
This patch doesn't touch this problem.
Quoted 5 lines
2.
CPU_DEAD->cleanup_workqueue_thread->(cwq->thread = NULL)->kthread_stop() ..
^^^^^^^^^^^^^^^^^^^^ |___ Problematic
Hmm... This should not be possible? cwq->thread != NULL on CPU_DEAD event. Event IF it was NULL, we don't call kthread_stop() in that case.
Quoted 4 lines
Now while we are blocked here, if a
work->func()callsflush_workqueue->flush_cpu_workqueue, we clearly cant identify that event thread is trying to flush its own queue (cwq->thread== current test fails) and hence we will deadlock.
Could you clarify? I believe cwq->thread == current test always works, we never "substitute" cwq->thread.
Quoted 3 lines
A
lock_cpu_hotplug(), or any other ability to block concurrent hotplug operations from happening, inrun_workqueuewould have avoided both the above races.
I still don't think this is a good idea. We also need
is_cpu_down_waits_for_lock_cpu_hotplug()
helper, otherwise we have a deadlock if work->func() sleeps and re-queues itself.
Quoted 2 lines
Alternatively, for the second race, I guess we can avoid setting
cwq->thread= NULL incleanup_workqueue_thread()till the thread has exited,
Yes, http://marc.theaimsgroup.com/?l=linux-kernel&m=116818097927685, I believe we can do this later. This way workqueue will have almost zero interaction with cpu-hotplug, and cpu UP/DOWN event won't be delayed by sleeping work.func(). take_over_work() can go away, this also allows us to simplify things.
Thanks!
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 11 lines · Oleg Nesterov
Quoted 7 lines
Spotted atleast these problems:
1. run_workqueue()->work.func()->flush_work()->mutex_lock(workqueue_mutex) deadlocks if we are blocked in cleanup_workqueue_thread()->kthread_stop() for the same worker thread to exit.
Looks possible in practice to me.
Yes, this is the same (old) problem as we have/had with
flush_workqueue(). We can convertflush_work()to usepreempt_disable(this is not straightforward, but easy), or forbid to callflush_work()fromwork.func().
I think I noticed other problems of avoiding workqueue_mutex() in flush_work() ..don't recall the exact problem.
Quoted 7 lines
Quoted 5 lines
2.
CPU_DEAD->cleanup_workqueue_thread->(cwq->thread = NULL)->kthread_stop() ..
^^^^^^^^^^^^^^^^^^^^ |___ ProblematicHmm... This should not be possible?
cwq->thread!= NULL onCPU_DEADevent.
sure, cwq->thread != NULL at CPU_DEAD event. However cleanup_workqueue_thread() will set it to NULL and block in kthread_stop(), waiting for the kthread to finish run_workqueue and exit. If one of the work functions being run by run_workqueue() calls flush_workqueue()->flush_cpu_workqueue() now, flush_cpu_workqueue() can fail to recognize that "keventd is trying to flush its own queue" which can cause deadlocks.
Quoted 7 lines
Quoted 4 lines
Now while we are blocked here, if a
work->func()callsflush_workqueue->flush_cpu_workqueue, we clearly cant identify that event thread is trying to flush its own queue (cwq->thread== current test fails) and hence we will deadlock.Could you clarify? I believe
cwq->thread== current test always works, we never "substitute"cwq->thread.
The test fails in the window described above.
Quoted 8 lines
Quoted 3 lines
A
lock_cpu_hotplug(), or any other ability to block concurrent hotplug operations from happening, inrun_workqueuewould have avoided both the above races.I still don't think this is a good idea. We also need
is_cpu_down_waits_for_lock_cpu_hotplug()helper, otherwise we have a deadlock if
work->func()sleeps and re-queues itself.
Can you elaborate this a bit?
Quoted 7 lines
Quoted 2 lines
Alternatively, for the second race, I guess we can avoid setting
cwq->thread= NULL incleanup_workqueue_thread()till the thread has exited,Yes, http://marc.theaimsgroup.com/?l=linux-kernel&m=116818097927685, I believe we can do this later. This way workqueue will have almost zero interaction with cpu-hotplug, and cpu UP/DOWN event won't be delayed by sleeping
work.func().take_over_work()can go away, this also allows us to simplify things.
I agree it minimizes the interactions. Maybe worth attempting. However I suspect it may not be as simple as it appears :)
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 8 lines · Srivatsa Vaddagiri
Quoted 6 lines · Oleg Nesterov
Quoted 5 lines
2.
CPU_DEAD->cleanup_workqueue_thread->(cwq->thread = NULL)->kthread_stop() ..
^^^^^^^^^^^^^^^^^^^^ |___ Problematic
Quoted 6 lines
Quoted 1 line
Hmm... This should not be possible?
cwq->thread!= NULL onCPU_DEADevent.sure,
cwq->thread!= NULL atCPU_DEADevent. Howevercleanup_workqueue_thread()will set it to NULL and block inkthread_stop(), waiting for the kthread to finishrun_workqueueand exit.
Ah, missed you point, thanks. Yet another old problem which was not introduced by recent changes. And yet another indication we should avoid kthread_stop() on CPU_DEAD event :) I believe this is easy to fix, but need to think more.
Quoted 10 lines
Quoted 8 lines
Quoted 3 lines
A
lock_cpu_hotplug(), or any other ability to block concurrent hotplug operations from happening, inrun_workqueuewould have avoided both the above races.I still don't think this is a good idea. We also need
is_cpu_down_waits_for_lock_cpu_hotplug()helper, otherwise we have a deadlock if
work->func()sleeps and re-queues itself.Can you elaborate this a bit?
If work->func() re-queues itself, run_workqueue() never returns because ->worklist is never empty. This means we should somehow check and detect that cpu-hotplug blocked because we hold lock_cpu_hotplug(). In that case run_workqueue() should return, and drop the lock. This will complicate worker_thread/run_workqueue further.
run_workqueue:
while (!list_empty(&cwq->worklist)) {
...
// We hold lock_cpu_hotplug(), cpu event can't make
// progress.
...
}Quoted 7 lines
Quoted 4 lines
Yes, http://marc.theaimsgroup.com/?l=linux-kernel&m=116818097927685, I believe we can do this later. This way workqueue will have almost zero interaction with cpu-hotplug, and cpu UP/DOWN event won't be delayed by sleeping
work.func().take_over_work()can go away, this also allows us to simplify things.I agree it minimizes the interactions. Maybe worth attempting. However I suspect it may not be as simple as it appears :)
Yes, that is why this patch only does the first step: flush_workqueue() checks the dead CPUs as well, this change is minimal.
Do you see any problems this patch adds?
Oleg.
RE: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 29 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Oleg Nesterov
Sent: Monday, January 08, 2007 9:07 AM
To: Srivatsa Vaddagiri
Cc: Andrew Morton; David Howells; Christoph Hellwig; Ingo Molnar; Linus Torvalds; [email protected]; Gautham shenoy
Subject: Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-updateQuoted 5 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
Quoted 3 lines
2.
CPU_DEAD->cleanup_workqueue_thread->(cwq->thread =
NULL)->kthread_stop() ..
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
^^^^^^^^^^^^^^^^^^^^ |___ ProblematicHmm... This should not be possible?
cwq->thread!= NULL on
CPU_DEADevent.Quoted 5 lines
sure,
cwq->thread!= NULL atCPU_DEADevent. Howevercleanup_workqueue_thread()will set it to NULL and block inkthread_stop(), waiting for the kthread to finishrun_workqueueand exit.
Quoted 6 lines
Ah, missed you point, thanks. Yet another old problem which was not introduced
by recent changes. And yet another indication we should avoidkthread_stop()
onCPU_DEADevent :) I believe this is easy to fix, but need to think more.
The current code is workqueue-hptplug path is full of races. I stumbled upon atleast couple of different deadlock situations being discussed here with ondemand governor using workqueue and trying to flush during cpu hot remove.
Specifically, a three way deadlock involving kthread_stop() with workqueue_mutex held and work itself blocked on some other mutex held by another task trying to flush the workqueue.
One other approach I was thinking about, was to do all the hardwork in workqueue CPU_DOWN_PREPARE callback rather than in CPU_DEAD. We can call cleanup_workqueue_thread and take_over_work in DOWN_PREPARE, With that, I don't think we need to hold the workqueue_mutex across these two callbacks and eliminate the deadlocks related to flush_workqueue.
Do you think this approach would simply things around here?
Thanks,
Venki
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 2 lines · Venkatesh
One other approach I was thinking about, was to do all the hardwork in workqueue
CPU_DOWN_PREPAREcallback rather than inCPU_DEAD.
Between DOWN_PREPARE and DEAD, more work can get added to the cpu's workqueue. So DOWN_PREPARE is kind of early to take_over_work ..
Quoted 5 lines
We can call
cleanup_workqueue_threadandtake_over_workinDOWN_PREPARE, With that, I don't think we need to hold theworkqueue_mutexacross these two callbacks and eliminate the deadlocks related toflush_workqueue.
Do you think this approach would simply things around here?
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Oleg NesterovQuoted 3 lines · Oleg Nesterov
Ah, missed you point, thanks. Yet another old problem which was not introduced by recent changes. And yet another indication we should avoid
kthread_stop()onCPU_DEADevent :) I believe this is easy to fix, but need to think more.
I think the problem is not just with CPU_DEAD. Anyone who calls cleanup_workqueue_thread (say destroy_workqueue?) will see this race.
Do you see any problems if cleanup_workqueue_thread is changed as:
cleanup_workqueue_thread()
{
kthread_stop(p);
spin_lock(cwq->lock);
cwq->thread = NULL;
spin_unlock(cwq->lock);
}Quoted 8 lines
run_workqueue:
while (!list_empty(&cwq->worklist)) {
...
// We hold lock_cpu_hotplug(), cpu event can't make
// progress.
...
}Ok ..yes a cpu_event_waits_for_lock() helper will help here.
Quoted 7 lines
Quoted 2 lines
I agree it minimizes the interactions. Maybe worth attempting. However I suspect it may not be as simple as it appears :)
Yes, that is why this patch only does the first step:
flush_workqueue()checks the dead CPUs as well, this change is minimal.Do you see any problems this patch adds?
I dont see as of now. I suspect we will know better when we implement the patch to eliminate CPU_DEAD handling in workqueue.c
Signature · 2 lines
Regards, vatsa
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
Quoted 8 lines · Srivatsa Vaddagiri
Quoted 3 lines · Oleg Nesterov
Ah, missed you point, thanks. Yet another old problem which was not introduced by recent changes. And yet another indication we should avoid
kthread_stop()onCPU_DEADevent :) I believe this is easy to fix, but need to think more.I think the problem is not just with
CPU_DEAD. Anyone who callscleanup_workqueue_thread(saydestroy_workqueue?) will see this race.
destroy_workqueue() first does flush_workqueue(), so it should be ok.
Anyway I agree with you, we shouldn't clear cwq->thread until it exits,
Quoted 9 lines
Do you see any problems if
cleanup_workqueue_threadis changed as:
cleanup_workqueue_thread(){ kthread_stop(p); spin_lock(cwq->lock); cwq->thread = NULL; spin_unlock(cwq->lock); }
I think the same. In fact I suspect we even don't need spin_lock, but didn't have a time to read the code since our discussion.
Oleg.
Re: [PATCH] fix-flush_workqueue-vs-cpu_dead-race-update
In reply to Andrew MortonQuoted 8 lines · Andrew Morton
Quoted 2 lines
How would this provide a stable access to
cpu_online_mapin functions that need to block while accessing it (asflush_workqueuerequires)?If a thread simply blocks, that will not permit a cpu plug/unplug to proceed.
The thread had to explicitly call
try_to_freeze(). CPU plug/unplug will not occur (andcpu_online_mapwill not change) until every process in the machine has calledtry_to_freeze()).
Maybe my misunderstanding of the code, but:
Looking at the code, it seems to me that try_to_freeze() will be called very likely from the signal-delivery path (get_signal_to_deliver()). Isnt that correct? If so, consider a thread as below:
Thread1 :
for_each_online_cpu() { online_map = 0x1111 at this point }
do_some_thing();
kmalloc(); <- blocksCan't Thread1 be frozen in the above blocking state w/o it voluntarily calling try_to_freeze? If so, online_map can change when it returns from kmalloc() ..
Quoted 9 lines
So the problem which you're referring to will only occur if a workqueue callback function calls
try_to_freeze(), which would be mad.Plus
flush_workqueue()is on the way out. We're slowly edging towards a workingcancel_work()which will only block if the work which you're trying to cancel is presently running. With that, pretty much all theflush_workqueue()calls go away, and all these accidental rarely-occurring deadlocks go away too.
Fundamentally, I think it is important to give the ability to block concurrent hotplug operations from happening ..
Signature · 2 lines
Regards, vatsa
Quoted 5 lines · Oleg Nesterov
void fastcall
flush_workqueue(struct workqueue_struct *wq){kernel/workqueue.c renamed from kernel/workqueue.c~1_flush_q +1
-2-might_sleep (); - +mutex_lock (&workqueue_mutex);
kernel/workqueue.c static void flush_cpu_workqueue(struct c · line 440
Quoted 10 lines
kernel/workqueue.c renamed from kernel/workqueue.c~1_flush_q +1
-3if (is_single_threaded (wq)) {/* Always use first cpu's area. */ -flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu), --1 ); +flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu)); }else {int cpu; -mutex_lock (&workqueue_mutex);for_each_online_cpu (cpu)
Can compiler optimizations lead to cpu_online_map being cached in a register while running this loop? AFAICS cpu_online_map is not declared to be volatile. If it can be cached, then we have the danger of invoking flush_cpu_workqueue() on a dead cpu (because flush_cpu_workqueue drops workqueue_mutex, cpu hp events can change cpu_online_map while we are in flush_cpu_workqueue).
Quoted 3 lines
kernel/workqueue.c renamed from kernel/workqueue.c~1_flush_q +1
-2-flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu), cpu); -mutex_unlock (&workqueue_mutex); +flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, cpu));
Signature · 2 lines
Regards, vatsa
Quoted 15 lines · Srivatsa Vaddagiri
Quoted 13 lines · Oleg Nesterov
void fastcall
flush_workqueue(struct workqueue_struct *wq){kernel/workqueue.c renamed from kernel/workqueue.c~1_flush_q +2
-4-might_sleep (); - +mutex_lock (&workqueue_mutex);if (is_single_threaded (wq)) {/* Always use first cpu's area. */ -flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu), --1 ); +flush_cpu_workqueue (per_cpu_ptr (wq->cpu_wq, singlethread_cpu)); }else {int cpu;
Quoted 7 lines
kernel/workqueue.c static void flush_cpu_workqueue(struct c · line 440
Quoted 2 lines
kernel/workqueue.c renamed from kernel/workqueue.c~1_flush_q +0
-1-mutex_lock (&workqueue_mutex);for_each_online_cpu (cpu)Can compiler optimizations lead to
cpu_online_mapbeing cached in a register while running this loop? AFAICScpu_online_mapis not declared to be volatile.
But it is not const either,
Quoted 1 line
If it can be cached,
I believe this would be a compiler's bug. Let's take a more simple example,
while (!condition) schedule();
What if compiler will cache the value of global 'condition' ?
then we have the danger of invoking
Quoted 3 lines
flush_cpu_workqueue()on a dead cpu (becauseflush_cpu_workqueuedropsworkqueue_mutex, cpu hp events can changecpu_online_mapwhile we are inflush_cpu_workqueue).
Oleg.