sched/completion: Add fast path in __wait_for_common()
PATCH 1/1 sched/completion: Add fast path in __wait_for_common()
We may optimize __wait_for_common() in case of it's waken up by complete{,_all}() if we do not take the spinlock.
New function completion_wake_function() is now used to wake waiters. It a case of successful waking it deletes a waiter from the task list. The waiter checks wait.task_list and skips the locking if it's empty. In case of a single waiter this prevents from unnecessary spin_{,un}_lock_irq().
In case of several waiters this improves parallelism in the obvious way.
--- include/linux/wait.h | 1 + kernel/sched/completion.c | 57 ++++++++++++++++++++++++++++++++------------- kernel/sched/wait.c | 7 ++++++ 3 files changed, 48 insertions(+), 17 deletions(-)
include/linux/wait.h +1 -0
@@ -148,6 +148,7 @@ __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)typedef int wait_bit_action_f (struct wait_bit_key *);void __wake_up (wait_queue_head_t *q,unsigned int mode,int nr,void *key);void __wake_up_locked_key (wait_queue_head_t *q,unsigned int mode,void *key); +void __wake_up_locked_nr_key (wait_queue_head_t *q,unsigned int mode,int nr,void *key);void __wake_up_sync_key (wait_queue_head_t *q,unsigned int mode,int nr,void *key);void __wake_up_locked (wait_queue_head_t *q,unsigned int mode,int nr);void __wake_up_sync (wait_queue_head_t *q,unsigned int mode,int nr);
kernel/sched/completion.c +40 -17
90 lines not on this page · raw message
kernel/sched/wait.c +7 -0
@@ -112,6 +112,13 @@ void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) }EXPORT_SYMBOL_GPL (__wake_up_locked_key); +void __wake_up_locked_nr_key (wait_queue_head_t *q,unsigned int mode, +int nr,void *key) +{ +__wake_up_common (q, mode, nr,0 , key); +} +EXPORT_SYMBOL_GPL (__wake_up_locked_nr_key); +/** * __wake_up_sync_key - wake up threads blocked on a waitqueue. * @q: the waitqueue
Please, ignore this patch.
This brings the problem with exiting threads like in
https://lkml.org/lkml/2015/2/4/761
В Ср, 04/02/2015 в 19:50 +0300, Kirill Tkhai пишет:
Quoted 148 lines
We may optimize
__wait_for_common()in case of it's waken up by complete{,_all}() if we do not take the spinlock.New function
completion_wake_function()is now used to wake waiters. It a case of successful waking it deletes a waiter from the task list. The waiter checks wait.task_list and skips the locking if it's empty. In case of a single waiter this prevents from unnecessary spin_{,un}_lock_irq().In case of several waiters this improves parallelism in the obvious way.
Signed-off-by Kirill Tkhai <[email protected]>--- include/linux/wait.h | 1 + kernel/sched/completion.c | 57 ++++++++++++++++++++++++++++++++------------- kernel/sched/wait.c | 7 ++++++ 3 files changed, 48 insertions(+), 17 deletions(-)include/linux/wait.h +1
-0@@ -148,6 +148,7 @@ __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)typedef int wait_bit_action_f (struct wait_bit_key *);void __wake_up (wait_queue_head_t *q,unsigned int mode,int nr,void *key);void __wake_up_locked_key (wait_queue_head_t *q,unsigned int mode,void *key); +void __wake_up_locked_nr_key (wait_queue_head_t *q,unsigned int mode,int nr,void *key);void __wake_up_sync_key (wait_queue_head_t *q,unsigned int mode,int nr,void *key);void __wake_up_locked (wait_queue_head_t *q,unsigned int mode,int nr);void __wake_up_sync (wait_queue_head_t *q,unsigned int mode,int nr);kernel/sched/completion.c +40
-17@@ -32,7 +32,7 @@ void complete(struct completion *x)spin_lock_irqsave (&x->wait.lock, flags); x->done++; -__wake_up_locked (&x->wait,TASK_NORMAL ,1 ); +__wake_up_locked_nr_key (&x->wait,TASK_NORMAL ,1 , x);spin_unlock_irqrestore (&x->wait.lock, flags); }EXPORT_SYMBOL (complete);@@ -52,17 +52,36 @@ void complete_all(struct completion *x)spin_lock_irqsave (&x->wait.lock, flags); x->done +=UINT_MAX /2 ; -__wake_up_locked (&x->wait,TASK_NORMAL ,0 ); +__wake_up_locked_nr_key (&x->wait,TASK_NORMAL ,0 , x);spin_unlock_irqrestore (&x->wait.lock, flags); }EXPORT_SYMBOL (complete_all); +static int completion_wake_function (wait_queue_t *wait,unsigned mode, +int sync,void *key) +{ +int ret =default_wake_function (wait, mode, sync, key); +struct completion *x = key; + +if (ret) { +list_del_init (&wait->task_list); + x->done--; + } +return ret; +} +static inline long __sched -do_wait_for_common (struct completion *x, -long (*action)(long ),long timeout,int state) +__wait_for_common (struct completion *x, +long (*action)(long ),long timeout,int state) { +might_sleep (); + +spin_lock_irq (&x->wait.lock);if (!x->done) { -DECLARE_WAITQUEUE (wait, current); +wait_queue_t wait = { + .private = current, + .func = completion_wake_function, + };__add_wait_queue_tail_exclusive (&x->wait, &wait); do {@@ -73,26 +92,30 @@ do_wait_for_common(struct completion *x,__set_current_state (state);spin_unlock_irq (&x->wait.lock); timeout =action (timeout); +/* +* This is the fast check whether we are woken up by +* completion_wake_function(). No spinlock held here. +*/ +if (list_empty (&wait.task_list)) + goto out;spin_lock_irq (&x->wait.lock); +/* +* The above check is unlocked and racy with the wake +* function. Test again to be sure, we haven't missed +* the sign of its work. +*/ +if (unlikely (list_empty (&wait.task_list))) + goto unlock; }while (!x->done && timeout);__remove_wait_queue (&x->wait, &wait);if (!x->done)return timeout; } x->done--; -return timeout ?:1 ; -} - -static inline long __sched -__wait_for_common (struct completion *x, -long (*action)(long ),long timeout,int state) -{ -might_sleep (); - -spin_lock_irq (&x->wait.lock); - timeout =do_wait_for_common (x, action, timeout, state); +unlock:spin_unlock_irq (&x->wait.lock); -return timeout; +out: +return timeout ?:1 ; }static long __schedkernel/sched/wait.c +7
-0@@ -112,6 +112,13 @@ void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) }EXPORT_SYMBOL_GPL (__wake_up_locked_key); +void __wake_up_locked_nr_key (wait_queue_head_t *q,unsigned int mode, +int nr,void *key) +{ +__wake_up_common (q, mode, nr,0 , key); +} +EXPORT_SYMBOL_GPL (__wake_up_locked_nr_key); +/** * __wake_up_sync_key - wake up threads blocked on a waitqueue. * @q: the waitqueue