locking/rwsem: Miscellaneous cleanup and fix
Waiman Long
reviewerv2 cover locking/rwsem: Miscellaneous cleanup and fix
Patch 1 is just a simple cleanup patch. Patches 2 and 3 applies the same consistent rules to optionally wake up waiters in reader & writers slowpaths as well as in the out_nolock paths.
Waiman Long (3):
locking/rwsem: No need to check for handoff bit if wait queue empty locking/rwsem: Conditionally wake waiters in reader/writer slowpaths locking/rwsem: Always try to wake waiters in out_nolock path
kernel/locking/rwsem.c | 121 ++++++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 51 deletions(-)
Signature · 1 line
2.27.0
Waiman Long
reviewerPATCH v2 1/3 locking/rwsem: No need to check for handoff bit if wait queue empty
Since commit d257cc8cb8d5 ("locking/rwsem: Make handoff bit handling more consistent"), the handoff bit is always cleared if the wait queue becomes empty. There is no need to check for RWSEM_FLAG_HANDOFF when the wait list is known to be empty.
--- kernel/locking/rwsem.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
kernel/locking/rwsem.c +4 -5
@@ -977,12 +977,11 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int statif (list_empty (&sem->wait_list)) {/* * In case the wait queue is empty and the lock isn't owned -* by a writer or has the handoff bit set, this reader can -* exit the slowpath and return immediately as its -* RWSEM_READER_BIAS has already been set in the count. +* by a writer, this reader can exit the slowpath and return +* immediately as its RWSEM_READER_BIAS has already been set +* in the count. */ -if (!(atomic_long_read (&sem->count) & - (RWSEM_WRITER_MASK |RWSEM_FLAG_HANDOFF ))) { +if (!(atomic_long_read (&sem->count) &RWSEM_WRITER_MASK )) {/* Provide lock ACQUIRE */ smp_acquire__after_ctrl_dep ();raw_spin_unlock_irq (&sem->wait_lock);
Signature · 1 line
2.27.0
[tip: locking/core] locking/rwsem: No need to check for handoff bit if wait queue empty
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: f9e21aa9e6fb11355e54c8949a390d49ca21cde1 Gitweb: https://git.kernel.org/tip/f9e21aa9e6fb11355e54c8949a390d49ca21cde1 Author: Waiman Long <[email protected]> AuthorDate: Tue, 22 Mar 2022 11:20:57 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 05 Apr 2022 10:24:34 +02:00
locking/rwsem: No need to check for handoff bit if wait queue empty
Since commit d257cc8cb8d5 ("locking/rwsem: Make handoff bit handling more consistent"), the handoff bit is always cleared if the wait queue becomes empty. There is no need to check for RWSEM_FLAG_HANDOFF when the wait list is known to be empty.
--- kernel/locking/rwsem.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
kernel/locking/rwsem.c +4 -5
@@ -977,12 +977,11 @@ queue:if (list_empty (&sem->wait_list)) {/* * In case the wait queue is empty and the lock isn't owned -* by a writer or has the handoff bit set, this reader can -* exit the slowpath and return immediately as its -* RWSEM_READER_BIAS has already been set in the count. +* by a writer, this reader can exit the slowpath and return +* immediately as its RWSEM_READER_BIAS has already been set +* in the count. */ -if (!(atomic_long_read (&sem->count) & - (RWSEM_WRITER_MASK |RWSEM_FLAG_HANDOFF ))) { +if (!(atomic_long_read (&sem->count) &RWSEM_WRITER_MASK )) {/* Provide lock ACQUIRE */ smp_acquire__after_ctrl_dep ();raw_spin_unlock_irq (&sem->wait_lock);
Waiman Long
reviewerPATCH v2 2/3 locking/rwsem: Conditionally wake waiters in reader/writer slowpaths
In reply to Waiman LongIn an analysis of a recent vmcore, a reader-owned rwsem was found with 385 readers but no writer in the wait queue. That is kind of unusual but it may be caused by some race conditions that we have not fully understood yet. In such a case, all the readers in the wait queue should join the other reader-owners and acquire the read lock.
In rwsem_down_write_slowpath(), an incoming writer will try to wake up the front readers under such circumstance. That is not the case for rwsem_down_read_slowpath(), add a new helper function rwsem_cond_wake_waiter() to do wakeup and use it in both reader and writer slowpaths to have a consistent and correct behavior.
--- kernel/locking/rwsem.c | 68 ++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 36 deletions(-)
kernel/locking/rwsem.c +32 -36
@@ -901,7 +901,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)*/ static inline void clear_nonspinnable (struct rw_semaphore *sem) { -if (rwsem_test_oflags (sem,RWSEM_NONSPINNABLE )) +if (unlikely (rwsem_test_oflags (sem,RWSEM_NONSPINNABLE )))atomic_long_andnot (RWSEM_NONSPINNABLE , &sem->owner); }
@@ -925,6 +925,31 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) }#endif +/* +* Prepare to wake up waiter(s) in the wait queue by putting them into the +* given wake_q if the rwsem lock owner isn't a writer. If rwsem is likely +* reader-owned, wake up read lock waiters in queue front or wake up any +* front waiter otherwise. + +* This is being called from both reader and writer slow paths. +*/ +static inline void rwsem_cond_wake_waiter (struct rw_semaphore *sem,long count, +struct wake_q_head *wake_q) +{ +enum rwsem_wake_type wake_type; + +if (count &RWSEM_WRITER_MASK ) +return ; + +if (count &RWSEM_READER_MASK ) { + wake_type =RWSEM_WAKE_READERS ; + }else { + wake_type =RWSEM_WAKE_ANY ; +clear_nonspinnable (sem); + } +rwsem_mark_wake (sem, wake_type, wake_q); +} +/* * Wait for the read lock to be granted */
@@ -935,7 +960,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int statlong rcnt = (count >>RWSEM_READER_SHIFT );struct rwsem_waiter waiter;DEFINE_WAKE_Q (wake_q); -bool wake = false;/* * To prevent a constant stream of readers from starving a sleeping
@@ -996,22 +1020,11 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat/* we're now waiting on the lock, but no longer actively locking */ count =atomic_long_add_return (adjustment, &sem->count); -/* -* If there are no active locks, wake the front queued process(es). -* -* If there are no writers and we are first in the queue, -* wake our own waiter to join the existing active readers ! -*/ -if (!(count &RWSEM_LOCK_MASK )) { -clear_nonspinnable (sem); - wake = true; - } -if (wake || (!(count &RWSEM_WRITER_MASK ) && - (adjustment &RWSEM_FLAG_WAITERS ))) -rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); - +rwsem_cond_wake_waiter (sem, count, &wake_q);raw_spin_unlock_irq (&sem->wait_lock); -wake_up_q (&wake_q); + +if (!wake_q_empty (&wake_q)) +wake_up_q (&wake_q);/* wait to be given the lock */ for (;;) {
@@ -1050,7 +1063,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int statstatic struct rw_semaphore *rwsem_down_write_slowpath (struct rw_semaphore *sem,int state) { -long count;struct rwsem_waiter waiter;DEFINE_WAKE_Q (wake_q);
@@ -1074,23 +1086,8 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)/* we're now waiting on the lock */ if (rwsem_first_waiter (sem) != &waiter) { - count =atomic_long_read (&sem->count); - -/* -* If there were already threads queued before us and: -* 1) there are no active locks, wake the front -* queued process(es) as the handoff bit might be set. -* 2) there are no active writers and some readers, the lock -* must be read owned; so we try to wake any read lock -* waiters that were queued ahead of us. -*/ -if (count &RWSEM_WRITER_MASK ) - goto wait; - -rwsem_mark_wake (sem, (count &RWSEM_READER_MASK ) - ?RWSEM_WAKE_READERS - :RWSEM_WAKE_ANY , &wake_q); - +rwsem_cond_wake_waiter (sem,atomic_long_read (&sem->count), + &wake_q);if (!wake_q_empty (&wake_q)) {/* * We want to minimize wait_lock hold time especially
@@ -1105,7 +1102,6 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)atomic_long_or (RWSEM_FLAG_WAITERS , &sem->count); } -wait:/* wait until we successfully acquire the lock */ set_current_state (state);for (;;) {
Signature · 1 line
2.27.0
[tip: locking/core] locking/rwsem: Conditionally wake waiters in reader/writer slowpaths
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 54c1ee4d614d52844cf24c46d8415bf1392021d0 Gitweb: https://git.kernel.org/tip/54c1ee4d614d52844cf24c46d8415bf1392021d0 Author: Waiman Long <[email protected]> AuthorDate: Tue, 22 Mar 2022 11:20:58 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 05 Apr 2022 10:24:35 +02:00
locking/rwsem: Conditionally wake waiters in reader/writer slowpaths
In an analysis of a recent vmcore, a reader-owned rwsem was found with 385 readers but no writer in the wait queue. That is kind of unusual but it may be caused by some race conditions that we have not fully understood yet. In such a case, all the readers in the wait queue should join the other reader-owners and acquire the read lock.
In rwsem_down_write_slowpath(), an incoming writer will try to wake up the front readers under such circumstance. That is not the case for rwsem_down_read_slowpath(), add a new helper function rwsem_cond_wake_waiter() to do wakeup and use it in both reader and writer slowpaths to have a consistent and correct behavior.
--- kernel/locking/rwsem.c | 68 +++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 36 deletions(-)
kernel/locking/rwsem.c +32 -36
@@ -901,7 +901,7 @@ done:*/ static inline void clear_nonspinnable (struct rw_semaphore *sem) { -if (rwsem_test_oflags (sem,RWSEM_NONSPINNABLE )) +if (unlikely (rwsem_test_oflags (sem,RWSEM_NONSPINNABLE )))atomic_long_andnot (RWSEM_NONSPINNABLE , &sem->owner); }
@@ -926,6 +926,31 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)#endif /* +* Prepare to wake up waiter(s) in the wait queue by putting them into the +* given wake_q if the rwsem lock owner isn't a writer. If rwsem is likely +* reader-owned, wake up read lock waiters in queue front or wake up any +* front waiter otherwise. + +* This is being called from both reader and writer slow paths. +*/ +static inline void rwsem_cond_wake_waiter (struct rw_semaphore *sem,long count, +struct wake_q_head *wake_q) +{ +enum rwsem_wake_type wake_type; + +if (count &RWSEM_WRITER_MASK ) +return ; + +if (count &RWSEM_READER_MASK ) { + wake_type =RWSEM_WAKE_READERS ; + }else { + wake_type =RWSEM_WAKE_ANY ; +clear_nonspinnable (sem); + } +rwsem_mark_wake (sem, wake_type, wake_q); +} + +/* * Wait for the read lock to be granted */ static struct rw_semaphore __sched *
@@ -935,7 +960,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int statlong rcnt = (count >>RWSEM_READER_SHIFT );struct rwsem_waiter waiter;DEFINE_WAKE_Q (wake_q); -bool wake = false;/* * To prevent a constant stream of readers from starving a sleeping
@@ -996,22 +1020,11 @@ queue:/* we're now waiting on the lock, but no longer actively locking */ count =atomic_long_add_return (adjustment, &sem->count); -/* -* If there are no active locks, wake the front queued process(es). -* -* If there are no writers and we are first in the queue, -* wake our own waiter to join the existing active readers ! -*/ -if (!(count &RWSEM_LOCK_MASK )) { -clear_nonspinnable (sem); - wake = true; - } -if (wake || (!(count &RWSEM_WRITER_MASK ) && - (adjustment &RWSEM_FLAG_WAITERS ))) -rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); - +rwsem_cond_wake_waiter (sem, count, &wake_q);raw_spin_unlock_irq (&sem->wait_lock); -wake_up_q (&wake_q); + +if (!wake_q_empty (&wake_q)) +wake_up_q (&wake_q);/* wait to be given the lock */ for (;;) {
@@ -1050,7 +1063,6 @@ out_nolock: static struct rw_semaphore __sched *rwsem_down_write_slowpath (struct rw_semaphore *sem,int state) { -long count;struct rwsem_waiter waiter;DEFINE_WAKE_Q (wake_q);
@@ -1074,23 +1086,8 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)/* we're now waiting on the lock */ if (rwsem_first_waiter (sem) != &waiter) { - count =atomic_long_read (&sem->count); - -/* -* If there were already threads queued before us and: -* 1) there are no active locks, wake the front -* queued process(es) as the handoff bit might be set. -* 2) there are no active writers and some readers, the lock -* must be read owned; so we try to wake any read lock -* waiters that were queued ahead of us. -*/ -if (count &RWSEM_WRITER_MASK ) - goto wait; - -rwsem_mark_wake (sem, (count &RWSEM_READER_MASK ) - ?RWSEM_WAKE_READERS - :RWSEM_WAKE_ANY , &wake_q); - +rwsem_cond_wake_waiter (sem,atomic_long_read (&sem->count), + &wake_q);if (!wake_q_empty (&wake_q)) {/* * We want to minimize wait_lock hold time especially
@@ -1105,7 +1102,6 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)atomic_long_or (RWSEM_FLAG_WAITERS , &sem->count); } -wait:/* wait until we successfully acquire the lock */ set_current_state (state);for (;;) {
Waiman Long
reviewerPATCH v2 3/3 locking/rwsem: Always try to wake waiters in out_nolock path
In reply to Waiman LongFor writers, the out_nolock path will always attempt to wake up waiters. This may not be really necessary if the waiter to be removed is not the first one.
For readers, no attempt to wake up waiter is being made. However, if the HANDOFF bit is set and the reader to be removed is the first waiter, the waiter behind it will inherit the HANDOFF bit and for a write lock waiter waking it up will allow it to spin on the lock to acquire it faster. So it can be beneficial to do a wakeup in this case.
Add a new rwsem_del_wake_waiter() helper function to do that consistently for both reader and writer out_nolock paths.
--- kernel/locking/rwsem.c | 44 ++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-)
kernel/locking/rwsem.c +34 -10
@@ -375,16 +375,19 @@ rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)* * Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of * this function. Modify with care. +* +* Return: true if wait_list isn't empty and false otherwise */ -static inline void +static inline bool rwsem_del_waiter (struct rw_semaphore *sem,struct rwsem_waiter *waiter) {lockdep_assert_held (&sem->wait_lock);list_del (&waiter->list);if (likely (!list_empty (&sem->wait_list))) -return ; +return true;atomic_long_andnot (RWSEM_FLAG_HANDOFF |RWSEM_FLAG_WAITERS , &sem->count); +return false; }/*
@@ -558,6 +561,33 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, } } +/* +* Remove a waiter and try to wake up other waiters in the wait queue +* This function is called from the out_nolock path of both the reader and +* writer slowpaths with wait_lock held. It releases the wait_lock and +* optionally wake up waiters before it returns. +*/ +static inline void +rwsem_del_wake_waiter (struct rw_semaphore *sem,struct rwsem_waiter *waiter, +struct wake_q_head *wake_q) + __releases(&sem->wait_lock) +{ +bool first =rwsem_first_waiter (sem) == waiter; + +wake_q_init (wake_q); + +/* +* If the wait_list isn't empty and the waiter to be deleted is +* the first waiter, we wake up the remaining waiters as they may +* be eligible to acquire or spin on the lock. +*/ +if (rwsem_del_waiter (sem, waiter) && first) +rwsem_mark_wake (sem,RWSEM_WAKE_ANY , wake_q); +raw_spin_unlock_irq (&sem->wait_lock); +if (!wake_q_empty (wake_q)) +wake_up_q (wake_q); +} +/* * This function must be called with the sem->wait_lock held to prevent * race conditions between checking the rwsem wait list and setting the
@@ -1050,8 +1080,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int statreturn sem; out_nolock: -rwsem_del_waiter (sem, &waiter); -raw_spin_unlock_irq (&sem->wait_lock); +rwsem_del_wake_waiter (sem, &waiter, &wake_q);__set_current_state (TASK_RUNNING );lockevent_inc (rwsem_rlock_fail);return ERR_PTR (-EINTR );
@@ -1095,7 +1124,6 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)*/ raw_spin_unlock_irq (&sem->wait_lock);wake_up_q (&wake_q); -wake_q_init (&wake_q);/* Used again, reinit */ raw_spin_lock_irq (&sem->wait_lock); } }else {
@@ -1148,11 +1176,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) out_nolock:__set_current_state (TASK_RUNNING );raw_spin_lock_irq (&sem->wait_lock); -rwsem_del_waiter (sem, &waiter); -if (!list_empty (&sem->wait_list)) -rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); -raw_spin_unlock_irq (&sem->wait_lock); -wake_up_q (&wake_q); +rwsem_del_wake_waiter (sem, &waiter, &wake_q);lockevent_inc (rwsem_wlock_fail);return ERR_PTR (-EINTR ); }
Signature · 1 line
2.27.0
[tip: locking/core] locking/rwsem: Always try to wake waiters in out_nolock path
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 1ee326196c66583006b0c95356a4b7dc51bf3531 Gitweb: https://git.kernel.org/tip/1ee326196c66583006b0c95356a4b7dc51bf3531 Author: Waiman Long <[email protected]> AuthorDate: Tue, 22 Mar 2022 11:20:59 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 05 Apr 2022 10:24:35 +02:00
locking/rwsem: Always try to wake waiters in out_nolock path
For writers, the out_nolock path will always attempt to wake up waiters. This may not be really necessary if the waiter to be removed is not the first one.
For readers, no attempt to wake up waiter is being made. However, if the HANDOFF bit is set and the reader to be removed is the first waiter, the waiter behind it will inherit the HANDOFF bit and for a write lock waiter waking it up will allow it to spin on the lock to acquire it faster. So it can be beneficial to do a wakeup in this case.
Add a new rwsem_del_wake_waiter() helper function to do that consistently for both reader and writer out_nolock paths.
--- kernel/locking/rwsem.c | 44 +++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-)
kernel/locking/rwsem.c +34 -10
@@ -375,16 +375,19 @@ rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)* * Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of * this function. Modify with care. +* +* Return: true if wait_list isn't empty and false otherwise */ -static inline void +static inline bool rwsem_del_waiter (struct rw_semaphore *sem,struct rwsem_waiter *waiter) {lockdep_assert_held (&sem->wait_lock);list_del (&waiter->list);if (likely (!list_empty (&sem->wait_list))) -return ; +return true;atomic_long_andnot (RWSEM_FLAG_HANDOFF |RWSEM_FLAG_WAITERS , &sem->count); +return false; }/*
@@ -559,6 +562,33 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, }/* +* Remove a waiter and try to wake up other waiters in the wait queue +* This function is called from the out_nolock path of both the reader and +* writer slowpaths with wait_lock held. It releases the wait_lock and +* optionally wake up waiters before it returns. +*/ +static inline void +rwsem_del_wake_waiter (struct rw_semaphore *sem,struct rwsem_waiter *waiter, +struct wake_q_head *wake_q) + __releases(&sem->wait_lock) +{ +bool first =rwsem_first_waiter (sem) == waiter; + +wake_q_init (wake_q); + +/* +* If the wait_list isn't empty and the waiter to be deleted is +* the first waiter, we wake up the remaining waiters as they may +* be eligible to acquire or spin on the lock. +*/ +if (rwsem_del_waiter (sem, waiter) && first) +rwsem_mark_wake (sem,RWSEM_WAKE_ANY , wake_q); +raw_spin_unlock_irq (&sem->wait_lock); +if (!wake_q_empty (wake_q)) +wake_up_q (wake_q); +} + +/* * This function must be called with the sem->wait_lock held to prevent * race conditions between checking the rwsem wait list and setting the * sem->count accordingly.
@@ -1050,8 +1080,7 @@ queue:return sem; out_nolock: -rwsem_del_waiter (sem, &waiter); -raw_spin_unlock_irq (&sem->wait_lock); +rwsem_del_wake_waiter (sem, &waiter, &wake_q);__set_current_state (TASK_RUNNING );lockevent_inc (rwsem_rlock_fail);return ERR_PTR (-EINTR );
@@ -1095,7 +1124,6 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)*/ raw_spin_unlock_irq (&sem->wait_lock);wake_up_q (&wake_q); -wake_q_init (&wake_q);/* Used again, reinit */ raw_spin_lock_irq (&sem->wait_lock); } }else {
@@ -1148,11 +1176,7 @@ trylock_again: out_nolock:__set_current_state (TASK_RUNNING );raw_spin_lock_irq (&sem->wait_lock); -rwsem_del_waiter (sem, &waiter); -if (!list_empty (&sem->wait_list)) -rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); -raw_spin_unlock_irq (&sem->wait_lock); -wake_up_q (&wake_q); +rwsem_del_wake_waiter (sem, &waiter, &wake_q);lockevent_inc (rwsem_wlock_fail);return ERR_PTR (-EINTR ); }