locking/rwsem: Rwsem rearchitecture part 1
Subsystems (2)
Waiman Long
reviewerv2 cover locking/rwsem: Rwsem rearchitecture part 1
v2:
- Sync up to v4 of the part 0 patch.
- Remove the rwsem.h->rwsem-xadd.h renaming patch & change patches to modify rwsem.h instead of rwsem-xadd.h.
- Add a new patch to micro-optimize rwsem_try_read_lock_unqueued().
This is part 1 of a 3-part (0/1/2) series to rearchitect the internal operation of rwsem. This depends on the v4 part 0 patches sent out previously
https://lore.kernel.org/lkml/[email protected]
This part lays the foundation for part 2 without making any functional changes. This part includes the following changes:
1) Move code around and micro-optimize rwsem_try_read_lock_unqueued()
(patches 1-4).
2) Enhance the DEBUG_RWSEMS_WARN_ON() macro to provide more information and add additional checks (patches 5 & 6).
3) Make the core qspinlock_stat.h code generic (lock event counting) so that it can be used by all the architectures as well as other locking subsystems such as rwsem (patches 7-10). Lock event counting help us visualize how frequently a code path is being used as well as spotting abnormal behavior due to bugs in the code without noticeably affecting kernel performance and hence behavior.
Both (2) and (3) are useful debugging aids.
Waiman Long (10):
locking/rwsem: Relocate rwsem_down_read_failed() locking/rwsem: Move owner setting code from rwsem.c to rwsem.h locking/rwsem: Move rwsem internal function declarations to
rwsem-xadd.h
locking/rwsem: Micro-optimize rwsem_try_read_lock_unqueued() locking/rwsem: Add debug check for __down_read*() locking/rwsem: Enhance DEBUG_RWSEMS_WARN_ON() macro locking/qspinlock_stat: Introduce a generic lockevent counting APIs locking/lock_events: Make lock_events available for all archs & other
locks
locking/lock_events: Don't show pvqspinlock events on bare metal locking/rwsem: Enable lock event counting
arch/Kconfig | 9 ++ arch/x86/Kconfig | 8 -- include/linux/rwsem.h | 7 -- kernel/locking/Makefile | 1 + kernel/locking/lock_events.c | 179 ++++++++++++++++++++++++++ kernel/locking/lock_events.h | 59 +++++++++ kernel/locking/lock_events_list.h | 67 ++++++++++ kernel/locking/qspinlock.c | 8 +- kernel/locking/qspinlock_paravirt.h | 19 +-- kernel/locking/qspinlock_stat.h | 242 +++++++----------------------------- kernel/locking/rwsem-xadd.c | 204 +++++++++++++++--------------- kernel/locking/rwsem.c | 25 +--- kernel/locking/rwsem.h | 47 ++++++- 13 files changed, 523 insertions(+), 352 deletions(-) create mode 100644 kernel/locking/lock_events.c create mode 100644 kernel/locking/lock_events.h create mode 100644 kernel/locking/lock_events_list.h
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH v2 1/10 locking/rwsem: Relocate rwsem_down_read_failed()
The rwsem_down_read_failed*() functions were relocted from above the optimistic spinning section to below that section. This enables the reader functions to use optimisitic spinning in future patches. There is no code change.
--- kernel/locking/rwsem-xadd.c | 172 ++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 86 deletions(-)
kernel/locking/rwsem-xadd.c +86 -86
@@ -225,92 +225,6 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem, }/* -* Wait for the read lock to be granted -*/ -static inline struct rw_semaphore __sched * -__rwsem_down_read_failed_common (struct rw_semaphore *sem,int state) -{ -long count, adjustment = -RWSEM_ACTIVE_READ_BIAS ; -struct rwsem_waiter waiter; -DEFINE_WAKE_Q (wake_q); - - waiter.task = current; - waiter.type =RWSEM_WAITING_FOR_READ ; - -raw_spin_lock_irq (&sem->wait_lock); -if (list_empty (&sem->wait_list)) { -/* -* In case the wait queue is empty and the lock isn't owned -* by a writer, this reader can exit the slowpath and return -* immediately as its RWSEM_ACTIVE_READ_BIAS has already -* been set in the count. -*/ -if (atomic_long_read (&sem->count) >=0 ) { -raw_spin_unlock_irq (&sem->wait_lock); -return sem; - } - adjustment +=RWSEM_WAITING_BIAS ; - } -list_add_tail (&waiter.list, &sem->wait_list); - -/* we're now waiting on the lock, but no longer actively locking */ - count =atomic_long_add_return (adjustment, &sem->count); - -/* -* If there are no active locks, wake the front queued process(es). -* -* If there are no writers and we are first in the queue, -* wake our own waiter to join the existing active readers ! -*/ -if (count ==RWSEM_WAITING_BIAS || - (count >RWSEM_WAITING_BIAS && - adjustment != -RWSEM_ACTIVE_READ_BIAS )) -__rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); - -raw_spin_unlock_irq (&sem->wait_lock); -wake_up_q (&wake_q); - -/* wait to be given the lock */ -while (true) { -set_current_state (state); -if (!waiter.task) -break ; -if (signal_pending_state (state, current)) { -raw_spin_lock_irq (&sem->wait_lock); -if (waiter.task) - goto out_nolock; -raw_spin_unlock_irq (&sem->wait_lock); -break ; - } -schedule (); - } - -__set_current_state (TASK_RUNNING ); -return sem; -out_nolock: -list_del (&waiter.list); -if (list_empty (&sem->wait_list)) -atomic_long_add (-RWSEM_WAITING_BIAS , &sem->count); -raw_spin_unlock_irq (&sem->wait_lock); -__set_current_state (TASK_RUNNING ); -return ERR_PTR (-EINTR ); -} - -__visible struct rw_semaphore * __sched -rwsem_down_read_failed(struct rw_semaphore *sem) -{ -return __rwsem_down_read_failed_common (sem,TASK_UNINTERRUPTIBLE ); -} -EXPORT_SYMBOL (rwsem_down_read_failed); - -__visible struct rw_semaphore * __sched -rwsem_down_read_failed_killable(struct rw_semaphore *sem) -{ -return __rwsem_down_read_failed_common (sem,TASK_KILLABLE ); -} -EXPORT_SYMBOL (rwsem_down_read_failed_killable); - -/* * This function must be called with the sem->wait_lock held to prevent * race conditions between checking the rwsem wait list and setting the * sem->count accordingly.
@@ -505,6 +419,92 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)#endif /* +* Wait for the read lock to be granted +*/ +static inline struct rw_semaphore __sched * +__rwsem_down_read_failed_common (struct rw_semaphore *sem,int state) +{ +long count, adjustment = -RWSEM_ACTIVE_READ_BIAS ; +struct rwsem_waiter waiter; +DEFINE_WAKE_Q (wake_q); + + waiter.task = current; + waiter.type =RWSEM_WAITING_FOR_READ ; + +raw_spin_lock_irq (&sem->wait_lock); +if (list_empty (&sem->wait_list)) { +/* +* In case the wait queue is empty and the lock isn't owned +* by a writer, this reader can exit the slowpath and return +* immediately as its RWSEM_ACTIVE_READ_BIAS has already +* been set in the count. +*/ +if (atomic_long_read (&sem->count) >=0 ) { +raw_spin_unlock_irq (&sem->wait_lock); +return sem; + } + adjustment +=RWSEM_WAITING_BIAS ; + } +list_add_tail (&waiter.list, &sem->wait_list); + +/* we're now waiting on the lock, but no longer actively locking */ + count =atomic_long_add_return (adjustment, &sem->count); + +/* +* If there are no active locks, wake the front queued process(es). +* +* If there are no writers and we are first in the queue, +* wake our own waiter to join the existing active readers ! +*/ +if (count ==RWSEM_WAITING_BIAS || + (count >RWSEM_WAITING_BIAS && + adjustment != -RWSEM_ACTIVE_READ_BIAS )) +__rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); + +raw_spin_unlock_irq (&sem->wait_lock); +wake_up_q (&wake_q); + +/* wait to be given the lock */ +while (true) { +set_current_state (state); +if (!waiter.task) +break ; +if (signal_pending_state (state, current)) { +raw_spin_lock_irq (&sem->wait_lock); +if (waiter.task) + goto out_nolock; +raw_spin_unlock_irq (&sem->wait_lock); +break ; + } +schedule (); + } + +__set_current_state (TASK_RUNNING ); +return sem; +out_nolock: +list_del (&waiter.list); +if (list_empty (&sem->wait_list)) +atomic_long_add (-RWSEM_WAITING_BIAS , &sem->count); +raw_spin_unlock_irq (&sem->wait_lock); +__set_current_state (TASK_RUNNING ); +return ERR_PTR (-EINTR ); +} + +__visible struct rw_semaphore * __sched +rwsem_down_read_failed(struct rw_semaphore *sem) +{ +return __rwsem_down_read_failed_common (sem,TASK_UNINTERRUPTIBLE ); +} +EXPORT_SYMBOL (rwsem_down_read_failed); + +__visible struct rw_semaphore * __sched +rwsem_down_read_failed_killable(struct rw_semaphore *sem) +{ +return __rwsem_down_read_failed_common (sem,TASK_KILLABLE ); +} +EXPORT_SYMBOL (rwsem_down_read_failed_killable); + +/* * Wait until we successfully acquire the write lock */ static inline struct rw_semaphore *
Signature · 1 line
1.8.3.1
Re: [PATCH-tip v2 01/10] locking/rwsem: Relocate rwsem_down_read_failed()
Quoted 9 lines · Waiman Long
The rwsem_down_read_failed*() functions were relocted from above the optimistic spinning section to below that section. This enables the reader functions to use optimisitic spinning in future patches. There is no code change.
Signed-off-by Waiman Long <[email protected]>--- kernel/locking/rwsem-xadd.c | 172 ++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 86 deletions(-)
Just a straightforward code move:
Will
Waiman Long
reviewerPATCH v2 2/10 locking/rwsem: Move owner setting code from rwsem.c to rwsem.h
In reply to Waiman LongMoves all the owner setting code closer to the rwsem-xadd fast paths directly within rwsem.h file.
For __down_read() and __down_read_killable(), it is assumed that rwsem will be marked as reader-owned when the functions return. That is currently true except in the transient case that the waiter queue just becomes empty. So a rwsem_set_reader_owned() call is added for this case. The __rwsem_set_reader_owned() call in __rwsem_mark_wake() is now necessary.
--- kernel/locking/rwsem-xadd.c | 6 +++--- kernel/locking/rwsem.c | 19 ++----------------- kernel/locking/rwsem.h | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 22 deletions(-)
kernel/locking/rwsem-xadd.c +3 -3
@@ -176,9 +176,8 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem, goto try_reader_grant; }/* -* It is not really necessary to set it to reader-owned here, -* but it gives the spinners an early indication that the -* readers now have the lock. +* Set it to reader-owned to give spinners an early +* indication that readers now have the lock. */ __rwsem_set_reader_owned (sem, waiter->task); }
@@ -441,6 +440,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)*/ if (atomic_long_read (&sem->count) >=0 ) {raw_spin_unlock_irq (&sem->wait_lock); +rwsem_set_reader_owned (sem);return sem; } adjustment +=RWSEM_WAITING_BIAS ;
kernel/locking/rwsem.c +2 -17
@@ -24,7 +24,6 @@ void __sched down_read(struct rw_semaphore *sem)rwsem_acquire_read (&sem->dep_map,0 ,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_read_trylock, __down_read); -rwsem_set_reader_owned (sem); }EXPORT_SYMBOL (down_read);
@@ -39,7 +38,6 @@ int __sched down_read_killable(struct rw_semaphore *sem)return -EINTR ; } -rwsem_set_reader_owned (sem);return 0 ; }
@@ -52,10 +50,8 @@ int down_read_trylock(struct rw_semaphore *sem) {int ret =__down_read_trylock (sem); -if (ret ==1 ) { +if (ret ==1 )rwsem_acquire_read (&sem->dep_map,0 ,1 , _RET_IP_); -rwsem_set_reader_owned (sem); - }return ret; }
@@ -70,7 +66,6 @@ void __sched down_write(struct rw_semaphore *sem)rwsem_acquire (&sem->dep_map,0 ,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_write_trylock, __down_write); -rwsem_set_owner (sem); }EXPORT_SYMBOL (down_write);
@@ -88,7 +83,6 @@ int __sched down_write_killable(struct rw_semaphore *sem)return -EINTR ; } -rwsem_set_owner (sem);return 0 ; }
@@ -101,10 +95,8 @@ int down_write_trylock(struct rw_semaphore *sem) {int ret =__down_write_trylock (sem); -if (ret ==1 ) { +if (ret ==1 )rwsem_acquire (&sem->dep_map,0 ,1 , _RET_IP_); -rwsem_set_owner (sem); - }return ret; }
@@ -119,7 +111,6 @@ void up_read(struct rw_semaphore *sem)rwsem_release (&sem->dep_map,1 , _RET_IP_);DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED )); -rwsem_clear_reader_owned (sem);__up_read (sem); }
@@ -133,7 +124,6 @@ void up_write(struct rw_semaphore *sem)rwsem_release (&sem->dep_map,1 , _RET_IP_);DEBUG_RWSEMS_WARN_ON (sem->owner != current); -rwsem_clear_owner (sem);__up_write (sem); }
@@ -147,7 +137,6 @@ void downgrade_write(struct rw_semaphore *sem)lock_downgrade (&sem->dep_map, _RET_IP_);DEBUG_RWSEMS_WARN_ON (sem->owner != current); -rwsem_set_reader_owned (sem);__downgrade_write (sem); }
@@ -161,7 +150,6 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)rwsem_acquire_read (&sem->dep_map, subclass,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_read_trylock, __down_read); -rwsem_set_reader_owned (sem); }EXPORT_SYMBOL (down_read_nested);
@@ -172,7 +160,6 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)rwsem_acquire_nest (&sem->dep_map,0 ,0 , nest, _RET_IP_);LOCK_CONTENDED (sem, __down_write_trylock, __down_write); -rwsem_set_owner (sem); }EXPORT_SYMBOL (_down_write_nest_lock);
@@ -193,7 +180,6 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)rwsem_acquire (&sem->dep_map, subclass,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_write_trylock, __down_write); -rwsem_set_owner (sem); }EXPORT_SYMBOL (down_write_nested);
@@ -208,7 +194,6 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)return -EINTR ; } -rwsem_set_owner (sem);return 0 ; }
kernel/locking/rwsem.h +15 -2
@@ -160,6 +160,8 @@ static inline void __down_read(struct rw_semaphore *sem) {if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 ))rwsem_down_read_failed (sem); +else +rwsem_set_reader_owned (sem); }static inline int __down_read_killable (struct rw_semaphore *sem)
@@ -167,8 +169,9 @@ static inline int __down_read_killable(struct rw_semaphore *sem)if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {if (IS_ERR (rwsem_down_read_failed_killable (sem)))return -EINTR ; + }else { +rwsem_set_reader_owned (sem); } -return 0 ; }
@@ -182,6 +185,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)do {if (atomic_long_try_cmpxchg_acquire (&sem->count, &tmp, tmp +RWSEM_ACTIVE_READ_BIAS )) { +rwsem_set_reader_owned (sem);return 1 ; } }while (tmp >=0 );
@@ -199,6 +203,7 @@ static inline void __down_write(struct rw_semaphore *sem) &sem->count);if (unlikely (tmp !=RWSEM_ACTIVE_WRITE_BIAS ))rwsem_down_write_failed (sem); +rwsem_set_owner (sem); }static inline int __down_write_killable (struct rw_semaphore *sem)
@@ -210,6 +215,7 @@ static inline int __down_write_killable(struct rw_semaphore *sem)if (unlikely (tmp !=RWSEM_ACTIVE_WRITE_BIAS ))if (IS_ERR (rwsem_down_write_failed_killable (sem)))return -EINTR ; +rwsem_set_owner (sem);return 0 ; }
@@ -219,7 +225,11 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) tmp =atomic_long_cmpxchg_acquire (&sem->count,RWSEM_UNLOCKED_VALUE ,RWSEM_ACTIVE_WRITE_BIAS ); -return tmp ==RWSEM_UNLOCKED_VALUE ; +if (tmp ==RWSEM_UNLOCKED_VALUE ) { +rwsem_set_owner (sem); +return true; + } +return false; }/*
@@ -229,6 +239,7 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; +rwsem_clear_reader_owned (sem); tmp =atomic_long_dec_return_release (&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))rwsem_wake (sem);
@@ -239,6 +250,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { +rwsem_clear_owner (sem);if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , &sem->count) <0 ))rwsem_wake (sem);
@@ -259,6 +271,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)* write side. As such, rely on RELEASE semantics. */ tmp =atomic_long_add_return_release (-RWSEM_WAITING_BIAS , &sem->count); +rwsem_set_reader_owned (sem);if (tmp <0 )rwsem_downgrade_wake (sem); }
Signature · 1 line
1.8.3.1
Re: [PATCH-tip v2 02/10] locking/rwsem: Move owner setting code from rwsem.c to rwsem.h
Hi Waiman,
Quoted 8 lines · Waiman Long
Moves all the owner setting code closer to the rwsem-xadd fast paths directly within
rwsem.hfile.For
__down_read()and__down_read_killable(), it is assumed that rwsem will be marked as reader-owned when the functions return. That is currently true except in the transient case that the waiter queue just becomes empty. So arwsem_set_reader_owned()call is added for this case.
Sorry, but I'm having a really hard time understanding the paragraph above. You make it sound like you're fixing a bug here, but that doesn't appear to be the case. Please can you elaborate?
Quoted 9 lines
The
__rwsem_set_reader_owned()call in__rwsem_mark_wake()is now necessary.Signed-off-by Waiman Long <[email protected]>--- kernel/locking/rwsem-xadd.c | 6 +++--- kernel/locking/rwsem.c | 19 ++----------------- kernel/locking/rwsem.h | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 22 deletions(-)
I personally find the existing code easier to read, but I don't know if that's just because I'm used to it...
Will
Waiman Long
reviewerRe: [PATCH-tip v2 02/10] locking/rwsem: Move owner setting code from rwsem.c to rwsem.h
Quoted 6 lines · Will Deacon
Hi Waiman,
Quoted 3 lines · Waiman Long
Moves all the owner setting code closer to the rwsem-xadd fast paths directly within
rwsem.hfile.
Quoted 8 lines
Quoted 5 lines
For
__down_read()and__down_read_killable(), it is assumed that rwsem will be marked as reader-owned when the functions return. That is currently true except in the transient case that the waiter queue just becomes empty. So arwsem_set_reader_owned()call is added for this case.Sorry, but I'm having a really hard time understanding the paragraph above. You make it sound like you're fixing a bug here, but that doesn't appear to be the case. Please can you elaborate?
Sorry for the confusion. I am referring to the code
if (list_empty(&sem->wait_list)) {
/*
* In case the wait queue is empty and the lock isn't owned
* by a writer, this reader can exit the slowpath and return
* immediately as its RWSEM_ACTIVE_READ_BIAS has already
* been set in the count.
*/
if (atomic_long_read(&sem->count) >= 0) { raw_spin_unlock_irq(&sem->wait_lock); return sem;
}
adjustment += RWSEM_WAITING_BIAS;
}
Here, a reader will return without going through the wakeup path. It is here that I need to insert an explicit owner setting call.
Quoted 11 lines
Quoted 9 lines
The
__rwsem_set_reader_owned()call in__rwsem_mark_wake()is now necessary.Signed-off-by Waiman Long <[email protected]>--- kernel/locking/rwsem-xadd.c | 6 +++--- kernel/locking/rwsem.c | 19 ++----------------- kernel/locking/rwsem.h | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 22 deletions(-)I personally find the existing code easier to read, but I don't know if that's just because I'm used to it...
The check reason for this patch is because I want to add a RWSEM_DEBUG check when a reader returns from the slowpath to make sure that the wakeup path is being call and because of some kind race condition. This check may be able to spot the wake_q race condition that was fixed earlier. I will update the commit log to make my intention clearer.
Cheers,
Longman
Waiman Long
reviewerPATCH v2 3/10 locking/rwsem: Move rwsem internal function declarations to rwsem-xadd.h
In reply to Waiman LongWe don't need to expose rwsem internal functions which are not supposed to be called directly from other kernel code.
--- include/linux/rwsem.h | 7 ------- kernel/locking/rwsem.h | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
include/linux/rwsem.h +0 -7
@@ -46,13 +46,6 @@ struct rw_semaphore {*/ #define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-2L) -extern struct rw_semaphore *rwsem_down_read_failed (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_read_failed_killable (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed_killable (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake (struct rw_semaphore *); -extern struct rw_semaphore *rwsem_downgrade_wake (struct rw_semaphore *sem); -/* In all implementations count != 0 means locked */ static inline int rwsem_is_locked (struct rw_semaphore *sem) {
kernel/locking/rwsem.h +7 -0
@@ -153,6 +153,13 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) }#endif +extern struct rw_semaphore *rwsem_down_read_failed (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_down_read_failed_killable (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_down_write_failed (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_down_write_failed_killable (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_wake (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_downgrade_wake (struct rw_semaphore *sem); +/* * lock for reading */
Signature · 1 line
1.8.3.1
Re: [PATCH-tip v2 03/10] locking/rwsem: Move rwsem internal function declarations to rwsem-xadd.h
Quoted 8 lines · Waiman Long
We don't need to expose rwsem internal functions which are not supposed to be called directly from other kernel code.
Signed-off-by Waiman Long <[email protected]>--- include/linux/rwsem.h | 7 ------- kernel/locking/rwsem.h | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
Fair enough,
Will
Waiman Long
reviewerPATCH v2 4/10 locking/rwsem: Micro-optimize rwsem_try_read_lock_unqueued()
In reply to Waiman LongThe atomic_long_cmpxchg_acquire() in rwsem_try_read_lock_unqueued() is replaced by atomic_long_try_cmpxchg_acquire() to simpify the code and generate slightly better assembly code. There is no functional change.
--- kernel/locking/rwsem-xadd.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-)
kernel/locking/rwsem-xadd.c +5 -10
@@ -259,21 +259,16 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)*/ static inline bool rwsem_try_write_lock_unqueued (struct rw_semaphore *sem) { -long old, count =atomic_long_read (&sem->count); +long count =atomic_long_read (&sem->count); -while (true) { -if (!(count ==0 || count ==RWSEM_WAITING_BIAS )) -return false; - - old =atomic_long_cmpxchg_acquire (&sem->count, count, - count +RWSEM_ACTIVE_WRITE_BIAS ); -if (old == count) { +while (!count || count ==RWSEM_WAITING_BIAS ) { +if (atomic_long_try_cmpxchg_acquire (&sem->count, &count, + count +RWSEM_ACTIVE_WRITE_BIAS )) {rwsem_set_owner (sem);return true; } - - count = old; } +return false; }static inline bool owner_on_cpu (struct task_struct *owner)
Signature · 1 line
1.8.3.1
Re: [PATCH-tip v2 04/10] locking/rwsem: Micro-optimize rwsem_try_read_lock_unqueued()
Quoted 3 lines · Waiman Long
The
atomic_long_cmpxchg_acquire()inrwsem_try_read_lock_unqueued()is replaced byatomic_long_try_cmpxchg_acquire()to simpify the code and generate slightly better assembly code. There is no functional change.
Might be nice to illustrate the improvement in the commit message, but it seems that try_cmpxchg is generally preferred for x86, so:
Will
Waiman Long
reviewerPATCH v2 5/10 locking/rwsem: Add debug check for __down_read*()
In reply to Waiman LongWhen rwsem_down_read_failed*() return, the read lock is acquired indirectly by others. So debug checks are added in __down_read() and __down_read_killable() to make sure the rwsem is really reader-owned.
The other debug check calls in kernel/locking/rwsem.c except the one in up_read_non_owner() are also moved over to rwsem-xadd.h.
--- kernel/locking/rwsem.c | 3 --- kernel/locking/rwsem.h | 12 ++++++++++-- 2 files changed, 10 insertions(+), 5 deletions(-)
kernel/locking/rwsem.c +0 -3
@@ -109,7 +109,6 @@ int down_write_trylock(struct rw_semaphore *sem)void up_read (struct rw_semaphore *sem) {rwsem_release (&sem->dep_map,1 , _RET_IP_); -DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ));__up_read (sem); }
@@ -122,7 +121,6 @@ void up_read(struct rw_semaphore *sem)void up_write (struct rw_semaphore *sem) {rwsem_release (&sem->dep_map,1 , _RET_IP_); -DEBUG_RWSEMS_WARN_ON (sem->owner != current);__up_write (sem); }
@@ -135,7 +133,6 @@ void up_write(struct rw_semaphore *sem)void downgrade_write (struct rw_semaphore *sem) {lock_downgrade (&sem->dep_map, _RET_IP_); -DEBUG_RWSEMS_WARN_ON (sem->owner != current);__downgrade_write (sem); }
kernel/locking/rwsem.h +10 -2
@@ -165,10 +165,13 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)*/ static inline void __down_read (struct rw_semaphore *sem) { -if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) +if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {rwsem_down_read_failed (sem); -else +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & +RWSEM_READER_OWNED )); + }else {rwsem_set_reader_owned (sem); + } }static inline int __down_read_killable (struct rw_semaphore *sem)
@@ -176,6 +179,8 @@ static inline int __down_read_killable(struct rw_semaphore *sem)if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {if (IS_ERR (rwsem_down_read_failed_killable (sem)))return -EINTR ; +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & +RWSEM_READER_OWNED )); }else {rwsem_set_reader_owned (sem); }
@@ -246,6 +251,7 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ));rwsem_clear_reader_owned (sem); tmp =atomic_long_dec_return_release (&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))
@@ -257,6 +263,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { +DEBUG_RWSEMS_WARN_ON (sem->owner != current);rwsem_clear_owner (sem);if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , &sem->count) <0 ))
@@ -277,6 +284,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)* read-locked region is ok to be re-ordered into the * write side. As such, rely on RELEASE semantics. */ +DEBUG_RWSEMS_WARN_ON (sem->owner != current); tmp =atomic_long_add_return_release (-RWSEM_WAITING_BIAS , &sem->count);rwsem_set_reader_owned (sem);if (tmp <0 )
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH v2 6/10 locking/rwsem: Enhance DEBUG_RWSEMS_WARN_ON() macro
In reply to Waiman LongCurrently, the DEBUG_RWSEMS_WARN_ON() macro just dumps a stack trace when the rwsem isn't in the right state. It does not show the actual states of the rwsem. This may not be that helpful in the debugging process.
Enhance the DEBUG_RWSEMS_WARN_ON() macro to also show the current content of the rwsem count and owner fields to give more information about what is wrong with the rwsem.
--- kernel/locking/rwsem.c | 3 ++- kernel/locking/rwsem.h | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-)
kernel/locking/rwsem.c +2 -1
@@ -198,7 +198,8 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)void up_read_non_owner (struct rw_semaphore *sem) { -DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED )); +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ), + sem);__up_read (sem); }
kernel/locking/rwsem.h +12 -7
@@ -27,9 +27,13 @@#define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)#ifdef CONFIG_DEBUG_RWSEMS -# define DEBUG_RWSEMS_WARN_ON (c) DEBUG_LOCKS_WARN_ON(c) +# define DEBUG_RWSEMS_WARN_ON (c, sem) \ + WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\ + #c, atomic_long_read(&(sem)->count), \ + (long)((sem)->owner), (long)current, \ + list_empty(&(sem)->wait_list) ? "" : "not ")#else -# define DEBUG_RWSEMS_WARN_ON (c) +# define DEBUG_RWSEMS_WARN_ON (c, sem)#endif /*
@@ -168,7 +172,7 @@ static inline void __down_read(struct rw_semaphore *sem)if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {rwsem_down_read_failed (sem);DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & -RWSEM_READER_OWNED )); +RWSEM_READER_OWNED ), sem); }else {rwsem_set_reader_owned (sem); }
@@ -180,7 +184,7 @@ static inline int __down_read_killable(struct rw_semaphore *sem)if (IS_ERR (rwsem_down_read_failed_killable (sem)))return -EINTR ;DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & -RWSEM_READER_OWNED )); +RWSEM_READER_OWNED ), sem); }else {rwsem_set_reader_owned (sem); }
@@ -251,7 +255,8 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; -DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED )); +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ), + sem);rwsem_clear_reader_owned (sem); tmp =atomic_long_dec_return_release (&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))
@@ -263,7 +268,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { -DEBUG_RWSEMS_WARN_ON (sem->owner != current); +DEBUG_RWSEMS_WARN_ON (sem->owner != current, sem);rwsem_clear_owner (sem);if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , &sem->count) <0 ))
@@ -284,7 +289,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)* read-locked region is ok to be re-ordered into the * write side. As such, rely on RELEASE semantics. */ -DEBUG_RWSEMS_WARN_ON (sem->owner != current); +DEBUG_RWSEMS_WARN_ON (sem->owner != current, sem); tmp =atomic_long_add_return_release (-RWSEM_WAITING_BIAS , &sem->count);rwsem_set_reader_owned (sem);if (tmp <0 )
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH v2 7/10 locking/qspinlock_stat: Introduce a generic lockevent counting APIs
In reply to Waiman LongThe percpu event counts used by qspinlock code can be useful for other locking code as well. So a new set of lockevent_* counting APIs is introduced with the lock event names extracted out into the new lock_events_list.h header file for easier addition in the future.
The existing qstat_inc() calls are replaced by either lockevent_inc() or lockevent_cond_inc() calls.
The qstat_hop() call is renamed to lockevent_pv_hop(). The "reset_counters" debugfs file is also renamed to ".reset_counts".
--- kernel/locking/lock_events.h | 55 ++++++++++++ kernel/locking/lock_events_list.h | 50 +++++++++++ kernel/locking/qspinlock.c | 8 +- kernel/locking/qspinlock_paravirt.h | 19 +++-- kernel/locking/qspinlock_stat.h | 163 ++++++++++++++---------------------- 5 files changed, 181 insertions(+), 114 deletions(-) create mode 100644 kernel/locking/lock_events.h create mode 100644 kernel/locking/lock_events_list.h
kernel/locking/lock_events.h new file +55 -0
@@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* Authors: Waiman Long <[email protected]> +*/ + +enum lock_events { + +#include "lock_events_list.h" + + lockevent_num,/* Total number of lock event counts */ + LOCKEVENT_reset_cnts = lockevent_num, +}; + +#ifdef CONFIG_QUEUED_LOCK_STAT +/* +* Per-cpu counters +*/ +DECLARE_PER_CPU(unsigned long, lockevents[lockevent_num]); + +/* +* Increment the PV qspinlock statistical counters +*/ +static inline void __lockevent_inc (enum lock_events event,bool cond) +{ +if (cond) +__this_cpu_inc (lockevents[event]); +} + +#define lockevent_inc (ev) __lockevent_inc(LOCKEVENT_ ##ev, true) +#define lockevent_cond_inc (ev, c) __lockevent_inc(LOCKEVENT_ ##ev, c) + +static inline void __lockevent_add (enum lock_events event,int inc) +{ +__this_cpu_add (lockevents[event], inc); +} + +#define lockevent_add (ev, c) __lockevent_add(LOCKEVENT_ ##ev, c) + +#else /* CONFIG_QUEUED_LOCK_STAT */ + +#define lockevent_inc (ev) +#define lockevent_add (ev, c) +#define lockevent_cond_inc (ev, c) + +#endif /* CONFIG_QUEUED_LOCK_STAT */
kernel/locking/lock_events_list.h new file +50 -0
50 lines not on this page · raw message
kernel/locking/qspinlock.c +4 -4
@@ -398,7 +398,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)* 0,1,0 -> 0,0,1 */ clear_pending_set_locked (lock); -qstat_inc (qstat_lock_pending, true); +lockevent_inc (lock_pending);return ;/*
@@ -406,7 +406,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)* queuing. */ queue: -qstat_inc (qstat_lock_slowpath, true); +lockevent_inc (lock_slowpath); pv_queue: node =this_cpu_ptr (&qnodes[0 ].mcs); idx = node->count++;
@@ -422,7 +422,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)* simple enough. */ if (unlikely (idx >=MAX_NODES )) { -qstat_inc (qstat_lock_no_node, true); +lockevent_inc (lock_no_node);while (!queued_spin_trylock (lock))cpu_relax (); goto release;
@@ -433,7 +433,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)/* * Keep counts of non-zero index values: */ -qstat_inc (qstat_lock_use_node2 + idx -1 , idx); +lockevent_cond_inc (lock_use_node2 + idx -1 , idx);/* * Ensure that we increment the head node->count before initialising
kernel/locking/qspinlock_paravirt.h +10 -9
@@ -89,7 +89,7 @@ static inline bool pv_hybrid_queued_unfair_trylock(struct qspinlock *lock)if (!(val & _Q_LOCKED_PENDING_MASK) && (cmpxchg_acquire (&lock->locked,0 , _Q_LOCKED_VAL) ==0 )) { -qstat_inc (qstat_pv_lock_stealing, true); +lockevent_inc (pv_lock_stealing);return true; }if (!(val & _Q_TAIL_MASK) || (val & _Q_PENDING_MASK))
@@ -219,7 +219,7 @@ static struct qspinlock **pv_hash(struct qspinlock *lock, struct pv_node *node) hopcnt++;if (!cmpxchg (&he->lock,NULL , lock)) {WRITE_ONCE (he->node, node); -qstat_hop (hopcnt); +lockevent_pv_hop (hopcnt);return &he->lock; } }
@@ -320,8 +320,8 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)smp_store_mb (pn->state, vcpu_halted);if (!READ_ONCE (node->locked)) { -qstat_inc (qstat_pv_wait_node, true); -qstat_inc (qstat_pv_wait_early, wait_early); +lockevent_inc (pv_wait_node); +lockevent_cond_inc (pv_wait_early, wait_early);pv_wait (&pn->state, vcpu_halted); }
@@ -339,7 +339,8 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)* So it is better to spin for a while in the hope that the * MCS lock will be released soon. */ -qstat_inc (qstat_pv_spurious_wakeup, !READ_ONCE (node->locked)); +lockevent_cond_inc (pv_spurious_wakeup, + !READ_ONCE (node->locked)); }/*
@@ -416,7 +417,7 @@ static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node)/* * Tracking # of slowpath locking operations */ -qstat_inc (qstat_lock_slowpath, true); +lockevent_inc (lock_slowpath);for (;; waitcnt++) {/*
@@ -464,8 +465,8 @@ static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node) } }WRITE_ONCE (pn->state, vcpu_hashed); -qstat_inc (qstat_pv_wait_head, true); -qstat_inc (qstat_pv_wait_again, waitcnt); +lockevent_inc (pv_wait_head); +lockevent_cond_inc (pv_wait_again, waitcnt);pv_wait (&lock->locked, _Q_SLOW_VAL);/*
@@ -528,7 +529,7 @@ static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node)* vCPU is harmless other than the additional latency in completing * the unlock. */ -qstat_inc (qstat_pv_kick_unlock, true); +lockevent_inc (pv_kick_unlock);pv_kick (node->cpu); }
kernel/locking/qspinlock_stat.h +62 -101
301 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH v2 8/10 locking/lock_events: Make lock_events available for all archs & other locks
In reply to Waiman LongThe QUEUED_LOCK_STAT option to report queued spinlocks event counts was previously allowed only on x86 architecture. To make the locking event counting code more useful, it is now renamed to a more generic LOCK_EVENT_COUNTS config option. This new option will be available to all the architectures that use qspinlock at the moment.
Other locking code can now start to use the generic locking event counting code by including lock_events.h and put the new locking event names into the lock_events_list.h header file.
My experience with lock event counting is that it gives valuable insight on how the locking code works and what can be done to make it better. I would like to extend this benefit to other locking code like mutex and rwsem in the near future.
The PV qspinlock specific code will stay in qspinlock_stat.h. The locking event counters will now reside in the <debugfs>/lock_event_counts directory.
--- arch/Kconfig | 10 +++ arch/x86/Kconfig | 8 --- kernel/locking/Makefile | 1 + kernel/locking/lock_events.c | 153 ++++++++++++++++++++++++++++++++++++++++ kernel/locking/lock_events.h | 10 ++- kernel/locking/qspinlock_stat.h | 141 ++++-------------------------------- 6 files changed, 183 insertions(+), 140 deletions(-) create mode 100644 kernel/locking/lock_events.c
arch/Kconfig +10 -0
@@ -888,6 +888,16 @@ config HAVE_ARCH_PREL32_RELOCATIONS config ARCH_USE_MEMREMAP_PROT bool +config LOCK_EVENT_COUNTS + bool "Locking event counts collection" + depends on DEBUG_FS + depends on QUEUED_SPINLOCKS + ---help--- + Enable light-weight counting of various locking related events + in the system with minimal performance impact. This reduces + the chance of application behavior change because of timing + differences. The counts are reported via debugfs. + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
arch/x86/Kconfig +0 -8
@@ -781,14 +781,6 @@ config PARAVIRT_SPINLOCKS If you are unsure how to answer this question, answer Y. -config QUEUED_LOCK_STAT - bool "Paravirt queued spinlock statistics" - depends on PARAVIRT_SPINLOCKS && DEBUG_FS - ---help--- - Enable the collection of statistical data on the slowpath - behavior of paravirtualized queued spinlocks and report - them on debugfs. - source "arch/x86/xen/Kconfig" config KVM_GUEST
kernel/locking/Makefile +1 -0
@@ -28,3 +28,4 @@ obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o obj-$(CONFIG_QUEUED_RWLOCKS) += qrwlock.o obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o obj-$(CONFIG_WW_MUTEX_SELFTEST) += test-ww_mutex.o +obj-$(CONFIG_LOCK_EVENT_COUNTS) += lock_events.o
kernel/locking/lock_events.c new file +153 -0
153 lines not on this page · raw message
kernel/locking/lock_events.h +7 -3
@@ -13,6 +13,9 @@* Authors: Waiman Long <[email protected]> */ +#ifndef __LOCKING_LOCK_EVENTS_H +#define __LOCKING_LOCK_EVENTS_H +enum lock_events {#include "lock_events_list.h"
@@ -21,7 +24,7 @@ enum lock_events { LOCKEVENT_reset_cnts = lockevent_num, }; -#ifdef CONFIG_QUEUED_LOCK_STAT +#ifdef CONFIG_LOCK_EVENT_COUNTS /* * Per-cpu counters */
@@ -46,10 +49,11 @@ static inline void __lockevent_add(enum lock_events event, int inc)#define lockevent_add (ev, c) __lockevent_add(LOCKEVENT_ ##ev, c) -#else /* CONFIG_QUEUED_LOCK_STAT */ +#else /* CONFIG_LOCK_EVENT_COUNTS */ #define lockevent_inc (ev)#define lockevent_add (ev, c)#define lockevent_cond_inc (ev, c) -#endif /* CONFIG_QUEUED_LOCK_STAT */ +#endif /* CONFIG_LOCK_EVENT_COUNTS */ +#endif /* __LOCKING_LOCK_EVENTS_H */
kernel/locking/qspinlock_stat.h +12 -129
182 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH v2 9/10 locking/lock_events: Don't show pvqspinlock events on bare metal
In reply to Waiman LongOn bare metail, the pvqspinlock event counts will always be 0. So there is no point in showing their corresponding debugfs files. So they are skipped in this case.
--- kernel/locking/lock_events.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-)
kernel/locking/lock_events.c +27 -1
@@ -115,6 +115,29 @@ static ssize_t lockevent_write(struct file *file, const char __user *user_buf, .llseek = default_llseek, }; +#ifdef CONFIG_PARAVIRT_SPINLOCKS +#include <asm/paravirt.h> + +static bool __init skip_lockevent (const char *name) +{ +static int pv_on__initdata =-1 ; + +if (pv_on <0 ) + pv_on = !pv_is_native_spin_unlock (); +/* +* Skip PV qspinlock events on bare metal. +*/ +if (!pv_on && !memcmp (name,"pv_" ,3 )) +return true; +return false; +} +#else +static inline bool skip_lockevent (const char *name) +{ +return false; +} +#endif +/* * Initialize debugfs for the locking event counts. */
@@ -133,10 +156,13 @@ static int __init init_lockevent_counts(void)* root is allowed to do the read/write to limit impact to system * performance. */ -for (i =0 ; i < lockevent_num; i++) +for (i =0 ; i < lockevent_num; i++) { +if (skip_lockevent (lockevent_names[i])) +continue ;if (!debugfs_create_file (lockevent_names[i],0400 , d_counts, (void *)(long )i, &fops_lockevent)) goto fail_undo; + } if (!debugfs_create_file(lockevent_names[LOCKEVENT_reset_cnts], 0200, d_counts, (void *)(long)LOCKEVENT_reset_cnts,
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH v2 10/10 locking/rwsem: Enable lock event counting
In reply to Waiman LongAdd lock event counting calls so that we can track the number of lock events happening in the rwsem code.
With CONFIG_LOCK_EVENT_COUNTS on and booting a 4-socket 112-thread x86-64 system, the rwsem counts after system bootup were as follows:
rwsem_opt_fail=261
rwsem_opt_wlock=50636
rwsem_rlock=445
rwsem_rlock_fail=0
rwsem_rlock_fast=22
rwsem_rtrylock=810144
rwsem_sleep_reader=441
rwsem_sleep_writer=310
rwsem_wake_reader=355
rwsem_wake_writer=2335
rwsem_wlock=261
rwsem_wlock_fail=0
rwsem_wtrylock=20583
It can be seen that most of the lock acquisitions in the slowpath were write-locks in the optimistic spinning code path with no sleeping at all. For this system, over 97% of the locks are acquired via optimistic spinning. It illustrates the importance of optimistic spinning in improving the performance of rwsem.
--- arch/Kconfig | 1 - kernel/locking/lock_events_list.h | 17 +++++++++++++++++ kernel/locking/rwsem-xadd.c | 11 +++++++++++ kernel/locking/rwsem.h | 4 ++++ 4 files changed, 32 insertions(+), 1 deletion(-)
arch/Kconfig +0 -1
@@ -891,7 +891,6 @@ config ARCH_USE_MEMREMAP_PROT config LOCK_EVENT_COUNTS bool "Locking event counts collection" depends on DEBUG_FS - depends on QUEUED_SPINLOCKS ---help--- Enable light-weight counting of various locking related events in the system with minimal performance impact. This reduces
kernel/locking/lock_events_list.h +17 -0
@@ -48,3 +48,20 @@ LOCK_EVENT(lock_use_node4) /* # of locking ops that use 4th percpu node */ LOCK_EVENT(lock_no_node) /* # of locking ops w/o using percpu node */ #endif /* CONFIG_QUEUED_SPINLOCKS */ + +/* + * Locking events for rwsem + */ +LOCK_EVENT(rwsem_sleep_reader) /* # of reader sleeps */ +LOCK_EVENT(rwsem_sleep_writer) /* # of writer sleeps */ +LOCK_EVENT(rwsem_wake_reader) /* # of reader wakeups */ +LOCK_EVENT(rwsem_wake_writer) /* # of writer wakeups */ +LOCK_EVENT(rwsem_opt_wlock) /* # of write locks opt-spin acquired */ +LOCK_EVENT(rwsem_opt_fail) /* # of failed opt-spinnings */ +LOCK_EVENT(rwsem_rlock) /* # of read locks acquired */ +LOCK_EVENT(rwsem_rlock_fast) /* # of fast read locks acquired */ +LOCK_EVENT(rwsem_rlock_fail) /* # of failed read lock acquisitions */ +LOCK_EVENT(rwsem_rtrylock) /* # of read trylock calls */ +LOCK_EVENT(rwsem_wlock) /* # of write locks acquired */ +LOCK_EVENT(rwsem_wlock_fail) /* # of failed write lock acquisitions */ +LOCK_EVENT(rwsem_wtrylock) /* # of write trylock calls */
kernel/locking/rwsem-xadd.c +11 -0
@@ -147,6 +147,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,* will notice the queued writer. */ wake_q_add (wake_q, waiter->task); +lockevent_inc (rwsem_wake_writer); }return ;
@@ -214,6 +215,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem, } adjustment = woken *RWSEM_ACTIVE_READ_BIAS - adjustment; +lockevent_cond_inc (rwsem_wake_reader, woken);if (list_empty (&sem->wait_list)) {/* hit end of list above */ adjustment -=RWSEM_WAITING_BIAS ;
@@ -265,6 +267,7 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)if (atomic_long_try_cmpxchg_acquire (&sem->count, &count, count +RWSEM_ACTIVE_WRITE_BIAS )) {rwsem_set_owner (sem); +lockevent_inc (rwsem_opt_wlock);return true; } }
@@ -389,6 +392,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)osq_unlock (&sem->osq); done:preempt_enable (); +lockevent_cond_inc (rwsem_opt_fail, !taken);return taken; }
@@ -436,6 +440,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)if (atomic_long_read (&sem->count) >=0 ) {raw_spin_unlock_irq (&sem->wait_lock);rwsem_set_reader_owned (sem); +lockevent_inc (rwsem_rlock_fast);return sem; } adjustment +=RWSEM_WAITING_BIAS ;
@@ -472,9 +477,11 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)break ; }schedule (); +lockevent_inc (rwsem_sleep_reader); }__set_current_state (TASK_RUNNING ); +lockevent_inc (rwsem_rlock);return sem; out_nolock:list_del (&waiter.list);
@@ -482,6 +489,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)atomic_long_add (-RWSEM_WAITING_BIAS , &sem->count);raw_spin_unlock_irq (&sem->wait_lock);__set_current_state (TASK_RUNNING ); +lockevent_inc (rwsem_rlock_fail);return ERR_PTR (-EINTR ); }
@@ -575,6 +583,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem) goto out_nolock;schedule (); +lockevent_inc (rwsem_sleep_writer);set_current_state (state); }while ((count =atomic_long_read (&sem->count)) &RWSEM_ACTIVE_MASK );
@@ -583,6 +592,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)__set_current_state (TASK_RUNNING );list_del (&waiter.list);raw_spin_unlock_irq (&sem->wait_lock); +lockevent_inc (rwsem_wlock);return ret;
@@ -596,6 +606,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)__rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q);raw_spin_unlock_irq (&sem->wait_lock);wake_up_q (&wake_q); +lockevent_inc (rwsem_wlock_fail);return ERR_PTR (-EINTR ); }
kernel/locking/rwsem.h +4 -0
@@ -23,6 +23,8 @@* is involved. Ideally we would like to track all the readers that own * a rwsem, but the overhead is simply too big. */ +#include "lock_events.h" +#define RWSEM_READER_OWNED (1UL << 0)#define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
@@ -198,6 +200,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)*/ long tmp =RWSEM_UNLOCKED_VALUE ; +lockevent_inc (rwsem_rtrylock); do { if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, tmp + RWSEM_ACTIVE_READ_BIAS)) {
@@ -239,6 +242,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) {long tmp; +lockevent_inc (rwsem_wtrylock); tmp =atomic_long_cmpxchg_acquire (&sem->count,RWSEM_UNLOCKED_VALUE ,RWSEM_ACTIVE_WRITE_BIAS );if (tmp ==RWSEM_UNLOCKED_VALUE ) {
Signature · 1 line
1.8.3.1
Quoted 26 lines · Waiman Long
v2:
- Sync up to v4 of the part 0 patch.
- Remove the rwsem.h->rwsem-xadd.h renaming patch & change patches to modifyrwsem.hinstead ofrwsem-xadd.h.
- Add a new patch to micro-optimizerwsem_try_read_lock_unqueued().This is part 1 of a 3-part (0/1/2) series to rearchitect the internal operation of rwsem. This depends on the v4 part 0 patches sent out previously
https://lore.kernel.org/lkml/[email protected]
This part lays the foundation for part 2 without making any functional changes. This part includes the following changes:
1) Move code around and micro-optimize
rwsem_try_read_lock_unqueued()(patches 1-4).
2) Enhance theDEBUG_RWSEMS_WARN_ON()macro to provide more information and add additional checks (patches 5 & 6).
3) Make the coreqspinlock_stat.hcode generic (lock event counting) so that it can be used by all the architectures as well as other locking subsystems such as rwsem (patches 7-10). Lock event counting help us visualize how frequently a code path is being used as well as spotting abnormal behavior due to bugs in the code without noticeably affecting kernel performance and hence behavior.
Quoted 1 line
Both (2) and (3) are useful debugging aids.
Yes, this will come in handy in the future. Feel free to add my:
Thanks.