locking/rwsem: Rwsem rearchitecture part 1
Subsystems (2)
Waiman Long
reviewercover locking/rwsem: Rwsem rearchitecture part 1
This is part 1 of a 3-part (0/1/2) series to rearchitect the internal operation of rwsem. This depends on the part 0 patches sent out previously
https://lore.kernel.org/lkml/[email protected]
This part lays the foundation for part 2 without making any functional changes. This part includes the following changes:
1) Move code around and rename kernel/locking/rwsem.h to kernel/locking/rwsem-xadd.h as it will be specific to rwsem-xadd.c (patches 1-4).
2) Enhance the DEBUG_RWSEMS_WARN_ON() macro to provide more information and add additional checks (patches 5 & 6).
3) Make the core qspinlock_stat.h code generic (lock event counting) so that it can be used by all the architectures as well as other locking subsystems such as rwsem (patches 7-10). Lock event counting help us visualize how frequently a code path is being used as well as spotting abnormal behavior due to bugs in the code without noticeably affecting kernel performance and hence behavior.
Both (2) and (3) are useful debugging aids.
Waiman Long (10):
locking/rwsem: Relocate rwsem_down_read_failed() locking/rwsem: Move owner setting code from rwsem.c to rwsem.h locking/rwsem: Rename kernel/locking/rwsem.h locking/rwsem: Move rwsem internal function declarations to
rwsem-xadd.h
locking/rwsem: Add debug check for __down_read*() locking/rwsem: Enhance DEBUG_RWSEMS_WARN_ON() macro locking/qspinlock_stat: Introduce a generic lockevent counting APIs locking/lock_events: Make lock_events available for all archs & other
locks
locking/lock_events: Don't show pvqspinlock events on bare metal locking/rwsem: Enable lock event counting
arch/Kconfig | 10 ++ arch/x86/Kconfig | 8 - include/linux/rwsem.h | 7 - kernel/locking/Makefile | 1 + kernel/locking/lock_events.c | 179 ++++++++++++++++++++++ kernel/locking/lock_events.h | 55 +++++++ kernel/locking/lock_events_list.h | 67 ++++++++ kernel/locking/percpu-rwsem.c | 4 +- kernel/locking/qspinlock.c | 8 +- kernel/locking/qspinlock_paravirt.h | 19 +-- kernel/locking/qspinlock_stat.h | 242 ++++++----------------------- kernel/locking/rwsem-xadd.c | 192 ++++++++++++----------- kernel/locking/rwsem-xadd.h | 294 ++++++++++++++++++++++++++++++++++++ kernel/locking/rwsem.c | 32 ++-- kernel/locking/rwsem.h | 264 -------------------------------- 15 files changed, 778 insertions(+), 604 deletions(-) create mode 100644 kernel/locking/lock_events.c create mode 100644 kernel/locking/lock_events.h create mode 100644 kernel/locking/lock_events_list.h create mode 100644 kernel/locking/rwsem-xadd.h delete mode 100644 kernel/locking/rwsem.h
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 1/10 locking/rwsem: Relocate rwsem_down_read_failed()
The rwsem_down_read_failed*() functions were relocted from above the optimistic spinning section to below that section. This enables the reader functions to use optimisitic spinning in future patches. There is no code change.
--- kernel/locking/rwsem-xadd.c | 172 ++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 86 deletions(-)
kernel/locking/rwsem-xadd.c +86 -86
@@ -225,92 +225,6 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem, }/* -* Wait for the read lock to be granted -*/ -static inline struct rw_semaphore __sched * -__rwsem_down_read_failed_common (struct rw_semaphore *sem,int state) -{ -long count, adjustment = -RWSEM_ACTIVE_READ_BIAS ; -struct rwsem_waiter waiter; -DEFINE_WAKE_Q (wake_q); - - waiter.task = current; - waiter.type =RWSEM_WAITING_FOR_READ ; - -raw_spin_lock_irq (&sem->wait_lock); -if (list_empty (&sem->wait_list)) { -/* -* In case the wait queue is empty and the lock isn't owned -* by a writer, this reader can exit the slowpath and return -* immediately as its RWSEM_ACTIVE_READ_BIAS has already -* been set in the count. -*/ -if (atomic_long_read (&sem->count) >=0 ) { -raw_spin_unlock_irq (&sem->wait_lock); -return sem; - } - adjustment +=RWSEM_WAITING_BIAS ; - } -list_add_tail (&waiter.list, &sem->wait_list); - -/* we're now waiting on the lock, but no longer actively locking */ - count =atomic_long_add_return (adjustment, &sem->count); - -/* -* If there are no active locks, wake the front queued process(es). -* -* If there are no writers and we are first in the queue, -* wake our own waiter to join the existing active readers ! -*/ -if (count ==RWSEM_WAITING_BIAS || - (count >RWSEM_WAITING_BIAS && - adjustment != -RWSEM_ACTIVE_READ_BIAS )) -__rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); - -raw_spin_unlock_irq (&sem->wait_lock); -wake_up_q (&wake_q); - -/* wait to be given the lock */ -while (true) { -set_current_state (state); -if (!waiter.task) -break ; -if (signal_pending_state (state, current)) { -raw_spin_lock_irq (&sem->wait_lock); -if (waiter.task) - goto out_nolock; -raw_spin_unlock_irq (&sem->wait_lock); -break ; - } -schedule (); - } - -__set_current_state (TASK_RUNNING ); -return sem; -out_nolock: -list_del (&waiter.list); -if (list_empty (&sem->wait_list)) -atomic_long_add (-RWSEM_WAITING_BIAS , &sem->count); -raw_spin_unlock_irq (&sem->wait_lock); -__set_current_state (TASK_RUNNING ); -return ERR_PTR (-EINTR ); -} - -__visible struct rw_semaphore * __sched -rwsem_down_read_failed(struct rw_semaphore *sem) -{ -return __rwsem_down_read_failed_common (sem,TASK_UNINTERRUPTIBLE ); -} -EXPORT_SYMBOL (rwsem_down_read_failed); - -__visible struct rw_semaphore * __sched -rwsem_down_read_failed_killable(struct rw_semaphore *sem) -{ -return __rwsem_down_read_failed_common (sem,TASK_KILLABLE ); -} -EXPORT_SYMBOL (rwsem_down_read_failed_killable); - -/* * This function must be called with the sem->wait_lock held to prevent * race conditions between checking the rwsem wait list and setting the * sem->count accordingly.
@@ -505,6 +419,92 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)#endif /* +* Wait for the read lock to be granted +*/ +static inline struct rw_semaphore __sched * +__rwsem_down_read_failed_common (struct rw_semaphore *sem,int state) +{ +long count, adjustment = -RWSEM_ACTIVE_READ_BIAS ; +struct rwsem_waiter waiter; +DEFINE_WAKE_Q (wake_q); + + waiter.task = current; + waiter.type =RWSEM_WAITING_FOR_READ ; + +raw_spin_lock_irq (&sem->wait_lock); +if (list_empty (&sem->wait_list)) { +/* +* In case the wait queue is empty and the lock isn't owned +* by a writer, this reader can exit the slowpath and return +* immediately as its RWSEM_ACTIVE_READ_BIAS has already +* been set in the count. +*/ +if (atomic_long_read (&sem->count) >=0 ) { +raw_spin_unlock_irq (&sem->wait_lock); +return sem; + } + adjustment +=RWSEM_WAITING_BIAS ; + } +list_add_tail (&waiter.list, &sem->wait_list); + +/* we're now waiting on the lock, but no longer actively locking */ + count =atomic_long_add_return (adjustment, &sem->count); + +/* +* If there are no active locks, wake the front queued process(es). +* +* If there are no writers and we are first in the queue, +* wake our own waiter to join the existing active readers ! +*/ +if (count ==RWSEM_WAITING_BIAS || + (count >RWSEM_WAITING_BIAS && + adjustment != -RWSEM_ACTIVE_READ_BIAS )) +__rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q); + +raw_spin_unlock_irq (&sem->wait_lock); +wake_up_q (&wake_q); + +/* wait to be given the lock */ +while (true) { +set_current_state (state); +if (!waiter.task) +break ; +if (signal_pending_state (state, current)) { +raw_spin_lock_irq (&sem->wait_lock); +if (waiter.task) + goto out_nolock; +raw_spin_unlock_irq (&sem->wait_lock); +break ; + } +schedule (); + } + +__set_current_state (TASK_RUNNING ); +return sem; +out_nolock: +list_del (&waiter.list); +if (list_empty (&sem->wait_list)) +atomic_long_add (-RWSEM_WAITING_BIAS , &sem->count); +raw_spin_unlock_irq (&sem->wait_lock); +__set_current_state (TASK_RUNNING ); +return ERR_PTR (-EINTR ); +} + +__visible struct rw_semaphore * __sched +rwsem_down_read_failed(struct rw_semaphore *sem) +{ +return __rwsem_down_read_failed_common (sem,TASK_UNINTERRUPTIBLE ); +} +EXPORT_SYMBOL (rwsem_down_read_failed); + +__visible struct rw_semaphore * __sched +rwsem_down_read_failed_killable(struct rw_semaphore *sem) +{ +return __rwsem_down_read_failed_common (sem,TASK_KILLABLE ); +} +EXPORT_SYMBOL (rwsem_down_read_failed_killable); + +/* * Wait until we successfully acquire the write lock */ static inline struct rw_semaphore *
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 2/10 locking/rwsem: Move owner setting code from rwsem.c to rwsem.h
In reply to Waiman LongThe setting of owner field is specific to rwsem-xadd code, it is not needed for rwsem-spinlock. This patch moves all the owner setting code closer to the rwsem-xadd fast paths directly within rwsem.h file.
For __down_read() and __down_read_killable(), it is assumed that rwsem will be marked as reader-owned when the functions return. That is currently true except in the transient case that the waiter queue just becomes empty. So a rwsem_set_reader_owned() call is added for this case. The __rwsem_set_reader_owned() call in __rwsem_mark_wake() is now necessary.
--- kernel/locking/rwsem-xadd.c | 6 +++--- kernel/locking/rwsem.c | 19 ++----------------- kernel/locking/rwsem.h | 17 +++++++++++++++-- 3 files changed, 20 insertions(+), 22 deletions(-)
kernel/locking/rwsem-xadd.c +3 -3
@@ -176,9 +176,8 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem, goto try_reader_grant; }/* -* It is not really necessary to set it to reader-owned here, -* but it gives the spinners an early indication that the -* readers now have the lock. +* Set it to reader-owned to give spinners an early +* indication that readers now have the lock. */ __rwsem_set_reader_owned (sem, waiter->task); }
@@ -441,6 +440,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)*/ if (atomic_long_read (&sem->count) >=0 ) {raw_spin_unlock_irq (&sem->wait_lock); +rwsem_set_reader_owned (sem);return sem; } adjustment +=RWSEM_WAITING_BIAS ;
kernel/locking/rwsem.c +2 -17
@@ -24,7 +24,6 @@ void __sched down_read(struct rw_semaphore *sem)rwsem_acquire_read (&sem->dep_map,0 ,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_read_trylock, __down_read); -rwsem_set_reader_owned (sem); }EXPORT_SYMBOL (down_read);
@@ -39,7 +38,6 @@ int __sched down_read_killable(struct rw_semaphore *sem)return -EINTR ; } -rwsem_set_reader_owned (sem);return 0 ; }
@@ -52,10 +50,8 @@ int down_read_trylock(struct rw_semaphore *sem) {int ret =__down_read_trylock (sem); -if (ret ==1 ) { +if (ret ==1 )rwsem_acquire_read (&sem->dep_map,0 ,1 , _RET_IP_); -rwsem_set_reader_owned (sem); - }return ret; }
@@ -70,7 +66,6 @@ void __sched down_write(struct rw_semaphore *sem)rwsem_acquire (&sem->dep_map,0 ,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_write_trylock, __down_write); -rwsem_set_owner (sem); }EXPORT_SYMBOL (down_write);
@@ -88,7 +83,6 @@ int __sched down_write_killable(struct rw_semaphore *sem)return -EINTR ; } -rwsem_set_owner (sem);return 0 ; }
@@ -101,10 +95,8 @@ int down_write_trylock(struct rw_semaphore *sem) {int ret =__down_write_trylock (sem); -if (ret ==1 ) { +if (ret ==1 )rwsem_acquire (&sem->dep_map,0 ,1 , _RET_IP_); -rwsem_set_owner (sem); - }return ret; }
@@ -119,7 +111,6 @@ void up_read(struct rw_semaphore *sem)rwsem_release (&sem->dep_map,1 , _RET_IP_);DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED )); -rwsem_clear_reader_owned (sem);__up_read (sem); }
@@ -133,7 +124,6 @@ void up_write(struct rw_semaphore *sem)rwsem_release (&sem->dep_map,1 , _RET_IP_);DEBUG_RWSEMS_WARN_ON (sem->owner != current); -rwsem_clear_owner (sem);__up_write (sem); }
@@ -147,7 +137,6 @@ void downgrade_write(struct rw_semaphore *sem)lock_downgrade (&sem->dep_map, _RET_IP_);DEBUG_RWSEMS_WARN_ON (sem->owner != current); -rwsem_set_reader_owned (sem);__downgrade_write (sem); }
@@ -161,7 +150,6 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)rwsem_acquire_read (&sem->dep_map, subclass,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_read_trylock, __down_read); -rwsem_set_reader_owned (sem); }EXPORT_SYMBOL (down_read_nested);
@@ -172,7 +160,6 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)rwsem_acquire_nest (&sem->dep_map,0 ,0 , nest, _RET_IP_);LOCK_CONTENDED (sem, __down_write_trylock, __down_write); -rwsem_set_owner (sem); }EXPORT_SYMBOL (_down_write_nest_lock);
@@ -193,7 +180,6 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)rwsem_acquire (&sem->dep_map, subclass,0 , _RET_IP_);LOCK_CONTENDED (sem, __down_write_trylock, __down_write); -rwsem_set_owner (sem); }EXPORT_SYMBOL (down_write_nested);
@@ -208,7 +194,6 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)return -EINTR ; } -rwsem_set_owner (sem);return 0 ; }
kernel/locking/rwsem.h +15 -2
@@ -161,6 +161,8 @@ static inline void __down_read(struct rw_semaphore *sem) {if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 ))rwsem_down_read_failed (sem); +else +rwsem_set_reader_owned (sem); }static inline int __down_read_killable (struct rw_semaphore *sem)
@@ -168,8 +170,9 @@ static inline int __down_read_killable(struct rw_semaphore *sem)if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {if (IS_ERR (rwsem_down_read_failed_killable (sem)))return -EINTR ; + }else { +rwsem_set_reader_owned (sem); } -return 0 ; }
@@ -180,6 +183,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)while (tmp >=0 ) {if (atomic_long_try_cmpxchg_acquire (&sem->count, &tmp, tmp +RWSEM_ACTIVE_READ_BIAS )) { +rwsem_set_reader_owned (sem);return 1 ; } }
@@ -197,6 +201,7 @@ static inline void __down_write(struct rw_semaphore *sem) &sem->count);if (unlikely (tmp !=RWSEM_ACTIVE_WRITE_BIAS ))rwsem_down_write_failed (sem); +rwsem_set_owner (sem); }static inline int __down_write_killable (struct rw_semaphore *sem)
@@ -208,6 +213,7 @@ static inline int __down_write_killable(struct rw_semaphore *sem)if (unlikely (tmp !=RWSEM_ACTIVE_WRITE_BIAS ))if (IS_ERR (rwsem_down_write_failed_killable (sem)))return -EINTR ; +rwsem_set_owner (sem);return 0 ; }
@@ -217,7 +223,11 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) tmp =atomic_long_cmpxchg_acquire (&sem->count,RWSEM_UNLOCKED_VALUE ,RWSEM_ACTIVE_WRITE_BIAS ); -return tmp ==RWSEM_UNLOCKED_VALUE ; +if (tmp ==RWSEM_UNLOCKED_VALUE ) { +rwsem_set_owner (sem); +return true; + } +return false; }/*
@@ -227,6 +237,7 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; +rwsem_clear_reader_owned (sem); tmp =atomic_long_dec_return_release (&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))rwsem_wake (sem);
@@ -237,6 +248,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { +rwsem_clear_owner (sem);if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , &sem->count) <0 ))rwsem_wake (sem);
@@ -257,6 +269,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)* write side. As such, rely on RELEASE semantics. */ tmp =atomic_long_add_return_release (-RWSEM_WAITING_BIAS , &sem->count); +rwsem_set_reader_owned (sem);if (tmp <0 )rwsem_downgrade_wake (sem); }
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 3/10 locking/rwsem: Rename kernel/locking/rwsem.h
In reply to Waiman LongThe content of kernel/locking/rwsem.h is now specific to rwsem-xadd only. Rename it to rwsem-xadd.h to indicate that it is specific to rwsem-xadd and include it only when CONFIG_RWSEM_XCHGADD_ALGORITHM is set. As a result, the CONFIG_RWSEM_XCHGADD_ALGORITHM conditional compilation directives can be removed. There is no functional change.
--- kernel/locking/percpu-rwsem.c | 4 +- kernel/locking/rwsem-xadd.c | 2 +- kernel/locking/rwsem-xadd.h | 274 +++++++++++++++++++++++++++++++++++++++++ kernel/locking/rwsem.c | 7 +- kernel/locking/rwsem.h | 277 ------------------------------------------ 5 files changed, 284 insertions(+), 280 deletions(-) create mode 100644 kernel/locking/rwsem-xadd.h delete mode 100644 kernel/locking/rwsem.h
kernel/locking/percpu-rwsem.c +3 -1
@@ -7,7 +7,9 @@#include <linux/sched.h> #include <linux/errno.h> -#include "rwsem.h" +#ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM +#include "rwsem-xadd.h" +#endif int __percpu_init_rwsem (struct percpu_rw_semaphore *sem,const char *name,struct lock_class_key *rwsem_key)
kernel/locking/rwsem-xadd.c +1 -1
@@ -19,7 +19,7 @@#include <linux/sched/debug.h> #include <linux/osq_lock.h> -#include "rwsem.h" +#include "rwsem-xadd.h" /* * Guide to the rw_semaphore's count field for common values.
kernel/locking/rwsem-xadd.h new file +274 -0
274 lines not on this page · raw message
kernel/locking/rwsem.c +6 -1
@@ -13,7 +13,12 @@#include <linux/rwsem.h> #include <linux/atomic.h> -#include "rwsem.h" +#ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM +#include "rwsem-xadd.h" +#else +#define __rwsem_set_reader_owned (a, b) +#define DEBUG_RWSEMS_WARN_ON (a) +#endif /* * lock for reading
kernel/locking/rwsem.h deleted +0 -277
277 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Peter Zijlstra
maintainerRe: [PATCH 03/10] locking/rwsem: Rename kernel/locking/rwsem.h
Quoted 5 lines · Waiman Long
The content of
kernel/locking/rwsem.his now specific to rwsem-xadd only. Rename it torwsem-xadd.hto indicate that it is specific to rwsem-xadd and include it only whenCONFIG_RWSEM_XCHGADD_ALGORITHMis set. As a result, theCONFIG_RWSEM_XCHGADD_ALGORITHMconditional compilation directives can be removed. There is no functional change.
Since all of rwsem-xadd is now generic code; how about we delete the spinlock thing and keep everything rwsem ?
We don't carry a special spinlock mutex implementation either. And arguably any arch that uses spinlock based atomics (afaict the only case where rwsem-spinlock makes any sense anyway) suck anyway.
Waiman Long
reviewerRe: [PATCH 03/10] locking/rwsem: Rename kernel/locking/rwsem.h
Quoted 6 lines · Peter Zijlstra
Quoted 5 lines · Waiman Long
The content of
kernel/locking/rwsem.his now specific to rwsem-xadd only. Rename it torwsem-xadd.hto indicate that it is specific to rwsem-xadd and include it only whenCONFIG_RWSEM_XCHGADD_ALGORITHMis set. As a result, theCONFIG_RWSEM_XCHGADD_ALGORITHMconditional compilation directives can be removed. There is no functional change.
Quoted 6 lines
Since all of rwsem-xadd is now generic code; how about we delete the spinlock thing and keep everything rwsem ?
We don't carry a special spinlock mutex implementation either. And arguably any arch that uses spinlock based atomics (afaict the only case where rwsem-spinlock makes any sense anyway) suck anyway.
I don't mind removing the rwsem-spinlock code and have just one implementation for all as long as there is no objection from others. I don't know the history of why we have 2 implementations of rwsem and so I didn't plan to do that.
Cheers,
Longman
Peter Zijlstra
maintainerRe: [PATCH 03/10] locking/rwsem: Rename kernel/locking/rwsem.h
Quoted 10 lines · Waiman Long
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Waiman Long
The content of
kernel/locking/rwsem.his now specific to rwsem-xadd only. Rename it torwsem-xadd.hto indicate that it is specific to rwsem-xadd and include it only whenCONFIG_RWSEM_XCHGADD_ALGORITHMis set. As a result, theCONFIG_RWSEM_XCHGADD_ALGORITHMconditional compilation directives can be removed. There is no functional change.Since all of rwsem-xadd is now generic code; how about we delete the spinlock thing and keep everything rwsem ?
Quoted 8 lines
Quoted 3 lines
We don't carry a special spinlock mutex implementation either. And arguably any arch that uses spinlock based atomics (afaict the only case where rwsem-spinlock makes any sense anyway) suck anyway.
I don't mind removing the rwsem-spinlock code and have just one implementation for all as long as there is no objection from others. I don't know the history of why we have 2 implementations of rwsem and so I didn't plan to do that.
It is from before my time too; but I think rwsem-spinlock is generic and didn't require arch asm helpers, where rwsem-xadd did, but we just fixed that :-)
Waiman Long
reviewerRe: [PATCH 03/10] locking/rwsem: Rename kernel/locking/rwsem.h
Quoted 14 lines · Peter Zijlstra
Quoted 13 lines · Waiman Long
Quoted 12 lines · Peter Zijlstra
Quoted 5 lines · Waiman Long
The content of
kernel/locking/rwsem.his now specific to rwsem-xadd only. Rename it torwsem-xadd.hto indicate that it is specific to rwsem-xadd and include it only whenCONFIG_RWSEM_XCHGADD_ALGORITHMis set. As a result, theCONFIG_RWSEM_XCHGADD_ALGORITHMconditional compilation directives can be removed. There is no functional change.Since all of rwsem-xadd is now generic code; how about we delete the spinlock thing and keep everything rwsem ?
We don't carry a special spinlock mutex implementation either. And arguably any arch that uses spinlock based atomics (afaict the only case where rwsem-spinlock makes any sense anyway) suck anyway.
Quoted 7 lines
Quoted 4 lines
I don't mind removing the rwsem-spinlock code and have just one implementation for all as long as there is no objection from others. I don't know the history of why we have 2 implementations of rwsem and so I didn't plan to do that.
It is from before my time too; but I think rwsem-spinlock is generic and didn't require arch asm helpers, where rwsem-xadd did, but we just fixed that :-)
OK, I will attempt to send a patch to remove rwsem-spinlock. That will touch a lot of arch specific files too.
Cheers,
Longman
Waiman Long
reviewerRe: [PATCH 03/10] locking/rwsem: Rename kernel/locking/rwsem.h
In reply to Peter ZijlstraQuoted 14 lines · Peter Zijlstra
Quoted 13 lines · Waiman Long
Quoted 12 lines · Peter Zijlstra
Quoted 5 lines · Waiman Long
The content of
kernel/locking/rwsem.his now specific to rwsem-xadd only. Rename it torwsem-xadd.hto indicate that it is specific to rwsem-xadd and include it only whenCONFIG_RWSEM_XCHGADD_ALGORITHMis set. As a result, theCONFIG_RWSEM_XCHGADD_ALGORITHMconditional compilation directives can be removed. There is no functional change.Since all of rwsem-xadd is now generic code; how about we delete the spinlock thing and keep everything rwsem ?
We don't carry a special spinlock mutex implementation either. And arguably any arch that uses spinlock based atomics (afaict the only case where rwsem-spinlock makes any sense anyway) suck anyway.
Quoted 7 lines
Quoted 4 lines
I don't mind removing the rwsem-spinlock code and have just one implementation for all as long as there is no objection from others. I don't know the history of why we have 2 implementations of rwsem and so I didn't plan to do that.
It is from before my time too; but I think rwsem-spinlock is generic and didn't require arch asm helpers, where rwsem-xadd did, but we just fixed that :-)
OK, I will attempt to send a patch to remove rwsem-spinlock. That will touch a lot of arch specific files too.
Cheers,
Longman
Waiman Long
reviewerRe: [PATCH 03/10] locking/rwsem: Rename kernel/locking/rwsem.h
In reply to Peter ZijlstraQuoted 14 lines · Peter Zijlstra
Quoted 13 lines · Waiman Long
Quoted 12 lines · Peter Zijlstra
Quoted 5 lines · Waiman Long
The content of
kernel/locking/rwsem.his now specific to rwsem-xadd only. Rename it torwsem-xadd.hto indicate that it is specific to rwsem-xadd and include it only whenCONFIG_RWSEM_XCHGADD_ALGORITHMis set. As a result, theCONFIG_RWSEM_XCHGADD_ALGORITHMconditional compilation directives can be removed. There is no functional change.Since all of rwsem-xadd is now generic code; how about we delete the spinlock thing and keep everything rwsem ?
We don't carry a special spinlock mutex implementation either. And arguably any arch that uses spinlock based atomics (afaict the only case where rwsem-spinlock makes any sense anyway) suck anyway.
Quoted 7 lines
Quoted 4 lines
I don't mind removing the rwsem-spinlock code and have just one implementation for all as long as there is no objection from others. I don't know the history of why we have 2 implementations of rwsem and so I didn't plan to do that.
It is from before my time too; but I think rwsem-spinlock is generic and didn't require arch asm helpers, where rwsem-xadd did, but we just fixed that :-)
OK, I will attempt to send a patch to remove rwsem-spinlock. That will touch a lot of arch specific files too.
Cheers,
Longman
Waiman Long
reviewerPATCH 4/10 locking/rwsem: Move rwsem internal function declarations to rwsem-xadd.h
In reply to Waiman LongWe don't need to expose rwsem internal functions which are not supposed to be called directly from other kernel code.
--- include/linux/rwsem.h | 7 ------- kernel/locking/rwsem-xadd.h | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
include/linux/rwsem.h +0 -7
@@ -50,13 +50,6 @@ struct rw_semaphore {*/ #define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-2L) -extern struct rw_semaphore *rwsem_down_read_failed (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_read_failed_killable (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed_killable (struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake (struct rw_semaphore *); -extern struct rw_semaphore *rwsem_downgrade_wake (struct rw_semaphore *sem); -/* In all implementations count != 0 means locked */ static inline int rwsem_is_locked (struct rw_semaphore *sem) {
kernel/locking/rwsem-xadd.h +7 -0
@@ -153,6 +153,13 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) }#endif +extern struct rw_semaphore *rwsem_down_read_failed (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_down_read_failed_killable (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_down_write_failed (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_down_write_failed_killable (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_wake (struct rw_semaphore *sem); +extern struct rw_semaphore *rwsem_downgrade_wake (struct rw_semaphore *sem); +/* * lock for reading */
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 5/10 locking/rwsem: Add debug check for __down_read*()
In reply to Waiman LongWhen rwsem_down_read_failed*() return, the read lock is acquired indirectly by others. So debug checks are added in __down_read() and __down_read_killable() to make sure the rwsem is really reader-owned.
The other debug check calls in kernel/locking/rwsem.c except the one in up_read_non_owner() are also moved over to rwsem-xadd.h.
--- kernel/locking/rwsem-xadd.h | 12 ++++++++++-- kernel/locking/rwsem.c | 3 --- 2 files changed, 10 insertions(+), 5 deletions(-)
kernel/locking/rwsem-xadd.h +10 -2
@@ -165,10 +165,13 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)*/ static inline void __down_read (struct rw_semaphore *sem) { -if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) +if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {rwsem_down_read_failed (sem); -else +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & +RWSEM_READER_OWNED )); + }else {rwsem_set_reader_owned (sem); + } }static inline int __down_read_killable (struct rw_semaphore *sem)
@@ -176,6 +179,8 @@ static inline int __down_read_killable(struct rw_semaphore *sem)if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {if (IS_ERR (rwsem_down_read_failed_killable (sem)))return -EINTR ; +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & +RWSEM_READER_OWNED )); }else {rwsem_set_reader_owned (sem); }
@@ -243,6 +248,7 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ));rwsem_clear_reader_owned (sem); tmp =atomic_long_dec_return_release (&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))
@@ -254,6 +260,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { +DEBUG_RWSEMS_WARN_ON (sem->owner != current);rwsem_clear_owner (sem);if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , &sem->count) <0 ))
@@ -274,6 +281,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)* read-locked region is ok to be re-ordered into the * write side. As such, rely on RELEASE semantics. */ +DEBUG_RWSEMS_WARN_ON (sem->owner != current); tmp =atomic_long_add_return_release (-RWSEM_WAITING_BIAS , &sem->count);rwsem_set_reader_owned (sem);if (tmp <0 )
kernel/locking/rwsem.c +0 -3
@@ -114,7 +114,6 @@ int down_write_trylock(struct rw_semaphore *sem)void up_read (struct rw_semaphore *sem) {rwsem_release (&sem->dep_map,1 , _RET_IP_); -DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ));__up_read (sem); }
@@ -127,7 +126,6 @@ void up_read(struct rw_semaphore *sem)void up_write (struct rw_semaphore *sem) {rwsem_release (&sem->dep_map,1 , _RET_IP_); -DEBUG_RWSEMS_WARN_ON (sem->owner != current);__up_write (sem); }
@@ -140,7 +138,6 @@ void up_write(struct rw_semaphore *sem)void downgrade_write (struct rw_semaphore *sem) {lock_downgrade (&sem->dep_map, _RET_IP_); -DEBUG_RWSEMS_WARN_ON (sem->owner != current);__downgrade_write (sem); }
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 6/10 locking/rwsem: Enhance DEBUG_RWSEMS_WARN_ON() macro
In reply to Waiman LongCurrently, the DEBUG_RWSEMS_WARN_ON() macro just dumps a stack trace when the rwsem isn't in the right state. It does not show the actual states of the rwsem. This may not be that helpful in the debugging process.
Enhance the DEBUG_RWSEMS_WARN_ON() macro to also show the current content of the rwsem count and owner fields to give more information about what is wrong with the rwsem.
--- kernel/locking/rwsem-xadd.h | 19 ++++++++++++------- kernel/locking/rwsem.c | 5 +++-- 2 files changed, 15 insertions(+), 9 deletions(-)
kernel/locking/rwsem-xadd.h +12 -7
@@ -27,9 +27,13 @@#define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)#ifdef CONFIG_DEBUG_RWSEMS -# define DEBUG_RWSEMS_WARN_ON (c) DEBUG_LOCKS_WARN_ON(c) +# define DEBUG_RWSEMS_WARN_ON (c, sem) \ + WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\ + #c, atomic_long_read(&(sem)->count), \ + (long)((sem)->owner), (long)current, \ + list_empty(&(sem)->wait_list) ? "" : "not ")#else -# define DEBUG_RWSEMS_WARN_ON (c) +# define DEBUG_RWSEMS_WARN_ON (c, sem)#endif /*
@@ -168,7 +172,7 @@ static inline void __down_read(struct rw_semaphore *sem)if (unlikely (atomic_long_inc_return_acquire (&sem->count) <=0 )) {rwsem_down_read_failed (sem);DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & -RWSEM_READER_OWNED )); +RWSEM_READER_OWNED ), sem); }else {rwsem_set_reader_owned (sem); }
@@ -180,7 +184,7 @@ static inline int __down_read_killable(struct rw_semaphore *sem)if (IS_ERR (rwsem_down_read_failed_killable (sem)))return -EINTR ;DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner & -RWSEM_READER_OWNED )); +RWSEM_READER_OWNED ), sem); }else {rwsem_set_reader_owned (sem); }
@@ -248,7 +252,8 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; -DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED )); +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ), + sem);rwsem_clear_reader_owned (sem); tmp =atomic_long_dec_return_release (&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))
@@ -260,7 +265,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { -DEBUG_RWSEMS_WARN_ON (sem->owner != current); +DEBUG_RWSEMS_WARN_ON (sem->owner != current, sem);rwsem_clear_owner (sem);if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , &sem->count) <0 ))
@@ -281,7 +286,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)* read-locked region is ok to be re-ordered into the * write side. As such, rely on RELEASE semantics. */ -DEBUG_RWSEMS_WARN_ON (sem->owner != current); +DEBUG_RWSEMS_WARN_ON (sem->owner != current, sem); tmp =atomic_long_add_return_release (-RWSEM_WAITING_BIAS , &sem->count);rwsem_set_reader_owned (sem);if (tmp <0 )
kernel/locking/rwsem.c +3 -2
@@ -17,7 +17,7 @@#include "rwsem-xadd.h" #else #define __rwsem_set_reader_owned (a, b) -#define DEBUG_RWSEMS_WARN_ON (a) +#define DEBUG_RWSEMS_WARN_ON (a, b)#endif /*
@@ -203,7 +203,8 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)void up_read_non_owner (struct rw_semaphore *sem) { -DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED )); +DEBUG_RWSEMS_WARN_ON (!((unsigned long )sem->owner &RWSEM_READER_OWNED ), + sem);__up_read (sem); }
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 7/10 locking/qspinlock_stat: Introduce a generic lockevent counting APIs
In reply to Waiman LongThe percpu event counts used by qspinlock code can be useful for other locking code as well. So a new set of lockevent_* counting APIs is introduced with the lock event names extracted out into the new lock_events_list.h header file for easier addition in the future.
The existing qstat_inc() calls are replaced by either lockevent_inc() or lockevent_cond_inc() calls.
The qstat_hop() call is renamed to lockevent_pv_hop(). The "reset_counters" debugfs file is also renamed to ".reset_counts".
--- kernel/locking/lock_events.h | 55 ++++++++++++ kernel/locking/lock_events_list.h | 50 +++++++++++ kernel/locking/qspinlock.c | 8 +- kernel/locking/qspinlock_paravirt.h | 19 +++-- kernel/locking/qspinlock_stat.h | 163 ++++++++++++++---------------------- 5 files changed, 181 insertions(+), 114 deletions(-) create mode 100644 kernel/locking/lock_events.h create mode 100644 kernel/locking/lock_events_list.h
kernel/locking/lock_events.h new file +55 -0
@@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* Authors: Waiman Long <[email protected]> +*/ + +enum lock_events { + +#include "lock_events_list.h" + + lockevent_num,/* Total number of lock event counts */ + LOCKEVENT_reset_cnts = lockevent_num, +}; + +#ifdef CONFIG_QUEUED_LOCK_STAT +/* +* Per-cpu counters +*/ +DECLARE_PER_CPU(unsigned long, lockevents[lockevent_num]); + +/* +* Increment the PV qspinlock statistical counters +*/ +static inline void __lockevent_inc (enum lock_events event,bool cond) +{ +if (cond) +__this_cpu_inc (lockevents[event]); +} + +#define lockevent_inc (ev) __lockevent_inc(LOCKEVENT_ ##ev, true) +#define lockevent_cond_inc (ev, c) __lockevent_inc(LOCKEVENT_ ##ev, c) + +static inline void __lockevent_add (enum lock_events event,int inc) +{ +__this_cpu_add (lockevents[event], inc); +} + +#define lockevent_add (ev, c) __lockevent_add(LOCKEVENT_ ##ev, c) + +#else /* CONFIG_QUEUED_LOCK_STAT */ + +#define lockevent_inc (ev) +#define lockevent_add (ev, c) +#define lockevent_cond_inc (ev, c) + +#endif /* CONFIG_QUEUED_LOCK_STAT */
kernel/locking/lock_events_list.h new file +50 -0
50 lines not on this page · raw message
kernel/locking/qspinlock.c +4 -4
@@ -398,7 +398,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)* 0,1,0 -> 0,0,1 */ clear_pending_set_locked (lock); -qstat_inc (qstat_lock_pending, true); +lockevent_inc (lock_pending);return ;/*
@@ -406,7 +406,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)* queuing. */ queue: -qstat_inc (qstat_lock_slowpath, true); +lockevent_inc (lock_slowpath); pv_queue: node =this_cpu_ptr (&qnodes[0 ].mcs); idx = node->count++;
@@ -422,7 +422,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)* simple enough. */ if (unlikely (idx >=MAX_NODES )) { -qstat_inc (qstat_lock_no_node, true); +lockevent_inc (lock_no_node);while (!queued_spin_trylock (lock))cpu_relax (); goto release;
@@ -433,7 +433,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)/* * Keep counts of non-zero index values: */ -qstat_inc (qstat_lock_use_node2 + idx -1 , idx); +lockevent_cond_inc (lock_use_node2 + idx -1 , idx);/* * Ensure that we increment the head node->count before initialising
kernel/locking/qspinlock_paravirt.h +10 -9
@@ -89,7 +89,7 @@ static inline bool pv_hybrid_queued_unfair_trylock(struct qspinlock *lock)if (!(val & _Q_LOCKED_PENDING_MASK) && (cmpxchg_acquire (&lock->locked,0 , _Q_LOCKED_VAL) ==0 )) { -qstat_inc (qstat_pv_lock_stealing, true); +lockevent_inc (pv_lock_stealing);return true; }if (!(val & _Q_TAIL_MASK) || (val & _Q_PENDING_MASK))
@@ -219,7 +219,7 @@ static struct qspinlock **pv_hash(struct qspinlock *lock, struct pv_node *node) hopcnt++;if (!cmpxchg (&he->lock,NULL , lock)) {WRITE_ONCE (he->node, node); -qstat_hop (hopcnt); +lockevent_pv_hop (hopcnt);return &he->lock; } }
@@ -320,8 +320,8 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)smp_store_mb (pn->state, vcpu_halted);if (!READ_ONCE (node->locked)) { -qstat_inc (qstat_pv_wait_node, true); -qstat_inc (qstat_pv_wait_early, wait_early); +lockevent_inc (pv_wait_node); +lockevent_cond_inc (pv_wait_early, wait_early);pv_wait (&pn->state, vcpu_halted); }
@@ -339,7 +339,8 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)* So it is better to spin for a while in the hope that the * MCS lock will be released soon. */ -qstat_inc (qstat_pv_spurious_wakeup, !READ_ONCE (node->locked)); +lockevent_cond_inc (pv_spurious_wakeup, + !READ_ONCE (node->locked)); }/*
@@ -416,7 +417,7 @@ static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node)/* * Tracking # of slowpath locking operations */ -qstat_inc (qstat_lock_slowpath, true); +lockevent_inc (lock_slowpath);for (;; waitcnt++) {/*
@@ -464,8 +465,8 @@ static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node) } }WRITE_ONCE (pn->state, vcpu_hashed); -qstat_inc (qstat_pv_wait_head, true); -qstat_inc (qstat_pv_wait_again, waitcnt); +lockevent_inc (pv_wait_head); +lockevent_cond_inc (pv_wait_again, waitcnt);pv_wait (&lock->locked, _Q_SLOW_VAL);/*
@@ -528,7 +529,7 @@ static void pv_kick_node(struct qspinlock *lock, struct mcs_spinlock *node)* vCPU is harmless other than the additional latency in completing * the unlock. */ -qstat_inc (qstat_pv_kick_unlock, true); +lockevent_inc (pv_kick_unlock);pv_kick (node->cpu); }
kernel/locking/qspinlock_stat.h +62 -101
301 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 8/10 locking/lock_events: Make lock_events available for all archs & other locks
In reply to Waiman LongThe QUEUED_LOCK_STAT option to report queued spinlocks event counts was previously allowed only on x86 architecture. To make the locking event counting code more useful, it is now renamed to a more generic LOCK_EVENT_COUNTS config option. This new option will be available to all the architectures that use qspinlock at the moment.
Other locking code can now start to use the generic locking event counting code by including lock_events.h and put the new locking event names into the lock_events_list.h header file.
My experience with lock event counting is that it gives valuable insight on how the locking code works and what can be done to make it better. I would like to extend this benefit to other locking code like mutex and rwsem in the near future.
The PV qspinlock specific code will stay in qspinlock_stat.h. The locking event counters will now reside in the <debugfs>/lock_event_counts directory.
--- arch/Kconfig | 10 +++ arch/x86/Kconfig | 8 --- kernel/locking/Makefile | 1 + kernel/locking/lock_events.c | 153 ++++++++++++++++++++++++++++++++++++++++ kernel/locking/lock_events.h | 6 +- kernel/locking/qspinlock_stat.h | 141 ++++-------------------------------- 6 files changed, 179 insertions(+), 140 deletions(-) create mode 100644 kernel/locking/lock_events.c
arch/Kconfig +10 -0
@@ -888,6 +888,16 @@ config HAVE_ARCH_PREL32_RELOCATIONS config ARCH_USE_MEMREMAP_PROT bool +config LOCK_EVENT_COUNTS + bool "Locking event counts collection" + depends on DEBUG_FS + depends on QUEUED_SPINLOCKS + ---help--- + Enable light-weight counting of various locking related events + in the system with minimal performance impact. This reduces + the chance of application behavior change because of timing + differences. The counts are reported via debugfs. + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
arch/x86/Kconfig +0 -8
@@ -784,14 +784,6 @@ config PARAVIRT_SPINLOCKS If you are unsure how to answer this question, answer Y. -config QUEUED_LOCK_STAT - bool "Paravirt queued spinlock statistics" - depends on PARAVIRT_SPINLOCKS && DEBUG_FS - ---help--- - Enable the collection of statistical data on the slowpath - behavior of paravirtualized queued spinlocks and report - them on debugfs. - source "arch/x86/xen/Kconfig" config KVM_GUEST
kernel/locking/Makefile +1 -0
@@ -30,3 +30,4 @@ obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o obj-$(CONFIG_QUEUED_RWLOCKS) += qrwlock.o obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o obj-$(CONFIG_WW_MUTEX_SELFTEST) += test-ww_mutex.o +obj-$(CONFIG_LOCK_EVENT_COUNTS) += lock_events.o
kernel/locking/lock_events.c new file +153 -0
153 lines not on this page · raw message
kernel/locking/lock_events.h +3 -3
@@ -21,7 +21,7 @@ enum lock_events { LOCKEVENT_reset_cnts = lockevent_num, }; -#ifdef CONFIG_QUEUED_LOCK_STAT +#ifdef CONFIG_LOCK_EVENT_COUNTS /* * Per-cpu counters */
@@ -46,10 +46,10 @@ static inline void __lockevent_add(enum lock_events event, int inc)#define lockevent_add (ev, c) __lockevent_add(LOCKEVENT_ ##ev, c) -#else /* CONFIG_QUEUED_LOCK_STAT */ +#else /* CONFIG_LOCK_EVENT_COUNTS */ #define lockevent_inc (ev)#define lockevent_add (ev, c)#define lockevent_cond_inc (ev, c) -#endif /* CONFIG_QUEUED_LOCK_STAT */ +#endif /* CONFIG_LOCK_EVENT_COUNTS */
kernel/locking/qspinlock_stat.h +12 -129
182 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 9/10 locking/lock_events: Don't show pvqspinlock events on bare metal
In reply to Waiman LongOn bare metail, the pvqspinlock event counts will always be 0. So there is no point in showing their corresponding debugfs files. So they are skipped in this case.
--- kernel/locking/lock_events.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-)
kernel/locking/lock_events.c +27 -1
@@ -115,6 +115,29 @@ static ssize_t lockevent_write(struct file *file, const char __user *user_buf, .llseek = default_llseek, }; +#ifdef CONFIG_PARAVIRT_SPINLOCKS +#include <asm/paravirt.h> + +static bool __init skip_lockevent (const char *name) +{ +static int pv_on__initdata =-1 ; + +if (pv_on <0 ) + pv_on = !pv_is_native_spin_unlock (); +/* +* Skip PV qspinlock events on bare metal. +*/ +if (!pv_on && !memcmp (name,"pv_" ,3 )) +return true; +return false; +} +#else +static inline bool skip_lockevent (const char *name) +{ +return false; +} +#endif +/* * Initialize debugfs for the locking event counts. */
@@ -133,10 +156,13 @@ static int __init init_lockevent_counts(void)* root is allowed to do the read/write to limit impact to system * performance. */ -for (i =0 ; i < lockevent_num; i++) +for (i =0 ; i < lockevent_num; i++) { +if (skip_lockevent (lockevent_names[i])) +continue ;if (!debugfs_create_file (lockevent_names[i],0400 , d_counts, (void *)(long )i, &fops_lockevent)) goto fail_undo; + } if (!debugfs_create_file(lockevent_names[LOCKEVENT_reset_cnts], 0200, d_counts, (void *)(long)LOCKEVENT_reset_cnts,
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 10/10 locking/rwsem: Enable lock event counting
In reply to Waiman LongAdd lock event counting calls so that we can track the number of lock events happening in the rwsem code.
With CONFIG_LOCK_EVENT_COUNTS on and booting a 4-socket 112-thread x86-64 system, the rwsem counts after system bootup were as follows:
rwsem_opt_fail=634
rwsem_opt_wlock=53735
rwsem_rlock=824
rwsem_rlock_fail=0
rwsem_rlock_fast=20
rwsem_sleep_reader=814
rwsem_sleep_writer=852
rwsem_wake_reader=462
rwsem_wake_writer=3747
rwsem_wlock=634
rwsem_wlock_fail=0
It can be seen that most of the lock acquisitions in the slowpath were write-locks in the optimistic spinning code path with no sleeping at all. For this system, over 97% of the locks are acquired via optimistic spinning. It illustrates the importance of optimistic spinning in improving the performance of rwsem.
--- arch/Kconfig | 2 +- kernel/locking/lock_events_list.h | 17 +++++++++++++++++ kernel/locking/rwsem-xadd.c | 12 ++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-)
arch/Kconfig +1 -1
@@ -891,7 +891,7 @@ config ARCH_USE_MEMREMAP_PROT config LOCK_EVENT_COUNTS bool "Locking event counts collection" depends on DEBUG_FS - depends on QUEUED_SPINLOCKS + depends on (QUEUED_SPINLOCKS || RWSEM_XCHGADD_ALGORITHM) ---help--- Enable light-weight counting of various locking related events in the system with minimal performance impact. This reduces
kernel/locking/lock_events_list.h +17 -0
@@ -48,3 +48,20 @@ LOCK_EVENT(lock_use_node4) /* # of locking ops that use 4th percpu node */ LOCK_EVENT(lock_no_node) /* # of locking ops w/o using percpu node */ #endif /* CONFIG_QUEUED_SPINLOCKS */ + +#ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM +/* + * Locking events for rwsem + */ +LOCK_EVENT(rwsem_sleep_reader) /* # of reader sleeps */ +LOCK_EVENT(rwsem_sleep_writer) /* # of writer sleeps */ +LOCK_EVENT(rwsem_wake_reader) /* # of reader wakeups */ +LOCK_EVENT(rwsem_wake_writer) /* # of writer wakeups */ +LOCK_EVENT(rwsem_opt_wlock) /* # of write locks opt-spin acquired */ +LOCK_EVENT(rwsem_opt_fail) /* # of failed opt-spinnings */ +LOCK_EVENT(rwsem_rlock) /* # of read locks acquired */ +LOCK_EVENT(rwsem_rlock_fast) /* # of fast read locks acquired */ +LOCK_EVENT(rwsem_rlock_fail) /* # of failed read lock acquisitions */ +LOCK_EVENT(rwsem_wlock) /* # of write locks acquired */ +LOCK_EVENT(rwsem_wlock_fail) /* # of failed write lock acquisitions */ +#endif /* CONFIG_RWSEM_XCHGADD_ALGORITHM */
kernel/locking/rwsem-xadd.c +12 -0
@@ -20,6 +20,7 @@#include <linux/osq_lock.h> #include "rwsem-xadd.h" +#include "lock_events.h" /* * Guide to the rw_semaphore's count field for common values.
@@ -147,6 +148,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem,* will notice the queued writer. */ wake_q_add (wake_q, waiter->task); +lockevent_inc (rwsem_wake_writer); }return ;
@@ -214,6 +216,7 @@ static void __rwsem_mark_wake(struct rw_semaphore *sem, } adjustment = woken *RWSEM_ACTIVE_READ_BIAS - adjustment; +lockevent_cond_inc (rwsem_wake_reader, woken);if (list_empty (&sem->wait_list)) {/* hit end of list above */ adjustment -=RWSEM_WAITING_BIAS ;
@@ -269,6 +272,7 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) count + RWSEM_ACTIVE_WRITE_BIAS);if (old == count) {rwsem_set_owner (sem); +lockevent_inc (rwsem_opt_wlock);return true; }
@@ -394,6 +398,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)osq_unlock (&sem->osq); done:preempt_enable (); +lockevent_cond_inc (rwsem_opt_fail, !taken);return taken; }
@@ -441,6 +446,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)if (atomic_long_read (&sem->count) >=0 ) {raw_spin_unlock_irq (&sem->wait_lock);rwsem_set_reader_owned (sem); +lockevent_inc (rwsem_rlock_fast);return sem; } adjustment +=RWSEM_WAITING_BIAS ;
@@ -477,9 +483,11 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)break ; }schedule (); +lockevent_inc (rwsem_sleep_reader); }__set_current_state (TASK_RUNNING ); +lockevent_inc (rwsem_rlock);return sem; out_nolock:list_del (&waiter.list);
@@ -487,6 +495,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)atomic_long_add (-RWSEM_WAITING_BIAS , &sem->count);raw_spin_unlock_irq (&sem->wait_lock);__set_current_state (TASK_RUNNING ); +lockevent_inc (rwsem_rlock_fail);return ERR_PTR (-EINTR ); }
@@ -580,6 +589,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem) goto out_nolock;schedule (); +lockevent_inc (rwsem_sleep_writer);set_current_state (state); }while ((count =atomic_long_read (&sem->count)) &RWSEM_ACTIVE_MASK );
@@ -588,6 +598,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)__set_current_state (TASK_RUNNING );list_del (&waiter.list);raw_spin_unlock_irq (&sem->wait_lock); +lockevent_inc (rwsem_wlock);return ret;
@@ -601,6 +612,7 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)__rwsem_mark_wake (sem,RWSEM_WAKE_ANY , &wake_q);raw_spin_unlock_irq (&sem->wait_lock);wake_up_q (&wake_q); +lockevent_inc (rwsem_wlock_fail);return ERR_PTR (-EINTR ); }
Signature · 1 line
1.8.3.1
Quoted 7 lines · Waiman Long
This is part 1 of a 3-part (0/1/2) series to rearchitect the internal operation of rwsem. This depends on the part 0 patches sent out previously
https://lore.kernel.org/lkml/[email protected]
This part lays the foundation for part 2 without making any functional changes. This part includes the following changes:
Did you post part 2 anywhere? I can't seem to find it for some reason.
Cheers,
Will
Waiman Long
reviewerQuoted 6 lines · Will Deacon
Quoted 5 lines · Waiman Long
This is part 1 of a 3-part (0/1/2) series to rearchitect the internal operation of rwsem. This depends on the part 0 patches sent out previously
Quoted 7 lines
Quoted 2 lines
This part lays the foundation for part 2 without making any functional changes. This part includes the following changes:
Did you post part 2 anywhere? I can't seem to find it for some reason.
Cheers,
Will
I haven't posted the part 2 yet. I will need to repost part 1 to work with part 0 v4.
-Longman