locking/rwlocks: Add contention detection for rwlocks
Subsystems (4)
PATCH 1/3 locking/rwlocks: Add contention detection for rwlocks
rwlocks do not currently have any facility to detect contention like spinlocks do. In order to allow users of rwlocks to better manage latency, add contention detection for queued rwlocks.
--- include/asm-generic/qrwlock.h | 23 +++++++++++++++++------ include/linux/rwlock.h | 7 +++++++ 2 files changed, 24 insertions(+), 6 deletions(-)
include/asm-generic/qrwlock.h +17 -6
@@ -116,15 +116,26 @@ static inline void queued_write_unlock(struct qrwlock *lock) smp_store_release(&lock->wlocked, 0); } +/** +* queued_rwlock_is_contended - check if the lock is contended +* @lock : Pointer to queue rwlock structure +* Return: 1 if lock contended, 0 otherwise +*/ +static inline int queued_rwlock_is_contended (struct qrwlock *lock) +{ +return arch_spin_is_locked (&lock->wait_lock); +} +/* * Remapping rwlock architecture specific functions to the corresponding * queue rwlock functions. */ -#define arch_read_lock (l) queued_read_lock(l) -#define arch_write_lock (l) queued_write_lock(l) -#define arch_read_trylock (l) queued_read_trylock(l) -#define arch_write_trylock (l) queued_write_trylock(l) -#define arch_read_unlock (l) queued_read_unlock(l) -#define arch_write_unlock (l) queued_write_unlock(l) +#define arch_read_lock (l) queued_read_lock(l) +#define arch_write_lock (l) queued_write_lock(l) +#define arch_read_trylock (l) queued_read_trylock(l) +#define arch_write_trylock (l) queued_write_trylock(l) +#define arch_read_unlock (l) queued_read_unlock(l) +#define arch_write_unlock (l) queued_write_unlock(l) +#define arch_rwlock_is_contended (l) queued_rwlock_is_contended(l)#endif /* __ASM_GENERIC_QRWLOCK_H */
include/linux/rwlock.h +7 -0
@@ -128,4 +128,11 @@ do { \ 1 : ({ local_irq_restore(flags); 0; }); \ }) +#ifdef arch_rwlock_is_contended +#define rwlock_is_contended (lock) \ + arch_rwlock_is_contended(&(lock)->raw_lock) +#else +#define rwlock_is_contended (lock) ((void)(lock), 0) +#endif /* arch_rwlock_is_contended */ +#endif /* __LINUX_RWLOCK_H */
Signature · 1 line
2.29.0.rc2.309.g374f81d7ae-goog
PATCH 2/3 sched: Add needbreak for rwlocks
Contention awareness while holding a spin lock is essential for reducing latency when long running kernel operations can hold that lock. Add the same contention detection interface for read/write spin locks.
--- include/linux/sched.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
include/linux/sched.h +17 -0
@@ -1870,6 +1870,23 @@ static inline int spin_needbreak(spinlock_t *lock)#endif } +/* +* Check if a rwlock is contended. +* Returns non-zero if there is another task waiting on the rwlock. +* Returns zero if the lock is not contended or the system / underlying +* rwlock implementation does not support contention detection. +* Technically does not depend on CONFIG_PREEMPTION, but a general need +* for low latency. +*/ +static inline int rwlock_needbreak (rwlock_t *lock) +{ +#ifdef CONFIG_PREEMPTION +return rwlock_is_contended (lock); +#else +return 0 ; +#endif +} + static __always_inline bool need_resched(void) {return unlikely (tif_need_resched ());
Signature · 1 line
2.29.0.rc2.309.g374f81d7ae-goog
PATCH 3/3 sched: Add cond_resched_rwlock
In reply to Ben GardonRescheduling while holding a spin lock is essential for keeping long running kernel operations running smoothly. Add the facility to cond_resched rwlocks.
--- include/linux/sched.h | 12 ++++++++++++ kernel/sched/core.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+)
include/linux/sched.h +12 -0
@@ -1841,12 +1841,24 @@ static inline int _cond_resched(void) { return 0; } })extern int __cond_resched_lock (spinlock_t *lock); +extern int __cond_resched_rwlock_read (rwlock_t *lock); +extern int __cond_resched_rwlock_write (rwlock_t *lock);#define cond_resched_lock (lock) ({ \ ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ __cond_resched_lock(lock); \ }) +#define cond_resched_rwlock_read (lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_read(lock); \ +}) + +#define cond_resched_rwlock_write (lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_write(lock); \ +}) +static inline void cond_resched_rcu (void ) {#if defined(CONFIG_DEBUG_ATOMIC_SLEEP ) || !defined(CONFIG_PREEMPT_RCU )
kernel/sched/core.c +40 -0
@@ -6152,6 +6152,46 @@ int __cond_resched_lock(spinlock_t *lock) }EXPORT_SYMBOL (__cond_resched_lock); +int __cond_resched_rwlock_read (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock); + +if (rwlock_needbreak (lock) || resched) { +read_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +read_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock); + +if (rwlock_needbreak (lock) || resched) { +write_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +write_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_write); +/** * yield - yield the current processor to other threads. *
Signature · 1 line
2.29.0.rc2.309.g374f81d7ae-goog
Re: [PATCH 3/3] sched: Add cond_resched_rwlock
Quoted 3 lines · Ben Gardon
Rescheduling while holding a spin lock is essential for keeping long running kernel operations running smoothly. Add the facility to
cond_reschedrwlocks.
This adds two new exports and two new macros without any in-tree users, which is generally frowned upon. You and I know these will be used by KVM's new TDP MMU, but the non-KVM folks, and more importantly the maintainers of this code, are undoubtedly going to ask "why". I.e. these patches probably belong in the KVM series to switch to a rwlock for the TDP MMU.
Regarding the code, it's all copy-pasted from the spinlock code and darn near identical. It might be worth adding builder macros for these.
Quoted 47 lines
Signed-off-by Ben Gardon <[email protected]>--- include/linux/sched.h | 12 ++++++++++++ kernel/sched/core.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+)include/linux/sched.h +12
-0@@ -1841,12 +1841,24 @@ static inline int _cond_resched(void) { return 0; } })extern int __cond_resched_lock (spinlock_t *lock); +extern int __cond_resched_rwlock_read (rwlock_t *lock); +extern int __cond_resched_rwlock_write (rwlock_t *lock);#define cond_resched_lock (lock) ({ \ ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ __cond_resched_lock(lock); \ }) +#define cond_resched_rwlock_read (lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_read(lock); \ +}) + +#define cond_resched_rwlock_write (lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_write(lock); \ +}) +static inline void cond_resched_rcu (void ) {#if defined(CONFIG_DEBUG_ATOMIC_SLEEP ) || !defined(CONFIG_PREEMPT_RCU )kernel/sched/core.c +4
-0@@ -6152,6 +6152,46 @@ int __cond_resched_lock(spinlock_t *lock) }EXPORT_SYMBOL (__cond_resched_lock); +int __cond_resched_rwlock_read (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ;
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6168
Quoted 10 lines
kernel/sched/core.c +10
-0+ +lockdep_assert_held (lock); + +if (rwlock_needbreak (lock) || resched) { +read_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ;
AFAICT, this rather odd code flow from __cond_resched_lock() is an artifact of code changes over the years and not intentionally weird. IMO, it would be cleaner and easier to read as:
int resched = should_resched(PREEMPT_LOCK_OFFSET); lockdep_assert_held(lock); if (!rwlock_needbreak(lock) && !resched) return 0; read_unlock(lock); if (resched) preempt_schedule_common(); else cpu_relax(); read_lock(lock) return 1;
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6180
Quoted 12 lines
kernel/sched/core.c +12
-0+read_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock);
This shoulid be lockdep_assert_held_write.
Quoted 20 lines
kernel/sched/core.c +14
-0+ +if (rwlock_needbreak (lock) || resched) { +write_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +write_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_write); +/** * yield - yield the current processor to other threads. * Signature · 1 line
2.29.0.rc2.309.g374f81d7ae-goog
Peter Zijlstra
maintainerRe: [PATCH 3/3] sched: Add cond_resched_rwlock
Quoted 10 lines · Sean Christopherson
Quoted 3 lines · Ben Gardon
Rescheduling while holding a spin lock is essential for keeping long running kernel operations running smoothly. Add the facility to
cond_reschedrwlocks.This adds two new exports and two new macros without any in-tree users, which is generally frowned upon. You and I know these will be used by KVM's new TDP MMU, but the non-KVM folks, and more importantly the maintainers of this code, are undoubtedly going to ask "why". I.e. these patches probably belong in the KVM series to switch to a rwlock for the TDP MMU.
I was informed about this ;-)
Quoted 2 lines
Regarding the code, it's all copy-pasted from the spinlock code and darn near identical. It might be worth adding builder macros for these.
I considered mentioning them; I'm typically a fan of them, but I'm not quite sure it's worth the effort here.
Quoted 19 lines
Quoted 4 lines
kernel/sched/core.c +4
-0+int __cond_resched_rwlock_read (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ;kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6168
Quoted 10 lines
kernel/sched/core.c +10
-0+ +lockdep_assert_held (lock); + +if (rwlock_needbreak (lock) || resched) { +read_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ;AFAICT, this rather odd code flow from
__cond_resched_lock()is an artifact of code changes over the years and not intentionally weird. IMO, it would be cleaner and easier to read as:
Quoted 3 lines
int resched = should_resched(PREEMPT_LOCK_OFFSET); lockdep_assert_held(lock);
lockdep_assert_held_read() :-)
Quoted 12 lines
if (!rwlock_needbreak(lock) && !resched) return 0; read_unlock(lock); if (resched) preempt_schedule_common(); else cpu_relax(); read_lock(lock) return 1;
I suppose that works, but then also change the existing one.
Davidlohr Bueso
Acked-byRe: [PATCH 3/3] sched: Add cond_resched_rwlock
In reply to Sean ChristophersonQuoted 4 lines · Sean Christopherson
Quoted 3 lines · Ben Gardon
Rescheduling while holding a spin lock is essential for keeping long running kernel operations running smoothly. Add the facility to
cond_reschedrwlocks.
Nit: I would start the paragraph with 'Safely rescheduling ...' While obvious when reading the code, 'Rescheduling while holding a spin lock' can throw the reader off.
Quoted 9 lines
This adds two new exports and two new macros without any in-tree users, which is generally frowned upon. You and I know these will be used by KVM's new TDP MMU, but the non-KVM folks, and more importantly the maintainers of this code, are undoubtedly going to ask "why". I.e. these patches probably belong in the KVM series to switch to a rwlock for the TDP MMU.
Regarding the code, it's all copy-pasted from the spinlock code and darn near identical. It might be worth adding builder macros for these.
Agreed, all three could be nicely consolidated. Otherwise this series looks sane, feel free to add my:
Peter Zijlstra
maintainerRe: [PATCH 3/3] sched: Add cond_resched_rwlock
In reply to Ben GardonQuoted 5 lines · Ben Gardon
kernel/sched/core.c +0
-0
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6160
Quoted 9 lines
kernel/sched/core.c +6
-0@@ -6152,6 +6152,46 @@ int __cond_resched_lock(spinlock_t *lock) }EXPORT_SYMBOL (__cond_resched_lock); +int __cond_resched_rwlock_read (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock);
lockdep_assert_held_read(lock);
Quoted 10 lines
kernel/sched/core.c +10
-0+ +if (rwlock_needbreak (lock) || resched) { +read_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +read_lock (lock); + }
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6180
Quoted 10 lines
kernel/sched/core.c +10
-0+return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock);
lockdep_assert_held_write(lock);
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6193
Quoted 13 lines
kernel/sched/core.c +13
-0+ +if (rwlock_needbreak (lock) || resched) { +write_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +write_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_write);
If this is the only feedback (the patches look fine to me), don't bother resending, I'll edit them when applying.
Paolo Bonzini
maintainerRe: [PATCH 3/3] sched: Add cond_resched_rwlock
Quoted 42 lines · Peter Zijlstra
Quoted 5 lines · Ben Gardon
kernel/sched/core.c +0
-0kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6160
Quoted 9 lines
kernel/sched/core.c +6
-0@@ -6152,6 +6152,46 @@ int __cond_resched_lock(spinlock_t *lock) }EXPORT_SYMBOL (__cond_resched_lock); +int __cond_resched_rwlock_read (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock);lockdep_assert_held_read(lock);Quoted 10 lines
kernel/sched/core.c +10
-0+ +if (rwlock_needbreak (lock) || resched) { +read_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +read_lock (lock); + }kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6180
Quoted 10 lines
kernel/sched/core.c +10
-0+return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock);lockdep_assert_held_write(lock);
Quoted 17 lines
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6193
Quoted 13 lines
kernel/sched/core.c +13
-0+ +if (rwlock_needbreak (lock) || resched) { +write_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +write_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_write);If this is the only feedback (the patches look fine to me), don't bother resending, I'll edit them when applying.
If that is an Acked-by, I'll merge them through the KVM tree when time comes.
Paolo
Re: [PATCH 3/3] sched: Add cond_resched_rwlock
In reply to Ben GardonQuoted 80 lines · Ben Gardon
Rescheduling while holding a spin lock is essential for keeping long running kernel operations running smoothly. Add the facility to
cond_reschedrwlocks.Signed-off-by Ben Gardon <[email protected]>--- include/linux/sched.h | 12 ++++++++++++ kernel/sched/core.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+)include/linux/sched.h +12
-0@@ -1841,12 +1841,24 @@ static inline int _cond_resched(void) { return 0; } })extern int __cond_resched_lock (spinlock_t *lock); +extern int __cond_resched_rwlock_read (rwlock_t *lock); +extern int __cond_resched_rwlock_write (rwlock_t *lock);#define cond_resched_lock (lock) ({ \ ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ __cond_resched_lock(lock); \ }) +#define cond_resched_rwlock_read (lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_read(lock); \ +}) + +#define cond_resched_rwlock_write (lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_write(lock); \ +}) +static inline void cond_resched_rcu (void ) {#if defined(CONFIG_DEBUG_ATOMIC_SLEEP ) || !defined(CONFIG_PREEMPT_RCU )kernel/sched/core.c +33
-0@@ -6152,6 +6152,46 @@ int __cond_resched_lock(spinlock_t *lock) }EXPORT_SYMBOL (__cond_resched_lock); +int __cond_resched_rwlock_read (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock); + +if (rwlock_needbreak (lock) || resched) { +read_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax (); + ret =1 ; +read_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write (rwlock_t *lock) +{ +int resched =should_resched (PREEMPT_LOCK_OFFSET ); +int ret =0 ; + +lockdep_assert_held (lock); + +if (rwlock_needbreak (lock) || resched) { +write_unlock (lock); +if (resched) +preempt_schedule_common (); +else +cpu_relax ();
kernel/sched/core.c int __cond_resched_lock(spinlock_t *lock) · line 6197
Quoted 10 lines
kernel/sched/core.c +7
-0+ ret =1 ; +write_lock (lock); + } +return ret; +} +EXPORT_SYMBOL (__cond_resched_rwlock_write); +/** * yield - yield the current processor to other threads. *
Other than the lockdep_assert_held() changes spotted by others, this patch looks good to me.
kernel test robot
build botThank you for the patch! Yet something to improve:
Hi Ben,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on tip/sched/core]
[also build test ERROR on tip/master linus/master linux/master v5.10-rc1 next-20201027] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Ben-Gardon/locking-rwlocks-Add-contention-detection-for-rwlocks/20201028-005208 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git feff2e65efd8d84cf831668e182b2ce73c604bbb config: arm64-randconfig-r021-20201027 (attached as .config) compiler: clang version 12.0.0 (https://github.com/llvm/llvm-project f2c25c70791de95d2466e09b5b58fc37f6ccd7a4)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# install arm64 cross compiling tool for clang build
# apt-get install binutils-aarch64-linux-gnu
# https://github.com/0day-ci/linux/commit/0cb6f36b82a21b6ce261382a68e09b1b4614a2ed
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Ben-Gardon/locking-rwlocks-Add-contention-detection-for-rwlocks/20201028-005208 git checkout 0cb6f36b82a21b6ce261382a68e09b1b4614a2ed
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=arm64
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
In file included from arch/arm64/kernel/asm-offsets.c:10:
In file included from include/linux/arm_sdei.h:8:
In file included from include/acpi/ghes.h:5:
In file included from include/acpi/apei.h:9:
In file included from include/linux/acpi.h:13:
In file included from include/linux/irqdomain.h:35:
In file included from include/linux/of.h:17:
In file included from include/linux/kobject.h:20:
In file included from include/linux/sysfs.h:16:
In file included from include/linux/kernfs.h:13:
In file included from include/linux/idr.h:15:
In file included from include/linux/radix-tree.h:16:
In file included from include/linux/spinlock.h:90:
In file included from arch/arm64/include/asm/spinlock.h:8:
In file included from ./arch/arm64/include/generated/asm/qrwlock.h:1:
>> include/asm-generic/qrwlock.h:126:9: error: implicit declaration of function 'arch_spin_is_locked' [-Werror,-Wimplicit-function-declaration]
return arch_spin_is_locked(&lock->wait_lock);
^ 1 error generated.
make[2]: *** [scripts/Makefile.build:117: arch/arm64/kernel/asm-offsets.s] Error 1 make[2]: Target '__build' not remade because of errors.
make[1]: *** [Makefile:1203: prepare0] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:185: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +/arch_spin_is_locked +126 include/asm-generic/qrwlock.h
118
119 /**
120 * queued_rwlock_is_contended - check if the lock is contended
121 * @lock : Pointer to queue rwlock structure
122 * Return: 1 if lock contended, 0 otherwise
123 */
124 static inline int queued_rwlock_is_contended(struct qrwlock *lock)
125 {
> 126 return arch_spin_is_locked(&lock->wait_lock);
127 }
128
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org