enable irqs when waiting for rwlocks
Subsystems (8)
cover enable irqs when waiting for rwlocks
Hello,
SGI has observed that on large systems, interrupts are not serviced for a long period of time when waiting for a rwlock. The following patch series re-enables irqs while waiting for the lock, resembling the code which is already there for spinlocks.
I only made the ia64 version, because the patch adds some overhead to the fast path. In particular, I wasn't able to make a reasonably lightweight implementation for x86, so I assume there are not such large systems based on x86(_64). Of course, the possibility to implement raw_{read|write}_lock_flags for any architecture is still there.
Petr Tesarik
PATCH 1/2 Allow rwlocks to re-enable interrupts
Pass the original flags to rwlock arch-code, so that it can re-enable interrupts if implemented for that architecture.
Initially, make __raw_read_lock_flags and __raw_write_lock_flags stubs which just do the same thing as their non-flags variants.
--- arch/alpha/include/asm/spinlock.h | 3 +++ arch/arm/include/asm/spinlock.h | 3 +++ arch/ia64/include/asm/spinlock.h | 3 +++ arch/mips/include/asm/spinlock.h | 2 ++ arch/powerpc/include/asm/spinlock.h | 3 +++ arch/s390/include/asm/spinlock.h | 3 +++ arch/sh/include/asm/spinlock.h | 3 +++ arch/sparc/include/asm/spinlock_32.h | 2 ++ arch/sparc/include/asm/spinlock_64.h | 2 ++ include/asm-cris/arch-v32/spinlock.h | 2 ++ include/asm-m32r/spinlock.h | 3 +++ include/asm-parisc/spinlock.h | 3 +++ include/asm-x86/spinlock.h | 3 +++ include/linux/spinlock.h | 6 ++++++ kernel/spinlock.c | 8 ++++++++ 15 files changed, 49 insertions(+), 0 deletions(-)
arch/alpha/include/asm/spinlock.h +3 -0
@@ -166,6 +166,9 @@ static inline void __raw_write_unlock(raw_rwlock_t * lock) lock->lock =0 ; } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
arch/arm/include/asm/spinlock.h +3 -0
@@ -217,6 +217,9 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw)/* read_can_lock - would read_trylock() succeed? */ #define __raw_read_can_lock (x) ((x)->lock < 0x80000000) +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
arch/ia64/include/asm/spinlock.h +3 -0
@@ -213,6 +213,9 @@ static inline int __raw_read_trylock(raw_rwlock_t *x)return (u32 )ia64_cmpxchg4_acq ((__u32 *)(x), new.word, old.word) = old.word; } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
arch/mips/include/asm/spinlock.h +2 -0
@@ -479,6 +479,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)return ret; } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock)#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()
arch/powerpc/include/asm/spinlock.h +3 -0
@@ -287,6 +287,9 @@ static inline void __raw_write_unlock(raw_rwlock_t *rw) rw->lock =0 ; } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) __spin_yield(lock)#define _raw_read_relax (lock) __rw_yield(lock)#define _raw_write_relax (lock) __rw_yield(lock)
arch/s390/include/asm/spinlock.h +3 -0
@@ -172,6 +172,9 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)return _raw_write_trylock_retry (rw); } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
arch/sh/include/asm/spinlock.h +3 -0
@@ -216,6 +216,9 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)return (oldval > (RW_LOCK_BIAS -1 )); } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
arch/sparc/include/asm/spinlock_32.h +2 -0
@@ -177,6 +177,8 @@ static inline int __read_trylock(raw_rwlock_t *rw)#define __raw_write_unlock (rw) do { (rw)->lock = 0; } while(0)#define __raw_spin_lock_flags (lock, flags) __raw_spin_lock(lock) +#define __raw_read_lock_flags (rw, flags) __raw_read_lock(rw) +#define __raw_write_lock_flags (rw, flags) __raw_write_lock(rw)#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()
arch/sparc/include/asm/spinlock_64.h +2 -0
@@ -230,9 +230,11 @@ static int inline __write_trylock(raw_rwlock_t *lock) }#define __raw_read_lock (p) __read_lock(p) +#define __raw_read_lock_flags (p, f) __read_lock(p)#define __raw_read_trylock (p) __read_trylock(p)#define __raw_read_unlock (p) __read_unlock(p)#define __raw_write_lock (p) __write_lock(p) +#define __raw_write_lock_flags (p, f) __write_lock(p)#define __raw_write_unlock (p) __write_unlock(p)#define __raw_write_trylock (p) __write_trylock(p)
include/asm-cris/arch-v32/spinlock.h +2 -0
@@ -121,6 +121,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)return 1 ; } +#define _raw_read_lock_flags (lock, flags) _raw_read_lock(lock) +#define _raw_write_lock_flags (lock, flags) _raw_write_lock(lock)#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()
include/asm-m32r/spinlock.h +3 -0
@@ -316,6 +316,9 @@ static inline int __raw_write_trylock(raw_rwlock_t *lock)return 0 ; } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
include/asm-parisc/spinlock.h +3 -0
@@ -187,6 +187,9 @@ static __inline__ int __raw_write_can_lock(raw_rwlock_t *rw)return !rw->counter; } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
include/asm-x86/spinlock.h +3 -0
@@ -357,6 +357,9 @@ static inline void __raw_write_unlock(raw_rwlock_t *rw) : "+m" (rw->lock) : "i" (RW_LOCK_BIAS) : "memory"); } +#define __raw_read_lock_flags (lock, flags) __raw_read_lock(lock) +#define __raw_write_lock_flags (lock, flags) __raw_write_lock(lock) +#define _raw_spin_relax (lock) cpu_relax()#define _raw_read_relax (lock) cpu_relax()#define _raw_write_relax (lock) cpu_relax()
include/linux/spinlock.h +6 -0
@@ -148,9 +148,11 @@ do { \extern int _raw_spin_trylock (spinlock_t *lock);extern void _raw_spin_unlock (spinlock_t *lock);extern void _raw_read_lock (rwlock_t *lock); +#define _raw_read_lock_flags (lock, flags) _raw_read_lock(lock)extern int _raw_read_trylock (rwlock_t *lock);extern void _raw_read_unlock (rwlock_t *lock);extern void _raw_write_lock (rwlock_t *lock); +#define _raw_write_lock_flags (lock, flags) _raw_write_lock(lock)extern int _raw_write_trylock (rwlock_t *lock);extern void _raw_write_unlock (rwlock_t *lock);#else
@@ -160,9 +162,13 @@ do { \# define _raw_spin_trylock (lock) __raw_spin_trylock(&(lock)->raw_lock)# define _raw_spin_unlock (lock) __raw_spin_unlock(&(lock)->raw_lock)# define _raw_read_lock (rwlock) __raw_read_lock(&(rwlock)->raw_lock) +# define _raw_read_lock_flags (lock, flags) \ + __raw_read_lock_flags(&(lock)->raw_lock, *(flags))# define _raw_read_trylock (rwlock) __raw_read_trylock(&(rwlock)->raw_lock)# define _raw_read_unlock (rwlock) __raw_read_unlock(&(rwlock)->raw_lock)# define _raw_write_lock (rwlock) __raw_write_lock(&(rwlock)->raw_lock) +# define _raw_write_lock_flags (lock, flags) \ + __raw_write_lock_flags(&(lock)->raw_lock, *(flags))# define _raw_write_trylock (rwlock) __raw_write_trylock(&(rwlock)->raw_lock)# define _raw_write_unlock (rwlock) __raw_write_unlock(&(rwlock)->raw_lock)#endif
kernel/spinlock.c +8 -0
@@ -121,7 +121,11 @@ unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)local_irq_save (flags);preempt_disable ();rwlock_acquire_read (&lock->dep_map,0 ,0 , _RET_IP_); +#ifdef CONFIG_LOCKDEP LOCK_CONTENDED (lock, _raw_read_trylock, _raw_read_lock); +#else +_raw_read_lock_flags (lock, &flags); +#endif return flags; }EXPORT_SYMBOL (_read_lock_irqsave);
@@ -151,7 +155,11 @@ unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)local_irq_save (flags);preempt_disable ();rwlock_acquire (&lock->dep_map,0 ,0 , _RET_IP_); +#ifdef CONFIG_LOCKDEP LOCK_CONTENDED (lock, _raw_write_trylock, _raw_write_lock); +#else +_raw_write_lock_flags (lock, &flags); +#endif return flags; }EXPORT_SYMBOL (_write_lock_irqsave);
Peter Zijlstra
maintainerRe: [PATCH 1/2] Allow rwlocks to re-enable interrupts
Quoted 5 lines · Petr Tesarik
c
b/kernel/spinlock.c
index 29ab207..f769d8a 100644
kernel/spinlock.c unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) · line 130
Quoted 10 lines
kernel/spinlock.c +4
-0@@ -121,7 +121,11 @@ unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)local_irq_save (flags);preempt_disable ();rwlock_acquire_read (&lock->dep_map,0 ,0 , _RET_IP_); +#ifdef CONFIG_LOCKDEP LOCK_CONTENDED (lock, _raw_read_trylock, _raw_read_lock); +#else +_raw_read_lock_flags (lock, &flags); +#endif return flags; }
That should be CONFIG_LOCK_STAT.
But aside from that, I really don't like this change, I'd rather you'd create a LOCK_CONTENDED_FLAGS() that can deal with this.
Re: [PATCH 1/2] Allow rwlocks to re-enable interrupts
Quoted 10 lines · Peter Zijlstra
Quoted 9 lines · Petr Tesarik
c
b/kernel/spinlock.c
index 29ab207..f769d8a 100644kernel/spinlock.c +1
-0@@ -121,7 +121,11 @@ unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)local_irq_save (flags);preempt_disable ();rwlock_acquire_read (&lock->dep_map,0 ,0 , _RET_IP_); +#ifdef CONFIG_LOCKDEP
Quoted 8 lines
kernel/spinlock.c unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) · line 130
Quoted 6 lines
kernel/spinlock.c +3
-0LOCK_CONTENDED (lock, _raw_read_trylock, _raw_read_lock); +#else +_raw_read_lock_flags (lock, &flags); +#endif return flags; }That should be
CONFIG_LOCK_STAT.
Ugh. Fine with me, but is CONFIG_LOCKDEP correct in _spin_lock_irqsave, or should it also read CONFIG_LOCK_STAT?
Quoted 2 lines
But aside from that, I really don't like this change, I'd rather you'd create a
LOCK_CONTENDED_FLAGS()that can deal with this.
No problem. I could then also use it for _spin_lock_irqsave, if the answer to the above question is use CONFIG_LOCK_STAT there as well.
Petr Tesarik
Peter Zijlstra
maintainerRe: [PATCH 1/2] Allow rwlocks to re-enable interrupts
Quoted 20 lines · Petr Tesarik
Quoted 10 lines · Peter Zijlstra
Quoted 9 lines · Petr Tesarik
c
b/kernel/spinlock.c
index 29ab207..f769d8a 100644kernel/spinlock.c +1
-0@@ -121,7 +121,11 @@ unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)local_irq_save (flags);preempt_disable ();rwlock_acquire_read (&lock->dep_map,0 ,0 , _RET_IP_); +#ifdef CONFIG_LOCKDEP Quoted 8 lines
kernel/spinlock.c unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) · line 130
Quoted 6 lines
kernel/spinlock.c +3
-0LOCK_CONTENDED (lock, _raw_read_trylock, _raw_read_lock); +#else +_raw_read_lock_flags (lock, &flags); +#endif return flags; }That should be
CONFIG_LOCK_STAT.
Quoted 2 lines
Ugh. Fine with me, but is
CONFIG_LOCKDEPcorrect in_spin_lock_irqsave, or should it also readCONFIG_LOCK_STAT?
Yep.
Quoted 5 lines
Quoted 2 lines
But aside from that, I really don't like this change, I'd rather you'd create a
LOCK_CONTENDED_FLAGS()that can deal with this.No problem. I could then also use it for
_spin_lock_irqsave, if the answer to the above question is useCONFIG_LOCK_STATthere as well.
If you create LOCK_CONTEDED_FLAGS() the whole issue goes away nicely.
Re: [PATCH 1/2] Allow rwlocks to re-enable interrupts
Quoted 4 lines · Peter Zijlstra
Quoted 2 lines
No problem. I could then also use it for
_spin_lock_irqsave, if the answer to the above question is useCONFIG_LOCK_STATthere as well.If you create
LOCK_CONTEDED_FLAGS()the whole issue goes away nicely.
Should it also be used for _spin_lock_irq()? I'm puzzled why it's only used for _irqsave().
(should _spin_lock_bh() re-enable BHs while waiting? Is it just not big enough of a deal?)
Signature · 4 lines
Matthew Wilcox Intel Open Source Technology Centre "Bill, look, we understand that you're interested in selling us this operating system, but compare it to ours. We can't possibly take such a retrograde step."
Peter Zijlstra
maintainerRe: [PATCH 1/2] Allow rwlocks to re-enable interrupts
Quoted 5 lines · Matthew Wilcox
Quoted 4 lines · Peter Zijlstra
Quoted 2 lines
No problem. I could then also use it for
_spin_lock_irqsave, if the answer to the above question is useCONFIG_LOCK_STATthere as well.If you create
LOCK_CONTEDED_FLAGS()the whole issue goes away nicely.
Gah, I looked at it again, and that #ifdef isn't only to select between LOCK_CONTENDED and not, but we can't actually have the re-enable for anything lockdep.
So I was wrong.
Quoted 2 lines
Should it also be used for
_spin_lock_irq()? I'm puzzled why it's only used for_irqsave().
Right, not sure how this maze is done.
The thing is, with spin_lock_irq() you know the irq state and can do the enable unconditionally - then again, with ticket locks we cannot do it at all.
The _flags() version needs the flags to see if irqs was enabled before we entered the op, if it wasn't we cannot go around enabling them.
Quoted 2 lines
(should
_spin_lock_bh()re-enable BHs while waiting? Is it just not big enough of a deal?)
Doubt it.. dunno.. personally I'd rather see softirqs die sooner rather than later.
Quoted 6 lines · Petr Tesarik
Hello,
SGI has observed that on large systems, interrupts are not serviced for a long period of time when waiting for a rwlock. The following patch series re-enables irqs while waiting for the lock, resembling the code which is already there for spinlocks.
Perhaps I'm just out in left field, but that (and the similar behaviour for obtaining a spinlock?) feels like treating a symptom rather than a root cause where the root cause would appear to be long lock hold times/contention?
rick jones
Quoted 8 lines · Rick Jones
Quoted 6 lines · Petr Tesarik
Hello,
SGI has observed that on large systems, interrupts are not serviced for a long period of time when waiting for a rwlock. The following patch series re-enables irqs while waiting for the lock, resembling the code which is already there for spinlocks.
Quoted 4 lines
Perhaps I'm just out in left field, but that (and the similar behaviour for obtaining a spinlock?) feels like treating a symptom rather than a root cause where the root cause would appear to be long lock hold times/contention?
Sometimes lock contention on large systems will take a few seconds to pass. This is normal behavior which simply can not be eliminated. For those cases, we need to rely upon being able to re-enable interrupts and allowing other operations to continue normally. Simply put, in some cases, nothing more can be done.
Thanks,
Robin