locking: Adopt relaxed barrier semantics
cover locking: Adopt relaxed barrier semantics
Hi,
This series continues porting users to Will's new _{acquire|release|relaxed} optimizations for weakly ordered archs -- and in practice, arm64 being the only actual user we have, thus could use a _lot_ more testing. While we have osq in -tip recently, along with Waiman's one for native qspinlock, the following patches pretty much complete the move over for all things kernel/locking/*. Of course, particular primitives can be found in the obvious patch titles. Note that patch 1 adds the necessary machinery to fully support some locks, such as rwsems.
Compiles on x86 and x86-64.
Thanks!
Davidlohr Bueso (5):
asm-generic: Add _{relaxed|acquire|release}() variants for inc/dec
atomics
locking/mutex: Relax to atomic/release semantics locking/rtmutex: Relax to acquire/release semantics locking/rwsem: Relax to acquire/release semantics locking/mcs: Relax to atomic/release semantics
include/asm-generic/atomic-long.h | 29 +++++++----- include/asm-generic/mutex-dec.h | 8 ++-- include/asm-generic/mutex-xchg.h | 10 ++-- include/asm-generic/rwsem.h | 14 +++--- include/linux/atomic.h | 97 +++++++++++++++++++++++++++++++++++++++ kernel/locking/mcs_spinlock.h | 4 +- kernel/locking/mutex.c | 9 ++-- kernel/locking/rtmutex.c | 30 ++++++++---- kernel/locking/rwsem-xadd.c | 5 +- 9 files changed, 161 insertions(+), 45 deletions(-)
Signature · 1 line
2.1.4
PATCH 1/5 asm-generic: Add _{relaxed|acquire|release}() variants for inc/dec atomics
Similar to what we have for regular add/sub calls. For now, no actual arch implements them, so everyone falls back to the default atomics... iow, nothing changes. These will be used in future primitives.
--- include/asm-generic/atomic-long.h | 29 +++++++----- include/linux/atomic.h | 97 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 12 deletions(-)
include/asm-generic/atomic-long.h +17 -12
@@ -154,19 +154,24 @@ static inline int atomic_long_add_negative(long i, atomic_long_t *l)return ATOMIC_LONG_PFX (_add_negative)(i, v); } -static inline long atomic_long_inc_return (atomic_long_t *l) -{ -ATOMIC_LONG_PFX (_t) *v = (ATOMIC_LONG_PFX (_t) *)l; - -return (long )ATOMIC_LONG_PFX (_inc_return)(v); -} - -static inline long atomic_long_dec_return (atomic_long_t *l) -{ -ATOMIC_LONG_PFX (_t) *v = (ATOMIC_LONG_PFX (_t) *)l; - -return (long )ATOMIC_LONG_PFX (_dec_return)(v); +#define ATOMIC_LONG_INC_DEC_OP (op, mo) \ +static inline long \ +atomic_long_##op##_return##mo(atomic_long_t *l) \ +{ \ + ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ + \ + return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(v); \ } +ATOMIC_LONG_INC_DEC_OP(inc,) +ATOMIC_LONG_INC_DEC_OP(inc, _relaxed) +ATOMIC_LONG_INC_DEC_OP(inc, _acquire) +ATOMIC_LONG_INC_DEC_OP(inc, _release) +ATOMIC_LONG_INC_DEC_OP(dec,) +ATOMIC_LONG_INC_DEC_OP(dec, _relaxed) +ATOMIC_LONG_INC_DEC_OP(dec, _acquire) +ATOMIC_LONG_INC_DEC_OP(dec, _release) + +#undef ATOMIC_LONG_INC_DEC_OP static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) {
include/linux/atomic.h +97 -0
121 lines not on this page · raw message
Signature · 1 line
2.1.4
PATCH 2/5 locking/mutex: Relax to atomic/release semantics
In reply to Davidlohr Bueso... by using acquire semantics where appropriate As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
--- include/asm-generic/mutex-dec.h | 8 ++++---- include/asm-generic/mutex-xchg.h | 10 +++++----- kernel/locking/mutex.c | 9 +++++---- 3 files changed, 14 insertions(+), 13 deletions(-)
include/asm-generic/mutex-dec.h +4 -4
@@ -20,7 +20,7 @@static inline void __mutex_fastpath_lock (atomic_t *count,void (*fail_fn)(atomic_t *)) { -if (unlikely (atomic_dec_return (count) <0 )) +if (unlikely (atomic_dec_return_acquire (count) <0 ))fail_fn (count); }
@@ -35,7 +35,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))static inline int __mutex_fastpath_lock_retval (atomic_t *count) { -if (unlikely (atomic_dec_return (count) <0 )) +if (unlikely (atomic_dec_return_acquire (count) <0 ))return -1 ;return 0 ; }
@@ -56,7 +56,7 @@ __mutex_fastpath_lock_retval(atomic_t *count)static inline void __mutex_fastpath_unlock (atomic_t *count,void (*fail_fn)(atomic_t *)) { -if (unlikely (atomic_inc_return (count) <=0 )) +if (unlikely (atomic_inc_return_release (count) <=0 ))fail_fn (count); }
@@ -80,7 +80,7 @@ __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *))static inline int __mutex_fastpath_trylock (atomic_t *count,int (*fail_fn)(atomic_t *)) { -if (likely (atomic_cmpxchg (count,1 ,0 ) ==1 )) +if (likely (atomic_cmpxchg_acquire (count,1 ,0 ) ==1 ))return 1 ;return 0 ; }
include/asm-generic/mutex-xchg.h +5 -5
@@ -31,7 +31,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))* to ensure that any waiting tasks are woken up by the * unlock slow path. */ -if (likely (atomic_xchg (count,-1 ) !=1 )) +if (likely (atomic_xchg_acquire (count,-1 ) !=1 ))fail_fn (count); }
@@ -46,7 +46,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))static inline int __mutex_fastpath_lock_retval (atomic_t *count) { -if (unlikely (atomic_xchg (count,0 ) !=1 )) +if (unlikely (atomic_xchg_acquire (count,0 ) !=1 ))if (likely (atomic_xchg (count,-1 ) !=1 ))return -1 ;return 0 ;
@@ -67,7 +67,7 @@ __mutex_fastpath_lock_retval(atomic_t *count)static inline void __mutex_fastpath_unlock (atomic_t *count,void (*fail_fn)(atomic_t *)) { -if (unlikely (atomic_xchg (count,1 ) !=0 )) +if (unlikely (atomic_xchg_release (count,1 ) !=0 ))fail_fn (count); }
@@ -91,7 +91,7 @@ __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *))static inline int __mutex_fastpath_trylock (atomic_t *count,int (*fail_fn)(atomic_t *)) { -int prev =atomic_xchg (count,0 ); +int prev =atomic_xchg_acquire (count,0 );if (unlikely (prev <0 )) {/*
@@ -105,7 +105,7 @@ __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))* owner's unlock path needlessly, but that's not a problem * in practice. ] */ - prev =atomic_xchg (count, prev); + prev =atomic_xchg_acquire (count, prev);if (prev <0 ) prev =0 ; }
kernel/locking/mutex.c +5 -4
@@ -277,7 +277,7 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock)static inline bool mutex_try_to_acquire (struct mutex *lock) {return !mutex_is_locked (lock) && - (atomic_cmpxchg (&lock->count,1 ,0 ) ==1 ); + (atomic_cmpxchg_acquire (&lock->count,1 ,0 ) ==1 ); }/*
@@ -529,7 +529,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,* Once more, try to acquire the lock. Only try-lock the mutex if * it is unlocked to reduce unnecessary xchg() operations. */ -if (!mutex_is_locked (lock) && (atomic_xchg (&lock->count,0 ) ==1 )) +if (!mutex_is_locked (lock) && + (atomic_xchg_acquire (&lock->count,0 ) ==1 )) goto skip_wait;debug_mutex_lock_common (lock, &waiter);
@@ -553,7 +554,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,* non-negative in order to avoid unnecessary xchg operations: */ if (atomic_read (&lock->count) >=0 && - (atomic_xchg (&lock->count,-1 ) ==1 )) + (atomic_xchg_acquire (&lock->count,-1 ) ==1 ))break ;/*
@@ -867,7 +868,7 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)spin_lock_mutex (&lock->wait_lock, flags); - prev =atomic_xchg (&lock->count,-1 ); + prev =atomic_xchg_acquire (&lock->count,-1 );if (likely (prev ==1 )) {mutex_set_owner (lock);mutex_acquire (&lock->dep_map,0 ,1 , _RET_IP_);
Signature · 1 line
2.1.4
Re: [PATCH 2/5] locking/mutex: Relax to atomic/release semantics
Quoted 3 lines · Davidlohr Bueso
... by using acquire semantics where appropriate As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
Subject line is wrong. I'm assuming it should be acquire/release rather than atomic/release.
Linus
Re: [PATCH 2/5] locking/mutex: Relax to atomic/release semantics
Quoted 7 lines · Linus Torvalds
Quoted 3 lines · Davidlohr Bueso
... by using acquire semantics where appropriate As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
Subject line is wrong. I'm assuming it should be acquire/release rather than atomic/release.
Dear me, yes.
PATCH 3/5 locking/rtmutex: Relax to acquire/release semantics
In reply to Davidlohr Bueso... by using acquire semantics where appropiate. As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
--- kernel/locking/rtmutex.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-)
kernel/locking/rtmutex.c +21 -9
@@ -74,14 +74,23 @@ static void fixup_rt_mutex_waiters(struct rt_mutex *lock)* set up. */ #ifndef CONFIG_DEBUG_RT_MUTEXES -# define rt_mutex_cmpxchg (l,c,n) (cmpxchg(&l->owner, c, n) == c) +# define rt_mutex_cmpxchg_relaxed (l,c,n) (cmpxchg_relaxed(&l->owner, c, n) == c) +# define rt_mutex_cmpxchg_acquire (l,c,n) (cmpxchg_acquire(&l->owner, c, n) == c) +# define rt_mutex_cmpxchg_release (l,c,n) (cmpxchg_release(&l->owner, c, n) == c) + +/* +* Callers must hold the ->wait_lock -- which is the whole purpose as we force +* all future threads that attempt to [Rmw] the lock to the slowpath. As such +* relaxed semantics suffice. +*/ static inline void mark_rt_mutex_waiters (struct rt_mutex *lock) {unsigned long owner, *p = (unsigned long *) &lock->owner;do { owner = *p; - }while (cmpxchg (p, owner, owner |RT_MUTEX_HAS_WAITERS ) != owner); + }while (cmpxchg_relaxed (p, owner, + owner |RT_MUTEX_HAS_WAITERS ) != owner); }/*
@@ -121,11 +130,14 @@ static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)* lock(wait_lock); * acquire(lock); */ -return rt_mutex_cmpxchg (lock, owner,NULL ); +return rt_mutex_cmpxchg_acquire (lock, owner,NULL ); }#else -# define rt_mutex_cmpxchg (l,c,n) (0) +# define rt_mutex_cmpxchg_relaxed (l,c,n) (0) +# define rt_mutex_cmpxchg_acquire (l,c,n) (0) +# define rt_mutex_cmpxchg_release (l,c,n) (0) +static inline void mark_rt_mutex_waiters (struct rt_mutex *lock) { lock->owner = (struct task_struct *)
@@ -1321,7 +1333,7 @@ rt_mutex_fastlock(struct rt_mutex *lock, int state,struct hrtimer_sleeper *timeout, enum rtmutex_chainwalk chwalk)) { - if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { + if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {rt_mutex_deadlock_account_lock (lock ,current );return 0 ; } else
@@ -1337,7 +1349,7 @@ rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, enum rtmutex_chainwalk chwalk)) {if (chwalk ==RT_MUTEX_MIN_CHAINWALK && -likely (rt_mutex_cmpxchg (lock,NULL , current))) { +likely (rt_mutex_cmpxchg_acquire (lock,NULL , current))) {rt_mutex_deadlock_account_lock (lock, current);return 0 ; }else
@@ -1348,7 +1360,7 @@ static inline int rt_mutex_fasttrylock(struct rt_mutex *lock, int (*slowfn)(struct rt_mutex *lock)) { - if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { + if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {rt_mutex_deadlock_account_lock (lock, current);return 1 ; }
@@ -1362,7 +1374,7 @@ rt_mutex_fastunlock(struct rt_mutex *lock, {WAKE_Q (wake_q); -if (likely (rt_mutex_cmpxchg (lock, current,NULL ))) { +if (likely (rt_mutex_cmpxchg_release (lock, current,NULL ))) {rt_mutex_deadlock_account_unlock (current); }else {
@@ -1484,7 +1496,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_unlock); bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock,struct wake_q_head *wqh) { -if (likely (rt_mutex_cmpxchg (lock, current,NULL ))) { +if (likely (rt_mutex_cmpxchg_release (lock, current,NULL ))) {rt_mutex_deadlock_account_unlock (current);return false; }
Signature · 1 line
2.1.4
PATCH 4/5 locking/rwsem: Relax to acquire/release semantics
In reply to Davidlohr Bueso... by using acquire semantics where appropiate. As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
--- include/asm-generic/rwsem.h | 14 +++++++------- kernel/locking/rwsem-xadd.c | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-)
include/asm-generic/rwsem.h +7 -7
@@ -33,7 +33,7 @@*/ static inline void __down_read (struct rw_semaphore *sem) { -if (unlikely (atomic_long_inc_return ((atomic_long_t *)&sem->count) <=0 )) +if (unlikely (atomic_long_inc_return_acquire ((atomic_long_t *)&sem->count) <=0 ))rwsem_down_read_failed (sem); }
@@ -42,7 +42,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)long tmp;while ((tmp = sem->count) >=0 ) { -if (tmp ==cmpxchg (&sem->count, tmp, +if (tmp ==cmpxchg_acquire (&sem->count, tmp, tmp +RWSEM_ACTIVE_READ_BIAS )) {return 1 ; }
@@ -57,7 +57,7 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) {long tmp; - tmp =atomic_long_add_return (RWSEM_ACTIVE_WRITE_BIAS , + tmp =atomic_long_add_return_acquire (RWSEM_ACTIVE_WRITE_BIAS , (atomic_long_t *)&sem->count);if (unlikely (tmp !=RWSEM_ACTIVE_WRITE_BIAS ))rwsem_down_write_failed (sem);
@@ -72,7 +72,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) {long tmp; - tmp =cmpxchg (&sem->count,RWSEM_UNLOCKED_VALUE , + tmp =cmpxchg_acquire (&sem->count,RWSEM_UNLOCKED_VALUE ,RWSEM_ACTIVE_WRITE_BIAS );return tmp ==RWSEM_UNLOCKED_VALUE ; }
@@ -84,7 +84,7 @@ static inline void __up_read(struct rw_semaphore *sem) {long tmp; - tmp =atomic_long_dec_return ((atomic_long_t *)&sem->count); + tmp =atomic_long_dec_return_release ((atomic_long_t *)&sem->count);if (unlikely (tmp <-1 && (tmp &RWSEM_ACTIVE_MASK ) ==0 ))rwsem_wake (sem); }
@@ -94,7 +94,7 @@ static inline void __up_read(struct rw_semaphore *sem)*/ static inline void __up_write (struct rw_semaphore *sem) { -if (unlikely (atomic_long_sub_return (RWSEM_ACTIVE_WRITE_BIAS , +if (unlikely (atomic_long_sub_return_release (RWSEM_ACTIVE_WRITE_BIAS , (atomic_long_t *)&sem->count) <0 ))rwsem_wake (sem); }
@@ -114,7 +114,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem) {long tmp; - tmp =atomic_long_add_return (-RWSEM_WAITING_BIAS , + tmp =atomic_long_add_return_acquire (-RWSEM_WAITING_BIAS , (atomic_long_t *)&sem->count);if (tmp <0 )rwsem_downgrade_wake (sem);
kernel/locking/rwsem-xadd.c +3 -2
@@ -262,7 +262,7 @@ static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)* to reduce unnecessary expensive cmpxchg() operations. */ if (count ==RWSEM_WAITING_BIAS && -cmpxchg (&sem->count,RWSEM_WAITING_BIAS , +cmpxchg_acquire (&sem->count,RWSEM_WAITING_BIAS ,RWSEM_ACTIVE_WRITE_BIAS ) ==RWSEM_WAITING_BIAS ) {if (!list_is_singular (&sem->wait_list))rwsem_atomic_update (RWSEM_WAITING_BIAS , sem);
@@ -285,7 +285,8 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)if (!(count ==0 || count ==RWSEM_WAITING_BIAS ))return false; - old =cmpxchg (&sem->count, count, count +RWSEM_ACTIVE_WRITE_BIAS ); + old =cmpxchg_acquire (&sem->count, count, + count +RWSEM_ACTIVE_WRITE_BIAS );if (old == count) {rwsem_set_owner (sem);return true;
Signature · 1 line
2.1.4
PATCH 5/5 locking/mcs: Relax to atomic/release semantics
In reply to Davidlohr Bueso... by using acquire semantics where appropriate As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
--- kernel/locking/mcs_spinlock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/locking/mcs_spinlock.h +2 -2
@@ -67,7 +67,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node) node->locked =0 ; node->next =NULL ; - prev =xchg (lock, node); + prev =xchg_acquire (lock, node);if (likely (prev ==NULL )) {/* * Lock acquired, don't need to set node->locked to 1. Threads
@@ -98,7 +98,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)/* * Release the lock by setting it to NULL */ -if (likely (cmpxchg (lock, node,NULL ) == node)) +if (likely (cmpxchg_release (lock, node,NULL ) == node))return ;/* Wait until the next pointer is set */ while (!(next =READ_ONCE (node->next)))
Signature · 1 line
2.1.4
Re: [PATCH 5/5] locking/mcs: Relax to atomic/release semantics
Quoted 3 lines · Davidlohr Bueso
... by using acquire semantics where appropriate As such, weakly ordered archs can benefit from more relaxed use of barriers when issuing atomics.
Again, subject is wrong.
Also, I actually think you should avoid using the word "relax", since that actually has a separate meaning for the atomics, and it is *not* using those relaxed semantics, and they wouldn't work anyway.
So you might as well just say something like
"locking: use acquire/release semantics"
or similar. Ok?
Linus