arm64: spinlock and lockref updates for 3.13
Subsystems (2)
cover arm64: spinlock and lockref updates for 3.13
Hello,
This patch series adds a ticket-lock implementation for arm64, and then builds on top of this to implement the lockless lockref implementation in a very similar fashion to what I previously posted for ARM.
All feedback welcome,
Will
Will Deacon (3):
arm64: locks: introduce ticket-based spinlock implementation
arm64: lockref: add support for lockless lockrefs using cmpxchg
arm64: cmpxchg: implement cmpxchg64_relaxed
arch/arm64/Kconfig | 1 + arch/arm64/include/asm/cmpxchg.h | 2 + arch/arm64/include/asm/spinlock.h | 83 ++++++++++++++++++++++----------- arch/arm64/include/asm/spinlock_types.h | 10 ++-- 4 files changed, 65 insertions(+), 31 deletions(-)
Signature · 1 line
1.8.2.2
PATCH 1/3 arm64: locks: introduce ticket-based spinlock implementation
This patch introduces a ticket lock implementation for arm64, along the same lines as the implementation for arch/arm/.
--- arch/arm64/include/asm/spinlock.h | 79 ++++++++++++++++++++++----------- arch/arm64/include/asm/spinlock_types.h | 10 ++--- 2 files changed, 58 insertions(+), 31 deletions(-)
arch/arm64/include/asm/spinlock.h +53 -26
@@ -22,17 +22,10 @@/* * Spinlock implementation. * -* The old value is read exclusively and the new one, if unlocked, is written -* exclusively. In case of failure, the loop is restarted. -* * The memory barriers are implicit with the load-acquire and store-release * instructions. -* -* Unlocked value: 0 -* Locked value: 1 */ -#define arch_spin_is_locked (x) ((x)->lock != 0)#define arch_spin_unlock_wait (lock) \ do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
@@ -41,32 +34,51 @@static inline void arch_spin_lock (arch_spinlock_t *lock) {unsigned int tmp; +arch_spinlock_t lockval, newval; asmvolatile ( -" sevl\n" -"1: wfe\n" -"2: ldaxr %w0, %1\n" -" cbnz %w0, 1b\n" -" stxr %w0, %w2, %1\n" -" cbnz %w0, 2b\n" - :"=&r" (tmp),"+Q" (lock->lock) - :"r" (1 ) - :"cc" ,"memory" ); +/* Atomically increment the next ticket. */ +" prfm pstl1strm, %3\n" +"1: ldaxr %w0, %3\n" +" add %w1, %w0, %w5\n" +" stxr %w2, %w1, %3\n" +" cbnz %w2, 1b\n" +/* Did we get the lock? */ +" eor %w1, %w0, %w0, ror #16\n" +" cbz %w1, 3f\n" +/* +* No: spin on the owner. Send a local event to avoid missing an +* unlock before the exclusive load. +*/ +" sevl\n" +"2: wfe\n" +" ldaxrh %w2, %4\n" +" eor %w1, %w2, %w0, lsr #16\n" +" cbnz %w1, 2b\n" +/* We got the lock. Critical section starts here. */ +"3:" + :"=&r" (lockval),"=&r" (newval),"=&r" (tmp),"+Q" (*lock) + :"Q" (lock->owner),"I" (1 <<TICKET_SHIFT ) + :"memory" ); }static inline int arch_spin_trylock (arch_spinlock_t *lock) {unsigned int tmp; +arch_spinlock_t lockval; asmvolatile ( -"2: ldaxr %w0, %1\n" -" cbnz %w0, 1f\n" -" stxr %w0, %w2, %1\n" -" cbnz %w0, 2b\n" -"1:\n" - :"=&r" (tmp),"+Q" (lock->lock) - :"r" (1 ) - :"cc" ,"memory" ); +" prfm pstl1strm, %2\n" +"1: ldaxr %w0, %2\n" +" eor %w1, %w0, %w0, ror #16\n" +" cbnz %w1, 2f\n" +" add %w0, %w0, %3\n" +" stxr %w1, %w0, %2\n" +" cbnz %w1, 1b\n" +"2:" + :"=&r" (lockval),"=&r" (tmp),"+Q" (*lock) + :"I" (1 <<TICKET_SHIFT ) + :"memory" );return !tmp; }
@@ -74,9 +86,24 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)static inline void arch_spin_unlock (arch_spinlock_t *lock) { asmvolatile ( -" stlr %w1, %0\n" - :"=Q" (lock->lock) :"r" (0 ) :"memory" ); +" stlrh %w1, %0\n" + :"=Q" (lock->owner) + :"r" (lock->owner +1 ) + :"memory" ); +} + +static inline int arch_spin_is_locked (arch_spinlock_t *lock) +{ +arch_spinlock_t lockval =ACCESS_ONCE (*lock); +return lockval.owner != lockval.next; +} + +static inline int arch_spin_is_contended (arch_spinlock_t *lock) +{ +arch_spinlock_t lockval =ACCESS_ONCE (*lock); +return (lockval.next - lockval.owner) >1 ; } +#define arch_spin_is_contended arch_spin_is_contended/* * Write lock implementation.
arch/arm64/include/asm/spinlock_types.h +5 -5
@@ -20,14 +20,14 @@# error "please don't include this file directly"#endif -/* We only require natural alignment for exclusive accesses. */ -#define __lock_aligned +#define TICKET_SHIFT 16typedef struct { -volatile unsigned int lock; -}arch_spinlock_t ; +u16 owner; +u16 next; +}__aligned(4) arch_spinlock_t ; -#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } +#define __ARCH_SPIN_LOCK_UNLOCKED { 0 , 0 } typedef struct { volatile unsigned int lock;
Signature · 1 line
1.8.2.2
PATCH 2/3 arm64: lockref: add support for lockless lockrefs using cmpxchg
In reply to Will DeaconOur spinlocks are only 32-bit (2x16-bit tickets) and our cmpxchg can deal with 8-bytes (as one would hope!).
This patch wires up the cmpxchg-based lockless lockref implementation for arm64.
--- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/spinlock.h | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-)
arch/arm64/Kconfig +1 -0
@@ -1,6 +1,7 @@ config ARM64 def_bool y select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE + select ARCH_USE_CMPXCHG_LOCKREF select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_WANT_COMPAT_IPC_PARSE_VERSION select ARCH_WANT_FRAME_POINTERS
arch/arm64/include/asm/spinlock.h +6 -2
@@ -92,10 +92,14 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) : "memory"); } +static inline int arch_spin_value_unlocked (arch_spinlock_t lock) +{ +return lock.owner == lock.next; +} +static inline int arch_spin_is_locked (arch_spinlock_t *lock) { -arch_spinlock_t lockval =ACCESS_ONCE (*lock); -return lockval.owner != lockval.next; +return !arch_spin_value_unlocked (ACCESS_ONCE (*lock)); }static inline int arch_spin_is_contended (arch_spinlock_t *lock)
Signature · 1 line
1.8.2.2
PATCH 3/3 arm64: cmpxchg: implement cmpxchg64_relaxed
In reply to Will DeaconThis patch introduces cmpxchg64_relaxed for arm64 using the existing cmpxchg_local macro, which performs a cmpxchg operation (up to 64 bits) without barrier semantics.
--- arch/arm64/include/asm/cmpxchg.h | 2 ++ 1 file changed, 2 insertions(+)
arch/arm64/include/asm/cmpxchg.h +2 -0
@@ -173,4 +173,6 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,#define cmpxchg64 (ptr,o,n) cmpxchg((ptr),(o),(n))#define cmpxchg64_local (ptr,o,n) cmpxchg_local((ptr),(o),(n)) +#define cmpxchg64_relaxed (ptr,o,n) cmpxchg_local((ptr),(o),(n)) +#endif /* __ASM_CMPXCHG_H */
Signature · 1 line
1.8.2.2