Wire up lockless lockrefs for ARM
Subsystems (2)
v2 cover Wire up lockless lockrefs for ARM
Hello,
This is version two of the patch I originally posted here:
http://lists.infradead.org/pipermail/linux-arm-kernel/2013-September/200109.html
The main changes since then are:
* Separate cmpxchg64 from atomic64_cmpxchg, so we can implement a barrier-less version.
* Use this new barrier-less function to implement cmpxchg64_relaxed
* Finally, use cmpxchg64_relaxed to optimise cmpxchg64_local and implement lockless lockrefs.
This results in ~100% improvement when running Linus's t.c test (essentially stating a file in a loop for 10 seconds).
Feedback welcome,
Will
Will Deacon (3):
ARM: cmpxchg: implement barrier-less cmpxchg64_local
ARM: cmpxchg: implement cmpxchg64_relaxed
ARM: lockref: add support for lockless lockrefs using cmpxchg64
arch/arm/Kconfig | 1 + arch/arm/include/asm/cmpxchg.h | 58 ++++++++++++++++++++++++++++++++--------- arch/arm/include/asm/spinlock.h | 8 ++++-- 3 files changed, 53 insertions(+), 14 deletions(-)
Signature · 1 line
1.8.2.2
PATCH v2 1/3 ARM: cmpxchg: implement barrier-less cmpxchg64_local
Our cmpxchg64 macros are wrappers around atomic64_cmpxchg. Whilst this is great for code re-use, there is a case for barrier-less cmpxchg where it is known to be safe (for example cmpxchg64_local and cmpxchg-based lockrefs).
This patch introduces a 64-bit cmpxchg implementation specifically for the cmpxchg64_* macros, so that it can be later used by the lockref code.
--- arch/arm/include/asm/cmpxchg.h | 52 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-)
arch/arm/include/asm/cmpxchg.h +42 -10
@@ -223,6 +223,42 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr,return ret; } +static inline unsigned long long __cmpxchg64 (unsigned long long *ptr, +unsigned long long old, +unsigned long long new) +{ +unsigned long long oldval; +unsigned long res; + + __asm__ __volatile__( +"1: ldrexd %1, %H1, [%3]\n" +" teq %1, %4\n" +" teqeq %H1, %H4\n" +" bne 2f\n" +" strexd %0, %5, %H5, [%3]\n" +" teq %0, #0\n" +" bne 1b\n" +"2:" + :"=&r" (res),"=&r" (oldval),"+Qo" (*ptr) + :"r" (ptr),"r" (old),"r" (new) + :"cc" ); + +return oldval; +} + +static inline unsigned long long __cmpxchg64_mb (unsigned long long *ptr, +unsigned long long old, +unsigned long long new) +{ +unsigned long long ret; + +smp_mb (); + ret =__cmpxchg64 (ptr, old, new); +smp_mb (); + +return ret; +} +#define cmpxchg_local (ptr,o,n) \ ((__typeof__(*(ptr)))__cmpxchg_local((ptr), \ (unsigned long)(o), \
@@ -230,18 +266,14 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, sizeof(*(ptr))))#define cmpxchg64 (ptr, o, n) \ - ((__typeof__(*(ptr)))atomic64_cmpxchg(container_of((ptr), \ - atomic64_t, \ - counter), \ - (unsigned long long)(o), \ - (unsigned long long)(n))) + ((__typeof__(*(ptr)))__cmpxchg64_mb((ptr), \ + (unsigned long long)(o), \ + (unsigned long long)(n)))#define cmpxchg64_local (ptr, o, n) \ - ((__typeof__(*(ptr)))local64_cmpxchg(container_of((ptr), \ - local64_t, \ - a), \ - (unsigned long long)(o), \ - (unsigned long long)(n))) + ((__typeof__(*(ptr)))__cmpxchg64((ptr), \ + (unsigned long long)(o), \ + (unsigned long long)(n)))#endif /* __LINUX_ARM_ARCH__ >= 6 */
Signature · 1 line
1.8.2.2
PATCH v2 2/3 ARM: cmpxchg: implement cmpxchg64_relaxed
In reply to Will DeaconThis patch introduces cmpxchg64_relaxed for arm, which performs a 64-bit cmpxchg operation without barrier semantics. cmpxchg64_local is updated to use the new operation.
--- arch/arm/include/asm/cmpxchg.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
arch/arm/include/asm/cmpxchg.h +3 -1
@@ -270,11 +270,13 @@ static inline unsigned long long __cmpxchg64_mb(unsigned long long *ptr, (unsigned long long)(o), \ (unsigned long long)(n))) -#define cmpxchg64_local (ptr, o, n) \ +#define cmpxchg64_relaxed (ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg64((ptr), \ (unsigned long long)(o), \ (unsigned long long)(n))) +#define cmpxchg64_local (ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) +#endif /* __LINUX_ARM_ARCH__ >= 6 */ #endif /* __ASM_ARM_CMPXCHG_H */
Signature · 1 line
1.8.2.2
PATCH v2 3/3 ARM: lockref: add support for lockless lockrefs using cmpxchg64
In reply to Will DeaconOur spinlocks are only 32-bit (2x16-bit tickets) and, on processors with 64-bit atomic instructions, cmpxchg64 makes use of the double-word exclusive accessors.
This patch wires up the cmpxchg-based lockless lockref implementation for ARM.
--- arch/arm/Kconfig | 1 + arch/arm/include/asm/spinlock.h | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-)
arch/arm/Kconfig +1 -0
@@ -5,6 +5,7 @@ config ARM select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAVE_CUSTOM_GPIO_H + select ARCH_USE_CMPXCHG_LOCKREF select ARCH_WANT_IPC_PARSE_VERSION select BUILDTIME_EXTABLE_SORT if MMU select CLONE_BACKWARDS
arch/arm/include/asm/spinlock.h +6 -2
@@ -127,10 +127,14 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)dsb_sev (); } +static inline int arch_spin_value_unlocked (arch_spinlock_t lock) +{ +return lock.tickets.owner == lock.tickets.next; +} +static inline int arch_spin_is_locked (arch_spinlock_t *lock) { -struct __raw_tickets tickets =ACCESS_ONCE (lock->tickets); -return tickets.owner != tickets.next; +return !arch_spin_value_unlocked (ACCESS_ONCE (*lock)); }static inline int arch_spin_is_contended (arch_spinlock_t *lock)
Signature · 1 line
1.8.2.2