powerpc: queued spinlocks and rwlocks
Subsystems (5)
Nicholas Piggin
reviewerv4 cover powerpc: queued spinlocks and rwlocks
Updated with everybody's feedback (thanks all), and more performance results.
What I've found is I might have been measuring the worst load point for the paravirt case, and by looking at a range of loads it's clear that queued spinlocks are overall better even on PV, doubly so when you look at the generally much improved worst case latencies.
I have defaulted it to N even though I'm less concerned about the PV numbers now, just because I think it needs more stress testing. But it's very nicely selectable so should be low risk to include.
All in all this is a very cool technology and great results especially on the big systems but even on smaller ones there are nice gains. Thanks Waiman and everyone who developed it.
Thanks,
Nick
Nicholas Piggin (6):
powerpc/pseries: move some PAPR paravirt functions to their own file
powerpc: move spinlock implementation to simple_spinlock powerpc/64s: implement queued spinlocks and rwlocks powerpc/pseries: implement paravirt qspinlocks for SPLPAR powerpc/qspinlock: optimised atomic_try_cmpxchg_lock that adds the
lock hint
powerpc: implement smp_cond_load_relaxed
arch/powerpc/Kconfig | 15 + arch/powerpc/include/asm/Kbuild | 1 + arch/powerpc/include/asm/atomic.h | 28 ++ arch/powerpc/include/asm/barrier.h | 14 + arch/powerpc/include/asm/paravirt.h | 87 +++++ arch/powerpc/include/asm/qspinlock.h | 91 ++++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 7 + arch/powerpc/include/asm/simple_spinlock.h | 288 ++++++++++++++++ .../include/asm/simple_spinlock_types.h | 21 ++ arch/powerpc/include/asm/spinlock.h | 308 +----------------- arch/powerpc/include/asm/spinlock_types.h | 17 +- arch/powerpc/lib/Makefile | 3 + arch/powerpc/lib/locks.c | 12 +- arch/powerpc/platforms/pseries/Kconfig | 9 +- arch/powerpc/platforms/pseries/setup.c | 4 +- include/asm-generic/qspinlock.h | 4 + 16 files changed, 588 insertions(+), 321 deletions(-) create mode 100644 arch/powerpc/include/asm/paravirt.h create mode 100644 arch/powerpc/include/asm/qspinlock.h create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt.h create mode 100644 arch/powerpc/include/asm/simple_spinlock.h create mode 100644 arch/powerpc/include/asm/simple_spinlock_types.h
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v4 1/6 powerpc/pseries: move some PAPR paravirt functions to their own file
These functions will be used by queued spinlock implementation, and may be useful elsewhere too, so move them out of spinlock.h.
--- arch/powerpc/include/asm/paravirt.h | 59 +++++++++++++++++++++++++++++ arch/powerpc/include/asm/spinlock.h | 24 +----------- arch/powerpc/lib/locks.c | 12 +++--- 3 files changed, 66 insertions(+), 29 deletions(-) create mode 100644 arch/powerpc/include/asm/paravirt.h
arch/powerpc/include/asm/paravirt.h new file +59 -0
@@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_PARAVIRT_H +#define _ASM_POWERPC_PARAVIRT_H + +#include <linux/jump_label.h> +#include <asm/smp.h> +#ifdef CONFIG_PPC64 +#include <asm/paca.h> +#include <asm/hvcall.h> +#endif + +#ifdef CONFIG_PPC_SPLPAR +DECLARE_STATIC_KEY_FALSE (shared_processor); + +static inline bool is_shared_processor (void ) +{ +return static_branch_unlikely (&shared_processor); +} + +/* If bit 0 is set, the cpu has been preempted */ +static inline u32 yield_count_of (int cpu) +{ +__be32 yield_count =READ_ONCE (lppaca_of (cpu).yield_count); +return be32_to_cpu (yield_count); +} + +static inline void yield_to_preempted (int cpu,u32 yield_count) +{ +plpar_hcall_norets (H_CONFER ,get_hard_smp_processor_id (cpu), yield_count); +} +#else +static inline bool is_shared_processor (void ) +{ +return false; +} + +static inline u32 yield_count_of (int cpu) +{ +return 0 ; +} + +extern void ___bad_yield_to_preempted (void ); +static inline void yield_to_preempted (int cpu,u32 yield_count) +{ +___bad_yield_to_preempted ();/* This would be a bug */ +} +#endif + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted (int cpu) +{ +if (!is_shared_processor ()) +return false; +if (yield_count_of (cpu) &1 ) +return true; +return false; +} + +#endif /* _ASM_POWERPC_PARAVIRT_H */
arch/powerpc/include/asm/spinlock.h +1 -23
@@ -15,11 +15,10 @@* * (the type definitions are in asm/spinlock_types.h) */ -#include <linux/jump_label.h> #include <linux/irqflags.h> +#include <asm/paravirt.h> #ifdef CONFIG_PPC64 #include <asm/paca.h> -#include <asm/hvcall.h> #endif #include <asm/synch.h> #include <asm/ppc-opcode.h>
@@ -35,18 +34,6 @@#define LOCK_TOKEN 1#endif -#ifdef CONFIG_PPC_PSERIES -DECLARE_STATIC_KEY_FALSE (shared_processor); - -#define vcpu_is_preempted vcpu_is_preempted -static inline bool vcpu_is_preempted (int cpu) -{ -if (!static_branch_unlikely (&shared_processor)) -return false; -return !!(be32_to_cpu (lppaca_of (cpu).yield_count) &1 ); -} -#endif - static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) {return lock.slock =0 ;
@@ -110,15 +97,6 @@ static inline void splpar_spin_yield(arch_spinlock_t *lock) {};static inline void splpar_rw_yield (arch_rwlock_t *lock) {};#endif -static inline bool is_shared_processor (void ) -{ -#ifdef CONFIG_PPC_SPLPAR -return static_branch_unlikely (&shared_processor); -#else -return false; -#endif -} -static inline void spin_yield (arch_spinlock_t *lock) {if (is_shared_processor ())
arch/powerpc/lib/locks.c +6 -6
@@ -27,14 +27,14 @@ void splpar_spin_yield(arch_spinlock_t *lock)return ; holder_cpu = lock_value &0xffff ;BUG_ON (holder_cpu >=NR_CPUS ); - yield_count =be32_to_cpu (lppaca_of (holder_cpu).yield_count); + + yield_count =yield_count_of (holder_cpu);if ((yield_count &1 ) =0 )return ;/* virtual cpu is currently running */ rmb ();if (lock->slock != lock_value)return ;/* something has changed */ -plpar_hcall_norets (H_CONFER , -get_hard_smp_processor_id (holder_cpu), yield_count); +yield_to_preempted (holder_cpu, yield_count); }EXPORT_SYMBOL_GPL (splpar_spin_yield);
@@ -53,13 +53,13 @@ void splpar_rw_yield(arch_rwlock_t *rw)return ;/* no write lock at present */ holder_cpu = lock_value &0xffff ;BUG_ON (holder_cpu >=NR_CPUS ); - yield_count =be32_to_cpu (lppaca_of (holder_cpu).yield_count); + + yield_count =yield_count_of (holder_cpu);if ((yield_count &1 ) =0 )return ;/* virtual cpu is currently running */ rmb ();if (rw->lock != lock_value)return ;/* something has changed */ -plpar_hcall_norets (H_CONFER , - get_hard_smp_processor_id(holder_cpu), yield_count); + yield_to_preempted(holder_cpu, yield_count); }#endif
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v4 2/6 powerpc: move spinlock implementation to simple_spinlock
In reply to Nicholas PigginTo prepare for queued spinlocks. This is a simple rename except to update preprocessor guard name and a file reference.
--- arch/powerpc/include/asm/simple_spinlock.h | 288 ++++++++++++++++++ .../include/asm/simple_spinlock_types.h | 21 ++ arch/powerpc/include/asm/spinlock.h | 285 +---------------- arch/powerpc/include/asm/spinlock_types.h | 12 +- 4 files changed, 311 insertions(+), 295 deletions(-) create mode 100644 arch/powerpc/include/asm/simple_spinlock.h create mode 100644 arch/powerpc/include/asm/simple_spinlock_types.h
arch/powerpc/include/asm/simple_spinlock.h new file +288 -0
@@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_H +#define _ASM_POWERPC_SIMPLE_SPINLOCK_H + +/* +* Simple spin lock operations. +* +* Copyright (C) 2001-2004 Paul Mackerras <[email protected]>, IBM +* Copyright (C) 2001 Anton Blanchard <[email protected]>, IBM +* Copyright (C) 2002 Dave Engebretsen <[email protected]>, IBM +* Rework to support virtual processors +* +* Type of int is used as a full 64b word is not necessary. +* +* (the type definitions are in asm/simple_spinlock_types.h) +*/ +#include <linux/irqflags.h> +#include <asm/paravirt.h> +#include <asm/paca.h> +#include <asm/synch.h> +#include <asm/ppc-opcode.h> + +#ifdef CONFIG_PPC64 +/* use 0x800000yy when locked, where yy = CPU number */ +#ifdef __BIG_ENDIAN__ +#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) +#else +#define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) +#endif +#else +#define LOCK_TOKEN 1 +#endif + +static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ +return lock.slock =0 ; +} + +static inline int arch_spin_is_locked (arch_spinlock_t *lock) +{ +smp_mb (); +return !arch_spin_value_unlocked (*lock); +} + +/* +* This returns the old value in the lock, so we succeeded +* in getting the lock if the return value is 0. +*/ +static inline unsigned long __arch_spin_trylock (arch_spinlock_t *lock) +{ +unsigned long tmp, token; + + token =LOCK_TOKEN ; + __asm__ __volatile__( +"1: " PPC_LWARX(%0,0,%2,1) "\n\ +cmpwi 0,%0,0\n\ +bne- 2f\n\ +stwcx. %1,0,%2\n\ +bne- 1b\n" +PPC_ACQUIRE_BARRIER +"2:" + : "=&r" (tmp) + : "r" (token), "r" (&lock->slock) + : "cr0", "memory"); + + return tmp; +} + +static inline int arch_spin_trylock (arch_spinlock_t *lock) +{ +return __arch_spin_trylock (lock) =0 ; +} + +/* +* On a system with shared processors (that is, where a physical +* processor is multiplexed between several virtual processors), +* there is no point spinning on a lock if the holder of the lock +* isn't currently scheduled on a physical processor. Instead +* we detect this situation and ask the hypervisor to give the +* rest of our timeslice to the lock holder. +* +* So that we can tell which virtual processor is holding a lock, +* we put 0x80000000 | smp_processor_id() in the lock when it is +* held. Conveniently, we have a word in the paca that holds this +* value. +*/ + +#if defined(CONFIG_PPC_SPLPAR ) +/* We only yield to the hypervisor if we are in shared processor mode */ +void splpar_spin_yield (arch_spinlock_t *lock); +void splpar_rw_yield (arch_rwlock_t *lock); +#else /* SPLPAR */ +static inline void splpar_spin_yield (arch_spinlock_t *lock) {}; +static inline void splpar_rw_yield (arch_rwlock_t *lock) {}; +#endif + +static inline void spin_yield (arch_spinlock_t *lock) +{ +if (is_shared_processor ()) +splpar_spin_yield (lock); +else +barrier (); +} + +static inline void rw_yield (arch_rwlock_t *lock) +{ +if (is_shared_processor ()) +splpar_rw_yield (lock); +else +barrier (); +} + +static inline void arch_spin_lock (arch_spinlock_t *lock) +{ +while (1 ) { +if (likely (__arch_spin_trylock (lock) =0 )) +break ; +do { +HMT_low (); +if (is_shared_processor ()) +splpar_spin_yield (lock); + }while (unlikely (lock->slock !=0 )); +HMT_medium (); + } +} + +static inline +void arch_spin_lock_flags (arch_spinlock_t *lock,unsigned long flags) +{ +unsigned long flags_dis; + +while (1 ) { +if (likely (__arch_spin_trylock (lock) =0 )) +break ; +local_save_flags (flags_dis); +local_irq_restore (flags); +do { +HMT_low (); +if (is_shared_processor ()) +splpar_spin_yield (lock); + }while (unlikely (lock->slock !=0 )); +HMT_medium (); +local_irq_restore (flags_dis); + } +} +#define arch_spin_lock_flags arch_spin_lock_flags + +static inline void arch_spin_unlock (arch_spinlock_t *lock) +{ + __asm__ __volatile__("# arch_spin_unlock\n\t" +PPC_RELEASE_BARRIER : : :"memory" ); + lock->slock =0 ; +} + +/* +* Read-write spinlocks, allowing multiple readers +* but only one writer. +* +* NOTE! it is quite common to have readers in interrupts +* but no interrupt writers. For those circumstances we +* can "mix" irq-safe locks - any writer needs to get a +* irq-safe write-lock, but readers can get non-irqsafe +* read-locks. +*/ + +#ifdef CONFIG_PPC64 +#define __DO_SIGN_EXTEND "extsw %0,%0\n" +#define WRLOCK_TOKEN LOCK_TOKEN/* it's negative */ +#else +#define __DO_SIGN_EXTEND +#define WRLOCK_TOKEN (-1) +#endif + +/* +* This returns the old value in the lock + 1, +* so we got a read lock if the return value is > 0. +*/ +static inline long __arch_read_trylock (arch_rwlock_t *rw) +{ +long tmp; + + __asm__ __volatile__( +"1: " PPC_LWARX(%0,0,%1,1) "\n" + __DO_SIGN_EXTEND +" addic. %0,%0,1\n\ +ble- 2f\n" +" stwcx. %0,0,%1\n\ +bne- 1b\n" +PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) + : "r" (&rw->lock) + : "cr0", "xer", "memory"); + + return tmp; +} + +/* +* This returns the old value in the lock, +* so we got the write lock if the return value is 0. +*/ +static inline long __arch_write_trylock (arch_rwlock_t *rw) +{ +long tmp, token; + + token =WRLOCK_TOKEN ; + __asm__ __volatile__( +"1: " PPC_LWARX(%0,0,%2,1) "\n\ +cmpwi 0,%0,0\n\ +bne- 2f\n" +" stwcx. %1,0,%2\n\ +bne- 1b\n" +PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) + : "r" (token), "r" (&rw->lock) + : "cr0", "memory"); + + return tmp; +} + +static inline void arch_read_lock (arch_rwlock_t *rw) +{ +while (1 ) { +if (likely (__arch_read_trylock (rw) >0 )) +break ; +do { +HMT_low (); +if (is_shared_processor ()) +splpar_rw_yield (rw); + }while (unlikely (rw->lock <0 )); +HMT_medium (); + } +} + +static inline void arch_write_lock (arch_rwlock_t *rw) +{ +while (1 ) { +if (likely (__arch_write_trylock (rw) =0 )) +break ; +do { +HMT_low (); +if (is_shared_processor ()) +splpar_rw_yield (rw); + }while (unlikely (rw->lock !=0 )); +HMT_medium (); + } +} + +static inline int arch_read_trylock (arch_rwlock_t *rw) +{ +return __arch_read_trylock (rw) >0 ; +} + +static inline int arch_write_trylock (arch_rwlock_t *rw) +{ +return __arch_write_trylock (rw) =0 ; +} + +static inline void arch_read_unlock (arch_rwlock_t *rw) +{ +long tmp; + + __asm__ __volatile__( +"# read_unlock\n\t" +PPC_RELEASE_BARRIER +"1: lwarx %0,0,%1\n\ +addic %0,%0,-1\n" +" stwcx. %0,0,%1\n\ +bne- 1b" + :"=&r" (tmp) + :"r" (&rw->lock) + :"cr0" ,"xer" ,"memory" ); +} + +static inline void arch_write_unlock (arch_rwlock_t *rw) +{ + __asm__ __volatile__("# write_unlock\n\t" +PPC_RELEASE_BARRIER : : :"memory" ); + rw->lock =0 ; +} + +#define arch_spin_relax (lock) spin_yield(lock) +#define arch_read_relax (lock) rw_yield(lock) +#define arch_write_relax (lock) rw_yield(lock) + +/* See include/linux/spinlock.h */ +#define smp_mb__after_spinlock () smp_mb() + +#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */
arch/powerpc/include/asm/simple_spinlock_types.h new file +21 -0
@@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H +#define _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H + +#ifndef __LINUX_SPINLOCK_TYPES_H +# error "please don't include this file directly" +#endif + +typedef struct { +volatile unsigned int slock; +}arch_spinlock_t ; + +#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } + +typedef struct { +volatile signed int lock; +}arch_rwlock_t ; + +#define __ARCH_RW_LOCK_UNLOCKED { 0 } + +#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H */
arch/powerpc/include/asm/spinlock.h +1 -284
291 lines not on this page · raw message
arch/powerpc/include/asm/spinlock_types.h +1 -11
@@ -6,16 +6,6 @@# error "please don't include this file directly"#endif -typedef struct { -volatile unsigned int slock; -}arch_spinlock_t ; - -#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } - -typedef struct { -volatile signed int lock; -}arch_rwlock_t ; - -#define __ARCH_RW_LOCK_UNLOCKED { 0 } +#include <asm/simple_spinlock_types.h> #endif
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v4 3/6 powerpc/64s: implement queued spinlocks and rwlocks
In reply to Nicholas PigginThese have shown significantly improved performance and fairness when spinlock contention is moderate to high on very large systems.
With this series including subsequent patches, on a 16 socket 1536 thread POWER9, a stress test such as same-file open/close from all CPUs gets big speedups, 11620op/s aggregate with simple spinlocks vs 384158op/s (33x faster), where the difference in throughput between the fastest and slowest thread goes from 7x to 1.4x.
Thanks to the fast path being identical in terms of atomics and barriers (after a subsequent optimisation patch), single threaded performance is not changed (no measurable difference).
On smaller systems, performance and fairness seems to be generally improved. Using dbench on tmpfs as a test (that starts to run into kernel spinlock contention), a 2-socket OpenPOWER POWER9 system was tested with bare metal and KVM guest configurations. Results can be found here:
https://github.com/linuxppc/issues/issues/305#issuecomment-663487453
Observations are:
- Queued spinlocks are equal when contention is insignificant, as expected and as measured with microbenchmarks.
- When there is contention, on bare metal queued spinlocks have better throughput and max latency at all points.
- When virtualised, queued spinlocks are slightly worse approaching peak throughput, but significantly better throughput and max latency at all points beyond peak, until queued spinlock maximum latency rises when clients are 2x vCPUs.
The regressions haven't been analysed very well yet, there are a lot of things that can be tuned, particularly the paravirtualised locking, but the numbers already look like a good net win even on relatively small systems.
--- arch/powerpc/Kconfig | 15 ++++++++++++++ arch/powerpc/include/asm/Kbuild | 1 + arch/powerpc/include/asm/qspinlock.h | 25 +++++++++++++++++++++++ arch/powerpc/include/asm/spinlock.h | 5 +++++ arch/powerpc/include/asm/spinlock_types.h | 5 +++++ arch/powerpc/lib/Makefile | 3 +++ include/asm-generic/qspinlock.h | 2 ++ 7 files changed, 56 insertions(+) create mode 100644 arch/powerpc/include/asm/qspinlock.h
arch/powerpc/Kconfig +15 -0
@@ -145,6 +145,8 @@ config PPC select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if PPC64 + select ARCH_USE_QUEUED_RWLOCKS if PPC_QUEUED_SPINLOCKS + select ARCH_USE_QUEUED_SPINLOCKS if PPC_QUEUED_SPINLOCKS select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WEAK_RELEASE_ACQUIRE select BINFMT_ELF
@@ -490,6 +492,19 @@ config HOTPLUG_CPU Say N if you are unsure. +config PPC_QUEUED_SPINLOCKS + bool "Queued spinlocks" + depends on SMP + help + Say Y here to use to use queued spinlocks which give better + scalability and fairness on large SMP and NUMA systems without + harming single threaded performance. + + This option is currently experimental, the code is more complex + and less tested so it defaults to "N" for the moment. + + If unsure, say "N". + config ARCH_CPU_PROBE_RELEASE def_bool y depends on HOTPLUG_CPU
arch/powerpc/include/asm/Kbuild +1 -0
@@ -6,5 +6,6 @@ generated-y += syscall_table_spu.h generic-y += export.h generic-y += local64.h generic-y += mcs_spinlock.h +generic-y += qrwlock.h generic-y += vtime.h generic-y += early_ioremap.h
arch/powerpc/include/asm/qspinlock.h new file +25 -0
@@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_QSPINLOCK_H +#define _ASM_POWERPC_QSPINLOCK_H + +#include <asm-generic/qspinlock_types.h> + +#define _Q_PENDING_LOOPS (1 << 9)/* not tuned */ + +#define smp_mb__after_spinlock () smp_mb() + +static __always_inline int queued_spin_is_locked(struct qspinlock *lock) +{ +/* +* This barrier was added to simple spinlocks by commit 51d7d5205d338, +* but it should now be possible to remove it, asm arm64 has done with +* commit c6f5d02b6a0f. +*/ +smp_mb (); +return atomic_read (&lock->val); +} +#define queued_spin_is_locked queued_spin_is_locked + +#include <asm-generic/qspinlock.h> + +#endif /* _ASM_POWERPC_QSPINLOCK_H */
arch/powerpc/include/asm/spinlock.h +5 -0
@@ -3,7 +3,12 @@#define __ASM_SPINLOCK_H#ifdef __KERNEL__ +#ifdef CONFIG_PPC_QUEUED_SPINLOCKS +#include <asm/qspinlock.h> +#include <asm/qrwlock.h> +#else #include <asm/simple_spinlock.h> +#endif #endif /* __KERNEL__ */ #endif /* __ASM_SPINLOCK_H */
arch/powerpc/include/asm/spinlock_types.h +5 -0
@@ -6,6 +6,11 @@# error "please don't include this file directly"#endif +#ifdef CONFIG_PPC_QUEUED_SPINLOCKS +#include <asm-generic/qspinlock_types.h> +#include <asm-generic/qrwlock_types.h> +#else #include <asm/simple_spinlock_types.h> +#endif #endif
arch/powerpc/lib/Makefile +3 -0
@@ -41,7 +41,10 @@ obj-$(CONFIG_PPC_BOOK3S_64) += copyuser_power7.o copypage_power7.o \ obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \ memcpy_64.o memcpy_mcsafe_64.o +ifndef CONFIG_PPC_QUEUED_SPINLOCKS obj64-$(CONFIG_SMP) += locks.o +endif + obj64-$(CONFIG_ALTIVEC) += vmx-helper.o obj64-$(CONFIG_KPROBES_SANITY_TEST) += test_emulate_step.o \ test_emulate_step_exec_instr.o
include/asm-generic/qspinlock.h +2 -0
@@ -12,6 +12,7 @@#include <asm-generic/qspinlock_types.h> +#ifndef queued_spin_is_locked/** * queued_spin_is_locked - is the spinlock locked? * @lock: Pointer to queued spinlock structure
@@ -25,6 +26,7 @@ static __always_inline int queued_spin_is_locked(struct qspinlock *lock)*/ return atomic_read(&lock->val); } +#endif /** * queued_spin_value_unlocked - is the spinlock structure unlocked?
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v4 4/6 powerpc/pseries: implement paravirt qspinlocks for SPLPAR
In reply to Nicholas PigginThis implements the generic paravirt qspinlocks using H_PROD and H_CONFER to kick and wait.
This uses an un-directed yield to any CPU rather than the directed yield to a pre-empted lock holder that paravirtualised simple spinlocks use, that requires no kick hcall. This is something that could be investigated and improved in future.
Performance results can be found in the commit which added queued spinlocks.
--- arch/powerpc/include/asm/paravirt.h | 28 ++++++++ arch/powerpc/include/asm/qspinlock.h | 66 +++++++++++++++++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 7 ++ arch/powerpc/include/asm/spinlock.h | 4 ++ arch/powerpc/platforms/pseries/Kconfig | 9 ++- arch/powerpc/platforms/pseries/setup.c | 4 +- include/asm-generic/qspinlock.h | 2 + 7 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt.h
arch/powerpc/include/asm/paravirt.h +28 -0
@@ -28,6 +28,16 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {plpar_hcall_norets (H_CONFER ,get_hard_smp_processor_id (cpu), yield_count); } + +static inline void prod_cpu (int cpu) +{ +plpar_hcall_norets (H_PROD ,get_hard_smp_processor_id (cpu)); +} + +static inline void yield_to_any (void ) +{ +plpar_hcall_norets (H_CONFER ,-1 ,0 ); +}#else static inline bool is_shared_processor (void ) {
@@ -44,6 +54,19 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {___bad_yield_to_preempted ();/* This would be a bug */ } + +extern void ___bad_yield_to_any (void ); +static inline void yield_to_any (void ) +{ +___bad_yield_to_any ();/* This would be a bug */ +} + +extern void ___bad_prod_cpu (void ); +static inline void prod_cpu (int cpu) +{ +___bad_prod_cpu ();/* This would be a bug */ +} +#endif #define vcpu_is_preempted vcpu_is_preempted
@@ -56,4 +79,9 @@ static inline bool vcpu_is_preempted(int cpu) return false; } +static inline bool pv_is_native_spin_unlock (void ) +{ +return !is_shared_processor (); +} +#endif /* _ASM_POWERPC_PARAVIRT_H */
arch/powerpc/include/asm/qspinlock.h +66 -0
81 lines not on this page · raw message
arch/powerpc/include/asm/qspinlock_paravirt.h new file +7 -0
@@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_QSPINLOCK_PARAVIRT_H +#define _ASM_POWERPC_QSPINLOCK_PARAVIRT_H + +EXPORT_SYMBOL (__pv_queued_spin_unlock); + +#endif /* _ASM_POWERPC_QSPINLOCK_PARAVIRT_H */
arch/powerpc/include/asm/spinlock.h +4 -0
@@ -10,5 +10,9 @@#include <asm/simple_spinlock.h> #endif +#ifndef CONFIG_PARAVIRT_SPINLOCKS +static inline void pv_spinlocks_init (void ) { } +#endif +#endif /* __KERNEL__ */ #endif /* __ASM_SPINLOCK_H */
arch/powerpc/platforms/pseries/Kconfig +8 -1
@@ -25,15 +25,22 @@ config PPC_PSERIES select SWIOTLB default y +config PARAVIRT_SPINLOCKS + bool + config PPC_SPLPAR - depends on PPC_PSERIES bool "Support for shared-processor logical partitions" + depends on PPC_PSERIES + select PARAVIRT_SPINLOCKS if PPC_QUEUED_SPINLOCKS + default y help Enabling this option will make the kernel run more efficiently on logically-partitioned pSeries systems which use shared processors, that is, which share physical processors between two or more partitions. + Say Y if you are unsure. + config DTL bool "Dispatch Trace Log" depends on PPC_SPLPAR && DEBUG_FS
arch/powerpc/platforms/pseries/setup.c +3 -1
@@ -771,8 +771,10 @@ static void __init pSeries_setup_arch(void)if (firmware_has_feature (FW_FEATURE_LPAR )) {vpa_init (boot_cpuid); -if (lppaca_shared_proc (get_lppaca ())) +if (lppaca_shared_proc (get_lppaca ())) {static_branch_enable (&shared_processor); +pv_spinlocks_init (); + } ppc_md.power_save = pseries_lpar_idle; ppc_md.enable_pmcs = pseries_lpar_enable_pmcs;
include/asm-generic/qspinlock.h +2 -0
@@ -69,6 +69,7 @@ static __always_inline int queued_spin_trylock(struct qspinlock *lock)extern void queued_spin_lock_slowpath (struct qspinlock *lock,u32 val); +#ifndef queued_spin_lock/** * queued_spin_lock - acquire a queued spinlock * @lock: Pointer to queued spinlock structure
@@ -82,6 +83,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) queued_spin_lock_slowpath(lock, val); } +#endif #ifndef queued_spin_unlock/**
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v4 5/6 powerpc/qspinlock: optimised atomic_try_cmpxchg_lock that adds the lock hint
In reply to Nicholas PigginThis brings the behaviour of the uncontended fast path back to roughly equivalent to simple spinlocks -- a single atomic op with lock hint.
--- arch/powerpc/include/asm/atomic.h | 28 ++++++++++++++++++++++++++++ arch/powerpc/include/asm/qspinlock.h | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/atomic.h +28 -0
@@ -193,6 +193,34 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v)#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_xchg_relaxed (v, new) xchg_relaxed(&((v)->counter), (new)) +/* +* Don't want to override the generic atomic_try_cmpxchg_acquire, because +* we add a lock hint to the lwarx, which may not be wanted for the +* _acquire case (and is not used by the other _acquire variants so it +* would be a surprise). +*/ +static __always_inline bool +atomic_try_cmpxchg_lock (atomic_t *v,int *old,int new) +{ +int r, o = *old; + + __asm__ __volatile__ ( +"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n" +" cmpw 0,%0,%3 \n" +" bne- 2f \n" +" stwcx. %4,0,%2 \n" +" bne- 1b \n" +"\t" PPC_ACQUIRE_BARRIER " \n" +"2: \n" + : "=&r" (r), "+m" (v->counter) + : "r" (&v->counter), "r" (o), "r" (new) + : "cr0", "memory"); + + if (unlikely (r != o)) + *old = r; +return likely (r = o); +} +/** * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t
arch/powerpc/include/asm/qspinlock.h +1 -1
@@ -37,7 +37,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) {u32 val =0 ; -if (likely (atomic_try_cmpxchg_acquire (&lock->val, &val, _Q_LOCKED_VAL))) +if (likely (atomic_try_cmpxchg_lock (&lock->val, &val, _Q_LOCKED_VAL)))return ;queued_spin_lock_slowpath (lock, val);
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v4 6/6 powerpc: implement smp_cond_load_relaxed
In reply to Nicholas PigginThis implements smp_cond_load_relaed with the slowpath busy loop using the preferred SMT priority pattern.
--- arch/powerpc/include/asm/barrier.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
arch/powerpc/include/asm/barrier.h +14 -0
@@ -76,6 +76,20 @@ do { \ ___p1; \ }) +#define smp_cond_load_relaxed (ptr, cond_expr) ({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + VAL = READ_ONCE(*__PTR); \ + if (unlikely(!(cond_expr))) { \ + spin_begin(); \ + do { \ + VAL = READ_ONCE(*__PTR); \ + } while (!(cond_expr)); \ + spin_end(); \ + } \ + (typeof(*ptr))VAL; \ +}) +#ifdef CONFIG_PPC_BOOK3S_64 #define NOSPEC_BARRIER_SLOT nop#elif defined(CONFIG_PPC_FSL_BOOK3E )
Signature · 1 line
2.23.0
Waiman Long
reviewerRe: [PATCH v4 6/6] powerpc: implement smp_cond_load_relaxed
Quoted 1 line · Nicholas Piggin
This implements
smp_cond_load_relaedwith the slowpath busy loop using the
Nit: "smp_cond_load_relaxed"
Cheers,
Longman
Quoted 45 lines · Nicholas Piggin
Updated with everybody's feedback (thanks all), and more performance results.
What I've found is I might have been measuring the worst load point for the paravirt case, and by looking at a range of loads it's clear that queued spinlocks are overall better even on PV, doubly so when you look at the generally much improved worst case latencies.
I have defaulted it to N even though I'm less concerned about the PV numbers now, just because I think it needs more stress testing. But it's very nicely selectable so should be low risk to include.
All in all this is a very cool technology and great results especially on the big systems but even on smaller ones there are nice gains. Thanks Waiman and everyone who developed it.
Thanks,
NickNicholas Piggin (6):
powerpc/pseries: move some PAPR paravirt functions to their own file
powerpc: move spinlock implementation tosimple_spinlockpowerpc/64s: implement queued spinlocks and rwlocks powerpc/pseries: implement paravirt qspinlocks for SPLPAR powerpc/qspinlock: optimisedatomic_try_cmpxchg_lockthat adds the
lock hint
powerpc: implementsmp_cond_load_relaxedarch/powerpc/Kconfig | 15 + arch/powerpc/include/asm/Kbuild | 1 + arch/powerpc/include/asm/atomic.h | 28 ++ arch/powerpc/include/asm/barrier.h | 14 + arch/powerpc/include/asm/paravirt.h | 87 +++++ arch/powerpc/include/asm/qspinlock.h | 91 ++++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 7 + arch/powerpc/include/asm/simple_spinlock.h | 288 ++++++++++++++++ .../include/asm/simple_spinlock_types.h | 21 ++ arch/powerpc/include/asm/spinlock.h | 308 +----------------- arch/powerpc/include/asm/spinlock_types.h | 17 +- arch/powerpc/lib/Makefile | 3 + arch/powerpc/lib/locks.c | 12 +- arch/powerpc/platforms/pseries/Kconfig | 9 +- arch/powerpc/platforms/pseries/setup.c | 4 +- include/asm-generic/qspinlock.h | 4 + 16 files changed, 588 insertions(+), 321 deletions(-)
Quoted 6 lines
create mode 100644
arch/powerpc/include/asm/paravirt.hcreate mode 100644arch/powerpc/include/asm/qspinlock.hcreate mode 100644arch/powerpc/include/asm/qspinlock_paravirt.hcreate mode 100644arch/powerpc/include/asm/simple_spinlock.hcreate mode 100644arch/powerpc/include/asm/simple_spinlock_types.h
That patch series looks good to me. Thanks for working on this.
For the series,
Michael Ellerman
AppliedQuoted 9 lines · Nicholas Piggin
Updated with everybody's feedback (thanks all), and more performance results.
What I've found is I might have been measuring the worst load point for the paravirt case, and by looking at a range of loads it's clear that queued spinlocks are overall better even on PV, doubly so when you look at the generally much improved worst case latencies.
[...]
Applied to powerpc/next.
[1/6] powerpc/pseries: Move some PAPR paravirt functions to their own file
https://git.kernel.org/powerpc/c/20d444d06f97504d165b08558678b4737dcefb02 [2/6] powerpc: Move spinlock implementation to simple_spinlock https://git.kernel.org/powerpc/c/12d0b9d6c843e7dbe739ebefcf16c7e4a45e4e78 [3/6] powerpc/64s: Implement queued spinlocks and rwlocks https://git.kernel.org/powerpc/c/aa65ff6b18e0366db1790609956a4ac7308c5668 [4/6] powerpc/pseries: Implement paravirt qspinlocks for SPLPAR https://git.kernel.org/powerpc/c/20c0e8269e9d515e677670902c7e1cc0209d6ad9 [5/6] powerpc/qspinlock: Optimised atomic_try_cmpxchg_lock() that adds the lock hint https://git.kernel.org/powerpc/c/2f6560e652dfdbdb59df28b45a3458bf36d3c580 [6/6] powerpc: Implement smp_cond_load_relaxed() https://git.kernel.org/powerpc/c/49a7d46a06c30c7beabbf9d1a8ea1de0f9e4fdfe
cheers