powerpc: queued spinlocks and rwlocks
Subsystems (6)
Nicholas Piggin
reviewerv2 cover powerpc: queued spinlocks and rwlocks
v2 is updated to account for feedback from Will, Peter, and Waiman (thank you), and trims off a couple of RFC and unrelated patches.
Thanks,
Nick
Nicholas Piggin (6):
powerpc/powernv: must include hvcall.h to get PAPR defines powerpc/pseries: move some PAPR paravirt functions to their own file
powerpc: move spinlock implementation to simple_spinlock powerpc/64s: implement queued spinlocks and rwlocks powerpc/pseries: implement paravirt qspinlocks for SPLPAR powerpc/qspinlock: optimised atomic_try_cmpxchg_lock that adds the
lock hint
arch/powerpc/Kconfig | 13 + arch/powerpc/include/asm/Kbuild | 2 + arch/powerpc/include/asm/atomic.h | 28 ++ arch/powerpc/include/asm/paravirt.h | 89 +++++ arch/powerpc/include/asm/qspinlock.h | 80 +++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 5 + arch/powerpc/include/asm/simple_spinlock.h | 292 +++++++++++++++++ .../include/asm/simple_spinlock_types.h | 21 ++ arch/powerpc/include/asm/spinlock.h | 308 +----------------- arch/powerpc/include/asm/spinlock_types.h | 17 +- arch/powerpc/lib/Makefile | 3 + arch/powerpc/lib/locks.c | 12 +- arch/powerpc/platforms/powernv/pci-ioda-tce.c | 1 + arch/powerpc/platforms/pseries/Kconfig | 5 + arch/powerpc/platforms/pseries/setup.c | 6 +- include/asm-generic/qspinlock.h | 4 + 16 files changed, 564 insertions(+), 322 deletions(-) create mode 100644 arch/powerpc/include/asm/paravirt.h create mode 100644 arch/powerpc/include/asm/qspinlock.h create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt.h create mode 100644 arch/powerpc/include/asm/simple_spinlock.h create mode 100644 arch/powerpc/include/asm/simple_spinlock_types.h
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v2 1/6 powerpc/powernv: must include hvcall.h to get PAPR defines
An include goes away in future patches which breaks compilation without this.
--- arch/powerpc/platforms/powernv/pci-ioda-tce.c | 1 + 1 file changed, 1 insertion(+)
arch/powerpc/platforms/powernv/pci-ioda-tce.c +1 -0
@@ -15,6 +15,7 @@#include <asm/iommu.h> #include <asm/tce.h> +#include <asm/hvcall.h> /* share error returns with PAPR */ #include "pci.h" unsigned long pnv_ioda_parse_tce_sizes (struct pnv_phb *phb)
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v2 2/6 powerpc/pseries: move some PAPR paravirt functions to their own file
In reply to Nicholas Piggin--- arch/powerpc/include/asm/paravirt.h | 61 +++++++++++++++++++++++++++++ arch/powerpc/include/asm/spinlock.h | 24 +----------- arch/powerpc/lib/locks.c | 12 +++--- 3 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 arch/powerpc/include/asm/paravirt.h
arch/powerpc/include/asm/paravirt.h new file +61 -0
@@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_PARAVIRT_H +#define __ASM_PARAVIRT_H +#ifdef __KERNEL__ + +#include <linux/jump_label.h> +#include <asm/smp.h> +#ifdef CONFIG_PPC64 +#include <asm/paca.h> +#include <asm/hvcall.h> +#endif + +#ifdef CONFIG_PPC_SPLPAR +DECLARE_STATIC_KEY_FALSE (shared_processor); + +static inline bool is_shared_processor (void ) +{ +return static_branch_unlikely (&shared_processor); +} + +/* If bit 0 is set, the cpu has been preempted */ +static inline u32 yield_count_of (int cpu) +{ +__be32 yield_count =READ_ONCE (lppaca_of (cpu).yield_count); +return be32_to_cpu (yield_count); +} + +static inline void yield_to_preempted (int cpu,u32 yield_count) +{ +plpar_hcall_norets (H_CONFER ,get_hard_smp_processor_id (cpu), yield_count); +} +#else +static inline bool is_shared_processor (void ) +{ +return false; +} + +static inline u32 yield_count_of (int cpu) +{ +return 0 ; +} + +extern void ___bad_yield_to_preempted (void ); +static inline void yield_to_preempted (int cpu,u32 yield_count) +{ +___bad_yield_to_preempted ();/* This would be a bug */ +} +#endif + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted (int cpu) +{ +if (!is_shared_processor ()) +return false; +if (yield_count_of (cpu) &1 ) +return true; +return false; +} + +#endif /* __KERNEL__ */ +#endif /* __ASM_PARAVIRT_H */
arch/powerpc/include/asm/spinlock.h +1 -23
@@ -15,11 +15,10 @@* * (the type definitions are in asm/spinlock_types.h) */ -#include <linux/jump_label.h> #include <linux/irqflags.h> +#include <asm/paravirt.h> #ifdef CONFIG_PPC64 #include <asm/paca.h> -#include <asm/hvcall.h> #endif #include <asm/synch.h> #include <asm/ppc-opcode.h>
@@ -35,18 +34,6 @@#define LOCK_TOKEN 1#endif -#ifdef CONFIG_PPC_PSERIES -DECLARE_STATIC_KEY_FALSE (shared_processor); - -#define vcpu_is_preempted vcpu_is_preempted -static inline bool vcpu_is_preempted (int cpu) -{ -if (!static_branch_unlikely (&shared_processor)) -return false; -return !!(be32_to_cpu (lppaca_of (cpu).yield_count) &1 ); -} -#endif - static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) {return lock.slock =0 ;
@@ -110,15 +97,6 @@ static inline void splpar_spin_yield(arch_spinlock_t *lock) {};static inline void splpar_rw_yield (arch_rwlock_t *lock) {};#endif -static inline bool is_shared_processor (void ) -{ -#ifdef CONFIG_PPC_SPLPAR -return static_branch_unlikely (&shared_processor); -#else -return false; -#endif -} -static inline void spin_yield (arch_spinlock_t *lock) {if (is_shared_processor ())
arch/powerpc/lib/locks.c +6 -6
@@ -27,14 +27,14 @@ void splpar_spin_yield(arch_spinlock_t *lock)return ; holder_cpu = lock_value &0xffff ;BUG_ON (holder_cpu >=NR_CPUS ); - yield_count =be32_to_cpu (lppaca_of (holder_cpu).yield_count); + + yield_count =yield_count_of (holder_cpu);if ((yield_count &1 ) =0 )return ;/* virtual cpu is currently running */ rmb ();if (lock->slock != lock_value)return ;/* something has changed */ -plpar_hcall_norets (H_CONFER , -get_hard_smp_processor_id (holder_cpu), yield_count); +yield_to_preempted (holder_cpu, yield_count); }EXPORT_SYMBOL_GPL (splpar_spin_yield);
@@ -53,13 +53,13 @@ void splpar_rw_yield(arch_rwlock_t *rw)return ;/* no write lock at present */ holder_cpu = lock_value &0xffff ;BUG_ON (holder_cpu >=NR_CPUS ); - yield_count =be32_to_cpu (lppaca_of (holder_cpu).yield_count); + + yield_count =yield_count_of (holder_cpu);if ((yield_count &1 ) =0 )return ;/* virtual cpu is currently running */ rmb ();if (rw->lock != lock_value)return ;/* something has changed */ -plpar_hcall_norets (H_CONFER , - get_hard_smp_processor_id(holder_cpu), yield_count); + yield_to_preempted(holder_cpu, yield_count); }#endif
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v2 3/6 powerpc: move spinlock implementation to simple_spinlock
In reply to Nicholas PigginTo prepare for queued spinlocks. This is a simple rename except to update preprocessor guard name and a file reference.
--- arch/powerpc/include/asm/simple_spinlock.h | 292 ++++++++++++++++++ .../include/asm/simple_spinlock_types.h | 21 ++ arch/powerpc/include/asm/spinlock.h | 285 +---------------- arch/powerpc/include/asm/spinlock_types.h | 12 +- 4 files changed, 315 insertions(+), 295 deletions(-) create mode 100644 arch/powerpc/include/asm/simple_spinlock.h create mode 100644 arch/powerpc/include/asm/simple_spinlock_types.h
arch/powerpc/include/asm/simple_spinlock.h new file +292 -0
@@ -0,0 +1,292 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_SIMPLE_SPINLOCK_H +#define __ASM_SIMPLE_SPINLOCK_H +#ifdef __KERNEL__ + +/* +* Simple spin lock operations. +* +* Copyright (C) 2001-2004 Paul Mackerras <[email protected]>, IBM +* Copyright (C) 2001 Anton Blanchard <[email protected]>, IBM +* Copyright (C) 2002 Dave Engebretsen <[email protected]>, IBM +* Rework to support virtual processors +* +* Type of int is used as a full 64b word is not necessary. +* +* (the type definitions are in asm/simple_spinlock_types.h) +*/ +#include <linux/irqflags.h> +#include <asm/paravirt.h> +#ifdef CONFIG_PPC64 +#include <asm/paca.h> +#endif +#include <asm/synch.h> +#include <asm/ppc-opcode.h> + +#ifdef CONFIG_PPC64 +/* use 0x800000yy when locked, where yy = CPU number */ +#ifdef __BIG_ENDIAN__ +#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) +#else +#define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) +#endif +#else +#define LOCK_TOKEN 1 +#endif + +static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ +return lock.slock =0 ; +} + +static inline int arch_spin_is_locked (arch_spinlock_t *lock) +{ +smp_mb (); +return !arch_spin_value_unlocked (*lock); +} + +/* +* This returns the old value in the lock, so we succeeded +* in getting the lock if the return value is 0. +*/ +static inline unsigned long __arch_spin_trylock (arch_spinlock_t *lock) +{ +unsigned long tmp, token; + + token =LOCK_TOKEN ; + __asm__ __volatile__( +"1: " PPC_LWARX(%0,0,%2,1) "\n\ +cmpwi 0,%0,0\n\ +bne- 2f\n\ +stwcx. %1,0,%2\n\ +bne- 1b\n" +PPC_ACQUIRE_BARRIER +"2:" + : "=&r" (tmp) + : "r" (token), "r" (&lock->slock) + : "cr0", "memory"); + + return tmp; +} + +static inline int arch_spin_trylock (arch_spinlock_t *lock) +{ +return __arch_spin_trylock (lock) =0 ; +} + +/* +* On a system with shared processors (that is, where a physical +* processor is multiplexed between several virtual processors), +* there is no point spinning on a lock if the holder of the lock +* isn't currently scheduled on a physical processor. Instead +* we detect this situation and ask the hypervisor to give the +* rest of our timeslice to the lock holder. +* +* So that we can tell which virtual processor is holding a lock, +* we put 0x80000000 | smp_processor_id() in the lock when it is +* held. Conveniently, we have a word in the paca that holds this +* value. +*/ + +#if defined(CONFIG_PPC_SPLPAR ) +/* We only yield to the hypervisor if we are in shared processor mode */ +void splpar_spin_yield (arch_spinlock_t *lock); +void splpar_rw_yield (arch_rwlock_t *lock); +#else /* SPLPAR */ +static inline void splpar_spin_yield (arch_spinlock_t *lock) {}; +static inline void splpar_rw_yield (arch_rwlock_t *lock) {}; +#endif + +static inline void spin_yield (arch_spinlock_t *lock) +{ +if (is_shared_processor ()) +splpar_spin_yield (lock); +else +barrier (); +} + +static inline void rw_yield (arch_rwlock_t *lock) +{ +if (is_shared_processor ()) +splpar_rw_yield (lock); +else +barrier (); +} + +static inline void arch_spin_lock (arch_spinlock_t *lock) +{ +while (1 ) { +if (likely (__arch_spin_trylock (lock) =0 )) +break ; +do { +HMT_low (); +if (is_shared_processor ()) +splpar_spin_yield (lock); + }while (unlikely (lock->slock !=0 )); +HMT_medium (); + } +} + +static inline +void arch_spin_lock_flags (arch_spinlock_t *lock,unsigned long flags) +{ +unsigned long flags_dis; + +while (1 ) { +if (likely (__arch_spin_trylock (lock) =0 )) +break ; +local_save_flags (flags_dis); +local_irq_restore (flags); +do { +HMT_low (); +if (is_shared_processor ()) +splpar_spin_yield (lock); + }while (unlikely (lock->slock !=0 )); +HMT_medium (); +local_irq_restore (flags_dis); + } +} +#define arch_spin_lock_flags arch_spin_lock_flags + +static inline void arch_spin_unlock (arch_spinlock_t *lock) +{ + __asm__ __volatile__("# arch_spin_unlock\n\t" +PPC_RELEASE_BARRIER : : :"memory" ); + lock->slock =0 ; +} + +/* +* Read-write spinlocks, allowing multiple readers +* but only one writer. +* +* NOTE! it is quite common to have readers in interrupts +* but no interrupt writers. For those circumstances we +* can "mix" irq-safe locks - any writer needs to get a +* irq-safe write-lock, but readers can get non-irqsafe +* read-locks. +*/ + +#ifdef CONFIG_PPC64 +#define __DO_SIGN_EXTEND "extsw %0,%0\n" +#define WRLOCK_TOKEN LOCK_TOKEN/* it's negative */ +#else +#define __DO_SIGN_EXTEND +#define WRLOCK_TOKEN (-1) +#endif + +/* +* This returns the old value in the lock + 1, +* so we got a read lock if the return value is > 0. +*/ +static inline long __arch_read_trylock (arch_rwlock_t *rw) +{ +long tmp; + + __asm__ __volatile__( +"1: " PPC_LWARX(%0,0,%1,1) "\n" + __DO_SIGN_EXTEND +" addic. %0,%0,1\n\ +ble- 2f\n" +" stwcx. %0,0,%1\n\ +bne- 1b\n" +PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) + : "r" (&rw->lock) + : "cr0", "xer", "memory"); + + return tmp; +} + +/* +* This returns the old value in the lock, +* so we got the write lock if the return value is 0. +*/ +static inline long __arch_write_trylock (arch_rwlock_t *rw) +{ +long tmp, token; + + token =WRLOCK_TOKEN ; + __asm__ __volatile__( +"1: " PPC_LWARX(%0,0,%2,1) "\n\ +cmpwi 0,%0,0\n\ +bne- 2f\n" +" stwcx. %1,0,%2\n\ +bne- 1b\n" +PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) + : "r" (token), "r" (&rw->lock) + : "cr0", "memory"); + + return tmp; +} + +static inline void arch_read_lock (arch_rwlock_t *rw) +{ +while (1 ) { +if (likely (__arch_read_trylock (rw) >0 )) +break ; +do { +HMT_low (); +if (is_shared_processor ()) +splpar_rw_yield (rw); + }while (unlikely (rw->lock <0 )); +HMT_medium (); + } +} + +static inline void arch_write_lock (arch_rwlock_t *rw) +{ +while (1 ) { +if (likely (__arch_write_trylock (rw) =0 )) +break ; +do { +HMT_low (); +if (is_shared_processor ()) +splpar_rw_yield (rw); + }while (unlikely (rw->lock !=0 )); +HMT_medium (); + } +} + +static inline int arch_read_trylock (arch_rwlock_t *rw) +{ +return __arch_read_trylock (rw) >0 ; +} + +static inline int arch_write_trylock (arch_rwlock_t *rw) +{ +return __arch_write_trylock (rw) =0 ; +} + +static inline void arch_read_unlock (arch_rwlock_t *rw) +{ +long tmp; + + __asm__ __volatile__( +"# read_unlock\n\t" +PPC_RELEASE_BARRIER +"1: lwarx %0,0,%1\n\ +addic %0,%0,-1\n" +" stwcx. %0,0,%1\n\ +bne- 1b" + :"=&r" (tmp) + :"r" (&rw->lock) + :"cr0" ,"xer" ,"memory" ); +} + +static inline void arch_write_unlock (arch_rwlock_t *rw) +{ + __asm__ __volatile__("# write_unlock\n\t" +PPC_RELEASE_BARRIER : : :"memory" ); + rw->lock =0 ; +} + +#define arch_spin_relax (lock) spin_yield(lock) +#define arch_read_relax (lock) rw_yield(lock) +#define arch_write_relax (lock) rw_yield(lock) + +/* See include/linux/spinlock.h */ +#define smp_mb__after_spinlock () smp_mb() + +#endif /* __KERNEL__ */ +#endif /* __ASM_SIMPLE_SPINLOCK_H */
arch/powerpc/include/asm/simple_spinlock_types.h new file +21 -0
@@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H +#define _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H + +#ifndef __LINUX_SPINLOCK_TYPES_H +# error "please don't include this file directly" +#endif + +typedef struct { +volatile unsigned int slock; +}arch_spinlock_t ; + +#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } + +typedef struct { +volatile signed int lock; +}arch_rwlock_t ; + +#define __ARCH_RW_LOCK_UNLOCKED { 0 } + +#endif
arch/powerpc/include/asm/spinlock.h +1 -284
291 lines not on this page · raw message
arch/powerpc/include/asm/spinlock_types.h +1 -11
@@ -6,16 +6,6 @@# error "please don't include this file directly"#endif -typedef struct { -volatile unsigned int slock; -}arch_spinlock_t ; - -#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } - -typedef struct { -volatile signed int lock; -}arch_rwlock_t ; - -#define __ARCH_RW_LOCK_UNLOCKED { 0 } +#include <asm/simple_spinlock_types.h> #endif
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v2 4/6 powerpc/64s: implement queued spinlocks and rwlocks
In reply to Nicholas PigginThese have shown significantly improved performance and fairness when spinlock contention is moderate to high on very large systems.
[ Numbers hopefully forthcoming after more testing, but initial
results look good ]
Thanks to the fast path, single threaded performance is not noticably hurt.
--- arch/powerpc/Kconfig | 13 ++++++++++++ arch/powerpc/include/asm/Kbuild | 2 ++ arch/powerpc/include/asm/qspinlock.h | 25 +++++++++++++++++++++++ arch/powerpc/include/asm/spinlock.h | 5 +++++ arch/powerpc/include/asm/spinlock_types.h | 5 +++++ arch/powerpc/lib/Makefile | 3 +++ include/asm-generic/qspinlock.h | 2 ++ 7 files changed, 55 insertions(+) create mode 100644 arch/powerpc/include/asm/qspinlock.h
arch/powerpc/Kconfig +13 -0
@@ -145,6 +145,8 @@ config PPC select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if PPC64 + select ARCH_USE_QUEUED_RWLOCKS if PPC_QUEUED_SPINLOCKS + select ARCH_USE_QUEUED_SPINLOCKS if PPC_QUEUED_SPINLOCKS select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WEAK_RELEASE_ACQUIRE select BINFMT_ELF
@@ -490,6 +492,17 @@ config HOTPLUG_CPU Say N if you are unsure. +config PPC_QUEUED_SPINLOCKS + bool "Queued spinlocks" + depends on SMP + default "y" if PPC_BOOK3S_64 + help + Say Y here to use to use queued spinlocks which are more complex + but give better salability and fairness on large SMP and NUMA + systems. + + If unsure, say "Y" if you have lots of cores, otherwise "N". + config ARCH_CPU_PROBE_RELEASE def_bool y depends on HOTPLUG_CPU
arch/powerpc/include/asm/Kbuild +2 -0
@@ -6,5 +6,7 @@ generated-y += syscall_table_spu.h generic-y += export.h generic-y += local64.h generic-y += mcs_spinlock.h +generic-y += qrwlock.h +generic-y += qspinlock.h generic-y += vtime.h generic-y += early_ioremap.h
arch/powerpc/include/asm/qspinlock.h new file +25 -0
@@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_QSPINLOCK_H +#define _ASM_POWERPC_QSPINLOCK_H + +#include <asm-generic/qspinlock_types.h> + +#define _Q_PENDING_LOOPS (1 << 9)/* not tuned */ + +#define smp_mb__after_spinlock () smp_mb() + +static __always_inline int queued_spin_is_locked(struct qspinlock *lock) +{ +/* +* This barrier was added to simple spinlocks by commit 51d7d5205d338, +* but it should now be possible to remove it, asm arm64 has done with +* commit c6f5d02b6a0f. +*/ +smp_mb (); +return atomic_read (&lock->val); +} +#define queued_spin_is_locked queued_spin_is_locked + +#include <asm-generic/qspinlock.h> + +#endif /* _ASM_POWERPC_QSPINLOCK_H */
arch/powerpc/include/asm/spinlock.h +5 -0
@@ -3,7 +3,12 @@#define __ASM_SPINLOCK_H#ifdef __KERNEL__ +#ifdef CONFIG_PPC_QUEUED_SPINLOCKS +#include <asm/qspinlock.h> +#include <asm/qrwlock.h> +#else #include <asm/simple_spinlock.h> +#endif #endif /* __KERNEL__ */ #endif /* __ASM_SPINLOCK_H */
arch/powerpc/include/asm/spinlock_types.h +5 -0
@@ -6,6 +6,11 @@# error "please don't include this file directly"#endif +#ifdef CONFIG_PPC_QUEUED_SPINLOCKS +#include <asm-generic/qspinlock_types.h> +#include <asm-generic/qrwlock_types.h> +#else #include <asm/simple_spinlock_types.h> +#endif #endif
arch/powerpc/lib/Makefile +3 -0
@@ -41,7 +41,10 @@ obj-$(CONFIG_PPC_BOOK3S_64) += copyuser_power7.o copypage_power7.o \ obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \ memcpy_64.o memcpy_mcsafe_64.o +ifndef CONFIG_PPC_QUEUED_SPINLOCKS obj64-$(CONFIG_SMP) += locks.o +endif + obj64-$(CONFIG_ALTIVEC) += vmx-helper.o obj64-$(CONFIG_KPROBES_SANITY_TEST) += test_emulate_step.o \ test_emulate_step_exec_instr.o
include/asm-generic/qspinlock.h +2 -0
@@ -12,6 +12,7 @@#include <asm-generic/qspinlock_types.h> +#ifndef queued_spin_is_locked/** * queued_spin_is_locked - is the spinlock locked? * @lock: Pointer to queued spinlock structure
@@ -25,6 +26,7 @@ static __always_inline int queued_spin_is_locked(struct qspinlock *lock)*/ return atomic_read(&lock->val); } +#endif /** * queued_spin_value_unlocked - is the spinlock structure unlocked?
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v2 5/6 powerpc/pseries: implement paravirt qspinlocks for SPLPAR
In reply to Nicholas Piggin--- arch/powerpc/include/asm/paravirt.h | 28 ++++++++++ arch/powerpc/include/asm/qspinlock.h | 55 +++++++++++++++++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 5 ++ arch/powerpc/platforms/pseries/Kconfig | 5 ++ arch/powerpc/platforms/pseries/setup.c | 6 +- include/asm-generic/qspinlock.h | 2 + 6 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt.h
arch/powerpc/include/asm/paravirt.h +28 -0
@@ -29,6 +29,16 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {plpar_hcall_norets (H_CONFER ,get_hard_smp_processor_id (cpu), yield_count); } + +static inline void prod_cpu (int cpu) +{ +plpar_hcall_norets (H_PROD ,get_hard_smp_processor_id (cpu)); +} + +static inline void yield_to_any (void ) +{ +plpar_hcall_norets (H_CONFER ,-1 ,0 ); +}#else static inline bool is_shared_processor (void ) {
@@ -45,6 +55,19 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {___bad_yield_to_preempted ();/* This would be a bug */ } + +extern void ___bad_yield_to_any (void ); +static inline void yield_to_any (void ) +{ +___bad_yield_to_any ();/* This would be a bug */ +} + +extern void ___bad_prod_cpu (void ); +static inline void prod_cpu (int cpu) +{ +___bad_prod_cpu ();/* This would be a bug */ +} +#endif #define vcpu_is_preempted vcpu_is_preempted
@@ -57,5 +80,10 @@ static inline bool vcpu_is_preempted(int cpu)return false; } +static inline bool pv_is_native_spin_unlock (void ) +{ +return !is_shared_processor (); +} +#endif /* __KERNEL__ */ #endif /* __ASM_PARAVIRT_H */
arch/powerpc/include/asm/qspinlock.h +55 -0
70 lines not on this page · raw message
arch/powerpc/include/asm/qspinlock_paravirt.h new file +5 -0
@@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_QSPINLOCK_PARAVIRT_H +#define __ASM_QSPINLOCK_PARAVIRT_H + +#endif /* __ASM_QSPINLOCK_PARAVIRT_H */
arch/powerpc/platforms/pseries/Kconfig +5 -0
@@ -25,9 +25,14 @@ config PPC_PSERIES select SWIOTLB default y +config PARAVIRT_SPINLOCKS + bool + default n + config PPC_SPLPAR depends on PPC_PSERIES bool "Support for shared-processor logical partitions" + select PARAVIRT_SPINLOCKS if PPC_QUEUED_SPINLOCKS help Enabling this option will make the kernel run more efficiently on logically-partitioned pSeries systems which use shared
arch/powerpc/platforms/pseries/setup.c +5 -1
@@ -771,8 +771,12 @@ static void __init pSeries_setup_arch(void)if (firmware_has_feature (FW_FEATURE_LPAR )) {vpa_init (boot_cpuid); -if (lppaca_shared_proc (get_lppaca ())) +if (lppaca_shared_proc (get_lppaca ())) {static_branch_enable (&shared_processor); +#ifdef CONFIG_PARAVIRT_SPINLOCKS +pv_spinlocks_init (); +#endif + } ppc_md.power_save = pseries_lpar_idle; ppc_md.enable_pmcs = pseries_lpar_enable_pmcs;
include/asm-generic/qspinlock.h +2 -0
@@ -69,6 +69,7 @@ static __always_inline int queued_spin_trylock(struct qspinlock *lock)extern void queued_spin_lock_slowpath (struct qspinlock *lock,u32 val); +#ifndef queued_spin_lock/** * queued_spin_lock - acquire a queued spinlock * @lock: Pointer to queued spinlock structure
@@ -82,6 +83,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) queued_spin_lock_slowpath(lock, val); } +#endif #ifndef queued_spin_unlock/**
Signature · 1 line
2.23.0
Waiman Long
reviewerRe: [PATCH v2 5/6] powerpc/pseries: implement paravirt qspinlocks for SPLPAR
Quoted 76 lines · Nicholas Piggin
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/paravirt.h | 28 ++++++++++ arch/powerpc/include/asm/qspinlock.h | 55 +++++++++++++++++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 5 ++ arch/powerpc/platforms/pseries/Kconfig | 5 ++ arch/powerpc/platforms/pseries/setup.c | 6 +- include/asm-generic/qspinlock.h | 2 + 6 files changed, 100 insertions(+), 1 deletion(-)create mode 100644
arch/powerpc/include/asm/qspinlock_paravirt.harch/powerpc/include/asm/paravirt.h +28
-0@@ -29,6 +29,16 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {plpar_hcall_norets (H_CONFER ,get_hard_smp_processor_id (cpu), yield_count); } + +static inline void prod_cpu (int cpu) +{ +plpar_hcall_norets (H_PROD ,get_hard_smp_processor_id (cpu)); +} + +static inline void yield_to_any (void ) +{ +plpar_hcall_norets (H_CONFER ,-1 ,0 ); +}#else static inline bool is_shared_processor (void ) {@@ -45,6 +55,19 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {___bad_yield_to_preempted ();/* This would be a bug */ } + +extern void ___bad_yield_to_any (void ); +static inline void yield_to_any (void ) +{ +___bad_yield_to_any ();/* This would be a bug */ +} + +extern void ___bad_prod_cpu (void ); +static inline void prod_cpu (int cpu) +{ +___bad_prod_cpu ();/* This would be a bug */ +} +#endif #define vcpu_is_preempted vcpu_is_preempted@@ -57,5 +80,10 @@ static inline bool vcpu_is_preempted(int cpu)return false; } +static inline bool pv_is_native_spin_unlock (void ) +{ +return !is_shared_processor (); +} +#endif /* __KERNEL__ */ #endif /* __ASM_PARAVIRT_H */ arch/powerpc/include/asm/qspinlock.h +2
-0@@ -3,9 +3,36 @@#define _ASM_POWERPC_QSPINLOCK_H#include <asm-generic/qspinlock_types.h> +#include <asm/paravirt.h> #define _Q_PENDING_LOOPS (1 << 9)/* not tuned */ +#ifdef CONFIG_PARAVIRT_SPINLOCKS
arch/powerpc/include/asm/qspinlock.h line 20
Quoted 10 lines
arch/powerpc/include/asm/qspinlock.h +10
-0+extern void native_queued_spin_lock_slowpath (struct qspinlock *lock,u32 val); +extern void __pv_queued_spin_lock_slowpath (struct qspinlock *lock,u32 val); + +static __always_inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ +if (!is_shared_processor ()) +native_queued_spin_lock_slowpath (lock, val); +else +__pv_queued_spin_lock_slowpath (lock, val); +}
In a previous mail, I said that:
You may need to match the use of __pv_queued_spin_lock_slowpath() with the corresponding __pv_queued_spin_unlock(), e.g.
#define queued_spin_unlock queued_spin_unlock static inline queued_spin_unlock(struct qspinlock *lock) {
if (!is_shared_processor()) smp_store_release(&lock->locked, 0); else
__pv_queued_spin_unlock(lock);
}
Otherwise, pv_kick() will never be called.
Maybe PowerPC HMT is different that the shared cpus can still process instruction, though slower, that cpu kicking like what was done in kvm is not really necessary. If that is the case, I think we should document that.
Cheers,
Longman
Nicholas Piggin
reviewerRe: [PATCH v2 5/6] powerpc/pseries: implement paravirt qspinlocks for SPLPAR
Excerpts from Waiman Long's message of July 6, 2020 5:00 am:
Quoted 76 lines · Nicholas Piggin
Quoted 75 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/paravirt.h | 28 ++++++++++ arch/powerpc/include/asm/qspinlock.h | 55 +++++++++++++++++++ arch/powerpc/include/asm/qspinlock_paravirt.h | 5 ++ arch/powerpc/platforms/pseries/Kconfig | 5 ++ arch/powerpc/platforms/pseries/setup.c | 6 +- include/asm-generic/qspinlock.h | 2 + 6 files changed, 100 insertions(+), 1 deletion(-)create mode 100644
arch/powerpc/include/asm/qspinlock_paravirt.harch/powerpc/include/asm/paravirt.h +28
-0@@ -29,6 +29,16 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {plpar_hcall_norets (H_CONFER ,get_hard_smp_processor_id (cpu), yield_count); } + +static inline void prod_cpu (int cpu) +{ +plpar_hcall_norets (H_PROD ,get_hard_smp_processor_id (cpu)); +} + +static inline void yield_to_any (void ) +{ +plpar_hcall_norets (H_CONFER ,-1 ,0 ); +}#else static inline bool is_shared_processor (void ) {@@ -45,6 +55,19 @@ static inline void yield_to_preempted(int cpu, u32 yield_count) {___bad_yield_to_preempted ();/* This would be a bug */ } + +extern void ___bad_yield_to_any (void ); +static inline void yield_to_any (void ) +{ +___bad_yield_to_any ();/* This would be a bug */ +} + +extern void ___bad_prod_cpu (void ); +static inline void prod_cpu (int cpu) +{ +___bad_prod_cpu ();/* This would be a bug */ +} +#endif #define vcpu_is_preempted vcpu_is_preempted@@ -57,5 +80,10 @@ static inline bool vcpu_is_preempted(int cpu)return false; } +static inline bool pv_is_native_spin_unlock (void ) +{ +return !is_shared_processor (); +} +#endif /* __KERNEL__ */ #endif /* __ASM_PARAVIRT_H */ arch/powerpc/include/asm/qspinlock.h +1
-0@@ -3,9 +3,36 @@#define _ASM_POWERPC_QSPINLOCK_H#include <asm-generic/qspinlock_types.h> +#include <asm/paravirt.h> #define _Q_PENDING_LOOPS (1 << 9)/* not tuned */
Quoted 13 lines
arch/powerpc/include/asm/qspinlock.h line 20
Quoted 11 lines
arch/powerpc/include/asm/qspinlock.h +11
-0+#ifdef CONFIG_PARAVIRT_SPINLOCKS +extern void native_queued_spin_lock_slowpath (struct qspinlock *lock,u32 val); +extern void __pv_queued_spin_lock_slowpath (struct qspinlock *lock,u32 val); + +static __always_inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ +if (!is_shared_processor ()) +native_queued_spin_lock_slowpath (lock, val); +else +__pv_queued_spin_lock_slowpath (lock, val); +}In a previous mail, I said that:
Hey, yeah I read that right after sending the series out. Thanks for the thorough review.
Quoted 12 lines
You may need to match the use of
__pv_queued_spin_lock_slowpath()with the corresponding__pv_queued_spin_unlock(), e.g.#define
queued_spin_unlockqueued_spin_unlockstatic inlinequeued_spin_unlock(struct qspinlock *lock){
if (!is_shared_processor())smp_store_release(&lock->locked, 0); else
__pv_queued_spin_unlock(lock);
}
Quoted 6 lines
Otherwise,
pv_kick()will never be called.Maybe PowerPC HMT is different that the shared cpus can still process instruction, though slower, that cpu kicking like what was done in kvm is not really necessary. If that is the case, I think we should document that.
It does stop dispatch, but it will wake up by itself after all other vCPUs have had a chance to dispatch. I will re-test with the fix in place and see if there's any significant performance differences.
Thanks,
Nick
Nicholas Piggin
reviewerPATCH v2 6/6 powerpc/qspinlock: optimised atomic_try_cmpxchg_lock that adds the lock hint
In reply to Nicholas PigginThis brings the behaviour of the uncontended fast path back to roughly equivalent to simple spinlocks -- a single atomic op with lock hint.
--- arch/powerpc/include/asm/atomic.h | 28 ++++++++++++++++++++++++++++ arch/powerpc/include/asm/qspinlock.h | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/atomic.h +28 -0
@@ -193,6 +193,34 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v)#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_xchg_relaxed (v, new) xchg_relaxed(&((v)->counter), (new)) +/* +* Don't want to override the generic atomic_try_cmpxchg_acquire, because +* we add a lock hint to the lwarx, which may not be wanted for the +* _acquire case (and is not used by the other _acquire variants so it +* would be a surprise). +*/ +static __always_inline bool +atomic_try_cmpxchg_lock (atomic_t *v,int *old,int new) +{ +int r, o = *old; + + __asm__ __volatile__ ( +"1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n" +" cmpw 0,%0,%3 \n" +" bne- 2f \n" +" stwcx. %4,0,%2 \n" +" bne- 1b \n" +"\t" PPC_ACQUIRE_BARRIER " \n" +"2: \n" + : "=&r" (r), "+m" (v->counter) + : "r" (&v->counter), "r" (o), "r" (new) + : "cr0", "memory"); + + if (unlikely (r != o)) + *old = r; +return likely (r = o); +} +/** * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t
arch/powerpc/include/asm/qspinlock.h +1 -1
@@ -26,7 +26,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) {u32 val =0 ; -if (likely (atomic_try_cmpxchg_acquire (&lock->val, &val, _Q_LOCKED_VAL))) +if (likely (atomic_try_cmpxchg_lock (&lock->val, &val, _Q_LOCKED_VAL)))return ;queued_spin_lock_slowpath (lock, val);
Signature · 1 line
2.23.0