Skip to contentSkip to conversation

locking changes for v4.5

GIT PULL Ingo Molnar 1 participant 1 message Jan 11, 2016

Subsystems (10)
Documentation x86 PowerPC Library code +6 more subsystems
lore
Share & tools
lore

Ingo Molnar

author
#0 lore raw

Linus,

Please pull the latest locking-core-for-linus git tree from:

git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking-core-for-linus

# HEAD: 337f13046ff03717a9e99675284a817527440a49 futex: Allow FUTEX_CLOCK_REALTIME with FUTEX_WAIT op

So we have a laundry list of locking subsystem changes:

- continuing barrier API and code improvements

- futex enhancements

- atomics API improvements

- pvqspinlock enhancements: in particular lock stealing and adaptive spinning.

- qspinlock micro-enhancements

Thanks,

Ingo

------------------>
Boqun Feng (1):
atomics: Add test for atomic operations with _relaxed variants

Darren Hart (1):
futex: Allow FUTEX_CLOCK_REALTIME with FUTEX_WAIT op

Davidlohr Bueso (3):
locking/cmpxchg, arch: Remove tas() definitions
lcoking/barriers, arch: Use smp barriers in smp_store_release() locking/barriers, arch: Remove ambiguous statement in the smp_store_mb() documentation

Peter Zijlstra (2):
locking, sched: Introduce smp_cond_acquire() and use it sched/core, locking: Document Program-Order guarantees

Thomas Gleixner (5):
futex: Drop refcount if requeue_pi() acquired the rtmutex
futex: Rename free_pi_state() to put_pi_state()
futex: Document pi_state refcounting in requeue code
futex: Remove pointless put_pi_state calls in requeue()
futex: Cleanup the goto confusion in requeue_pi()

Waiman Long (7):
locking/qspinlock: Use _acquire/_release() versions of cmpxchg() & xchg() locking/qspinlock: Prefetch the next node cacheline locking/qspinlock: Avoid redundant read of next pointer locking/pvqspinlock, x86: Optimize the PV unlock code path locking/pvqspinlock: Collect slowpath lock statistics locking/pvqspinlock: Allow limited lock stealing locking/pvqspinlock: Queue node adaptive spinning

 Documentation/memory-barriers.txt         |   4 +-
 arch/blackfin/include/asm/cmpxchg.h       |   1 -
 arch/c6x/include/asm/cmpxchg.h            |   2 -
 arch/frv/include/asm/cmpxchg.h            |   2 -
 arch/ia64/include/asm/barrier.h           |   2 +-
 arch/powerpc/include/asm/barrier.h        |   2 +-
 arch/s390/include/asm/barrier.h           |   2 +-
 arch/tile/include/asm/cmpxchg.h           |   2 -
 arch/x86/Kconfig                          |   8 +
 arch/x86/include/asm/qspinlock_paravirt.h |  59 ++++++
 include/asm-generic/barrier.h             |   2 +-
 include/asm-generic/qspinlock.h           |   9 +-
 include/linux/compiler.h                  |  17 ++
 kernel/futex.c                            |  83 ++++++---
 kernel/locking/qspinlock.c                |  82 ++++++--
 kernel/locking/qspinlock_paravirt.h       | 252 +++++++++++++++++++++----
 kernel/locking/qspinlock_stat.h           | 300 ++++++++++++++++++++++++++++++
 kernel/sched/core.c                       |  99 +++++++++-
 kernel/sched/sched.h                      |   2 +-
 lib/atomic64_test.c                       | 120 ++++++++----
 20 files changed, 904 insertions(+), 146 deletions(-)
 create mode 100644 kernel/locking/qspinlock_stat.h
Documentation/memory-barriers.txt +2 -2
index aef9487303d0..c85054dc4460 100644
@@ -1673,8 +1673,8 @@ CPU from reordering them.
  (*) smp_store_mb(var, value)
 
      This assigns the value to the variable and then inserts a full memory
-     barrier after it, depending on the function.  It isn't guaranteed to
-     insert anything more than a compiler barrier in a UP compilation.
+     barrier after it.  It isn't guaranteed to insert anything more than a
+     compiler barrier in a UP compilation.
 
 
  (*) smp_mb__before_atomic();
arch/blackfin/include/asm/cmpxchg.h +0 -1
index c05868cc61c1..253928854299 100644
@@ -128,6 +128,5 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
 #endif /* !CONFIG_SMP */
 
 #define xchg(ptr, x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))))
-#define tas(ptr) ((void)xchg((ptr), 1))
 
 #endif /* __ARCH_BLACKFIN_CMPXCHG__ */
arch/c6x/include/asm/cmpxchg.h +0 -2
index b27c8cefb8c3..93d0a5a047a2 100644
@@ -47,8 +47,6 @@ static inline unsigned int __xchg(unsigned int x, volatile void *ptr, int size)
 #define xchg(ptr, x) \
 	((__typeof__(*(ptr)))__xchg((unsigned int)(x), (void *) (ptr), \
 				    sizeof(*(ptr))))
-#define tas(ptr)    xchg((ptr), 1)
-
 
 #include <asm-generic/cmpxchg-local.h>
arch/frv/include/asm/cmpxchg.h +0 -2
index 5b04dd0aecab..a899765102ea 100644
@@ -69,8 +69,6 @@ extern uint32_t __xchg_32(uint32_t i, volatile void *v);
 
 #endif
 
-#define tas(ptr) (xchg((ptr), 1))
-
 /*****************************************************************************/
 /*
  * compare and conditionally exchange value with memory
arch/ia64/include/asm/barrier.h +1 -1
index df896a1c41d3..209c4b817c95 100644
@@ -77,7 +77,7 @@ do {									\
 	___p1;								\
 })
 
-#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 /*
  * The group barrier in front of the rsm & ssm are necessary to ensure
arch/powerpc/include/asm/barrier.h +1 -1
index 0eca6efc0631..a7af5fb7b914 100644
@@ -34,7 +34,7 @@
 #define rmb()  __asm__ __volatile__ ("sync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("sync" : : : "memory")
 
-#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #ifdef __SUBARCH_HAS_LWSYNC
 #    define SMPWMB      LWSYNC
arch/s390/include/asm/barrier.h +1 -1
index d68e11e0df5e..7ffd0b19135c 100644
@@ -36,7 +36,7 @@
 #define smp_mb__before_atomic()		smp_mb()
 #define smp_mb__after_atomic()		smp_mb()
 
-#define smp_store_mb(var, value)		do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_store_release(p, v)						\
 do {									\
arch/tile/include/asm/cmpxchg.h +0 -2
index 0ccda3c425be..25d5899497be 100644
@@ -127,8 +127,6 @@ long long _atomic64_cmpxchg(long long *v, long long o, long long n);
 
 #endif
 
-#define tas(ptr) xchg((ptr), 1)
-
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_TILE_CMPXCHG_H */
arch/x86/Kconfig +8 -0
index db3622f22b61..965fc4216f76 100644
@@ -687,6 +687,14 @@ config PARAVIRT_SPINLOCKS
 
 	  If you are unsure how to answer this question, answer Y.
 
+config QUEUED_LOCK_STAT
+	bool "Paravirt queued spinlock statistics"
+	depends on PARAVIRT_SPINLOCKS && DEBUG_FS && QUEUED_SPINLOCKS
+	---help---
+	  Enable the collection of statistical data on the slowpath
+	  behavior of paravirtualized queued spinlocks and report
+	  them on debugfs.
+
 source "arch/x86/xen/Kconfig"
 
 config KVM_GUEST
arch/x86/include/asm/qspinlock_paravirt.h +59 -0

65 lines not on this page · raw message

include/asm-generic/barrier.h +1 -1
index b42afada1280..0f45f93ef692 100644
@@ -93,7 +93,7 @@
 #endif	/* CONFIG_SMP */
 
 #ifndef smp_store_mb
-#define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 #endif
 
 #ifndef smp_mb__before_atomic
include/asm-generic/qspinlock.h +5 -4
index e2aadbc7151f..39e1cb201b8e 100644
@@ -12,8 +12,9 @@
  * GNU General Public License for more details.
  *
  * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
+ * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
  *
- * Authors: Waiman Long <[email protected]>
+ * Authors: Waiman Long <[email protected]>
  */
 #ifndef __ASM_GENERIC_QSPINLOCK_H
 #define __ASM_GENERIC_QSPINLOCK_H

@@ -62,7 +63,7 @@ static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
 static __always_inline int queued_spin_trylock(struct qspinlock *lock)
 {
 	if (!atomic_read(&lock->val) &&
-	   (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+	   (atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL) == 0))
 		return 1;
 	return 0;
 }

@@ -77,7 +78,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock)
 {
 	u32 val;
 
-	val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+	val = atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL);
 	if (likely(val == 0))
 		return;
 	queued_spin_lock_slowpath(lock, val);

@@ -93,7 +94,7 @@ static __always_inline void queued_spin_unlock(struct qspinlock *lock)
 	/*
 	 * smp_mb__before_atomic() in order to guarantee release semantics
 	 */
-	smp_mb__before_atomic_dec();
+	smp_mb__before_atomic();
 	atomic_sub(_Q_LOCKED_VAL, &lock->val);
 }
 #endif
include/linux/compiler.h +17 -0
index 4dac1036594f..00b042c49ccd 100644
@@ -299,6 +299,23 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
 	__u.__val;					\
 })
 
+/**
+ * smp_cond_acquire() - Spin wait for cond with ACQUIRE ordering
+ * @cond: boolean expression to wait for
+ *
+ * Equivalent to using smp_load_acquire() on the condition variable but employs
+ * the control dependency of the wait to reduce the barrier on many platforms.
+ *
+ * The control dependency provides a LOAD->STORE order, the additional RMB
+ * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
+ * aka. ACQUIRE.
+ */
+#define smp_cond_acquire(cond)	do {		\
+	while (!(cond))				\
+		cpu_relax();			\
+	smp_rmb(); /* ctrl + rmb := acquire */	\
+} while (0)
+
 #endif /* __KERNEL__ */
 
 #endif /* __ASSEMBLY__ */
kernel/futex.c +61 -22

159 lines not on this page · raw message

kernel/locking/qspinlock.c +64 -18

164 lines not on this page · raw message

kernel/locking/qspinlock_paravirt.h +212 -40

379 lines not on this page · raw message

kernel/locking/qspinlock_stat.h new file +300 -0

300 lines not on this page · raw message

kernel/sched/core.c +92 -7

117 lines not on this page · raw message

kernel/sched/sched.h +1 -1
index b242775bf670..1e0bb4afe3fd 100644
@@ -1076,7 +1076,7 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 	 * In particular, the load of prev->state in finish_task_switch() must
 	 * happen before this.
 	 *
-	 * Pairs with the control dependency and rmb in try_to_wake_up().
+	 * Pairs with the smp_cond_acquire() in try_to_wake_up().
 	 */
 	smp_store_release(&prev->on_cpu, 0);
 #endif
lib/atomic64_test.c +79 -41

149 lines not on this page · raw message