locking tree updates for v3.20
Subsystems (3)
Linus,
Please pull the latest locking-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking-core-for-linus
# HEAD: afffc6c1805d98e08e778cddb644a666e78cfcfd locking/rtmutex: Optimize setting task running after being blocked
The main changes are:
- mutex, completions and rtmutex micro-optimizations
- lock debugging fix
- various cleanups in the MCS and the futex code
Thanks,
Ingo
------------------>
Davidlohr Bueso (8):
locking/mutex: Checking the stamp is WW only locking/mutex: Move MCS related comments to proper location locking/mutex: Introduce ww_mutex_set_context_slowpath() locking/mcs: Better differentiate between MCS variants locking/osq: No need for load/acquire when acquire-polling locking/mutex: Explicitly mark task as running after wakeup locking/rwsem: Use task->state helpers locking/rtmutex: Optimize setting task running after being blocked
Heiko Carstens (1):
softirq/preempt: Add missing current->preempt_disable_ip update
Michael Kerrisk (2):
doc: Fix misnamed FUTEX_CMP_REQUEUE_PI op constants
futex: Fix argument handling in futex_lock_pi() calls
Nicholas Mc Guire (2):
sched/completion: Remove unnecessary ->wait.lock serialization when reading completion state sched/completion: Add lock-free checking of the blocking case
SeongJae Park (1):
locking/Documentation: Update code path
Documentation/futex-requeue-pi.txt | 8 ++--
Documentation/locking/lockdep-design.txt | 2 +-
include/linux/osq_lock.h | 12 +++++-
kernel/Kconfig.locks | 4 ++
kernel/futex.c | 6 +--
kernel/locking/Makefile | 3 +-
kernel/locking/mcs_spinlock.h | 16 -------
kernel/locking/mutex.c | 60 ++++++++++++++-------------
kernel/locking/{mcs_spinlock.c => osq_lock.c} | 9 +---
kernel/locking/rtmutex.c | 7 ++--
kernel/locking/rwsem-spinlock.c | 2 +-
kernel/locking/rwsem-xadd.c | 3 +-
kernel/sched/completion.c | 18 ++++----
kernel/softirq.c | 6 ++-
14 files changed, 78 insertions(+), 78 deletions(-)
rename kernel/locking/{mcs_spinlock.c => osq_lock.c} (98%)Documentation/futex-requeue-pi.txt +4 -4
@@ -98,7 +98,7 @@ rt_mutex_start_proxy_lock() and rt_mutex_finish_proxy_lock(), which allow the requeue code to acquire an uncontended rt_mutex on behalf of the waiter and to enqueue the waiter on a contended rt_mutex. Two new system calls provide the kernel<->user interface to -requeue_pi: FUTEX_WAIT_REQUEUE_PI and FUTEX_REQUEUE_CMP_PI. +requeue_pi: FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI. FUTEX_WAIT_REQUEUE_PI is called by the waiter (pthread_cond_wait() and pthread_cond_timedwait()) to block on the initial futex and wait
@@ -107,7 +107,7 @@ result of a high-speed collision between futex_wait() and futex_lock_pi(), with some extra logic to check for the additional wake-up scenarios. -FUTEX_REQUEUE_CMP_PI is called by the waker +FUTEX_CMP_REQUEUE_PI is called by the waker (pthread_cond_broadcast() and pthread_cond_signal()) to requeue and possibly wake the waiting tasks. Internally, this system call is still handled by futex_requeue (by passing requeue_pi=1). Before
@@ -120,12 +120,12 @@ task as a waiter on the underlying rt_mutex. It is possible that the lock can be acquired at this stage as well, if so, the next waiter is woken to finish the acquisition of the lock. -FUTEX_REQUEUE_PI accepts nr_wake and nr_requeue as arguments, but +FUTEX_CMP_REQUEUE_PI accepts nr_wake and nr_requeue as arguments, but their sum is all that really matters. futex_requeue() will wake or requeue up to nr_wake + nr_requeue tasks. It will wake only as many tasks as it can acquire the lock for, which in the majority of cases should be 0 as good programming practice dictates that the caller of either pthread_cond_broadcast() or pthread_cond_signal() acquire the -mutex prior to making the call. FUTEX_REQUEUE_PI requires that +mutex prior to making the call. FUTEX_CMP_REQUEUE_PI requires that nr_wake=1. nr_requeue should be INT_MAX for broadcast and 0 for signal.
Documentation/locking/lockdep-design.txt +1 -1
@@ -34,7 +34,7 @@ State - 'ever held with STATE enabled' - 'ever held as readlock with STATE enabled' -Where STATE can be either one of (kernel/lockdep_states.h) +Where STATE can be either one of (kernel/locking/lockdep_states.h) - hardirq - softirq - reclaim_fs
include/linux/osq_lock.h +10 -2
@@ -5,8 +5,11 @@* An MCS like lock especially tailored for optimistic spinning for sleeping * lock implementations (mutex, rwsem, etc). */ - -#define OSQ_UNLOCKED_VAL (0) +struct optimistic_spin_node { +struct optimistic_spin_node *next, *prev; +int locked;/* 1 if lock acquired */ +int cpu;/* encoded CPU # + 1 value */ +};struct optimistic_spin_queue {/*
@@ -16,6 +19,8 @@ struct optimistic_spin_queue {atomic_t tail; }; +#define OSQ_UNLOCKED_VAL (0) +/* Init macro and function. */ #define OSQ_LOCK_UNLOCKED { ATOMIC_INIT(OSQ_UNLOCKED_VAL) }
@@ -24,4 +29,7 @@ static inline void osq_lock_init(struct optimistic_spin_queue *lock) atomic_set(&lock->tail, OSQ_UNLOCKED_VAL); } +extern bool osq_lock (struct optimistic_spin_queue *lock); +extern void osq_unlock (struct optimistic_spin_queue *lock); +#endif
kernel/Kconfig.locks +4 -0
@@ -231,6 +231,10 @@ config RWSEM_SPIN_ON_OWNER def_bool y depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW +config LOCK_SPIN_ON_OWNER + def_bool y + depends on MUTEX_SPIN_ON_OWNER || RWSEM_SPIN_ON_OWNER + config ARCH_USE_QUEUE_RWLOCK bool
kernel/futex.c +3 -3
@@ -2258,7 +2258,7 @@ static long futex_wait_restart(struct restart_block *restart)* if there are waiters then it will block, it does PI, etc. (Due to * races the kernel might see a 0 value of the futex too.) */ -static int futex_lock_pi (u32 __user *uaddr,unsigned int flags,int detect, +static int futex_lock_pi (u32 __user *uaddr,unsigned int flags,ktime_t *time,int trylock) {struct hrtimer_sleeper timeout, *to =NULL ;
@@ -2953,11 +2953,11 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,case FUTEX_WAKE_OP :return futex_wake_op (uaddr, flags, uaddr2, val, val2, val3);case FUTEX_LOCK_PI : -return futex_lock_pi (uaddr, flags, val, timeout,0 ); +return futex_lock_pi (uaddr, flags, timeout,0 );case FUTEX_UNLOCK_PI :return futex_unlock_pi (uaddr, flags);case FUTEX_TRYLOCK_PI : -return futex_lock_pi (uaddr, flags,0 , timeout,1 ); +return futex_lock_pi (uaddr, flags,NULL ,1 );case FUTEX_WAIT_REQUEUE_PI : val3 =FUTEX_BITSET_MATCH_ANY ; return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
kernel/locking/Makefile +2 -1
@@ -1,5 +1,5 @@ -obj-y += mutex.o semaphore.o rwsem.o mcs_spinlock.o +obj-y += mutex.o semaphore.o rwsem.o ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_lockdep.o = -pg
@@ -14,6 +14,7 @@ ifeq ($(CONFIG_PROC_FS),y) obj-$(CONFIG_LOCKDEP) += lockdep_proc.o endif obj-$(CONFIG_SMP) += spinlock.o +obj-$(CONFIG_LOCK_SPIN_ON_OWNER) += osq_lock.o obj-$(CONFIG_SMP) += lglock.o obj-$(CONFIG_PROVE_LOCKING) += spinlock.o obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
kernel/locking/mcs_spinlock.h +0 -16
@@ -108,20 +108,4 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node) arch_mcs_spin_unlock_contended(&next->locked); } -/* -* Cancellable version of the MCS lock above. -* -* Intended for adaptive spinning of sleeping locks: -* mutex_lock()/rwsem_down_{read,write}() etc. -*/ - -struct optimistic_spin_node { -struct optimistic_spin_node *next, *prev; -int locked;/* 1 if lock acquired */ -int cpu;/* encoded CPU # value */ -}; - -extern bool osq_lock (struct optimistic_spin_queue *lock); -extern void osq_unlock (struct optimistic_spin_queue *lock); -#endif /* __LINUX_MCS_SPINLOCK_H */
kernel/locking/mutex.c +32 -28
107 lines not on this page · raw message
kernel/locking/osq_lock.c renamed from kernel/locking/mcs_spinlock.c +2 -7
@@ -1,8 +1,6 @@#include <linux/percpu.h> #include <linux/sched.h> -#include "mcs_spinlock.h" - -#ifdef CONFIG_SMP +#include <linux/osq_lock.h> /* * An MCS like lock especially tailored for optimistic spinning for sleeping
@@ -111,7 +109,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)* cmpxchg in an attempt to undo our queueing. */ -while (!smp_load_acquire (&node->locked)) { +while (!ACCESS_ONCE (node->locked)) {/* * If we need to reschedule bail... so we can block. */
@@ -203,6 +201,3 @@ void osq_unlock(struct optimistic_spin_queue *lock)if (next)ACCESS_ONCE (next->locked) =1 ; } - -#endif -
kernel/locking/rtmutex.c +3 -4
@@ -1130,6 +1130,7 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state,set_current_state (state); } +__set_current_state (TASK_RUNNING );return ret; }
@@ -1188,10 +1189,9 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, ret =task_blocks_on_rt_mutex (lock, &waiter, current, chwalk);if (likely (!ret)) +/* sleep on the mutex */ ret =__rt_mutex_slowlock (lock, state, timeout, &waiter); -set_current_state (TASK_RUNNING ); -if (unlikely (ret)) {remove_waiter (lock, &waiter);rt_mutex_handle_deadlock (ret, chwalk, &waiter);
@@ -1626,10 +1626,9 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,set_current_state (TASK_INTERRUPTIBLE ); +/* sleep on the mutex */ ret =__rt_mutex_slowlock (lock,TASK_INTERRUPTIBLE , to, waiter); -set_current_state (TASK_RUNNING ); -if (unlikely (ret))remove_waiter (lock, waiter);
kernel/locking/rwsem-spinlock.c +1 -1
@@ -154,7 +154,7 @@ void __sched __down_read(struct rw_semaphore *sem)set_task_state (tsk,TASK_UNINTERRUPTIBLE ); } - tsk->state =TASK_RUNNING ; +__set_task_state (tsk,TASK_RUNNING ); out: ; }
kernel/locking/rwsem-xadd.c +1 -2
@@ -242,8 +242,7 @@ struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)schedule (); } - tsk->state =TASK_RUNNING ; - +__set_task_state (tsk,TASK_RUNNING );return sem; }EXPORT_SYMBOL (rwsem_down_read_failed);
kernel/sched/completion.c +10 -8
@@ -268,6 +268,15 @@ bool try_wait_for_completion(struct completion *x)unsigned long flags;int ret =1 ; +/* +* Since x->done will need to be locked only +* in the non-blocking case, we check x->done +* first without taking the lock so we can +* return early in the blocking case. +*/ +if (!ACCESS_ONCE (x->done)) +return 0 ; +spin_lock_irqsave (&x->wait.lock, flags);if (!x->done) ret =0 ;
@@ -288,13 +297,6 @@ EXPORT_SYMBOL(try_wait_for_completion);*/ bool completion_done (struct completion *x) { -unsigned long flags; -int ret =1 ; - -spin_lock_irqsave (&x->wait.lock, flags); -if (!x->done) - ret =0 ; -spin_unlock_irqrestore (&x->wait.lock, flags); -return ret; +return !!ACCESS_ONCE (x->done); }EXPORT_SYMBOL (completion_done);
kernel/softirq.c +5 -1
@@ -114,8 +114,12 @@ void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)trace_softirqs_off (ip);raw_local_irq_restore (flags); -if (preempt_count () == cnt) +if (preempt_count () == cnt) { +#ifdef CONFIG_DEBUG_PREEMPT + current->preempt_disable_ip =get_parent_ip (CALLER_ADDR1 ); +#endif trace_preempt_off (CALLER_ADDR0 ,get_parent_ip (CALLER_ADDR1 )); + } }EXPORT_SYMBOL (__local_bh_disable_ip);#endif /* CONFIG_TRACE_IRQFLAGS */