locking tree updates for v3.17
Subsystems (19)
Linus,
Please pull the latest locking-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking-core-for-linus
# HEAD: 3a6bfbc91df04b081a44d419e0260bad54abddf7 arch, locking: Ciao arch_mutex_cpu_relax()
The main changes in this cycle are:
- big rtmutex and futex cleanup and robustification from Thomas Gleixner
- mutex optimizations and refinements from Jason Low
- arch_mutex_cpu_relax() removal and related cleanups
- smaller lockdep tweaks
Thanks,
Ingo
------------------>
Andreas Gruenbacher (1):
locking/lockdep: Only ask for /proc/lock_stat output when available
Davidlohr Bueso (1):
arch, locking: Ciao arch_mutex_cpu_relax()
Jason Low (4):
locking/mutexes: Correct documentation on mutex optimistic spinning locking/mutexes: Delete the MUTEX_SHOW_NO_WAITER macro locking/mutexes: Try to acquire mutex only if it is unlocked locking/mutexes: Optimize mutex trylock slowpath
Peter Zijlstra (1):
x86, locking: Use no more OOSTORE nonsense
Thomas Gleixner (15):
rtmutex: Simplify rtmutex_slowtrylock()
rtmutex: Simplify and document try_to_take_rtmutex()
rtmutex: No need to keep task ref for lock owner check
rtmutex: Clarify the boost/deboost part
rtmutex: Document pi chain walk
rtmutex: Simplify remove_waiter()
rtmutex: Confine deadlock logic to futex
rtmutex: Cleanup deadlock detector debug logic
rtmutex: Avoid pointless requeueing in the deadlock detection chain walk
futex: Make unlock_pi more robust
futex: Use futex_top_waiter() in lookup_pi_state()
futex: Split out the waiter check from lookup_pi_state()
futex: Split out the first waiter attachment from lookup_pi_state()
futex: Simplify futex_lock_pi_atomic() and make it more robust
rtmutex: Make the rtmutex tester depend on BROKEN
Trond Myklebust (1):
lockdep: Revert lockdep check in raw_seqcount_begin()
arch/alpha/include/asm/processor.h | 1 + arch/arc/include/asm/processor.h | 2 + arch/arm/include/asm/processor.h | 2 + arch/arm64/include/asm/processor.h | 1 + arch/avr32/include/asm/processor.h | 1 + arch/blackfin/include/asm/processor.h | 2 +- arch/c6x/include/asm/processor.h | 1 + arch/cris/include/asm/processor.h | 1 + arch/hexagon/include/asm/processor.h | 1 + arch/ia64/include/asm/processor.h | 1 + arch/m32r/include/asm/processor.h | 1 + arch/m68k/include/asm/processor.h | 1 + arch/metag/include/asm/processor.h | 1 + arch/microblaze/include/asm/processor.h | 1 + arch/mips/include/asm/processor.h | 1 + arch/mn10300/include/asm/processor.h | 2 + arch/openrisc/include/asm/processor.h | 1 + arch/parisc/include/asm/processor.h | 1 + arch/powerpc/include/asm/processor.h | 2 + arch/s390/include/asm/processor.h | 2 +- arch/score/include/asm/processor.h | 1 + arch/sh/include/asm/processor.h | 1 + arch/sparc/include/asm/processor_32.h | 2 + arch/sparc/include/asm/processor_64.h | 1 + arch/tile/include/asm/processor.h | 2 + arch/unicore32/include/asm/processor.h | 1 + arch/x86/include/asm/barrier.h | 2 +- arch/x86/include/asm/processor.h | 2 + arch/x86/include/asm/qrwlock.h | 2 +- arch/x86/um/asm/processor.h | 3 +- arch/xtensa/include/asm/processor.h | 1 + include/linux/mutex.h | 4 - include/linux/rtmutex.h | 6 +- include/linux/seqlock.h | 2 - kernel/futex.c | 402 +++++++++++------------ kernel/locking/lockdep.c | 2 + kernel/locking/mcs_spinlock.c | 8 +- kernel/locking/mcs_spinlock.h | 4 +- kernel/locking/mutex.c | 39 +-- kernel/locking/qrwlock.c | 9 +- kernel/locking/rtmutex-debug.c | 5 +- kernel/locking/rtmutex-debug.h | 7 +- kernel/locking/rtmutex.c | 562 ++++++++++++++++++++++++-------- kernel/locking/rtmutex.h | 7 +- kernel/locking/rtmutex_common.h | 22 +- kernel/locking/rwsem-xadd.c | 4 +- lib/Kconfig.debug | 2 +- lib/lockref.c | 3 +- 48 files changed, 707 insertions(+), 425 deletions(-)
arch/alpha/include/asm/processor.h +1 -0
@@ -57,6 +57,7 @@ unsigned long get_wchan(struct task_struct *p); ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp)#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW
arch/arc/include/asm/processor.h +2 -0
@@ -62,6 +62,8 @@ unsigned long thread_saved_pc(struct task_struct *t);#define cpu_relax () do { } while (0)#endif +#define cpu_relax_lowlatency () cpu_relax() +#define copy_segments (tsk, mm) do { } while (0)#define release_segments (mm) do { } while (0)
arch/arm/include/asm/processor.h +2 -0
@@ -82,6 +82,8 @@ unsigned long get_wchan(struct task_struct *p);#define cpu_relax () barrier()#endif +#define cpu_relax_lowlatency () cpu_relax() +#define task_pt_regs (p) \ ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
arch/arm64/include/asm/processor.h +1 -0
@@ -129,6 +129,7 @@ extern void release_thread(struct task_struct *);unsigned long get_wchan (struct task_struct *p);#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()/* Thread switching */ extern struct task_struct *cpu_switch_to(struct task_struct *prev,
arch/avr32/include/asm/processor.h +1 -0
@@ -92,6 +92,7 @@ extern struct avr32_cpuinfo boot_cpu_data;#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3))#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#define cpu_sync_pipeline () asm volatile("sub pc, -2" : : : "memory")struct cpu_context {
arch/blackfin/include/asm/processor.h +1 -1
@@ -99,7 +99,7 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk) == current ? rdusp() : (tsk)->thread.usp)#define cpu_relax () smp_mb() - +#define cpu_relax_lowlatency () cpu_relax()/* Get the Silicon Revision of the chip */ static inline uint32_t __pure bfin_revid (void )
arch/c6x/include/asm/processor.h +1 -0
@@ -121,6 +121,7 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (task) (task_pt_regs(task)->sp)#define cpu_relax () do { } while (0) +#define cpu_relax_lowlatency () cpu_relax()extern const struct seq_operations cpuinfo_op;
arch/cris/include/asm/processor.h +1 -0
@@ -63,6 +63,7 @@ static inline void release_thread(struct task_struct *dead_task)#define init_stack (init_thread_union.stack)#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()void default_idle (void );
arch/hexagon/include/asm/processor.h +1 -0
@@ -56,6 +56,7 @@ struct thread_struct { }#define cpu_relax () __vmyield() +#define cpu_relax_lowlatency () cpu_relax()/* * Decides where the kernel will search for a free chunk of vm space during
arch/ia64/include/asm/processor.h +1 -0
@@ -548,6 +548,7 @@ ia64_eoi (void) }#define cpu_relax () ia64_hint(ia64_hint_pause) +#define cpu_relax_lowlatency () cpu_relax()static inline int ia64_get_irr (unsigned int vector)
arch/m32r/include/asm/processor.h +1 -0
@@ -133,5 +133,6 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk)->thread.sp)#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#endif /* _ASM_M32R_PROCESSOR_H */
arch/m68k/include/asm/processor.h +1 -0
@@ -176,5 +176,6 @@ unsigned long get_wchan(struct task_struct *p);#define task_pt_regs (tsk) ((struct pt_regs *) ((tsk)->thread.esp0))#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#endif
arch/metag/include/asm/processor.h +1 -0
@@ -155,6 +155,7 @@ unsigned long get_wchan(struct task_struct *p);#define user_stack_pointer (regs) ((regs)->ctx.AX[0].U0)#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()extern void setup_priv (void );
arch/microblaze/include/asm/processor.h +1 -0
@@ -22,6 +22,7 @@extern const struct seq_operations cpuinfo_op;# define cpu_relax () barrier() +# define cpu_relax_lowlatency () cpu_relax()#define task_pt_regs (tsk) \ (((struct pt_regs *)(THREAD_SIZE + task_stack_page(tsk))) - 1)
arch/mips/include/asm/processor.h +1 -0
@@ -367,6 +367,7 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_STATUS (tsk) (task_pt_regs(tsk)->cp0_status)#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()/* * Return_address is a replacement for __builtin_return_address(count)
arch/mn10300/include/asm/processor.h +2 -0
@@ -68,7 +68,9 @@ extern struct mn10300_cpuinfo cpu_data[];extern void identify_cpu (struct mn10300_cpuinfo *);extern void print_cpu_info (struct mn10300_cpuinfo *);extern void dodgy_tsc (void ); +#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()/* * User space process size: 1.75GB (default).
arch/openrisc/include/asm/processor.h +1 -0
@@ -101,6 +101,7 @@ extern unsigned long thread_saved_pc(struct task_struct *t);#define init_stack (init_thread_union.stack)#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#endif /* __ASSEMBLY__ */ #endif /* __ASM_OPENRISC_PROCESSOR_H */
arch/parisc/include/asm/processor.h +1 -0
@@ -338,6 +338,7 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk)->thread.regs.gr[30])#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()/* Used as a macro to identify the combined VIPT/PIPT cached * CPUs which require a guarantee of coherency (no inequivalent
arch/powerpc/include/asm/processor.h +2 -0
@@ -400,6 +400,8 @@ static inline unsigned long __pack_fe01(unsigned int fpmode)#define cpu_relax () barrier()#endif +#define cpu_relax_lowlatency () cpu_relax() +/* Check that a certain kernel stack pointer is valid in task_struct p */ int validate_sp (unsigned long sp,struct task_struct *p,unsigned long nbytes);
arch/s390/include/asm/processor.h +1 -1
@@ -217,7 +217,7 @@ static inline void cpu_relax(void)barrier (); } -#define arch_mutex_cpu_relax () barrier() +#define cpu_relax_lowlatency () barrier()static inline void psw_set_key (unsigned int key) {
arch/score/include/asm/processor.h +1 -0
@@ -24,6 +24,7 @@ extern unsigned long get_wchan(struct task_struct *p);#define current_text_addr () ({ __label__ _l; _l: &&_l; })#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#define release_thread (thread) do {} while (0)/*
arch/sh/include/asm/processor.h +1 -0
@@ -97,6 +97,7 @@ extern struct sh_cpuinfo cpu_data[];#define cpu_sleep () __asm__ __volatile__ ("sleep" : : : "memory")#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()void default_idle (void );void stop_this_cpu (void *);
arch/sparc/include/asm/processor_32.h +2 -0
@@ -119,6 +119,8 @@ extern struct task_struct *last_task_used_math;int do_mathemu (struct pt_regs *regs,struct task_struct *fpt);#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax() +extern void (*sparc_idle)(void );#endif
arch/sparc/include/asm/processor_64.h +1 -0
@@ -216,6 +216,7 @@ unsigned long get_wchan(struct task_struct *task); "nop\n\t" \ ".previous" \ ::: "memory") +#define cpu_relax_lowlatency () cpu_relax()/* Prefetch support. This is tuned for UltraSPARC-III and later. * UltraSPARC-I will treat these as nops, and UltraSPARC-II has
arch/tile/include/asm/processor.h +2 -0
@@ -266,6 +266,8 @@ static inline void cpu_relax(void)barrier (); } +#define cpu_relax_lowlatency () cpu_relax() +/* Info on this processor (see fs/proc/cpuinfo.c) */ struct seq_operations ;extern const struct seq_operations cpuinfo_op;
arch/unicore32/include/asm/processor.h +1 -0
@@ -71,6 +71,7 @@ extern void release_thread(struct task_struct *);unsigned long get_wchan (struct task_struct *p);#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()#define task_pt_regs (p) \ ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
arch/x86/include/asm/barrier.h +1 -1
@@ -99,7 +99,7 @@#if defined(CONFIG_X86_PPRO_FENCE )/* -* For either of these options x86 doesn't have a strong TSO memory +* For this option x86 doesn't have a strong TSO memory * model and we should fall back to full barriers. */
arch/x86/include/asm/processor.h +2 -0
@@ -696,6 +696,8 @@ static inline void cpu_relax(void)rep_nop (); } +#define cpu_relax_lowlatency () cpu_relax() +/* Stop speculative execution and prefetching of modified code. */ static inline void sync_core (void ) {
arch/x86/include/asm/qrwlock.h +1 -1
@@ -3,7 +3,7 @@#include <asm-generic/qrwlock_types.h> -#if !defined(CONFIG_X86_OOSTORE ) && !defined(CONFIG_X86_PPRO_FENCE ) +#ifndef CONFIG_X86_PPRO_FENCE #define queue_write_unlock queue_write_unlockstatic inline void queue_write_unlock (struct qrwlock *lock) {
arch/x86/um/asm/processor.h +2 -1
@@ -25,7 +25,8 @@ static inline void rep_nop(void) __asm__ __volatile__("rep;nop" : : :"memory" ); } -#define cpu_relax () rep_nop() +#define cpu_relax () rep_nop() +#define cpu_relax_lowlatency () cpu_relax()#include <asm/processor-generic.h>
arch/xtensa/include/asm/processor.h +1 -0
@@ -182,6 +182,7 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) (task_pt_regs(tsk)->areg[1])#define cpu_relax () barrier() +#define cpu_relax_lowlatency () cpu_relax()/* Special register access. */
include/linux/mutex.h +0 -4
@@ -176,8 +176,4 @@ extern void mutex_unlock(struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); -#ifndef arch_mutex_cpu_relax -# define arch_mutex_cpu_relax () cpu_relax() -#endif -#endif /* __LINUX_MUTEX_H */
include/linux/rtmutex.h +2 -4
@@ -90,11 +90,9 @@ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name);extern void rt_mutex_destroy (struct rt_mutex *lock);extern void rt_mutex_lock (struct rt_mutex *lock); -extern int rt_mutex_lock_interruptible (struct rt_mutex *lock, -int detect_deadlock); +extern int rt_mutex_lock_interruptible (struct rt_mutex *lock);extern int rt_mutex_timed_lock (struct rt_mutex *lock, -struct hrtimer_sleeper *timeout, -int detect_deadlock); +struct hrtimer_sleeper *timeout);extern int rt_mutex_trylock (struct rt_mutex *lock);
include/linux/seqlock.h +0 -2
@@ -164,8 +164,6 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s)static inline unsigned raw_seqcount_begin (const seqcount_t *s) {unsigned ret =ACCESS_ONCE (s->sequence); - -seqcount_lockdep_reader_access (s);smp_rmb ();return ret & ~1 ; }
kernel/futex.c +183 -219
530 lines not on this page · raw message
kernel/locking/lockdep.c +2 -0
@@ -384,7 +384,9 @@ static void print_lockdep_off(const char *bug_msg) {printk (KERN_DEBUG "%s\n" , bug_msg);printk (KERN_DEBUG "turning off the locking correctness validator.\n" ); +#ifdef CONFIG_LOCK_STAT printk (KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n" ); +#endif }static int save_trace (struct stack_trace *trace)
kernel/locking/mcs_spinlock.c +3 -5
@@ -1,6 +1,4 @@ -#include <linux/percpu.h> -#include <linux/mutex.h> #include <linux/sched.h> #include "mcs_spinlock.h"
@@ -79,7 +77,7 @@ osq_wait_next(struct optimistic_spin_queue *lock,break ; } -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }return next;
@@ -120,7 +118,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)if (need_resched ()) goto unqueue; -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }return true;
@@ -146,7 +144,7 @@ unqueue:if (smp_load_acquire (&node->locked))return true; -arch_mutex_cpu_relax (); +cpu_relax_lowlatency ();/* * Or we race against a concurrent unqueue()'s step-B, in which
kernel/locking/mcs_spinlock.h +2 -2
@@ -27,7 +27,7 @@ struct mcs_spinlock {#define arch_mcs_spin_lock_contended (l) \ do { \ while (!(smp_load_acquire(l))) \ - arch_mutex_cpu_relax(); \ + cpu_relax_lowlatency(); \ } while (0)#endif
@@ -104,7 +104,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)return ;/* Wait until the next pointer is set */ while (!(next =ACCESS_ONCE (node->next))) -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }/* Pass lock to next waiter. */
kernel/locking/mutex.c +20 -19
@@ -46,12 +46,6 @@# include <asm/mutex.h> #endif -/* -* A negative mutex count indicates that waiters are sleeping waiting for the -* mutex. -*/ -#define MUTEX_SHOW_NO_WAITER (mutex) (atomic_read(&(mutex)->count) >= 0) -void __mutex_init (struct mutex *lock,const char *name,struct lock_class_key *key) {
@@ -152,7 +146,7 @@ int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)if (need_resched ())break ; -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }rcu_read_unlock ();
@@ -388,12 +382,10 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,/* * Optimistic spinning. * -* We try to spin for acquisition when we find that there are no -* pending waiters and the lock owner is currently running on a -* (different) CPU. -* -* The rationale is that if the lock owner is running, it is likely to -* release the lock soon. +* We try to spin for acquisition when we find that the lock owner +* is currently running on a (different) CPU and while we don't +* need to reschedule. The rationale is that if the lock owner is +* running, it is likely to release the lock soon. * * Since this needs the lock owner, and this mutex implementation * doesn't track the owner atomically in the lock field, we need to
@@ -440,7 +432,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,if (owner && !mutex_spin_on_owner (lock, owner))break ; -if ((atomic_read (&lock->count) ==1 ) && +/* Try to acquire the mutex if it is unlocked. */ +if (!mutex_is_locked (lock) && (atomic_cmpxchg (&lock->count,1 ,0 ) ==1 )) {lock_acquired (&lock->dep_map, ip);if (use_ww_ctx) {
@@ -471,7 +464,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,* memory barriers as we'll eventually observe the right * values at the cost of a few extra spins. */ -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }osq_unlock (&lock->osq); slowpath:
@@ -485,8 +478,11 @@ slowpath:#endif spin_lock_mutex (&lock->wait_lock, flags); -/* once more, can we acquire the lock? */ -if (MUTEX_SHOW_NO_WAITER (lock) && (atomic_xchg (&lock->count,0 ) ==1 )) +/* +* Once more, try to acquire the lock. Only try-lock the mutex if +* it is unlocked to reduce unnecessary xchg() operations. +*/ +if (!mutex_is_locked (lock) && (atomic_xchg (&lock->count,0 ) ==1 )) goto skip_wait;debug_mutex_lock_common (lock, &waiter);
@@ -506,9 +502,10 @@ slowpath:* it's unlocked. Later on, if we sleep, this is the * operation that gives us the lock. We xchg it to -1, so * that when we release the lock, we properly wake up the -* other waiters: +* other waiters. We only attempt the xchg if the count is +* non-negative in order to avoid unnecessary xchg operations: */ -if (MUTEX_SHOW_NO_WAITER (lock) && +if (atomic_read (&lock->count) >=0 && (atomic_xchg (&lock->count,-1 ) ==1 ))break ;
@@ -823,6 +820,10 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)unsigned long flags;int prev; +/* No need to trylock if the mutex is locked. */ +if (mutex_is_locked (lock)) +return 0 ; +spin_lock_mutex (&lock->wait_lock, flags); prev =atomic_xchg (&lock->count,-1 );
kernel/locking/qrwlock.c +4 -5
@@ -20,7 +20,6 @@#include <linux/cpumask.h> #include <linux/percpu.h> #include <linux/hardirq.h> -#include <linux/mutex.h> #include <asm/qrwlock.h> /**
@@ -35,7 +34,7 @@ static __always_inline void rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts) {while ((cnts & _QW_WMASK) == _QW_LOCKED) { -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); cnts =smp_load_acquire ((u32 *)&lock->cnts); } }
@@ -75,7 +74,7 @@ void queue_read_lock_slowpath(struct qrwlock *lock)* to make sure that the write lock isn't taken. */ while (atomic_read (&lock->cnts) & _QW_WMASK) -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); cnts =atomic_add_return (_QR_BIAS, &lock->cnts) - _QR_BIAS;rspin_until_writer_unlock (lock, cnts);
@@ -114,7 +113,7 @@ void queue_write_lock_slowpath(struct qrwlock *lock) cnts | _QW_WAITING) == cnts))break ; -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }/* When no more readers, set the locked flag */
@@ -125,7 +124,7 @@ void queue_write_lock_slowpath(struct qrwlock *lock) _QW_LOCKED) == _QW_WAITING))break ; -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); } unlock:arch_spin_unlock (&lock->lock);
kernel/locking/rtmutex-debug.c +3 -2
@@ -66,12 +66,13 @@ void rt_mutex_debug_task_free(struct task_struct *task)* the deadlock. We print when we return. act_waiter can be NULL in * case of a remove waiter operation. */ -void debug_rt_mutex_deadlock (int detect,struct rt_mutex_waiter *act_waiter, +void debug_rt_mutex_deadlock (enum rtmutex_chainwalk chwalk, +struct rt_mutex_waiter *act_waiter,struct rt_mutex *lock) {struct task_struct *task; -if (!debug_locks || detect || !act_waiter) +if (!debug_locks || chwalk ==RT_MUTEX_FULL_CHAINWALK || !act_waiter)return ; task =rt_mutex_owner (act_waiter->lock);
kernel/locking/rtmutex-debug.h +4 -3
@@ -20,14 +20,15 @@ extern void debug_rt_mutex_unlock(struct rt_mutex *lock);extern void debug_rt_mutex_proxy_lock (struct rt_mutex *lock,struct task_struct *powner);extern void debug_rt_mutex_proxy_unlock (struct rt_mutex *lock); -extern void debug_rt_mutex_deadlock (int detect,struct rt_mutex_waiter *waiter, +extern void debug_rt_mutex_deadlock (enum rtmutex_chainwalk chwalk, +struct rt_mutex_waiter *waiter,struct rt_mutex *lock);extern void debug_rt_mutex_print_deadlock (struct rt_mutex_waiter *waiter);# define debug_rt_mutex_reset_waiter (w) \ do { (w)->deadlock_lock = NULL; } while (0) -static inline int debug_rt_mutex_detect_deadlock (struct rt_mutex_waiter *waiter, -int detect) +static inline bool debug_rt_mutex_detect_deadlock (struct rt_mutex_waiter *waiter, +enum rtmutex_chainwalk walk) {return (waiter !=NULL ); }
kernel/locking/rtmutex.c +417 -145
881 lines not on this page · raw message
kernel/locking/rtmutex.h +6 -1
@@ -22,10 +22,15 @@#define debug_rt_mutex_init (m, n) do { } while (0)#define debug_rt_mutex_deadlock (d, a ,l) do { } while (0)#define debug_rt_mutex_print_deadlock (w) do { } while (0) -#define debug_rt_mutex_detect_deadlock (w,d) (d)#define debug_rt_mutex_reset_waiter (w) do { } while (0)static inline void rt_mutex_print_deadlock (struct rt_mutex_waiter *w) {WARN (1 ,"rtmutex deadlock detected\n" ); } + +static inline bool debug_rt_mutex_detect_deadlock (struct rt_mutex_waiter *w, +enum rtmutex_chainwalk walk) +{ +return walk ==RT_MUTEX_FULL_CHAINWALK ; +}
kernel/locking/rtmutex_common.h +18 -4
@@ -102,6 +102,21 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex *lock) }/* +* Constants for rt mutex functions which have a selectable deadlock +* detection. +* +* RT_MUTEX_MIN_CHAINWALK: Stops the lock chain walk when there are +* no further PI adjustments to be made. +* +* RT_MUTEX_FULL_CHAINWALK: Invoke deadlock detection with a full +* walk of the lock chain. +*/ +enum rtmutex_chainwalk { +RT_MUTEX_MIN_CHAINWALK , +RT_MUTEX_FULL_CHAINWALK , +}; + +/* * PI-futex support (proxy locking functions, etc.): */ extern struct task_struct *rt_mutex_next_owner (struct rt_mutex *lock);
@@ -111,12 +126,11 @@ extern void rt_mutex_proxy_unlock(struct rt_mutex *lock, struct task_struct *proxy_owner);extern int rt_mutex_start_proxy_lock (struct rt_mutex *lock,struct rt_mutex_waiter *waiter, -struct task_struct *task, -int detect_deadlock); +struct task_struct *task);extern int rt_mutex_finish_proxy_lock (struct rt_mutex *lock,struct hrtimer_sleeper *to, -struct rt_mutex_waiter *waiter, -int detect_deadlock); +struct rt_mutex_waiter *waiter); +extern int rt_mutex_timed_futex_lock (struct rt_mutex *l,struct hrtimer_sleeper *to);#ifdef CONFIG_DEBUG_RT_MUTEXES # include "rtmutex-debug.h"
kernel/locking/rwsem-xadd.c +2 -2
@@ -329,7 +329,7 @@ bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner)if (need_resched ())break ; -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }rcu_read_unlock ();
@@ -381,7 +381,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)* memory barriers as we'll eventually observe the right * values at the cost of a few extra spins. */ -arch_mutex_cpu_relax (); +cpu_relax_lowlatency (); }osq_unlock (&sem->osq); done:
lib/Kconfig.debug +1 -1
@@ -835,7 +835,7 @@ config DEBUG_RT_MUTEXES config RT_MUTEX_TESTER bool "Built-in scriptable tester for rt-mutexes" - depends on DEBUG_KERNEL && RT_MUTEXES + depends on DEBUG_KERNEL && RT_MUTEXES && BROKEN help This option enables a rt-mutex tester.
lib/lockref.c +1 -2
@@ -1,6 +1,5 @@#include <linux/export.h> #include <linux/lockref.h> -#include <linux/mutex.h> #if USE_CMPXCHG_LOCKREF
@@ -29,7 +28,7 @@if (likely (old.lock_count == prev.lock_count)) { \SUCCESS ; \ } \ -arch_mutex_cpu_relax (); \ +cpu_relax_lowlatency (); \ } \ }while (0 )