locking changes for v4.1
Subsystems (12)
Linus,
Please pull the latest locking-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking-core-for-linus
# HEAD: 58995a9a5b292458f94a2356b8c878230fa56fe0 powerpc, jump_label: Include linux/jump_label.h to get HAVE_JUMP_LABEL define
Main changes:
- jump label asm preparatory work for PowerPC (Anton Blanchard)
- rwsem optimizations and cleanups (Davidlohr Bueso)
- mutex optimizations and cleanups (Jason Low)
- futex fix (Oleg Nesterov)
- remove broken atomicity checks from {READ,WRITE}_ONCE() (Peter Zijlstra)
Thanks,
Ingo
------------------>
Anton Blanchard (3):
jump_label: Allow asm/jump_label.h to be included in assembly
jump_label: Allow jump labels to be used in assembly
powerpc, jump_label: Include linux/jump_label.h to get HAVE_JUMP_LABEL define
Davidlohr Bueso (5):
locking/rwsem: Document barrier need when waking tasks locking/rwsem: Set lock ownership ASAP locking/rwsem: Avoid deceiving lock spinners locking/rwsem: Check for active lock before bailing on spinning
locking: Remove ACCESS_ONCE() usage
Jason Low (4):
locking/mutex: In mutex_spin_on_owner(), return true when owner changes locking/mutex: Refactor mutex_spin_on_owner() locking/rwsem: Fix lock optimistic spinning when owner is not running locking/mutex: Further simplify mutex_spin_on_owner()
Oleg Nesterov (1):
locking/futex: Check PF_KTHREAD rather than !p->mm to filter out kthreads
Peter Zijlstra (1):
locking: Remove atomicy checks from {READ,WRITE}_ONCE
Tom(JeHyeon) Yeon (1):
locking/rtmutex: Rename argument in the rt_mutex_adjust_prio_chain() documentation as well
Makefile | 1 + arch/arm/include/asm/jump_label.h | 5 +- arch/arm64/include/asm/jump_label.h | 8 +-- arch/mips/include/asm/jump_label.h | 7 +- arch/powerpc/platforms/powernv/opal-wrappers.S | 2 +- arch/powerpc/platforms/pseries/hvCall.S | 2 +- arch/powerpc/platforms/pseries/lpar.c | 2 +- arch/s390/include/asm/jump_label.h | 3 + arch/sparc/include/asm/jump_label.h | 5 +- arch/x86/include/asm/jump_label.h | 5 +- include/linux/compiler.h | 16 ----- include/linux/jump_label.h | 21 ++++-- include/linux/seqlock.h | 6 +- kernel/futex.c | 2 +- kernel/locking/mcs_spinlock.h | 6 +- kernel/locking/mutex.c | 51 ++++++-------- kernel/locking/osq_lock.c | 14 ++-- kernel/locking/rtmutex.c | 2 +- kernel/locking/rwsem-spinlock.c | 7 ++ kernel/locking/rwsem-xadd.c | 98 +++++++++++++++----------- kernel/locking/rwsem.c | 22 +----- kernel/locking/rwsem.h | 20 ++++++ lib/lockref.c | 2 +- 23 files changed, 160 insertions(+), 147 deletions(-) create mode 100644 kernel/locking/rwsem.h
Makefile +1 -0
@@ -779,6 +779,7 @@ KBUILD_ARFLAGS := $(call ar-option,D) # check for 'asm goto' ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y) KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO + KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO endif include $(srctree)/scripts/Makefile.kasan
arch/arm/include/asm/jump_label.h +2 -3
@@ -1,7 +1,7 @@#ifndef _ASM_ARM_JUMP_LABEL_H#define _ASM_ARM_JUMP_LABEL_H -#ifdef __KERNEL__ +#ifndef __ASSEMBLY__#include <linux/types.h>
@@ -27,8 +27,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)return true; } -#endif /* __KERNEL__ */ -typedef u32 jump_label_t ;struct jump_entry {
@@ -37,4 +35,5 @@ struct jump_entry {jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif
arch/arm64/include/asm/jump_label.h +4 -4
@@ -18,11 +18,12 @@*/ #ifndef __ASM_JUMP_LABEL_H#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ +#include <linux/types.h> #include <asm/insn.h> -#ifdef __KERNEL__ -#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE static __always_inline bool arch_static_branch(struct static_key *key)
@@ -39,8 +40,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)return true; } -#endif /* __KERNEL__ */ -typedef u64 jump_label_t ;struct jump_entry {
@@ -49,4 +48,5 @@ struct jump_entry {jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif /* __ASM_JUMP_LABEL_H */
arch/mips/include/asm/jump_label.h +3 -4
@@ -8,9 +8,9 @@#ifndef _ASM_MIPS_JUMP_LABEL_H#define _ASM_MIPS_JUMP_LABEL_H -#include <linux/types.h> +#ifndef __ASSEMBLY__ -#ifdef __KERNEL__ +#include <linux/types.h> #define JUMP_LABEL_NOP_SIZE 4
@@ -39,8 +39,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)return true; } -#endif /* __KERNEL__ */ -#ifdef CONFIG_64BIT typedef u64 jump_label_t ;#else
@@ -53,4 +51,5 @@ struct jump_entry {jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif /* _ASM_MIPS_JUMP_LABEL_H */
arch/powerpc/platforms/powernv/opal-wrappers.S +1 -1
@@ -9,11 +9,11 @@ * 2 of the License, or (at your option) any later version. */ +#include <linux/jump_label.h> #include <asm/ppc_asm.h> #include <asm/hvcall.h> #include <asm/asm-offsets.h> #include <asm/opal.h> -#include <asm/jump_label.h> .section ".text"
arch/powerpc/platforms/pseries/hvCall.S +1 -1
@@ -7,12 +7,12 @@ * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ +#include <linux/jump_label.h> #include <asm/hvcall.h> #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> -#include <asm/jump_label.h> .section ".text"
arch/powerpc/platforms/pseries/lpar.c +1 -1
@@ -26,7 +26,7 @@#include <linux/dma-mapping.h> #include <linux/console.h> #include <linux/export.h> -#include <linux/static_key.h> +#include <linux/jump_label.h> #include <asm/processor.h> #include <asm/mmu.h> #include <asm/page.h>
arch/s390/include/asm/jump_label.h +3 -0
@@ -1,6 +1,8 @@#ifndef _ASM_S390_JUMP_LABEL_H#define _ASM_S390_JUMP_LABEL_H +#ifndef __ASSEMBLY__ +#include <linux/types.h> #define JUMP_LABEL_NOP_SIZE 6
@@ -39,4 +41,5 @@ struct jump_entry {jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif
arch/sparc/include/asm/jump_label.h +2 -3
@@ -1,7 +1,7 @@#ifndef _ASM_SPARC_JUMP_LABEL_H#define _ASM_SPARC_JUMP_LABEL_H -#ifdef __KERNEL__ +#ifndef __ASSEMBLY__#include <linux/types.h>
@@ -22,8 +22,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)return true; } -#endif /* __KERNEL__ */ -typedef u32 jump_label_t ;struct jump_entry {
@@ -32,4 +30,5 @@ struct jump_entry {jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif
arch/x86/include/asm/jump_label.h +2 -3
@@ -1,7 +1,7 @@#ifndef _ASM_X86_JUMP_LABEL_H#define _ASM_X86_JUMP_LABEL_H -#ifdef __KERNEL__ +#ifndef __ASSEMBLY__#include <linux/stringify.h> #include <linux/types.h>
@@ -30,8 +30,6 @@ static __always_inline bool arch_static_branch(struct static_key *key)return true; } -#endif /* __KERNEL__ */ -#ifdef CONFIG_X86_64 typedef u64 jump_label_t ;#else
@@ -44,4 +42,5 @@ struct jump_entry {jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif
include/linux/compiler.h +0 -16
@@ -192,29 +192,16 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);#include <uapi/linux/types.h> -static __always_inline void data_access_exceeds_word_size(void) -#ifdef __compiletime_warning -__compiletime_warning ("data access exceeds word size and won't be atomic" ) -#endif -; - -static __always_inline void data_access_exceeds_word_size(void) -{ -} - static __always_inline void __read_once_size(const volatile void *p, void *res, int size) {switch (size) {case 1 : *(__u8 *)res = *(volatile __u8 *)p;break ;case 2 : *(__u16 *)res = *(volatile __u16 *)p;break ;case 4 : *(__u32 *)res = *(volatile __u32 *)p;break ; -#ifdef CONFIG_64BIT case 8 : *(__u64 *)res = *(volatile __u64 *)p;break ; -#endif default :barrier ();__builtin_memcpy ((void *)res, (const void *)p, size); -data_access_exceeds_word_size ();barrier (); } }
@@ -225,13 +212,10 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int scase 1 : *(volatile __u8 *)p = *(__u8 *)res;break ;case 2 : *(volatile __u16 *)p = *(__u16 *)res;break ;case 4 : *(volatile __u32 *)p = *(__u32 *)res;break ; -#ifdef CONFIG_64BIT case 8 : *(volatile __u64 *)p = *(__u64 *)res;break ; -#endif default :barrier ();__builtin_memcpy ((void *)p, (const void *)res, size); -data_access_exceeds_word_size ();barrier (); } }
include/linux/jump_label.h +17 -4
@@ -45,6 +45,12 @@* same as using STATIC_KEY_INIT_FALSE. */ +#if defined(CC_HAVE_ASM_GOTO ) && defined(CONFIG_JUMP_LABEL ) +# define HAVE_JUMP_LABEL +#endif + +#ifndef __ASSEMBLY__ +#include <linux/types.h> #include <linux/compiler.h> #include <linux/bug.h>
@@ -55,7 +61,7 @@ extern bool static_key_initialized; "%s used before call to jump_label_init", \ __func__) -#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) +#ifdef HAVE_JUMP_LABEL struct static_key { atomic_t enabled;
@@ -66,13 +72,18 @@ struct static_key {#endif }; -# include <asm/jump_label.h> -# define HAVE_JUMP_LABEL #else struct static_key {atomic_t enabled; }; -#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ +#endif /* HAVE_JUMP_LABEL */ +#endif /* __ASSEMBLY__ */ + +#ifdef HAVE_JUMP_LABEL +#include <asm/jump_label.h> +#endif + +#ifndef __ASSEMBLY__enum jump_label_type { JUMP_LABEL_DISABLE = 0,
@@ -203,3 +214,5 @@ static inline bool static_key_enabled(struct static_key *key) }#endif /* _LINUX_JUMP_LABEL_H */ + +#endif /* __ASSEMBLY__ */
include/linux/seqlock.h +3 -3
@@ -108,7 +108,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s)unsigned ret; repeat: - ret =ACCESS_ONCE (s->sequence); + ret =READ_ONCE (s->sequence);if (unlikely (ret &1 )) {cpu_relax (); goto repeat;
@@ -127,7 +127,7 @@ static inline unsigned __read_seqcount_begin(const seqcount_t *s)*/ static inline unsigned raw_read_seqcount (const seqcount_t *s) { -unsigned ret =ACCESS_ONCE (s->sequence); +unsigned ret =READ_ONCE (s->sequence);smp_rmb ();return ret; }
@@ -179,7 +179,7 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s)*/ static inline unsigned raw_seqcount_begin (const seqcount_t *s) { -unsigned ret =ACCESS_ONCE (s->sequence); +unsigned ret =READ_ONCE (s->sequence);smp_rmb ();return ret & ~1 ; }
kernel/futex.c +1 -1
@@ -900,7 +900,7 @@ static int attach_to_pi_owner(u32 uval, union futex_key *key,if (!p)return -ESRCH ; -if (!p->mm) { +if (unlikely (p->flags &PF_KTHREAD )) {put_task_struct (p);return -EPERM ; }
kernel/locking/mcs_spinlock.h +3 -3
@@ -78,7 +78,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node)*/ return ; } -ACCESS_ONCE (prev->next) = node; +WRITE_ONCE (prev->next, node);/* Wait until the lock holder passes the lock down. */ arch_mcs_spin_lock_contended (&node->locked);
@@ -91,7 +91,7 @@ void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node)static inline void mcs_spin_unlock (struct mcs_spinlock **lock,struct mcs_spinlock *node) { -struct mcs_spinlock *next =ACCESS_ONCE (node->next); +struct mcs_spinlock *next =READ_ONCE (node->next);if (likely (!next)) {/*
@@ -100,7 +100,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)if (likely (cmpxchg (lock, node,NULL ) == node))return ;/* Wait until the next pointer is set */ -while (!(next =ACCESS_ONCE (node->next))) +while (!(next =READ_ONCE (node->next)))cpu_relax_lowlatency (); }
kernel/locking/mutex.c +21 -30
100 lines not on this page · raw message
kernel/locking/osq_lock.c +7 -7
@@ -98,7 +98,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) prev =decode_cpu (old); node->prev = prev; -ACCESS_ONCE (prev->next) = node; +WRITE_ONCE (prev->next, node);/* * Normally @prev is untouchable after the above store; because at that
@@ -109,7 +109,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)* cmpxchg in an attempt to undo our queueing. */ -while (!ACCESS_ONCE (node->locked)) { +while (!READ_ONCE (node->locked)) {/* * If we need to reschedule bail... so we can block. */
@@ -148,7 +148,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)* Or we race against a concurrent unqueue()'s step-B, in which * case its step-C will write us a new @node->prev pointer. */ - prev =ACCESS_ONCE (node->prev); + prev =READ_ONCE (node->prev); }/*
@@ -170,8 +170,8 @@ bool osq_lock(struct optimistic_spin_queue *lock)* it will wait in Step-A. */ -ACCESS_ONCE (next->prev) = prev; -ACCESS_ONCE (prev->next) = next; +WRITE_ONCE (next->prev, prev); +WRITE_ONCE (prev->next, next);return false; }
@@ -193,11 +193,11 @@ void osq_unlock(struct optimistic_spin_queue *lock) node =this_cpu_ptr (&osq_node); next =xchg (&node->next,NULL );if (next) { -ACCESS_ONCE (next->locked) =1 ; +WRITE_ONCE (next->locked,1 );return ; } next =osq_wait_next (lock, node,NULL );if (next) -ACCESS_ONCE (next->locked) =1 ; +WRITE_ONCE (next->locked,1 ); }
kernel/locking/rtmutex.c +1 -1
@@ -349,7 +349,7 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)* * @task: the task owning the mutex (owner) for which a chain walk is * probably needed -* @deadlock_detect: do we have to carry out deadlock detection? +* @chwalk: do we have to carry out deadlock detection? * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck * things for a task that has just got its priority adjusted, and * is waiting on a mutex)
kernel/locking/rwsem-spinlock.c +7 -0
@@ -85,6 +85,13 @@ __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)list_del (&waiter->list); tsk = waiter->task; +/* +* Make sure we do not wakeup the next reader before +* setting the nil condition to grant the next reader; +* otherwise we could miss the wakeup on the other +* side and end up sleeping again. See the pairing +* in rwsem_down_read_failed(). +*/ smp_mb (); waiter->task =NULL ;wake_up_process (tsk);
kernel/locking/rwsem-xadd.c +58 -40
167 lines not on this page · raw message
kernel/locking/rwsem.c +1 -21
@@ -9,29 +9,9 @@#include <linux/sched.h> #include <linux/export.h> #include <linux/rwsem.h> -#include <linux/atomic.h> -#ifdef CONFIG_RWSEM_SPIN_ON_OWNER -static inline void rwsem_set_owner (struct rw_semaphore *sem) -{ - sem->owner = current; -} - -static inline void rwsem_clear_owner (struct rw_semaphore *sem) -{ - sem->owner =NULL ; -} - -#else -static inline void rwsem_set_owner (struct rw_semaphore *sem) -{ -} - -static inline void rwsem_clear_owner (struct rw_semaphore *sem) -{ -} -#endif +#include "rwsem.h" /* * lock for reading
kernel/locking/rwsem.h new file +20 -0
@@ -0,0 +1,20 @@ +#ifdef CONFIG_RWSEM_SPIN_ON_OWNER +static inline void rwsem_set_owner (struct rw_semaphore *sem) +{ + sem->owner = current; +} + +static inline void rwsem_clear_owner (struct rw_semaphore *sem) +{ + sem->owner =NULL ; +} + +#else +static inline void rwsem_set_owner (struct rw_semaphore *sem) +{ +} + +static inline void rwsem_clear_owner (struct rw_semaphore *sem) +{ +} +#endif
lib/lockref.c +1 -1
@@ -18,7 +18,7 @@#define CMPXCHG_LOOP (CODE ,SUCCESS ) do { \ struct lockref old; \ BUILD_BUG_ON(sizeof(old) != 8); \ - old.lock_count = ACCESS_ONCE(lockref->lock_count); \ + old.lock_count = READ_ONCE(lockref->lock_count); \ while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ struct lockref new = old, prev = old; \ CODE \