mutex: Introduce arch_mutex_cpu_relax()
Subsystems (2)
PATCH 1/1 mutex: Introduce arch_mutex_cpu_relax()
From: Gerald Schaefer <[email protected]>
The spinning mutex implementation uses cpu_relax() in busy loops as a compiler barrier. Depending on the architecture, cpu_relax() may do more than needed in this specific mutex spin loops. On System z we also give up the time slice of the virtual cpu in cpu_relax(), which prevents effective spinning on the mutex.
This patch replaces cpu_relax() in the spinning mutex code with arch_mutex_cpu_relax(), which can be defined by each architecture that selects HAVE_ARCH_MUTEX_CPU_RELAX. The default is still cpu_relax(), so this patch should not affect other architectures than System z for now.
--- arch/Kconfig | 3 +++ arch/s390/Kconfig | 1 + arch/s390/include/asm/mutex.h | 2 ++ include/linux/mutex.h | 4 ++++ kernel/mutex.c | 2 +- kernel/sched.c | 3 ++- 6 files changed, 13 insertions(+), 2 deletions(-)
arch/Kconfig +3 -0
@@ -175,4 +175,7 @@ config HAVE_PERF_EVENTS_NMI config HAVE_ARCH_JUMP_LABEL bool +config HAVE_ARCH_MUTEX_CPU_RELAX + bool + source "kernel/gcov/Kconfig"
arch/s390/Kconfig +1 -0
@@ -99,6 +99,7 @@ config S390 select HAVE_KERNEL_LZMA select HAVE_KERNEL_LZO select HAVE_GET_USER_PAGES_FAST + select HAVE_ARCH_MUTEX_CPU_RELAX select ARCH_INLINE_SPIN_TRYLOCK select ARCH_INLINE_SPIN_TRYLOCK_BH select ARCH_INLINE_SPIN_LOCK
arch/s390/include/asm/mutex.h +2 -0
@@ -7,3 +7,5 @@*/ #include <asm-generic/mutex-dec.h> + +#define arch_mutex_cpu_relax () barrier()
include/linux/mutex.h +4 -0
@@ -160,4 +160,8 @@ extern int mutex_trylock(struct mutex *lextern void mutex_unlock (struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); +#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX +#define arch_mutex_cpu_relax () cpu_relax() +#endif +#endif
kernel/mutex.c +1 -1
@@ -199,7 +199,7 @@ __mutex_lock_common(struct mutex *lock,* memory barriers as we'll eventually observe the right * values at the cost of a few extra spins. */ - cpu_relax(); + arch_mutex_cpu_relax(); }#endif spin_lock_mutex (&lock->wait_lock, flags);
kernel/sched.c +2 -1
@@ -75,6 +75,7 @@#include <asm/tlb.h> #include <asm/irq_regs.h> +#include <asm/mutex.h> #include "sched_cpupri.h" #include "workqueue_sched.h"
@@ -4029,7 +4030,7 @@ int mutex_spin_on_owner(struct mutex *loif (task_thread_info (rq->curr) != owner ||need_resched ())return 0 ; -cpu_relax (); +arch_mutex_cpu_relax (); }return 1 ;
Quoted 18 lines · On Mon, 22 Nov 2010 15:47:36 +0100 Gerald Schaefer…
From: Gerald Schaefer <[email protected]>
The spinning mutex implementation uses
cpu_relax()in busy loops as a compiler barrier. Depending on the architecture,cpu_relax()may do more than needed in this specific mutex spin loops. On System z we also give up the time slice of the virtual cpu incpu_relax(), which prevents effective spinning on the mutex.This patch replaces
cpu_relax()in the spinning mutex code witharch_mutex_cpu_relax(), which can be defined by each architecture that selectsHAVE_ARCH_MUTEX_CPU_RELAX. The default is stillcpu_relax(), so this patch should not affect other architectures than System z for now....
include/linux/mutex.h extern int mutex_trylock(struct mutex *l · line 165
Quoted 6 lines
include/linux/mutex.h +3
-0@@ -160,4 +160,8 @@ extern int mutex_trylock(struct mutex *lextern void mutex_unlock (struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); +#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX +#define arch_mutex_cpu_relax () cpu_relax() +#endif
A simpler way of doing this is to remove the CONFIG_ variable altogether and do
#ifndef arch_mutex_cpu_relax
#define arch_mutex_cpu_relax() cpu_relax()
#endif
When doing this, one should be clear about _which_ arch file has the responsibility of defining arch_mutex_cpu_relax, and make sure that this arch file is reliably included in the .c file.
Quoted 31 lines · Andrew Morton
Quoted 18 lines · On Mon, 22 Nov 2010 15:47:36 +0100 Gerald Schaefer…
From: Gerald Schaefer <[email protected]>
The spinning mutex implementation uses
cpu_relax()in busy loops as a compiler barrier. Depending on the architecture,cpu_relax()may do more than needed in this specific mutex spin loops. On System z we also give up the time slice of the virtual cpu incpu_relax(), which prevents effective spinning on the mutex.This patch replaces
cpu_relax()in the spinning mutex code witharch_mutex_cpu_relax(), which can be defined by each architecture that selectsHAVE_ARCH_MUTEX_CPU_RELAX. The default is stillcpu_relax(), so this patch should not affect other architectures than System z for now....
include/linux/mutex.h extern int mutex_trylock(struct mutex *l · line 165
Quoted 6 lines
include/linux/mutex.h +3
-0@@ -160,4 +160,8 @@ extern int mutex_trylock(struct mutex *lextern void mutex_unlock (struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); +#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX +#define arch_mutex_cpu_relax () cpu_relax() +#endif A simpler way of doing this is to remove the
CONFIG_variable altogether and do
Quoted 7 lines
#ifndef
arch_mutex_cpu_relax
#definearch_mutex_cpu_relax()cpu_relax()
#endifWhen doing this, one should be clear about _which_ arch file has the responsibility of defining
arch_mutex_cpu_relax, and make sure that this arch file is reliably included in the .c file.
Well, I've tried that with my last approach, defining arch_mutex_cpu_relax() in <asm/mutex.h> and including that from <linux/mutex.h>. This didn't work well because of ugly header file dependencies, and Peter also commented that "including "asm/mutex.h" isn't advised". The problem is the following code in kernel/mutex.c (after including <linux/mutex.h>) when CONFIG_DEBUG_MUTEXES is set:
#ifdef CONFIG_DEBUG_MUTEXES
# include "mutex-debug.h" # include <asm-generic/mutex-null.h> #else # include "mutex.h" # include <asm/mutex.h> #endif
So I can only include <asm/mutex.h> from <linux/mutex.h> with an ugly "#ifndef CONFIG_DEBUG_MUTEXES" around it, or use a completely different or new arch header file (but <asm/mutex.h> seems like the right place for this). The CONFIG_ approach avoids all this header file dependency mess, or did I miss something (or maybe it's just me and it is not ugly at all)?
Quoted 55 lines · Gerald Schaefer
Quoted 31 lines · Andrew Morton
Quoted 18 lines · On Mon, 22 Nov 2010 15:47:36 +0100 Gerald Schaefer…
From: Gerald Schaefer <[email protected]>
The spinning mutex implementation uses
cpu_relax()in busy loops as a compiler barrier. Depending on the architecture,cpu_relax()may do more than needed in this specific mutex spin loops. On System z we also give up the time slice of the virtual cpu incpu_relax(), which prevents effective spinning on the mutex.This patch replaces
cpu_relax()in the spinning mutex code witharch_mutex_cpu_relax(), which can be defined by each architecture that selectsHAVE_ARCH_MUTEX_CPU_RELAX. The default is stillcpu_relax(), so this patch should not affect other architectures than System z for now....
include/linux/mutex.h extern int mutex_trylock(struct mutex *l · line 165
Quoted 6 lines
include/linux/mutex.h +3
-0@@ -160,4 +160,8 @@ extern int mutex_trylock(struct mutex *lextern void mutex_unlock (struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); +#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX +#define arch_mutex_cpu_relax () cpu_relax() +#endif A simpler way of doing this is to remove the
CONFIG_variable altogether and doQuoted 7 lines
#ifndef
arch_mutex_cpu_relax
#definearch_mutex_cpu_relax()cpu_relax()
#endifWhen doing this, one should be clear about _which_ arch file has the responsibility of defining
arch_mutex_cpu_relax, and make sure that this arch file is reliably included in the .c file.Well, I've tried that with my last approach, defining
arch_mutex_cpu_relax()in <asm/mutex.h> and including that from <linux/mutex.h>. This didn't work well because of ugly header file dependencies, and Peter also commented that "including "asm/mutex.h" isn't advised". The problem is the following code inkernel/mutex.c(after including <linux/mutex.h>) whenCONFIG_DEBUG_MUTEXESis set:#ifdef
CONFIG_DEBUG_MUTEXES# include "mutex-debug.h" # include <asm-generic/mutex-null.h> #else # include "mutex.h" # include <asm/mutex.h> #endif
Quoted 6 lines
So I can only include <
asm/mutex.h> from <linux/mutex.h> with an ugly "#ifndefCONFIG_DEBUG_MUTEXES" around it, or use a completely different or new arch header file (but <asm/mutex.h> seems like the right place for this). TheCONFIG_approach avoids all this header file dependency mess, or did I miss something (or maybe it's just me and it is not ugly at all)?
Yeah, that all cause massive grief.. I've applied your patch as is, assuming s390 already has the needed arch_mutex_cpu_relax() implementation (otherwise I've just broken my s390 build).
Thanks
Quoted 63 lines · Peter Zijlstra
Quoted 55 lines · Gerald Schaefer
Quoted 31 lines · Andrew Morton
Quoted 18 lines · On Mon, 22 Nov 2010 15:47:36 +0100 Gerald Schaefer…
From: Gerald Schaefer <[email protected]>
The spinning mutex implementation uses
cpu_relax()in busy loops as a compiler barrier. Depending on the architecture,cpu_relax()may do more than needed in this specific mutex spin loops. On System z we also give up the time slice of the virtual cpu incpu_relax(), which prevents effective spinning on the mutex.This patch replaces
cpu_relax()in the spinning mutex code witharch_mutex_cpu_relax(), which can be defined by each architecture that selectsHAVE_ARCH_MUTEX_CPU_RELAX. The default is stillcpu_relax(), so this patch should not affect other architectures than System z for now....
include/linux/mutex.h extern int mutex_trylock(struct mutex *l · line 165
Quoted 6 lines
include/linux/mutex.h +3
-0@@ -160,4 +160,8 @@ extern int mutex_trylock(struct mutex *lextern void mutex_unlock (struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); +#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX +#define arch_mutex_cpu_relax () cpu_relax() +#endif A simpler way of doing this is to remove the
CONFIG_variable altogether and doQuoted 7 lines
#ifndef
arch_mutex_cpu_relax
#definearch_mutex_cpu_relax()cpu_relax()
#endifWhen doing this, one should be clear about _which_ arch file has the responsibility of defining
arch_mutex_cpu_relax, and make sure that this arch file is reliably included in the .c file.Well, I've tried that with my last approach, defining
arch_mutex_cpu_relax()in <asm/mutex.h> and including that from <linux/mutex.h>. This didn't work well because of ugly header file dependencies, and Peter also commented that "including "asm/mutex.h" isn't advised". The problem is the following code inkernel/mutex.c(after including <linux/mutex.h>) whenCONFIG_DEBUG_MUTEXESis set:#ifdef
CONFIG_DEBUG_MUTEXES# include "mutex-debug.h" # include <asm-generic/mutex-null.h> #else # include "mutex.h" # include <asm/mutex.h> #endifQuoted 6 lines
So I can only include <
asm/mutex.h> from <linux/mutex.h> with an ugly "#ifndefCONFIG_DEBUG_MUTEXES" around it, or use a completely different or new arch header file (but <asm/mutex.h> seems like the right place for this). TheCONFIG_approach avoids all this header file dependency mess, or did I miss something (or maybe it's just me and it is not ugly at all)?
Quoted 3 lines
Yeah, that all cause massive grief.. I've applied your patch as is, assuming s390 already has the needed
arch_mutex_cpu_relax()implementation (otherwise I've just broken my s390 build).
Thanks, my patch already includes the s390 implementation for arch_mutex_cpu_relax() in arch/s390/include/asm/mutex.h (we use a simple barrier() in this case). We still have HAVE_DEFAULT_NO_SPIN_MUTEXES selected though, this will be removed with one of Martins next patches, once this preparation patch is upstream.
[tip:sched/core] mutexes, sched: Introduce arch_mutex_cpu_relax()
In reply to Gerald SchaeferCommit-ID: 335d7afbfb71faac833734a94240c1e07cf0ead8 Gitweb: http://git.kernel.org/tip/335d7afbfb71faac833734a94240c1e07cf0ead8 Author: Gerald Schaefer <[email protected]> AuthorDate: Mon, 22 Nov 2010 15:47:36 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 26 Nov 2010 15:05:34 +0100
mutexes, sched: Introduce arch_mutex_cpu_relax()
The spinning mutex implementation uses cpu_relax() in busy loops as a compiler barrier. Depending on the architecture, cpu_relax() may do more than needed in this specific mutex spin loops. On System z we also give up the time slice of the virtual cpu in cpu_relax(), which prevents effective spinning on the mutex.
This patch replaces cpu_relax() in the spinning mutex code with arch_mutex_cpu_relax(), which can be defined by each architecture that selects HAVE_ARCH_MUTEX_CPU_RELAX. The default is still cpu_relax(), so this patch should not affect other architectures than System z for now.
LKML-Reference: <1290437256.7455.4.camel@thinkpad>
--- arch/Kconfig | 3 +++ arch/s390/Kconfig | 1 + arch/s390/include/asm/mutex.h | 2 ++ include/linux/mutex.h | 4 ++++ kernel/mutex.c | 2 +- kernel/sched.c | 3 ++- 6 files changed, 13 insertions(+), 2 deletions(-)
arch/Kconfig +3 -0
@@ -175,4 +175,7 @@ config HAVE_PERF_EVENTS_NMI config HAVE_ARCH_JUMP_LABEL bool +config HAVE_ARCH_MUTEX_CPU_RELAX + bool + source "kernel/gcov/Kconfig"
arch/s390/Kconfig +1 -0
@@ -99,6 +99,7 @@ config S390 select HAVE_KERNEL_LZMA select HAVE_KERNEL_LZO select HAVE_GET_USER_PAGES_FAST + select HAVE_ARCH_MUTEX_CPU_RELAX select ARCH_INLINE_SPIN_TRYLOCK select ARCH_INLINE_SPIN_TRYLOCK_BH select ARCH_INLINE_SPIN_LOCK
arch/s390/include/asm/mutex.h +2 -0
@@ -7,3 +7,5 @@*/ #include <asm-generic/mutex-dec.h> + +#define arch_mutex_cpu_relax () barrier()
include/linux/mutex.h +4 -0
@@ -160,4 +160,8 @@ extern int mutex_trylock(struct mutex *lock);extern void mutex_unlock (struct mutex *lock);extern int atomic_dec_and_mutex_lock (atomic_t *cnt,struct mutex *lock); +#ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX +#define arch_mutex_cpu_relax () cpu_relax() +#endif +#endif
kernel/mutex.c +1 -1
@@ -199,7 +199,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,* memory barriers as we'll eventually observe the right * values at the cost of a few extra spins. */ - cpu_relax(); + arch_mutex_cpu_relax(); }#endif spin_lock_mutex (&lock->wait_lock, flags);
kernel/sched.c +2 -1
@@ -75,6 +75,7 @@#include <asm/tlb.h> #include <asm/irq_regs.h> +#include <asm/mutex.h> #include "sched_cpupri.h" #include "workqueue_sched.h"
@@ -3888,7 +3889,7 @@ int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)if (task_thread_info (rq->curr) != owner ||need_resched ())return 0 ; -cpu_relax (); +arch_mutex_cpu_relax (); }return 1 ;