2.6.6-rc2 Allow architectures to reenable interrupts on contended spinlocks

PATCH In review Keith Owens 6 participants 11 messages Apr 27, 2004 – Apr 30, 2004 Locking primitives

Share & tools

Keith Owens

author

2004-04-27 02:01 UTC #0 lore raw

PATCH 1/1 2.6.6-rc2 Allow architectures to reenable interrupts on contended spinlocks

Enable interrupts while waiting for a disabled spinlock, but only if interrupts were enabled before issuing spin_lock_irqsave(). It makes a measurable difference to interrupt servicing on large systems.

This patch consists of an ia64 specific change (David Mosberger is happy with it) and an architecture independent change. The latter has no effect unless the architecture implements this feature and defines __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS. IOW, this change has no effect on anything except ia64, unless the other architecture maintainers want to implement this feature for their architecture.

arch/ia64/kernel/head.S +19 -5

Index: 2.6.6-rc2/arch/ia64/kernel/head.S
===================================================================
@@ -866,12 +866,14 @@
 	 * Inputs:
 	 *   ar.pfs - saved CFM of caller
 	 *   ar.ccv - 0 (and available for use)
+	 *   r27    - flags from spin_lock_irqsave or 0.  Must be preserved.
 	 *   r28    - available for use.
 	 *   r29    - available for use.
 	 *   r30    - available for use.
 	 *   r31    - address of lock, available for use.
 	 *   b6     - return address
 	 *   p14    - available for use.
+	 *   p15    - used to track flag status.
 	 *
 	 * If you patch this code to use more registers, do not forget to update
 	 * the clobber lists for spin_lock() in include/asm-ia64/spinlock.h.

@@ -885,22 +887,27 @@
 	.save rp, r28
 	.body
 	nop 0
-	nop 0
+	tbit.nz p15,p0=r27,IA64_PSR_I_BIT
 	.restore sp		// pop existing prologue after next insn
 	mov b6 = r28
 	.prologue
 	.save ar.pfs, r0
 	.altrp b6
 	.body
+	;;
+(p15)	ssm psr.i		// reenable interrupts if they were on
+	;;
+(p15)	srlz.d
 .wait:
 	// exponential backoff, kdb, lockmeter etc. go in here
 	hint @pause
 	ld4 r30=[r31]		// don't use ld4.bias; if it's contended, we won't write the word
 	nop 0
 	;;
-	cmp4.eq p14,p0=r30,r0
-(p14)	br.cond.sptk.few b6	// lock is now free, try to acquire
-	br.cond.sptk.few .wait
+	cmp4.ne p14,p0=r30,r0
+(p14)	br.cond.sptk.few .wait
+(p15)	rsm psr.i		// disable interrupts if we reenabled them
+	br.cond.sptk.few b6	// lock is now free, try to acquire
 END(ia64_spinlock_contention_pre3_4)
 
 #else

@@ -909,14 +916,21 @@
 	.prologue
 	.altrp b6
 	.body
+	tbit.nz p15,p0=r27,IA64_PSR_I_BIT
+	;;
 .wait:
+(p15)	ssm psr.i		// reenable interrupts if they were on
+	;;
+(p15)	srlz.d
+.wait2:
 	// exponential backoff, kdb, lockmeter etc. go in here
 	hint @pause
 	ld4 r30=[r31]		// don't use ld4.bias; if it's contended, we won't write the word
 	;;
 	cmp4.ne p14,p0=r30,r0
 	mov r30 = 1
-(p14)	br.cond.sptk.few .wait
+(p14)	br.cond.sptk.few .wait2
+(p15)	rsm psr.i		// disable interrupts if we reenabled them
 	;;
 	cmpxchg4.acq r30=[r31], r30, ar.ccv
 	;;

include/asm-ia64/spinlock.h +15 -9

Index: 2.6.6-rc2/include/asm-ia64/spinlock.h
===================================================================
@@ -32,10 +32,12 @@
  * carefully coded to touch only those registers that spin_lock() marks "clobbered".
  */
 
-#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "r28", "r29", "r30", "b6", "memory"
+#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "p15", "r27", "r28", "r29", "r30", "b6", "memory"
+
+#define __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS
 
 static inline void
-_raw_spin_lock (spinlock_t *lock)
+_raw_spin_lock_flags (spinlock_t *lock, unsigned long flags)
 {
 	register volatile unsigned int *ptr asm ("r31") = &lock->lock;

@@ -50,9 +52,10 @@
 		      "cmpxchg4.acq r30 = [%1], r30, ar.ccv\n\t"
 		      "movl r29 = ia64_spinlock_contention_pre3_4;;\n\t"
 		      "cmp4.ne p14, p0 = r30, r0\n\t"
-		      "mov b6 = r29;;\n"
+		      "mov b6 = r29;;\n\t"
+		      "mov r27=%2\n\t"
 		      "(p14) br.cond.spnt.many b6"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # else
 	asm volatile ("{\n\t"
 		      "  mov ar.ccv = r0\n\t"

@@ -60,29 +63,32 @@
 		      "  mov r30 = 1;;\n\t"
 		      "}\n\t"
 		      "cmpxchg4.acq r30 = [%1], r30, ar.ccv;;\n\t"
-		      "cmp4.ne p14, p0 = r30, r0\n"
+		      "cmp4.ne p14, p0 = r30, r0\n\t"
+		      "mov r27=%2\n\t"
 		      "(p14) brl.cond.spnt.many ia64_spinlock_contention_pre3_4;;"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # endif /* CONFIG_MCKINLEY */
 #else
 # ifdef CONFIG_ITANIUM
 	/* don't use brl on Itanium... */
 	/* mis-declare, so we get the entry-point, not it's function descriptor: */
 	asm volatile ("mov r30 = 1\n\t"
+		      "mov r27=%2\n\t"
 		      "mov ar.ccv = r0;;\n\t"
 		      "cmpxchg4.acq r30 = [%0], r30, ar.ccv\n\t"
 		      "movl r29 = ia64_spinlock_contention;;\n\t"
 		      "cmp4.ne p14, p0 = r30, r0\n\t"
-		      "mov b6 = r29;;\n"
+		      "mov b6 = r29;;\n\t"
 		      "(p14) br.call.spnt.many b6 = b6"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # else
 	asm volatile ("mov r30 = 1\n\t"
+		      "mov r27=%2\n\t"
 		      "mov ar.ccv = r0;;\n\t"
 		      "cmpxchg4.acq r30 = [%0], r30, ar.ccv;;\n\t"
 		      "cmp4.ne p14, p0 = r30, r0\n\t"
 		      "(p14) brl.call.spnt.many b6=ia64_spinlock_contention;;"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # endif /* CONFIG_MCKINLEY */
 #endif
 }

include/linux/spinlock.h +7 -1

Index: 2.6.6-rc2/include/linux/spinlock.h
===================================================================
@@ -184,6 +184,12 @@
 
 #endif /* !SMP */
 
+#ifdef __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
+#else
+#define _raw_spin_lock_flags(lock, flags) do { (void)flags; _raw_spin_lock(lock); } while(0)
+#endif
+
 /*
  * Define the various spin_lock and rw_lock methods.  Note we define these
  * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various

@@ -257,7 +263,7 @@
 do { \
 	local_irq_save(flags); \
 	preempt_disable(); \
-	_raw_spin_lock(lock); \
+	_raw_spin_lock_flags(lock, flags); \
 } while (0)
 
 #define spin_lock_irq(lock) \

Andrew Morton

2004-04-27 04:50 UTC #1 lore raw

Quoted 4 lines · Keith Owens

Enable interrupts while waiting for a disabled spinlock, but only if interrupts were enabled before issuing spin_lock_irqsave(). It makes a measurable difference to interrupt servicing on large systems.

Do you know which are the offending locks?

How much difference, and how large are the systems?

Thanks.

Keith Owens

author

2004-04-27 05:20 UTC #2 lore raw

Quoted 7 lines · Andrew Morton

Quoted 4 lines · Keith Owens
Enable interrupts while waiting for a disabled spinlock, but only if interrupts were enabled before issuing spin_lock_irqsave(). It makes a measurable difference to interrupt servicing on large systems.
Do you know which are the offending locks?

Workload dependent. We already service interrupts while waiting for a non-disabled spinlock. The patch allows a cpu to do some useful work and service interrupts while waiting for disabled spinlocks as well.

Quoted 3 lines

How much difference, and how large are the systems?
From memory (September 2003) 3-5% improvement on an AIM7 run, with 64

processors.

Benjamin Herrenschmidt

2004-04-27 06:59 UTC #3 lore raw

In reply to Keith Owens

include/linux/spinlock.h line 191

Quoted 6 lines

include/linux/spinlock.h +5 -0

+#ifdef __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
+#else
+#define _raw_spin_lock_flags(lock, flags) do { (void)flags; _raw_spin_lock(lock); } while(0)
+#endif

Looks good, except as paulus noted that using 0 for flags in the _raw_spin_lock() case is wrong, since 0 is a valid flags value for some archs that could mean anything...

Ben.

Keith Owens

author

2004-04-27 07:46 UTC #4 lore raw

Quoted 10 lines · Benjamin Herrenschmidt

include/linux/spinlock.h line 191
Quoted 6 lines
include/linux/spinlock.h +5 -0
+#ifdef __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
+#else
+#define _raw_spin_lock_flags(lock, flags) do { (void)flags; _raw_spin_lock(lock); } while(0)
+#endif
Looks good, except as paulus noted that using 0 for flags in the _raw_spin_lock() case is wrong, since 0 is a valid flags value for some archs that could mean anything...

0 is valid for ia64, which is the only architecture that currently defines __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS. If other architectures want to define __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS and they need a different flag value to indicate 'no flags available' then the 0 can be changed to an arch defined value. Worry about that if it ever occurs.

Paul Mackerras

2004-04-27 11:36 UTC #5 lore raw

Quoted 11 lines · Keith Owens

Quoted 3 lines · Benjamin Herrenschmidt
Looks good, except as paulus noted that using 0 for flags in the _raw_spin_lock() case is wrong, since 0 is a valid flags value for some archs that could mean anything...
0 is valid for ia64, which is the only architecture that currently defines __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS. If other architectures want to define __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS and they need a different flag value to indicate 'no flags available' then the 0 can be changed to an arch defined value. Worry about that if it ever occurs.

I was just thinking yesterday that it would be good to reenable interrupts during spin_lock_irq on ppc64. I am hacking on the spinlocks for ppc64 at the moment and this looks like something worth adding.

Why not keep _raw_spin_lock as it is and only use _raw_spin_lock_flags in the spin_lock_irq{,save} case?

Paul.

Keith Owens

author

2004-04-27 12:05 UTC #6 lore raw

Quoted 7 lines · Paul Mackerras

I was just thinking yesterday that it would be good to reenable interrupts during spin_lock_irq on ppc64. I am hacking on the spinlocks for ppc64 at the moment and this looks like something worth adding.
Why not keep _raw_spin_lock as it is and only use _raw_spin_lock_flags in the spin_lock_irq{,save} case?

Using both _raw_spin_lock and _raw_spin_lock_flags doubles the amount of code to maintain.

Architectures like ia64 that use a common out of line contention path need a common interface for all spinlocks, for the cases with and without irq{,save}.

You can test __builtin_constant_p(flags) and fold out the case of the flags being a known 'no flags available' value to get separate contended paths if that is really what you want. But for architectures that want a common path for all contended locks, you have to start with common code.

Christoph Hellwig

2004-04-27 12:33 UTC #7 lore raw

Quoted 7 lines · Keith Owens

Quoted 5 lines · Paul Mackerras
I was just thinking yesterday that it would be good to reenable interrupts during spin_lock_irq on ppc64. I am hacking on the spinlocks for ppc64 at the moment and this looks like something worth adding.

Quoted 5 lines

Quoted 2 lines
Why not keep _raw_spin_lock as it is and only use _raw_spin_lock_flags in the spin_lock_irq{,save} case?
Using both _raw_spin_lock and _raw_spin_lock_flags doubles the amount of code to maintain.

So define _raw_spin_lock to _raw_spin_lock_flags(lock, 0) in ia64 code?

Linus Torvalds

2004-04-27 16:01 UTC #8 lore raw

In reply to Keith Owens

Quoted 7 lines · Keith Owens

This patch consists of an ia64 specific change (David Mosberger is happy with it) and an architecture independent change. The latter has no effect unless the architecture implements this feature and defines __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS. IOW, this change has no effect on anything except ia64, unless the other architecture maintainers want to implement this feature for their architecture.

Aargh. Ugly ugly. Can you instead _first_ do all the infrastructure, and just add the unused "flags" argument to all architectures, ie take this part of the patch:

Quoted 18 lines

Index: 2.6.6-rc2/include/linux/spinlock.h
===================================================================

include/linux/spinlock.h +6 -0

@@ -184,6 +184,12 @@
 
 #endif /* !SMP */
 
+#ifdef __HAVE_ARCH_RAW_SPIN_LOCK_FLAGS
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
+#else
+#define _raw_spin_lock_flags(lock, flags) do { (void)flags; _raw_spin_lock(lock); } while(0)
+#endif
+
 /*
  * Define the various spin_lock and rw_lock methods.  Note we define these
  * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various

include/linux/spinlock.h line 269

Quoted 8 lines

include/linux/spinlock.h +1 -1

@@ -257,7 +263,7 @@
 do { \
 	local_irq_save(flags); \
 	preempt_disable(); \
-	_raw_spin_lock(lock); \
+	_raw_spin_lock_flags(lock, flags); \
 } while (0)
 
 #define spin_lock_irq(lock) \

And remove the need for "__HAVE_ARCH_RAW_SPIN_LOCK_FLAGS", and instead create a patch where _all_ architectures have that

	_raw_spin_lock_flags()

define, it's just that they ignore the "flags" value.

I think the patch makes sense, but I'd rather make the infrastructure clean, than have another silly architecture flag.

Linus

Keith Owens

author

2004-04-28 03:30 UTC #9 lore raw

PATCH 1/1 2.6.6-rc3 Allow architectures to reenable interrupts on contended spinlocks

Quoted 2 lines · Linus Torvalds

Aargh. Ugly ugly. Can you instead _first_ do all the infrastructure, and just add the unused "flags" argument to all architectures ...

As requested by Linus, update all architectures to add the common infrastructure. Tested on ia64 and i386.

Enable interrupts while waiting for a disabled spinlock, but only if interrupts were enabled before issuing spin_lock_irqsave().

This patch consists of three sections :-

* An architecture independent change to call _raw_spin_lock_flags() instead of _raw_spin_lock() when the flags are available.

* An ia64 specific change to implement _raw_spin_lock_flags() and to define _raw_spin_lock(lock) as _raw_spin_lock_flags(lock, 0) for the ASM_SUPPORTED case.

* Patches for all other architectures and for ia64 with !ASM_SUPPORTED to map _raw_spin_lock_flags(lock, flags) to _raw_spin_lock(lock). Architecture maintainers can define _raw_spin_lock_flags() to do something useful if they want to enable interrupts while waiting for a disabled spinlock.

include/linux/spinlock.h +1 -1

Index: 2.6.6-rc3/include/linux/spinlock.h
===================================================================
@@ -257,7 +257,7 @@
 do { \
 	local_irq_save(flags); \
 	preempt_disable(); \
-	_raw_spin_lock(lock); \
+	_raw_spin_lock_flags(lock, flags); \
 } while (0)
 
 #define spin_lock_irq(lock) \

include/asm-ia64/spinlock.h +15 -9

Index: 2.6.6-rc3/include/asm-ia64/spinlock.h
===================================================================
@@ -32,10 +32,10 @@
  * carefully coded to touch only those registers that spin_lock() marks "clobbered".
  */
 
-#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "r28", "r29", "r30", "b6", "memory"
+#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "p15", "r27", "r28", "r29", "r30", "b6", "memory"
 
 static inline void
-_raw_spin_lock (spinlock_t *lock)
+_raw_spin_lock_flags (spinlock_t *lock, unsigned long flags)
 {
 	register volatile unsigned int *ptr asm ("r31") = &lock->lock;

@@ -50,9 +50,10 @@
 		      "cmpxchg4.acq r30 = [%1], r30, ar.ccv\n\t"
 		      "movl r29 = ia64_spinlock_contention_pre3_4;;\n\t"
 		      "cmp4.ne p14, p0 = r30, r0\n\t"
-		      "mov b6 = r29;;\n"
+		      "mov b6 = r29;;\n\t"
+		      "mov r27=%2\n\t"
 		      "(p14) br.cond.spnt.many b6"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # else
 	asm volatile ("{\n\t"
 		      "  mov ar.ccv = r0\n\t"

@@ -60,33 +61,38 @@
 		      "  mov r30 = 1;;\n\t"
 		      "}\n\t"
 		      "cmpxchg4.acq r30 = [%1], r30, ar.ccv;;\n\t"
-		      "cmp4.ne p14, p0 = r30, r0\n"
+		      "cmp4.ne p14, p0 = r30, r0\n\t"
+		      "mov r27=%2\n\t"
 		      "(p14) brl.cond.spnt.many ia64_spinlock_contention_pre3_4;;"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # endif /* CONFIG_MCKINLEY */
 #else
 # ifdef CONFIG_ITANIUM
 	/* don't use brl on Itanium... */
 	/* mis-declare, so we get the entry-point, not it's function descriptor: */
 	asm volatile ("mov r30 = 1\n\t"
+		      "mov r27=%2\n\t"
 		      "mov ar.ccv = r0;;\n\t"
 		      "cmpxchg4.acq r30 = [%0], r30, ar.ccv\n\t"
 		      "movl r29 = ia64_spinlock_contention;;\n\t"
 		      "cmp4.ne p14, p0 = r30, r0\n\t"
-		      "mov b6 = r29;;\n"
+		      "mov b6 = r29;;\n\t"
 		      "(p14) br.call.spnt.many b6 = b6"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # else
 	asm volatile ("mov r30 = 1\n\t"
+		      "mov r27=%2\n\t"
 		      "mov ar.ccv = r0;;\n\t"
 		      "cmpxchg4.acq r30 = [%0], r30, ar.ccv;;\n\t"
 		      "cmp4.ne p14, p0 = r30, r0\n\t"
 		      "(p14) brl.call.spnt.many b6=ia64_spinlock_contention;;"
-		      : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+		      : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
 # endif /* CONFIG_MCKINLEY */
 #endif
 }
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
 #else /* !ASM_SUPPORTED */
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 # define _raw_spin_lock(x)								\
 do {											\
 	__u32 *ia64_spinlock_ptr = (__u32 *) (x);					\

arch/ia64/kernel/head.S +17 -5

Index: 2.6.6-rc3/arch/ia64/kernel/head.S
===================================================================
@@ -866,12 +866,14 @@
 	 * Inputs:
 	 *   ar.pfs - saved CFM of caller
 	 *   ar.ccv - 0 (and available for use)
+	 *   r27    - flags from spin_lock_irqsave or 0.  Must be preserved.
 	 *   r28    - available for use.
 	 *   r29    - available for use.
 	 *   r30    - available for use.
 	 *   r31    - address of lock, available for use.
 	 *   b6     - return address
 	 *   p14    - available for use.
+	 *   p15    - used to track flag status.
 	 *
 	 * If you patch this code to use more registers, do not forget to update
 	 * the clobber lists for spin_lock() in include/asm-ia64/spinlock.h.

@@ -885,22 +887,26 @@
 	.save rp, r28
 	.body
 	nop 0
-	nop 0
+	tbit.nz p15,p0=r27,IA64_PSR_I_BIT
 	.restore sp		// pop existing prologue after next insn
 	mov b6 = r28
 	.prologue
 	.save ar.pfs, r0
 	.altrp b6
 	.body
+	;;
+(p15)	ssm psr.i		// reenable interrupts if they were on
+				// DavidM says that srlz.d is slow and is not required in this case
 .wait:
 	// exponential backoff, kdb, lockmeter etc. go in here
 	hint @pause
 	ld4 r30=[r31]		// don't use ld4.bias; if it's contended, we won't write the word
 	nop 0
 	;;
-	cmp4.eq p14,p0=r30,r0
-(p14)	br.cond.sptk.few b6	// lock is now free, try to acquire
-	br.cond.sptk.few .wait
+	cmp4.ne p14,p0=r30,r0
+(p14)	br.cond.sptk.few .wait
+(p15)	rsm psr.i		// disable interrupts if we reenabled them
+	br.cond.sptk.few b6	// lock is now free, try to acquire
 END(ia64_spinlock_contention_pre3_4)
 
 #else

@@ -909,14 +915,20 @@
 	.prologue
 	.altrp b6
 	.body
+	tbit.nz p15,p0=r27,IA64_PSR_I_BIT
+	;;
 .wait:
+(p15)	ssm psr.i		// reenable interrupts if they were on
+				// DavidM says that srlz.d is slow and is not required in this case
+.wait2:
 	// exponential backoff, kdb, lockmeter etc. go in here
 	hint @pause
 	ld4 r30=[r31]		// don't use ld4.bias; if it's contended, we won't write the word
 	;;
 	cmp4.ne p14,p0=r30,r0
 	mov r30 = 1
-(p14)	br.cond.sptk.few .wait
+(p14)	br.cond.sptk.few .wait2
+(p15)	rsm psr.i		// disable interrupts if we reenabled them
 	;;
 	cmpxchg4.acq r30=[r31], r30, ar.ccv
 	;;

include/asm-alpha/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-alpha/spinlock.h
===================================================================
@@ -36,6 +36,7 @@
 
 #define spin_is_locked(x)	((x)->lock != 0)
 #define spin_unlock_wait(x)	({ do { barrier(); } while ((x)->lock); })
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 #ifdef CONFIG_DEBUG_SPINLOCK
 extern void _raw_spin_unlock(spinlock_t * lock);

include/asm-arm/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-arm/spinlock.h
===================================================================
@@ -24,6 +24,7 @@
 #define spin_lock_init(x)	do { *(x) = SPIN_LOCK_UNLOCKED; } while (0)
 #define spin_is_locked(x)	((x)->lock != 0)
 #define spin_unlock_wait(x)	do { barrier(); } while (spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 static inline void _raw_spin_lock(spinlock_t *lock)
 {

include/asm-i386/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-i386/spinlock.h
===================================================================
@@ -42,6 +42,7 @@
 
 #define spin_is_locked(x)	(*(volatile signed char *)(&(x)->lock) <= 0)
 #define spin_unlock_wait(x)	do { barrier(); } while(spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 #define spin_lock_string \
 	"\n1:\t" \

include/asm-mips/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-mips/spinlock.h
===================================================================
@@ -23,6 +23,7 @@
 
 #define spin_is_locked(x)	((x)->lock != 0)
 #define spin_unlock_wait(x)	do { barrier(); } while ((x)->lock)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 /*
  * Simple spin lock operations.  There are two variants, one clears IRQ's

include/asm-parisc/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-parisc/spinlock.h
===================================================================
@@ -15,6 +15,7 @@
 #define spin_is_locked(x) ((x)->lock == 0)
 
 #define spin_unlock_wait(x)	do { barrier(); } while(((volatile spinlock_t *)(x))->lock == 0)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 #if 1
 #define _raw_spin_lock(x) do { \

include/asm-ppc/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-ppc/spinlock.h
===================================================================
@@ -27,6 +27,7 @@
 #define spin_lock_init(x) 	do { *(x) = SPIN_LOCK_UNLOCKED; } while(0)
 #define spin_is_locked(x)	((x)->lock != 0)
 #define spin_unlock_wait(x)	do { barrier(); } while(spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 #ifndef CONFIG_DEBUG_SPINLOCK

include/asm-ppc64/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-ppc64/spinlock.h
===================================================================
@@ -22,6 +22,7 @@
 #define SPIN_LOCK_UNLOCKED	(spinlock_t) { 0 }
 
 #define spin_is_locked(x)	((x)->lock != 0)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 static __inline__ int _raw_spin_trylock(spinlock_t *lock)
 {

include/asm-s390/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-s390/spinlock.h
===================================================================
@@ -42,6 +42,7 @@
 #define spin_lock_init(lp) do { (lp)->lock = 0; } while(0)
 #define spin_unlock_wait(lp)	do { barrier(); } while(((volatile spinlock_t *)(lp))->lock)
 #define spin_is_locked(x) ((x)->lock != 0)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 extern inline void _raw_spin_lock(spinlock_t *lp)
 {

include/asm-sh/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-sh/spinlock.h
===================================================================
@@ -25,6 +25,7 @@
 
 #define spin_is_locked(x)	((x)->lock != 0)
 #define spin_unlock_wait(x)	do { barrier(); } while (spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 /*
  * Simple spin lock operations.  There are two variants, one clears IRQ's

include/asm-sparc/spinlock.h +2 -0

Index: 2.6.6-rc3/include/asm-sparc/spinlock.h
===================================================================
@@ -216,6 +216,8 @@
 
 #endif /* CONFIG_DEBUG_SPINLOCK */
 
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
+
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* __SPARC_SPINLOCK_H */

include/asm-sparc64/spinlock.h +2 -0

Index: 2.6.6-rc3/include/asm-sparc64/spinlock.h
===================================================================
@@ -106,6 +106,8 @@
 
 #endif /* CONFIG_DEBUG_SPINLOCK */
 
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
+
 /* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */
 
 #ifndef CONFIG_DEBUG_SPINLOCK

include/asm-x86_64/spinlock.h +1 -0

Index: 2.6.6-rc3/include/asm-x86_64/spinlock.h
===================================================================
@@ -41,6 +41,7 @@
 
 #define spin_is_locked(x)	(*(volatile signed char *)(&(x)->lock) <= 0)
 #define spin_unlock_wait(x)	do { barrier(); } while(spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 #define spin_lock_string \
 	"\n1:\t" \

Andrew Morton

2004-04-30 07:08 UTC #10 lore raw

Re: [patch] 2.6.6-rc3 Allow architectures to reenable interrupts on contended spinlocks

Quoted 7 lines · Keith Owens

As requested by Linus, update all architectures to add the common infrastructure. Tested on ia64 and i386.
Enable interrupts while waiting for a disabled spinlock, but only if interrupts were enabled before issuing spin_lock_irqsave().

Please test stuff with CONFIG_SMP=n?

include/linux/sched.h: In function `dequeue_signal_lock':
include/linux/sched.h:795: warning: implicit declaration of function `_raw_spin_lock_flags'

include/linux/spinlock.h renamed from include/linux/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks-fix +2 -0

@@ -46,6 +46,8 @@
 
 #else
 
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
+
 #if !defined(CONFIG_PREEMPT) && !defined(CONFIG_DEBUG_SPINLOCK)
 # define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic)
 # define ATOMIC_DEC_AND_LOCK

The feature is only enabled for spin_lock_irqsave(). Should it not also do something about spin_lock_irq()?