Skip to contentSkip to conversation

lock bitops and some bitops fixes

PATCH New Nick Piggin 1 participant 9 messages Aug 28, 2007

Subsystems (5)
Documentation Generic include/asm header files Atomic infrastructure Bitops API +1 more subsystems
lore
Share & tools
lore

Nick Piggin

author
#0 lore raw

cover lock bitops and some bitops fixes

[Sent again with the correct date stamps, sorry for the noise :(]

Here is a new set of bitops patches that can hopefully get into the next merge window.

I'm not sending the intrusive part of the patchset (the page lock and buffer lock conversions), so that you'll hopefully apply these bits and get them some early testing. I've just converted bit_spinlock over to use the lock bitops, in order to get a bit of test coverage.

I've spotted some potential bugs in alpha and mips bitops functions, but would like acks from maintainers for _those_ specific patches.

Thanks,
Nick

author
#1 lore raw

PATCH 1/8 bitops: introduce lock ops

Introduce test_and_set_bit_lock / clear_bit_unlock bitops with lock semantics. Convert all architectures to use the generic implementation.

Signed-off-by Nick Piggin <[email protected]>
Acked-by David Howells <[email protected]>
---
 Documentation/atomic_ops.txt      |   14 +++++++++++
 Documentation/memory-barriers.txt |   14 ++++++++++-
 include/asm-alpha/bitops.h        |    1 
 include/asm-arm/bitops.h          |    1 
 include/asm-avr32/bitops.h        |    1 
 include/asm-blackfin/bitops.h     |    1 
 include/asm-cris/bitops.h         |    1 
 include/asm-frv/bitops.h          |    1 
 include/asm-generic/bitops.h      |    1 
 include/asm-generic/bitops/lock.h |   45 ++++++++++++++++++++++++++++++++++++++
 include/asm-h8300/bitops.h        |    1 
 include/asm-i386/bitops.h         |    1 
 include/asm-ia64/bitops.h         |    2 +
 include/asm-m32r/bitops.h         |    1 
 include/asm-m68k/bitops.h         |    1 
 include/asm-m68knommu/bitops.h    |    1 
 include/asm-mips/bitops.h         |    1 
 include/asm-parisc/bitops.h       |    1 
 include/asm-powerpc/bitops.h      |    1 
 include/asm-s390/bitops.h         |    1 
 include/asm-sh/bitops.h           |    1 
 include/asm-sh64/bitops.h         |    1 
 include/asm-sparc/bitops.h        |    1 
 include/asm-sparc64/bitops.h      |    1 
 include/asm-v850/bitops.h         |    1 
 include/asm-x86_64/bitops.h       |    1 
 include/asm-xtensa/bitops.h       |    1 
 27 files changed, 96 insertions(+), 2 deletions(-)
include/asm-alpha/bitops.h +1 -0
Index: linux-2.6/include/asm-alpha/bitops.h
===================================================================
@@ -367,6 +367,7 @@ static inline unsigned int hweight8(unsi
 #else
 #include <asm-generic/bitops/hweight.h>
 #endif
+#include <asm-generic/bitops/lock.h>
 
 #endif /* __KERNEL__ */
include/asm-arm/bitops.h +1 -0
Index: linux-2.6/include/asm-arm/bitops.h
===================================================================
@@ -286,6 +286,7 @@ static inline int constant_fls(int x)
 
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 /*
  * Ext2 is defined to use little-endian byte ordering.
include/asm-avr32/bitops.h +1 -0
Index: linux-2.6/include/asm-avr32/bitops.h
===================================================================
@@ -288,6 +288,7 @@ static inline int ffs(unsigned long word
 #include <asm-generic/bitops/fls64.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
include/asm-cris/bitops.h +1 -0
Index: linux-2.6/include/asm-cris/bitops.h
===================================================================
@@ -154,6 +154,7 @@ static inline int test_and_change_bit(in
 #include <asm-generic/bitops/fls64.h>
 #include <asm-generic/bitops/hweight.h>
 #include <asm-generic/bitops/find.h>
+#include <asm-generic/bitops/lock.h>
 
 #include <asm-generic/bitops/ext2-non-atomic.h>
include/asm-frv/bitops.h +1 -0
Index: linux-2.6/include/asm-frv/bitops.h
===================================================================
@@ -302,6 +302,7 @@ int __ilog2_u64(u64 n)
 
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #include <asm-generic/bitops/ext2-non-atomic.h>
include/asm-generic/bitops.h +1 -0
Index: linux-2.6/include/asm-generic/bitops.h
===================================================================
@@ -22,6 +22,7 @@
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/ffs.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
include/asm-generic/bitops/lock.h new file +45 -0

45 lines not on this page · raw message

include/asm-h8300/bitops.h +1 -0
Index: linux-2.6/include/asm-h8300/bitops.h
===================================================================
@@ -194,6 +194,7 @@ static __inline__ unsigned long __ffs(un
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
 #include <asm-generic/bitops/minix.h>
include/asm-i386/bitops.h +1 -0
Index: linux-2.6/include/asm-i386/bitops.h
===================================================================
@@ -402,6 +402,7 @@ static inline int fls(int x)
 }
 
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #endif /* __KERNEL__ */
include/asm-m32r/bitops.h +1 -0
Index: linux-2.6/include/asm-m32r/bitops.h
===================================================================
@@ -255,6 +255,7 @@ static __inline__ int test_and_change_bi
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/ffs.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #endif /* __KERNEL__ */
include/asm-m68k/bitops.h +1 -0
Index: linux-2.6/include/asm-m68k/bitops.h
===================================================================
@@ -314,6 +314,7 @@ static inline int fls(int x)
 #include <asm-generic/bitops/fls64.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 /* Bitmap functions for the minix filesystem */
include/asm-m68knommu/bitops.h +1 -0
Index: linux-2.6/include/asm-m68knommu/bitops.h
===================================================================
@@ -160,6 +160,7 @@ static __inline__ int __test_bit(int nr,
 
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 static __inline__ int ext2_set_bit(int nr, volatile void * addr)
 {
include/asm-mips/bitops.h +1 -0
Index: linux-2.6/include/asm-mips/bitops.h
===================================================================
@@ -556,6 +556,7 @@ static inline int ffs(int word)
 
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
 #include <asm-generic/bitops/minix.h>
include/asm-parisc/bitops.h +1 -0
Index: linux-2.6/include/asm-parisc/bitops.h
===================================================================
@@ -208,6 +208,7 @@ static __inline__ int fls(int x)
 
 #include <asm-generic/bitops/fls64.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/sched.h>
 
 #endif /* __KERNEL__ */
include/asm-s390/bitops.h +1 -0
Index: linux-2.6/include/asm-s390/bitops.h
===================================================================
@@ -746,6 +746,7 @@ static inline int sched_find_first_bit(u
 #include <asm-generic/bitops/fls64.h>
 
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 /*
  * ATTENTION: intel byte ordering convention for ext2 and minix !!
include/asm-sh/bitops.h +1 -0
Index: linux-2.6/include/asm-sh/bitops.h
===================================================================
@@ -137,6 +137,7 @@ static inline unsigned long __ffs(unsign
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/ffs.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
include/asm-sh64/bitops.h +1 -0
Index: linux-2.6/include/asm-sh64/bitops.h
===================================================================
@@ -136,6 +136,7 @@ static __inline__ unsigned long ffz(unsi
 #include <asm-generic/bitops/__ffs.h>
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/ffs.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
include/asm-sparc/bitops.h +1 -0
Index: linux-2.6/include/asm-sparc/bitops.h
===================================================================
@@ -96,6 +96,7 @@ static inline void change_bit(unsigned l
 #include <asm-generic/bitops/fls.h>
 #include <asm-generic/bitops/fls64.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
include/asm-sparc64/bitops.h +1 -0
Index: linux-2.6/include/asm-sparc64/bitops.h
===================================================================
@@ -81,6 +81,7 @@ static inline unsigned int hweight8(unsi
 #include <asm-generic/bitops/hweight.h>
 
 #endif
+#include <asm-generic/bitops/lock.h>
 #endif /* __KERNEL__ */
 
 #include <asm-generic/bitops/find.h>
include/asm-v850/bitops.h +1 -0
Index: linux-2.6/include/asm-v850/bitops.h
===================================================================
@@ -145,6 +145,7 @@ static inline int __test_bit (int nr, co
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #define ext2_set_bit_atomic(l,n,a)      test_and_set_bit(n,a)
include/asm-xtensa/bitops.h +1 -0
Index: linux-2.6/include/asm-xtensa/bitops.h
===================================================================
@@ -108,6 +108,7 @@ static inline int fls (unsigned int x)
 #endif
 
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/minix.h>
Documentation/atomic_ops.txt +14 -0
Index: linux-2.6/Documentation/atomic_ops.txt
===================================================================
@@ -369,6 +369,20 @@ brothers:
 	 */
 	 smp_mb__after_clear_bit();
 
+There are two special bitops with lock barrier semantics (acquire/release,
+same as spinlocks). These operate in the same way as their non-_lock/unlock
+postfixed variants, except that they are to provide acquire/release semantics,
+respectively. This means they can be used for bit_spin_trylock and
+bit_spin_unlock type operations without specifying any more barriers.
+
+	int test_and_set_bit_lock(unsigned long nr, unsigned long *addr);
+	void clear_bit_unlock(unsigned long nr, unsigned long *addr);
+	void __clear_bit_unlock(unsigned long nr, unsigned long *addr);
+
+The __clear_bit_unlock version is non-atomic, however it still implements
+unlock barrier semantics. This can be useful if the lock itself is protecting
+the other bits in the word.
+
 Finally, there are non-atomic versions of the bitmask operations
 provided.  They are used in contexts where some other higher-level SMP
 locking scheme is being used to protect the bitmask, and thus less
Documentation/memory-barriers.txt +12 -2
Index: linux-2.6/Documentation/memory-barriers.txt
===================================================================
@@ -1479,7 +1479,8 @@ kernel.
 
 Any atomic operation that modifies some state in memory and returns information
 about the state (old or new) implies an SMP-conditional general memory barrier
-(smp_mb()) on each side of the actual operation.  These include:
+(smp_mb()) on each side of the actual operation (with the exception of
+explicit lock operations, described later).  These include:
 
 	xchg();
 	cmpxchg();

@@ -1536,10 +1537,19 @@ If they're used for constructing a lock 
 do need memory barriers as a lock primitive generally has to do things in a
 specific order.
 
-
 Basically, each usage case has to be carefully considered as to whether memory
 barriers are needed or not.
 
+The following operations are special locking primitives:
+
+	test_and_set_bit_lock();
+	clear_bit_unlock();
+	__clear_bit_unlock();
+
+These implement LOCK-class and UNLOCK-class operations. These should be used in
+preference to other operations when implementing locking primitives, because
+their implementations can be optimised on many architectures.
+
 [!] Note that special memory barrier primitives are available for these
 situations because on some CPUs the atomic instructions used imply full memory
 barriers, and so barrier instructions are superfluous in conjunction with them,
include/asm-blackfin/bitops.h +1 -0
Index: linux-2.6/include/asm-blackfin/bitops.h
===================================================================
@@ -199,6 +199,7 @@ static __inline__ int __test_bit(int nr,
 
 #include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #include <asm-generic/bitops/ext2-atomic.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
include/asm-x86_64/bitops.h +1 -0
Index: linux-2.6/include/asm-x86_64/bitops.h
===================================================================
@@ -408,6 +408,7 @@ static __inline__ int fls(int x)
 #define ARCH_HAS_FAST_MULTIPLIER 1
 
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #endif /* __KERNEL__ */
include/asm-ia64/bitops.h +2 -0
Index: linux-2.6/include/asm-ia64/bitops.h
===================================================================
@@ -371,6 +371,8 @@ hweight64 (unsigned long x)
 #define hweight16(x)	(unsigned int) hweight64((x) & 0xfffful)
 #define hweight8(x)	(unsigned int) hweight64((x) & 0xfful)
 
+#include <asm-generic/bitops/lock.h>
+
 #endif /* __KERNEL__ */
 
 #include <asm-generic/bitops/find.h>
include/asm-powerpc/bitops.h +1 -0
Index: linux-2.6/include/asm-powerpc/bitops.h
===================================================================
@@ -266,6 +266,7 @@ static __inline__ int fls(unsigned int x
 #include <asm-generic/bitops/fls64.h>
 
 #include <asm-generic/bitops/hweight.h>
+#include <asm-generic/bitops/lock.h>
 
 #define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0)
 unsigned long find_next_zero_bit(const unsigned long *addr,
author
#2 lore raw

PATCH 2/8 alpha: fix bitops

In reply to Nick Piggin

Documentation/atomic_ops.txt defines these primitives must contain a memory barrier both before and after their memory operation. This is consistent with the atomic ops implementation on alpha.

Signed-off-by Nick Piggin <[email protected]>
---
 include/asm-alpha/bitops.h |    9 +++++++++
 1 file changed, 9 insertions(+)
include/asm-alpha/bitops.h +9 -0
Index: linux-2.6/include/asm-alpha/bitops.h
===================================================================
@@ -117,6 +117,9 @@ test_and_set_bit(unsigned long nr, volat
 	int *m = ((int *) addr) + (nr >> 5);
 
 	__asm__ __volatile__(
+#ifdef CONFIG_SMP
+	"	mb\n"
+#endif
 	"1:	ldl_l %0,%4\n"
 	"	and %0,%3,%2\n"
 	"	bne %2,2f\n"

@@ -158,6 +161,9 @@ test_and_clear_bit(unsigned long nr, vol
 	int *m = ((int *) addr) + (nr >> 5);
 
 	__asm__ __volatile__(
+#ifdef CONFIG_SMP
+	"	mb\n"
+#endif
 	"1:	ldl_l %0,%4\n"
 	"	and %0,%3,%2\n"
 	"	beq %2,2f\n"

@@ -199,6 +205,9 @@ test_and_change_bit(unsigned long nr, vo
 	int *m = ((int *) addr) + (nr >> 5);
 
 	__asm__ __volatile__(
+#ifdef CONFIG_SMP
+	"	mb\n"
+#endif
 	"1:	ldl_l %0,%4\n"
 	"	and %0,%3,%2\n"
 	"	xor %0,%3,%0\n"
author
#3 lore raw

PATCH 3/8 alpha: lock bitops

In reply to Nick Piggin

Alpha can avoid one mb when acquiring a lock with test_and_set_bit_lock.

Signed-off-by Nick Piggin <[email protected]>
---
 include/asm-alpha/bitops.h |   42 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)
include/asm-alpha/bitops.h +41 -1
Index: linux-2.6/include/asm-alpha/bitops.h
===================================================================
@@ -69,6 +69,13 @@ clear_bit(unsigned long nr, volatile voi
 	:"Ir" (1UL << (nr & 31)), "m" (*m));
 }
 
+static inline void
+clear_bit_unlock(unsigned long nr, volatile void * addr)
+{
+	smp_mb();
+	clear_bit(nr, addr);
+}
+
 /*
  * WARNING: non atomic version.
  */

@@ -81,6 +88,13 @@ __clear_bit(unsigned long nr, volatile v
 }
 
 static inline void
+__clear_bit_unlock(unsigned long nr, volatile void * addr)
+{
+	smp_mb();
+	__clear_bit(nr, addr);
+}
+
+static inline void
 change_bit(unsigned long nr, volatile void * addr)
 {
 	unsigned long temp;

@@ -139,6 +153,33 @@ test_and_set_bit(unsigned long nr, volat
 	return oldbit != 0;
 }
 
+static inline int
+test_and_set_bit_lock(unsigned long nr, volatile void *addr)
+{
+	unsigned long oldbit;
+	unsigned long temp;
+	int *m = ((int *) addr) + (nr >> 5);
+
+	__asm__ __volatile__(
+	"1:	ldl_l %0,%4\n"
+	"	and %0,%3,%2\n"
+	"	bne %2,2f\n"
+	"	xor %0,%3,%0\n"
+	"	stl_c %0,%1\n"
+	"	beq %0,3f\n"
+	"2:\n"
+#ifdef CONFIG_SMP
+	"	mb\n"
+#endif
+	".subsection 2\n"
+	"3:	br 1b\n"
+	".previous"
+	:"=&r" (temp), "=m" (*m), "=&r" (oldbit)
+	:"Ir" (1UL << (nr & 31)), "m" (*m) : "memory");
+
+	return oldbit != 0;
+}
+
 /*
  * WARNING: non atomic version.
  */

@@ -376,7 +417,6 @@ static inline unsigned int hweight8(unsi
 #else
 #include <asm-generic/bitops/hweight.h>
 #endif
-#include <asm-generic/bitops/lock.h>
 
 #endif /* __KERNEL__ */
author
#4 lore raw

PATCH 4/8 ia64: lock bitops

In reply to Nick Piggin

Convert ia64 to new bitops.

Signed-off-by Nick Piggin <[email protected]>
---
 include/asm-ia64/bitops.h |   43 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)
include/asm-ia64/bitops.h +41 -2
Index: linux-2.6/include/asm-ia64/bitops.h
===================================================================
@@ -94,6 +94,38 @@ clear_bit (int nr, volatile void *addr)
 }
 
 /**
+ * clear_bit_unlock - Clears a bit in memory with release
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * clear_bit_unlock() is atomic and may not be reordered.  It does
+ * contain a memory barrier suitable for unlock type operations.
+ */
+static __inline__ void
+clear_bit_unlock (int nr, volatile void *addr)
+{
+	__u32 mask, old, new;
+	volatile __u32 *m;
+	CMPXCHG_BUGCHECK_DECL
+
+	m = (volatile __u32 *) addr + (nr >> 5);
+	mask = ~(1 << (nr & 31));
+	do {
+		CMPXCHG_BUGCHECK(m);
+		old = *m;
+		new = old & mask;
+	} while (cmpxchg_rel(m, old, new) != old);
+}
+
+/**
+ * __clear_bit_unlock - Non-atomically clear a bit with release
+ *
+ * This is like clear_bit_unlock, but the implementation may use a non-atomic
+ * store (this one uses an atomic, however).
+ */
+#define __clear_bit_unlock clear_bit_unlock
+
+/**
  * __clear_bit - Clears a bit in memory (non-atomic version)
  */
 static __inline__ void

@@ -170,6 +202,15 @@ test_and_set_bit (int nr, volatile void 
 }
 
 /**
+ * test_and_set_bit_lock - Set a bit and return its old value for lock
+ * @nr: Bit to set
+ * @addr: Address to count from
+ *
+ * This is the same as test_and_set_bit on ia64
+ */
+#define test_and_set_bit_lock test_and_set_bit
+
+/**
  * __test_and_set_bit - Set a bit and return its old value
  * @nr: Bit to set
  * @addr: Address to count from

@@ -371,8 +412,6 @@ hweight64 (unsigned long x)
 #define hweight16(x)	(unsigned int) hweight64((x) & 0xfffful)
 #define hweight8(x)	(unsigned int) hweight64((x) & 0xfful)
 
-#include <asm-generic/bitops/lock.h>
-
 #endif /* __KERNEL__ */
 
 #include <asm-generic/bitops/find.h>
author
#5 lore raw

PATCH 5/8 mips: fix bitops

In reply to Nick Piggin

Documentation/atomic_ops.txt defines these primitives must contain a memory barrier both before and after their memory operation. This is consistent with the atomic ops implementation on mips.

Signed-off-by Nick Piggin <[email protected]>
---
 include/asm-mips/bitops.h |    6 ++++++
 1 file changed, 6 insertions(+)
include/asm-mips/bitops.h +6 -0
Index: linux-2.6/include/asm-mips/bitops.h
===================================================================
@@ -240,6 +240,8 @@ static inline int test_and_set_bit(unsig
 	unsigned short bit = nr & SZLONG_MASK;
 	unsigned long res;
 
+	smp_llsc_mb();
+
 	if (cpu_has_llsc && R10000_LLSC_WAR) {
 		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 		unsigned long temp;

@@ -308,6 +310,8 @@ static inline int test_and_clear_bit(uns
 	unsigned short bit = nr & SZLONG_MASK;
 	unsigned long res;
 
+	smp_llsc_mb();
+
 	if (cpu_has_llsc && R10000_LLSC_WAR) {
 		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 		unsigned long temp;

@@ -396,6 +400,8 @@ static inline int test_and_change_bit(un
 	unsigned short bit = nr & SZLONG_MASK;
 	unsigned long res;
 
+	smp_llsc_mb();
+
 	if (cpu_has_llsc && R10000_LLSC_WAR) {
 		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 		unsigned long temp;
author
#6 lore raw

PATCH 6/8 mips: lock bitops

In reply to Nick Piggin

mips can avoid one mb when acquiring a lock with test_and_set_bit_lock.

Signed-off-by Nick Piggin <[email protected]>

--

---
 include/asm-mips/bitops.h |   97 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 96 insertions(+), 1 deletion(-)
include/asm-mips/bitops.h +96 -1
Index: linux-2.6/include/asm-mips/bitops.h
===================================================================
@@ -172,6 +172,20 @@ static inline void clear_bit(unsigned lo
 }
 
 /*
+ * clear_bit_unlock - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * clear_bit() is atomic and implies release semantics before the memory
+ * operation. It can be used for an unlock.
+ */
+static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
+{
+	smp_mb__before_clear_bit();
+	clear_bit(nr, addr);
+}
+
+/*
  * change_bit - Toggle a bit in memory
  * @nr: Bit to change
  * @addr: Address to start counting from

@@ -297,6 +311,73 @@ static inline int test_and_set_bit(unsig
 }
 
 /*
+ * test_and_set_bit_lock - Set a bit and return its old value
+ * @nr: Bit to set
+ * @addr: Address to count from
+ *
+ * This operation is atomic and implies acquire ordering semantics
+ * after the memory operation.
+ */
+static inline int test_and_set_bit_lock(unsigned long nr,
+	volatile unsigned long *addr)
+{
+	unsigned short bit = nr & SZLONG_MASK;
+	unsigned long res;
+
+	if (cpu_has_llsc && R10000_LLSC_WAR) {
+		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
+		unsigned long temp;
+
+		__asm__ __volatile__(
+		"	.set	mips3					\n"
+		"1:	" __LL "%0, %1		# test_and_set_bit	\n"
+		"	or	%2, %0, %3				\n"
+		"	" __SC	"%2, %1					\n"
+		"	beqzl	%2, 1b					\n"
+		"	and	%2, %0, %3				\n"
+		"	.set	mips0					\n"
+		: "=&r" (temp), "=m" (*m), "=&r" (res)
+		: "r" (1UL << bit), "m" (*m)
+		: "memory");
+	} else if (cpu_has_llsc) {
+		unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
+		unsigned long temp;
+
+		__asm__ __volatile__(
+		"	.set	push					\n"
+		"	.set	noreorder				\n"
+		"	.set	mips3					\n"
+		"1:	" __LL "%0, %1		# test_and_set_bit	\n"
+		"	or	%2, %0, %3				\n"
+		"	" __SC	"%2, %1					\n"
+		"	beqz	%2, 2f					\n"
+		"	 and	%2, %0, %3				\n"
+		"	.subsection 2					\n"
+		"2:	b	1b					\n"
+		"	 nop						\n"
+		"	.previous					\n"
+		"	.set	pop					\n"
+		: "=&r" (temp), "=m" (*m), "=&r" (res)
+		: "r" (1UL << bit), "m" (*m)
+		: "memory");
+	} else {
+		volatile unsigned long *a = addr;
+		unsigned long mask;
+		unsigned long flags;
+
+		a += nr >> SZLONG_LOG;
+		mask = 1UL << bit;
+		raw_local_irq_save(flags);
+		res = (mask & *a);
+		*a |= mask;
+		raw_local_irq_restore(flags);
+	}
+
+	smp_llsc_mb();
+
+	return res != 0;
+}
+/*
  * test_and_clear_bit - Clear a bit and return its old value
  * @nr: Bit to clear
  * @addr: Address to count from

@@ -459,6 +540,21 @@ static inline int test_and_change_bit(un
 #include <asm-generic/bitops/non-atomic.h>
 
 /*
+ * __clear_bit_unlock - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * __clear_bit() is non-atomic and implies release semantics before the memory
+ * operation. It can be used for an unlock if no other CPUs can concurrently
+ * modify other bits in the word.
+ */
+static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
+{
+	smp_mb();
+	__clear_bit(nr, addr);
+}
+
+/*
  * Return the bit position (0..63) of the most significant 1 bit in a word
  * Returns -1 if no 1 bit exists
  */

@@ -562,7 +658,6 @@ static inline int ffs(int word)
 
 #include <asm-generic/bitops/sched.h>
 #include <asm-generic/bitops/hweight.h>
-#include <asm-generic/bitops/lock.h>
 #include <asm-generic/bitops/ext2-non-atomic.h>
 #include <asm-generic/bitops/ext2-atomic.h>
 #include <asm-generic/bitops/minix.h>
author
#7 lore raw

PATCH 7/8 powerpc: lock bitops

In reply to Nick Piggin

Add non-trivial lock bitops implementation for powerpc.

Signed-off-by Nick Piggin <[email protected]>
Acked-by Benjamin Herrenschmidt <[email protected]>
---
 include/asm-powerpc/bitops.h |   46 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)
include/asm-powerpc/bitops.h +45 -1
Index: linux-2.6/include/asm-powerpc/bitops.h
===================================================================
@@ -86,6 +86,24 @@ static __inline__ void clear_bit(int nr,
 	: "cc" );
 }
 
+static __inline__ void clear_bit_unlock(int nr, volatile unsigned long *addr)
+{
+	unsigned long old;
+	unsigned long mask = BITOP_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+
+	__asm__ __volatile__(
+	LWSYNC_ON_SMP
+"1:"	PPC_LLARX "%0,0,%3	# clear_bit_unlock\n"
+	"andc	%0,%0,%2\n"
+	PPC405_ERR77(0,%3)
+	PPC_STLCX "%0,0,%3\n"
+	"bne-	1b"
+	: "=&r" (old), "+m" (*p)
+	: "r" (mask), "r" (p)
+	: "cc", "memory");
+}
+
 static __inline__ void change_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long old;

@@ -125,6 +143,27 @@ static __inline__ int test_and_set_bit(u
 	return (old & mask) != 0;
 }
 
+static __inline__ int test_and_set_bit_lock(unsigned long nr,
+				       volatile unsigned long *addr)
+{
+	unsigned long old, t;
+	unsigned long mask = BITOP_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+
+	__asm__ __volatile__(
+"1:"	PPC_LLARX "%0,0,%3		# test_and_set_bit_lock\n"
+	"or	%1,%0,%2 \n"
+	PPC405_ERR77(0,%3)
+	PPC_STLCX "%1,0,%3 \n"
+	"bne-	1b"
+	ISYNC_ON_SMP
+	: "=&r" (old), "=&r" (t)
+	: "r" (mask), "r" (p)
+	: "cc", "memory");
+
+	return (old & mask) != 0;
+}
+
 static __inline__ int test_and_clear_bit(unsigned long nr,
 					 volatile unsigned long *addr)
 {

@@ -185,6 +224,12 @@ static __inline__ void set_bits(unsigned
 
 #include <asm-generic/bitops/non-atomic.h>
 
+static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr)
+{
+	__asm__ __volatile__(LWSYNC_ON_SMP ::: "memory");
+	__clear_bit(nr, addr);
+}
+
 /*
  * Return the zero-based bit position (LE, not IBM bit numbering) of
  * the most significant 1-bit in a double word.

@@ -266,7 +311,6 @@ static __inline__ int fls(unsigned int x
 #include <asm-generic/bitops/fls64.h>
 
 #include <asm-generic/bitops/hweight.h>
-#include <asm-generic/bitops/lock.h>
 
 #define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0)
 unsigned long find_next_zero_bit(const unsigned long *addr,
author
#8 lore raw

PATCH 8/8 bit_spin_lock: use lock bitops

In reply to Nick Piggin

Convert bit_spin_lock to new locking bitops. Slub can use the non-atomic store version to clear (Christoph?)

Signed-off-by Nick Piggin <[email protected]>
---
 include/linux/bit_spinlock.h |   26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)
include/linux/bit_spinlock.h +22 -4
Index: linux-2.6/include/linux/bit_spinlock.h
===================================================================
@@ -18,7 +18,7 @@ static inline void bit_spin_lock(int bit
 	 */
 	preempt_disable();
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
-	while (test_and_set_bit(bitnum, addr)) {
+	while (unlikely(test_and_set_bit_lock(bitnum, addr))) {
 		while (test_bit(bitnum, addr)) {
 			preempt_enable();
 			cpu_relax();

@@ -36,7 +36,7 @@ static inline int bit_spin_trylock(int b
 {
 	preempt_disable();
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
-	if (test_and_set_bit(bitnum, addr)) {
+	if (unlikely(test_and_set_bit_lock(bitnum, addr))) {
 		preempt_enable();
 		return 0;
 	}

@@ -50,10 +50,28 @@ static inline int bit_spin_trylock(int b
  */
 static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
 {
+#ifdef CONFIG_DEBUG_SPINLOCK
+	BUG_ON(!test_bit(bitnum, addr));
+#endif
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+	clear_bit_unlock(bitnum, addr);
+#endif
+	preempt_enable();
+	__release(bitlock);
+}
+
+/*
+ *  bit-based spin_unlock()
+ *  non-atomic version, which can be used eg. if the bit lock itself is
+ *  protecting the rest of the flags in the word.
+ */
+static inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
+{
+#ifdef CONFIG_DEBUG_SPINLOCK
 	BUG_ON(!test_bit(bitnum, addr));
-	smp_mb__before_clear_bit();
-	clear_bit(bitnum, addr);
+#endif
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+	__clear_bit_unlock(bitnum, addr);
 #endif
 	preempt_enable();
 	__release(bitlock);