x86: lock bitops
PATCH 1/1 x86: lock bitops
I missed an obvious one!
x86 CPUs are defined not to reorder stores past earlier loads, so there is no hardware memory barrier required to implement a release-consistent store (all stores are, by definition).
So ditch the generic lock bitops, and implement optimised versions for x86, which removes the mfence from __clear_bit_unlock (which is already a useful primitive for SLUB).
Signed-off-by Nick Piggin <[email protected]>
---
include/asm-x86/bitops_32.h +42 -1
@@ -76,6 +76,20 @@ static inline void clear_bit(int nr, vol :"Ir" (nr)); } +/* +* clear_bit_unlock - Clears a bit in memory +* @nr: Bit to clear +* @addr: Address to start counting from +* +* clear_bit() is atomic and implies release semantics before the memory +* operation. It can be used for an unlock. +*/ +static inline void clear_bit_unlock (unsigned long nr,volatile unsigned long *addr) +{ +barrier (); +clear_bit (nr, addr); +} +static inline void __clear_bit (int nr,volatile unsigned long * addr) { __asm__ __volatile__(
@@ -83,6 +97,25 @@ static inline void __clear_bit(int nr, v :"+m" (ADDR) :"Ir" (nr)); } + +/* +* __clear_bit_unlock - Clears a bit in memory +* @nr: Bit to clear +* @addr: Address to start counting from +* +* __clear_bit() is non-atomic and implies release semantics before the memory +* operation. It can be used for an unlock if no other CPUs can concurrently +* modify other bits in the word. +* +* No memory barrier is required here, because x86 cannot reorder stores past +* older loads. Same principle as spin_unlock. +*/ +static inline void __clear_bit_unlock (unsigned long nr,volatile unsigned long *addr) +{ +barrier (); +__clear_bit (nr, addr); +} +#define smp_mb__before_clear_bit () barrier()#define smp_mb__after_clear_bit () barrier()
@@ -142,6 +175,15 @@ static inline int test_and_set_bit(int n }/** +* test_and_set_bit_lock - Set a bit and return its old value for lock +* @nr: Bit to set +* @addr: Address to count from +* +* This is the same as test_and_set_bit on x86 +*/ +#define test_and_set_bit_lock test_and_set_bit + +/** * __test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from
@@ -402,7 +444,6 @@ static inline int fls(int x) }#include <asm-generic/bitops/hweight.h> -#include <asm-generic/bitops/lock.h> #endif /* __KERNEL__ */
include/asm-x86/bitops_64.h +41 -1
66 lines not on this page · raw message