atomic.h : standardizing atomic primitives
Subsystems (2)
cover atomic.h : standardizing atomic primitives
atomic.h : standardizing atomic primitives
It mainly adds support for missing 64 bits cmpxchg and 64 bits atomic add unless. Therefore, principally 64 bits architectures are targeted by these patches. It also adds the complete list of atomic operations on the atomic_long type.
These patches apply on 2.6.20-rc4-git3.
PATCH 1/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha
atomic.h : Add atomic64 cmpxchg, xchg and add_unless to alpha
include/asm-alpha/atomic.h +48 -3
@@ -175,19 +175,64 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)return result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +/** +* atomic_add_unless - add unless the number is a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)
PATCH 2/9 atomic.h : Complete atomic_long operations in asm-generic
In reply to Mathieu Desnoyersatomic.h : Complete atomic_long operations in asm-generic
include/asm-generic/atomic.h +168 -0
@@ -66,6 +66,90 @@ static inline void atomic_long_sub(long i, atomic_long_t *l)atomic64_sub (i, v); } +static inline int atomic_long_sub_and_test (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_sub_and_test (i, v); +} + +static inline int atomic_long_dec_and_test (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_dec_and_test (v); +} + +static inline int atomic_long_inc_and_test (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_inc_and_test (v); +} + +static inline int atomic_long_add_negative (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_add_negative (i, v); +} + +static inline long atomic_long_add_return (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_add_return (i, v); +} + +static inline long atomic_long_sub_return (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_sub_return (i, v); +} + +static inline long atomic_long_inc_return (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_inc_return (v); +} + +static inline long atomic_long_dec_return (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_dec_return (v); +} + +static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_add_unless (v, a, u); +} + +static inline long atomic_long_inc_not_zero (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_inc_not_zero (v); +} + +static inline long atomic_long_cmpxchg (atomic_long_t *l,long old,long new) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_cmpxchg (v, old, new); +} + +static inline long atomic_long_xchg (atomic_long_t *l,long new) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_xchg (v, new); +} +#else /* BITS_PER_LONG == 64 */ typedef atomic_t atomic_long_t ;
@@ -113,6 +197,90 @@ static inline void atomic_long_sub(long i, atomic_long_t *l)atomic_sub (i, v); } +static inline int atomic_long_sub_and_test (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_sub_and_test (i, v); +} + +static inline int atomic_long_dec_and_test (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_dec_and_test (v); +} + +static inline int atomic_long_inc_and_test (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_inc_and_test (v); +} + +static inline int atomic_long_add_negative (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_add_negative (i, v); +} + +static inline long atomic_long_add_return (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_add_return (i, v); +} + +static inline long atomic_long_sub_return (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_sub_return (i, v); +} + +static inline long atomic_long_inc_return (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_inc_return (v); +} + +static inline long atomic_long_dec_return (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_dec_return (v); +} + +static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_add_unless (v, a, u); +} + +static inline long atomic_long_inc_not_zero (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_inc_not_zero (v); +} + +static inline long atomic_long_cmpxchg (atomic_long_t *l,long old,long new) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_cmpxchg (v, old, new); +} + +static inline long atomic_long_xchg (atomic_long_t *l,long new) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_xchg (v, new); +} +#endif /* BITS_PER_LONG == 64 */ #endif /* _ASM_GENERIC_ATOMIC_H */
PATCH 3/9 atomic.h : i386 type safety fix
In reply to Mathieu Desnoyersatomic.h : i386 type safety fix
This patch removes an explicit cast to an integer type for the result returned by cmpxchg. It is not per se a problem on the i386 architecture, because sizeof(int) == sizeof(long), but whenever this code is cut'n'pasted to a different architecture (which happened at least for x86_64), it would simply accept passing an atomic64_t value as parameter to cmpxchg, xchg and add_unless, having 64 bits inputs casted to 32 bits.
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v)return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v)*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
PATCH 4/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64
In reply to Mathieu Desnoyersatomic.h : Add atomic64 cmpxchg, xchg and add_unless to ia64
include/asm-ia64/atomic.h +23 -2
@@ -88,12 +88,17 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v)return new; } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__(v->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -107,6 +112,22 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__(v->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \
PATCH 5/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips
In reply to Mathieu Desnoyersatomic.h : Add atomic64 cmpxchg, xchg and add_unless to mips
include/asm-mips/atomic.h +27 -3
@@ -291,8 +291,9 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)return result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -305,7 +306,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -651,6 +652,29 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)return result; } +#define atomic64_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))
PATCH 6/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc
In reply to Mathieu Desnoyersatomic.h : Add atomic64 cmpxchg, xchg and add_unless to parisc
include/asm-parisc/atomic.h +28 -2
@@ -163,7 +163,8 @@ static __inline__ int atomic_read(const atomic_t *v) }/* exported interface */ -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -177,7 +178,7 @@ static __inline__ int atomic_read(const atomic_t *v)*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -270,6 +271,31 @@ atomic64_read(const atomic64_t *v)#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0)#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i),(v)) == 0) +/* exported interface */ +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) + +#endif /* __LP64__ */ #include <asm-generic/atomic.h>
PATCH 7/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc
In reply to Mathieu Desnoyersatomic.h : Add atomic64 cmpxchg, xchg and add_unless to powerpc
include/asm-powerpc/atomic.h +39 -1
@@ -165,7 +165,8 @@ static __inline__ int atomic_dec_return(atomic_t *v)return t; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -411,6 +412,43 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) return t; } +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) +{ +long t; + + __asm__ __volatile__ ( +LWSYNC_ON_SMP +"1: ldarx %0,0,%1 # atomic_add_unless\n\ +cmpd 0,%0,%3 \n\ +beq- 2f \n\ +add %0,%2,%0 \n" +" stdcx. %0,0,%1 \n\ +bne- 1b \n" +ISYNC_ON_SMP +" subf %0,%2,%0 \n\ +2:" + :"=&r" (t) + :"r" (&v->counter),"r" (a),"r" (u) + :"cc" ,"memory" ); + +return t != u; +} + +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#endif /* __powerpc64__ */ #include <asm-generic/atomic.h>
PATCH 8/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64
In reply to Mathieu Desnoyersatomic.h : Add atomic64 cmpxchg, xchg and add_unless to sparc64
include/asm-sparc64/atomic.h +23 -2
@@ -70,12 +70,13 @@ extern int atomic64_sub_ret(int, atomic64_t *);#define atomic_add_negative (i, v) (atomic_add_ret(i, v) < 0)#define atomic64_add_negative (i, v) (atomic64_add_ret(i, v) < 0) -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -89,6 +90,26 @@ extern int atomic64_sub_ret(int, atomic64_t *); })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + likely(c != (u)); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();
PATCH 9/9 atomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64
In reply to Mathieu Desnoyersatomic.h : Add atomic64 cmpxchg, xchg and add_unless to x86_64
include/asm-x86_64/atomic.h +34 -4
@@ -375,8 +375,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t *v)long __i = i; __asm__ __volatile__(LOCK_PREFIX "xaddq %0, %1;" - :"=r" (i) - :"m" (v->counter),"0" (i)); + :"+r" (i),"+m" (v->counter) + : :"memory" );return i + __i; }
@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v)#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -402,7 +407,7 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v)*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -416,6 +421,31 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
PATCH 1/1 atomic.h : standardizing atomic primitives
In reply to Mathieu Desnoyersatomic.h : standardizing atomic primitives
It mainly adds support for missing 64 bits cmpxchg and 64 bits atomic add unless. Therefore, principally 64 bits architectures are targeted by these patches. It also adds the complete list of atomic operations on the atomic_long type.
build tested on:
arm-pxa255-idp_defconfig-xbuild i386-defconfig-xbuild
ia64-defconfig-xbuild mips-ip32_defconfig-xbuild mips-yosemite_defconfig-xbuild parisc-a500_defconfig-xbuild -- needs parisc-2.6 tree powerpc-ppc64_defconfig-xbuild sparc64-defconfig-xbuild sparc-defconfig-xbuild um-x86_64-defconfig-xbuild x86_64-defconfig-xbuild
notable exception: alpha (assembler errors, guess binutils funkyness)
--- include/asm-alpha/atomic.h | 51 ++++++++++++ include/asm-generic/atomic.h | 169 +++++++++++++++++++++++++++++++++++++++++++ include/asm-i386/atomic.h | 7 + include/asm-ia64/atomic.h | 25 +++++- include/asm-mips/atomic.h | 28 ++++++- include/asm-parisc/atomic.h | 29 ++++++- include/asm-powerpc/atomic.h | 40 +++++++++- include/asm-powerpc/bitops.h | 1 include/asm-powerpc/system.h | 1 include/asm-sparc64/atomic.h | 25 +++++- include/asm-x86_64/atomic.h | 38 ++++++++- 11 files changed, 392 insertions(+), 22 deletions(-)
include/asm-alpha/atomic.h +48 -3
@@ -175,19 +175,64 @@ static __inline__ long atomic64_sub_retureturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +/** +* atomic_add_unless - add unless the number is a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)
include/asm-generic/atomic.h +169 -0
187 lines not on this page · raw message
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ static __inline__ int atomic_sub_return(*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-ia64/atomic.h +23 -2
@@ -88,12 +88,17 @@ ia64_atomic64_sub (__s64 i, atomic64_t *return new; } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__(v->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -107,6 +112,22 @@ ia64_atomic64_sub (__s64 i, atomic64_t * })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__(v->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \
include/asm-mips/atomic.h +25 -3
@@ -291,8 +291,8 @@ static __inline__ int atomic_sub_if_posireturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -305,7 +305,7 @@ static __inline__ int atomic_sub_if_posi*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -651,6 +651,28 @@ static __inline__ long atomic64_sub_if_preturn result; } +#define atomic64_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))
include/asm-parisc/atomic.h +27 -2
@@ -163,7 +163,8 @@ static __inline__ int atomic_read(const }/* exported interface */ -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -177,7 +178,7 @@ static __inline__ int atomic_read(const*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -270,6 +271,30 @@ atomic64_read(const atomic64_t *v)#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0)#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i),(v)) == 0) +/* exported interface */ +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#endif /* CONFIG_64BIT */ #include <asm-generic/atomic.h>
include/asm-powerpc/atomic.h +39 -1
@@ -165,7 +165,8 @@ static __inline__ int atomic_dec_return(return t; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -413,6 +414,43 @@ static __inline__ long atomic64_dec_if_p return t; } +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) +{ +long t; + + __asm__ __volatile__ ( +LWSYNC_ON_SMP +"1: ldarx %0,0,%1 # atomic_add_unless\n\ +cmpd 0,%0,%3 \n\ +beq- 2f \n\ +add %0,%2,%0 \n" +" stdcx. %0,0,%1 \n\ +bne- 1b \n" +ISYNC_ON_SMP +" subf %0,%2,%0 \n\ +2:" + :"=&r" (t) + :"r" (&v->counter),"r" (a),"r" (u) + :"cc" ,"memory" ); + +return t != u; +} + +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#endif /* __powerpc64__ */ #include <asm-generic/atomic.h>
include/asm-powerpc/bitops.h +0 -1
@@ -39,7 +39,6 @@#ifdef __KERNEL__#include <linux/compiler.h> -#include <asm/atomic.h> #include <asm/asm-compat.h> #include <asm/synch.h>
include/asm-powerpc/system.h +0 -1
@@ -7,7 +7,6 @@#include <linux/kernel.h> #include <asm/hw_irq.h> -#include <asm/atomic.h> /* * Memory barrier.
include/asm-sparc64/atomic.h +23 -2
@@ -70,12 +70,13 @@ extern int atomic64_sub_ret(int, atomic6#define atomic_add_negative (i, v) (atomic_add_ret(i, v) < 0)#define atomic64_add_negative (i, v) (atomic64_add_ret(i, v) < 0) -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -89,6 +90,26 @@ extern int atomic64_sub_ret(int, atomic6 })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + likely(c != (u)); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();
include/asm-x86_64/atomic.h +34 -4
@@ -375,8 +375,8 @@ static __inline__ long atomic64_add_retulong __i = i; __asm__ __volatile__(LOCK_PREFIX "xaddq %0, %1;" - :"=r" (i) - :"m" (v->counter),"0" (i)); + :"+r" (i),"+m" (v->counter) + : :"memory" );return i + __i; }
@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_retu#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -402,7 +407,7 @@ static __inline__ long atomic64_sub_retu*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -416,6 +421,31 @@ static __inline__ long atomic64_sub_retu })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
Hi Peter,
Thanks for testing Andrew's fixes. I am wondering about what happened to the Powerpc recursive include problems Andrew experienced. Quoting him :
"OK, I fixed eight separate compile errors in this patch series and now powerpc is being very ugly with a twisty maze of include dependencies.
I'm giving up. Someone should publish a suite of cross-compilers for us so stuff like this doesn't need to happen."
I see that you have removed the include <asm/atomic.h> from bitops.h and system.h in powerpc. If it compiles on every architectures, then it's a good approach.
I planned to post a new patch which uses macros for cmpxchg and xchg in asm-generic/atomic.h instead of inline functions. It would remove the dependency on system.h. However, if your modifications work well on every architecture, my fix might not be needed. Anyone has a preferred solution ? I have not been able to setup my cross-compiler test bench yet due to some hardware issues and waited for it before I released further fixes, but if you want to try my macro-based fix, I could post it.
And about the alpha build, Does the assembler errors also happen without this patch ?
Regards,
Mathieu
Quoted 686 lines · * Peter Zijlstra ([email protected]) wrote:
atomic.h: standardizing atomic primitivesIt mainly adds support for missing 64 bits cmpxchg and 64 bits atomic add unless. Therefore, principally 64 bits architectures are targeted by these patches. It also adds the complete list of atomic operations on the
atomic_longtype.build tested on:
arm-pxa255-idp_defconfig-xbuild i386-defconfig-xbuild
ia64-defconfig-xbuild mips-ip32_defconfig-xbuild mips-yosemite_defconfig-xbuild parisc-a500_defconfig-xbuild -- needs parisc-2.6 tree powerpc-ppc64_defconfig-xbuild sparc64-defconfig-xbuild sparc-defconfig-xbuild um-x86_64-defconfig-xbuild x86_64-defconfig-xbuildnotable exception: alpha (assembler errors, guess binutils funkyness)
Signed-off-by Peter Zijlstra <[email protected]>Signed-off-by Mathieu Desnoyers <[email protected]>--- include/asm-alpha/atomic.h | 51 ++++++++++++ include/asm-generic/atomic.h | 169 +++++++++++++++++++++++++++++++++++++++++++ include/asm-i386/atomic.h | 7 + include/asm-ia64/atomic.h | 25 +++++- include/asm-mips/atomic.h | 28 ++++++- include/asm-parisc/atomic.h | 29 ++++++- include/asm-powerpc/atomic.h | 40 +++++++++- include/asm-powerpc/bitops.h | 1 include/asm-powerpc/system.h | 1 include/asm-sparc64/atomic.h | 25 +++++- include/asm-x86_64/atomic.h | 38 ++++++++- 11 files changed, 392 insertions(+), 22 deletions(-)Index:
linux-2.6/include/asm-alpha/atomic.h
===================================================================include/asm-alpha/atomic.h +48
-3@@ -175,19 +175,64 @@ static __inline__ long atomic64_sub_retureturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +/** +* atomic_add_unless - add unless the number is a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)Index:
linux-2.6/include/asm-generic/atomic.h
===================================================================include/asm-generic/atomic.h +169
-0@@ -8,6 +8,7 @@* edit all arch specific atomic.h files. */ +#include <asm/system.h> #include <asm/types.h> /* @@ -66,6 +67,90 @@ static inline void atomic_long_sub(longatomic64_sub (i, v); } +static inline int atomic_long_sub_and_test (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return atomic64_sub_and_test (i, v); +} + +static inline int atomic_long_dec_and_test (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return atomic64_dec_and_test (v); +} + +static inline int atomic_long_inc_and_test (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return atomic64_inc_and_test (v); +} + +static inline int atomic_long_add_negative (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return atomic64_add_negative (i, v); +} + +static inline long atomic_long_add_return (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_add_return (i, v); +} + +static inline long atomic_long_sub_return (long i,atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_sub_return (i, v); +} + +static inline long atomic_long_inc_return (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_inc_return (v); +} + +static inline long atomic_long_dec_return (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_dec_return (v); +} + +static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_add_unless (v, a, u); +} + +static inline long atomic_long_inc_not_zero (atomic_long_t *l) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_inc_not_zero (v); +} + +static inline long atomic_long_cmpxchg (atomic_long_t *l,long old,long new) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_cmpxchg (v, old, new); +} + +static inline long atomic_long_xchg (atomic_long_t *l,long new) +{ +atomic64_t *v = (atomic64_t *)l; + +return (long )atomic64_xchg (v, new); +} +#else /* BITS_PER_LONG == 64 */ typedef atomic_t atomic_long_t ;@@ -113,6 +198,90 @@ static inline void atomic_long_sub(longatomic_sub (i, v); } +static inline int atomic_long_sub_and_test (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_sub_and_test (i, v); +} + +static inline int atomic_long_dec_and_test (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_dec_and_test (v); +} + +static inline int atomic_long_inc_and_test (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_inc_and_test (v); +} + +static inline int atomic_long_add_negative (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_add_negative (i, v); +} + +static inline long atomic_long_add_return (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_add_return (i, v); +} + +static inline long atomic_long_sub_return (long i,atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_sub_return (i, v); +} + +static inline long atomic_long_inc_return (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_inc_return (v); +} + +static inline long atomic_long_dec_return (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_dec_return (v); +} + +static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_add_unless (v, a, u); +} + +static inline long atomic_long_inc_not_zero (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_inc_not_zero (v); +} + +static inline long atomic_long_cmpxchg (atomic_long_t *l,long old,long new) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_cmpxchg (v, old, new); +} + +static inline long atomic_long_xchg (atomic_long_t *l,long new) +{ +atomic_t *v = (atomic_t *)l; + +return (long )atomic_xchg (v, new); +} +#endif /* BITS_PER_LONG == 64 */ #endif /* _ASM_GENERIC_ATOMIC_H */ Index:
linux-2.6/include/asm-i386/atomic.h
===================================================================include/asm-i386/atomic.h +4
-3@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value @@ -221,7 +222,7 @@ static __inline__ int atomic_sub_return(*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \Index:
linux-2.6/include/asm-ia64/atomic.h
===================================================================include/asm-ia64/atomic.h +23
-2@@ -88,12 +88,17 @@ ia64_atomic64_sub (__s64 i, atomic64_t *return new; } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__(v->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \@@ -107,6 +112,22 @@ ia64_atomic64_sub (__s64 i, atomic64_t * })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__(v->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \Index:
linux-2.6/include/asm-mips/atomic.h
===================================================================include/asm-mips/atomic.h +25
-3@@ -291,8 +291,8 @@ static __inline__ int atomic_sub_if_posireturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value @@ -305,7 +305,7 @@ static __inline__ int atomic_sub_if_posi*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \@@ -651,6 +651,28 @@ static __inline__ long atomic64_sub_if_preturn result; } +#define atomic64_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))Index:
linux-2.6/include/asm-parisc/atomic.h
===================================================================include/asm-parisc/atomic.h +27
-2@@ -163,7 +163,8 @@ static __inline__ int atomic_read(const }/* exported interface */ -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/** @@ -177,7 +178,7 @@ static __inline__ int atomic_read(const*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \@@ -270,6 +271,30 @@ atomic64_read(const atomic64_t *v)#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0)#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i),(v)) == 0) +/* exported interface */ +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#endif /* CONFIG_64BIT */ #include <asm-generic/atomic.h> Index:
linux-2.6/include/asm-powerpc/atomic.h
===================================================================include/asm-powerpc/atomic.h +39
-1@@ -165,7 +165,8 @@ static __inline__ int atomic_dec_return(return t; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/** @@ -413,6 +414,43 @@ static __inline__ long atomic64_dec_if_p return t; } +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) +{ +long t; + + __asm__ __volatile__ ( +LWSYNC_ON_SMP +"1: ldarx %0,0,%1 # atomic_add_unless\n\ +cmpd 0,%0,%3 \n\ +beq- 2f \n\ +add %0,%2,%0 \n" +" stdcx. %0,0,%1 \n\ +bne- 1b \n" +ISYNC_ON_SMP +" subf %0,%2,%0 \n\ +2:" + :"=&r" (t) + :"r" (&v->counter),"r" (a),"r" (u) + :"cc" ,"memory" ); + +return t != u; +} + +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#endif /* __powerpc64__ */ #include <asm-generic/atomic.h> Index:
linux-2.6/include/asm-powerpc/bitops.h
===================================================================include/asm-powerpc/bitops.h +0
-1@@ -39,7 +39,6 @@#ifdef __KERNEL__#include <linux/compiler.h> -#include <asm/atomic.h> #include <asm/asm-compat.h> #include <asm/synch.h> Index:
linux-2.6/include/asm-powerpc/system.h
===================================================================include/asm-powerpc/system.h +0
-1@@ -7,7 +7,6 @@#include <linux/kernel.h> #include <asm/hw_irq.h> -#include <asm/atomic.h> /* * Memory barrier. Index:
linux-2.6/include/asm-sparc64/atomic.h
===================================================================include/asm-sparc64/atomic.h +23
-2@@ -70,12 +70,13 @@ extern int atomic64_sub_ret(int, atomic6#define atomic_add_negative (i, v) (atomic_add_ret(i, v) < 0)#define atomic64_add_negative (i, v) (atomic64_add_ret(i, v) < 0) -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \@@ -89,6 +90,26 @@ extern int atomic64_sub_ret(int, atomic6 })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + likely(c != (u)); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();Index:
linux-2.6/include/asm-x86_64/atomic.h
===================================================================include/asm-x86_64/atomic.h +34
-4@@ -375,8 +375,8 @@ static __inline__ long atomic64_add_retulong __i = i; __asm__ __volatile__(LOCK_PREFIX "xaddq %0, %1;" - :"=r" (i) - :"m" (v->counter),"0" (i)); + :"+r" (i),"+m" (v->counter) + : :"memory" );return i + __i; }@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_retu#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/** @@ -402,7 +407,7 @@ static __inline__ long atomic64_sub_retu*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \@@ -416,6 +421,31 @@ static __inline__ long atomic64_sub_retu })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
Signature · 2 lines
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Quoted 3 lines · Mathieu Desnoyers
Hi Peter,
Thanks for testing Andrew's fixes.
I haven't actually taken any from him. I just started afresh.
Quoted 7 lines
I am wondering about what happened to
the Powerpc recursive include problems Andrew experienced. Quoting him :"OK, I fixed eight separate compile errors in this patch series and now powerpc is being very ugly with a twisty maze of include dependencies.
Quoted 6 lines
I'm giving up. Someone should publish a suite of cross-compilers for us so stuff like this doesn't need to happen."
I see that you have removed the include <
asm/atomic.h> frombitops.handsystem.hin powerpc. If it compiles on every architectures, then it's a good approach.
Yeah, I fiddled around with those powerpc headers a bit until it stopped whining. Maybe the powerpc folks ought to eyeball it a bit, but it build a kernel image here.
Quoted 8 lines
I planned to post a new patch which uses macros for cmpxchg and xchg in
asm-generic/atomic.hinstead of inline functions. It would remove the dependency onsystem.h. However, if your modifications work well on every architecture, my fix might not be needed. Anyone has a preferred solution ? I have not been able to setup my cross-compiler test bench yet due to some hardware issues and waited for it before I released further fixes, but if you want to try my macro-based fix, I could post it.
Whatever people want; inlines are generally preferred due to the extra type checking.
I just needed atomic_long_cmpxchg to work so I kicked your patches about till they compiled.
Quoted 2 lines
And about the alpha build, Does the assembler errors also happen without this patch ?
Yes. Something fishy going on there... gcc-4.1.1 + binutils-2.17 Compiler seems to build fine, but kernel code makes it go belly up.