LTTng 0.6.36 for 2.6.18 : atomic UP operations on SMP
Subsystems (2)
PATCH 4/16 LTTng 0.6.36 for 2.6.18 : atomic UP operations on SMP
This patch adds a UP flavor of SMP operations which is intended to provide atomic modification of per-cpu data without suffering from the LOCK of memory barrier performance cost. Note that extreme care must be taken when accessing this data from different CPUs : smp_wmb() and smp_rmb() must be used explicitely. As this last scenario happens very rarely in LTTng, it provides an interesting performance gain.
Some tests to see the speedup given by using atomic-up.h on per cpu variables.
Non LOCKed atomic ops that I now use on SMP :
A test ran on a 3GHz Pentium 4 shows that (20000 loops) :
irq save/restore pair 210.60 ns
cmpxchg 49.46 ns
(76 % speedup)
cmpxchg-up (no lock prefix) 9.00 ns
(95 % speedup)On my 3GHz Pentium 4, it takes 255.83ns to log a 4 bytes event when the LOCK prefix is used (without atomic-up). When I enable my modified version, it drops to 205.63ns. Therefore, the speedup is :
(205.63-255.83)/255.83 * 100% = -19.62 %
(Test : 3*20000 loops of 4 bytes event log in flight recorder mode)
patch04-2.6.18-lttng-core-0.6.36-atomic_up.diff
--BEGIN--
include/asm-i386/atomic-up.h new file +229 -0
@@ -0,0 +1,229 @@ +#ifndef __ARCH_I386_ATOMIC_UP__ +#define __ARCH_I386_ATOMIC_UP__ + +#include <linux/compiler.h> +#include <asm/processor.h> +#include <asm/atomic.h> + +/* +* atomic_up variants insure operation atomicity only if the variable is not +* shared between cpus. This is useful to have per-cpu atomic operations to +* protect from contexts like non-maskable interrupts without the LOCK prefix +* performance cost. +*/ + +/** +* atomic_up_add - add integer to atomic variable +* @i: integer value to add +* @v: pointer of type atomic_t +* +* Atomically adds @i to @v. +*/ +static __inline__void atomic_up_add (int i,atomic_t *v) +{ + __asm__ __volatile__( +"addl %1,%0" + :"+m" (v->counter) + :"ir" (i)); +} + +/** +* atomic_up_sub - subtract the atomic variable +* @i: integer value to subtract +* @v: pointer of type atomic_t +* +* Atomically subtracts @i from @v. +*/ +static __inline__void atomic_up_sub (int i,atomic_t *v) +{ + __asm__ __volatile__( +"subl %1,%0" + :"+m" (v->counter) + :"ir" (i)); +} + +/** +* atomic_up_sub_and_test - subtract value from variable and test result +* @i: integer value to subtract +* @v: pointer of type atomic_t +* +* Atomically subtracts @i from @v and returns +* true if the result is zero, or false for all +* other cases. +*/ +static __inline__int atomic_up_sub_and_test (int i,atomic_t *v) +{ +unsigned char c; + + __asm__ __volatile__( +"subl %2,%0; sete %1" + :"+m" (v->counter),"=qm" (c) + :"ir" (i) :"memory" ); +return c; +} + +/** +* atomic_up_inc - increment atomic variable +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1. +*/ +static __inline__void atomic_up_inc (atomic_t *v) +{ + __asm__ __volatile__( +"incl %0" + :"+m" (v->counter)); +} + +/** +* atomic_up_dec - decrement atomic variable +* @v: pointer of type atomic_t +* +* Atomically decrements @v by 1. +*/ +static __inline__void atomic_up_dec (atomic_t *v) +{ + __asm__ __volatile__( +"decl %0" + :"+m" (v->counter)); +} + +/** +* atomic_up_dec_and_test - decrement and test +* @v: pointer of type atomic_t +* +* Atomically decrements @v by 1 and +* returns true if the result is 0, or false for all other +* cases. +*/ +static __inline__int atomic_up_dec_and_test (atomic_t *v) +{ +unsigned char c; + + __asm__ __volatile__( +"decl %0; sete %1" + :"+m" (v->counter),"=qm" (c) + : :"memory" ); +return c !=0 ; +} + +/** +* atomic_up_inc_and_test - increment and test +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1 +* and returns true if the result is zero, or false for all +* other cases. +*/ +static __inline__int atomic_up_inc_and_test (atomic_t *v) +{ +unsigned char c; + + __asm__ __volatile__( +"incl %0; sete %1" + :"+m" (v->counter),"=qm" (c) + : :"memory" ); +return c !=0 ; +} + +/** +* atomic_up_add_negative - add and test if negative +* @v: pointer of type atomic_t +* @i: integer value to add +* +* Atomically adds @i to @v and returns true +* if the result is negative, or false when +* result is greater than or equal to zero. +*/ +static __inline__int atomic_up_add_negative (int i,atomic_t *v) +{ +unsigned char c; + + __asm__ __volatile__( +"addl %2,%0; sets %1" + :"+m" (v->counter),"=qm" (c) + :"ir" (i) :"memory" ); +return c; +} + +/** +* atomic_up_add_return - add and return +* @v: pointer of type atomic_t +* @i: integer value to add +* +* Atomically adds @i to @v and returns @i + @v +*/ +static __inline__int atomic_up_add_return (int i,atomic_t *v) +{ +int __i; +#ifdef CONFIG_M386 +unsigned long flags; +if (unlikely (boot_cpu_data.x86==3 )) + goto no_xadd; +#endif +/* Modern 486+ processor */ + __i = i; + __asm__ __volatile__( +"xaddl %0, %1;" + :"=r" (i) + :"m" (v->counter),"0" (i)); +return i + __i; + +#ifdef CONFIG_M386 +no_xadd:/* Legacy 386 processor */ +local_irq_save (flags); + __i =atomic_up_read (v); +atomic_up_set (v, i + __i); +local_irq_restore (flags); +return i + __i; +#endif +} + +static __inline__int atomic_up_sub_return (int i,atomic_t *v) +{ +return atomic_up_add_return (-i,v); +} + +#define atomic_up_cmpxchg (v, old, new) ((int)cmpxchg_up(&((v)->counter), \ + old, new)) +/* xchg always has a LOCK prefix */ +#define atomic_up_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic_up_add_unless - add unless the number is a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic_up_add_unless (v, a, u) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_up_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic_up_inc_not_zero (v) atomic_up_add_unless((v), 1, 0) + +#define atomic_up_inc_return (v) (atomic_up_add_return(1,v)) +#define atomic_up_dec_return (v) (atomic_up_sub_return(1,v)) + +/* These are x86-specific, used by some header files */ +#define atomic_up_clear_mask (mask, addr) \ +__asm__ __volatile__("andl %0,%1" \ +: : "r" (~(mask)),"m" (*addr) : "memory") + +#define atomic_up_set_mask (mask, addr) \ +__asm__ __volatile__("orl %0,%1" \ +: : "r" (mask),"m" (*(addr)) : "memory") + +#endif
include/asm-i386/system.h +58 -1
86 lines not on this page · raw message
include/asm-x86_64/atomic-up.h new file +375 -0
375 lines not on this page · raw message
include/asm-x86_64/system.h +36 -0
@@ -208,9 +208,45 @@ static inline unsigned long __cmpxchg(voreturn old; } +static inline unsigned long __cmpxchg_up (volatile void *ptr,unsigned long old, +unsigned long new,int size) +{ +unsigned long prev; +switch (size) { +case 1 : + __asm__ __volatile__("cmpxchgb %b1,%2" + :"=a" (prev) + :"q" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 2 : + __asm__ __volatile__("cmpxchgw %w1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 4 : + __asm__ __volatile__("cmpxchgl %k1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 8 : + __asm__ __volatile__("cmpxchgq %1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; + } +return old; +} +#define cmpxchg (ptr,o,n)\ ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr)))) +#define cmpxchg_up (ptr,o,n)\ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ + (unsigned long)(n),sizeof(*(ptr))))#ifdef CONFIG_SMP #define smp_mb () mb()
include/asm-powerpc/atomic-up.h new file +380 -0
380 lines not on this page · raw message
include/asm-powerpc/system.h +126 -0
174 lines not on this page · raw message
include/asm-arm/atomic-up.h new file +6 -0
@@ -0,0 +1,6 @@ +#ifndef _ASM_ATOMIC_UP_H +#define _ASM_ATOMIC_UP_H + +#include <asm-generic/atomic-up.h> + +#endif
include/asm-mips/atomic-up.h new file +6 -0
@@ -0,0 +1,6 @@ +#ifndef _ASM_ATOMIC_UP_H +#define _ASM_ATOMIC_UP_H + +#include <asm-generic/atomic-up.h> + +#endif
include/asm-generic/atomic-up.h new file +49 -0
49 lines not on this page · raw message
--END--
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Quoted 5 lines · Mathieu Desnoyers
This patch adds a UP flavor of SMP operations which is intended to provide atomic modification of per-cpu data without suffering from the LOCK of memory barrier performance cost. Note that extreme care must be taken when accessing this data from different CPUs :
smp_wmb()andsmp_rmb()must be used explicitely. As this last scenario happens very rarely in LTTng, it provides a
We already have local_t in asm/local.h for this purposed. Unfortunately several architecture implementations are rather suboptimal, but I'm sure the architecture maintainers would be interested in patches to optimize the various implementations.
PATCH 1/2 atomic.h atomic64_t standardization
Quoted 5 lines · * Christoph Hellwig ([email protected]) wrote:
We already have
local_tinasm/local.hfor this purposed. Unfortunately several architecture implementations are rather suboptimal, but I'm sure the architecture maintainers would be interested in patches to optimize the various implementations.
Hi Christoph,
I just implemented some modifications over i386, x86_64, powerpc, mips and arm implementations (2.6.18) of atomic.h to add the atomic64_cmpxchg primitives. They are required for proper asm-generic/atomic.h atomic_long_t type. This patch also adds missing primitives to asm-generic/atomic.h.
local.h modifications for these architectures follows in the next post.
Mathieu
---BEGIN---
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-i386/system.h +58 -1
86 lines not on this page · raw message
include/asm-x86_64/atomic.h +10 -2
@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_retu#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -402,7 +407,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -416,6 +421,9 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) atomic_add_unless((v), (a), (u)) +#define atomic64_inc_not_zero (v) atomic_inc_not_zero((v)) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
include/asm-x86_64/system.h +36 -0
@@ -208,9 +208,45 @@ static inline unsigned long __cmpxchg(voreturn old; } +static inline unsigned long __cmpxchg_local (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; +switch (size) { +case 1 : + __asm__ __volatile__("cmpxchgb %b1,%2" + :"=a" (prev) + :"q" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 2 : + __asm__ __volatile__("cmpxchgw %w1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 4 : + __asm__ __volatile__("cmpxchgl %k1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 8 : + __asm__ __volatile__("cmpxchgq %1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; + } +return old; +} +#define cmpxchg (ptr,o,n)\ ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr)))) +#define cmpxchg_local (ptr,o,n)\ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ + (unsigned long)(n),sizeof(*(ptr))))#ifdef CONFIG_SMP #define smp_mb () mb()
include/asm-powerpc/atomic.h +40 -1
53 lines not on this page · raw message
include/asm-powerpc/system.h +129 -0
177 lines not on this page · raw message
include/asm-arm/atomic.h +1 -0
@@ -185,6 +185,7 @@ static inline int atomic_add_unless(atom c = old;return c != u; } +#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0)#define atomic_add (i, v) (void) atomic_add_return(i, v)
include/asm-mips/atomic.h +27 -3
@@ -292,8 +292,9 @@ static __inline__ int atomic_sub_if_posireturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -306,7 +307,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -646,6 +647,29 @@ static __inline__ long atomic64_sub_if_preturn result; } +#define atomic64_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))
include/asm-generic/atomic.h +148 -0
159 lines not on this page · raw message
---END---
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 1/2] atomic.h atomic64_t standardization
New version, fixes PowerPC typo.
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-i386/system.h +58 -1
86 lines not on this page · raw message
include/asm-x86_64/atomic.h +10 -2
@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_retu#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -402,7 +407,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -416,6 +421,9 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) atomic_add_unless((v), (a), (u)) +#define atomic64_inc_not_zero (v) atomic_inc_not_zero((v)) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
include/asm-x86_64/system.h +36 -0
@@ -208,9 +208,45 @@ static inline unsigned long __cmpxchg(voreturn old; } +static inline unsigned long __cmpxchg_local (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; +switch (size) { +case 1 : + __asm__ __volatile__("cmpxchgb %b1,%2" + :"=a" (prev) + :"q" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 2 : + __asm__ __volatile__("cmpxchgw %w1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 4 : + __asm__ __volatile__("cmpxchgl %k1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 8 : + __asm__ __volatile__("cmpxchgq %1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; + } +return old; +} +#define cmpxchg (ptr,o,n)\ ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr)))) +#define cmpxchg_local (ptr,o,n)\ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ + (unsigned long)(n),sizeof(*(ptr))))#ifdef CONFIG_SMP #define smp_mb () mb()
include/asm-powerpc/atomic.h +40 -1
53 lines not on this page · raw message
include/asm-powerpc/system.h +129 -0
177 lines not on this page · raw message
include/asm-arm/atomic.h +1 -0
@@ -185,6 +185,7 @@ static inline int atomic_add_unless(atom c = old;return c != u; } +#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0)#define atomic_add (i, v) (void) atomic_add_return(i, v)
include/asm-mips/atomic.h +27 -3
@@ -292,8 +292,9 @@ static __inline__ int atomic_sub_if_posireturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -306,7 +307,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -646,6 +647,29 @@ static __inline__ long atomic64_sub_if_preturn result; } +#define atomic64_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))
include/asm-generic/atomic.h +148 -0
159 lines not on this page · raw message
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 1/2] atomic.h atomic64_t standardization
Hi,
I finalized the work for atomic64_t cmpxchg and atomic64_add_unless on all architectures. asm-generic/atomic.h atomic_long_t is also streamlined.
Review is welcome.
Mathieu
---BEGIN---
include/asm-alpha/atomic.h +48 -3
@@ -175,19 +175,64 @@ static __inline__ long atomic64_sub_retureturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +/** +* atomic_add_unless - add unless the number is a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)
include/asm-alpha/system.h +227 -0
239 lines not on this page · raw message
include/asm-arm/atomic.h +1 -0
@@ -185,6 +185,7 @@ static inline int atomic_add_unless(atom c = old;return c != u; } +#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0)#define atomic_add (i, v) (void) atomic_add_return(i, v)
include/asm-generic/atomic.h +145 -0
156 lines not on this page · raw message
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-i386/system.h +58 -1
86 lines not on this page · raw message
include/asm-ia64/atomic.h +23 -2
@@ -88,12 +88,17 @@ ia64_atomic64_sub (__s64 i, atomic64_t *return new; } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__(v->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -107,6 +112,22 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__(v->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \
include/asm-mips/atomic.h +27 -3
@@ -292,8 +292,9 @@ static __inline__ int atomic_sub_if_posireturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -306,7 +307,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -646,6 +647,29 @@ static __inline__ long atomic64_sub_if_preturn result; } +#define atomic64_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))
include/asm-parisc/atomic.h +28 -2
@@ -163,7 +163,8 @@ static __inline__ int atomic_read(const }/* exported interface */ -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -177,7 +178,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -270,6 +271,31 @@ #define atomic64_inc_and_test(v) (atomi#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0)#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i),(v)) == 0) +/* exported interface */ +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) + +#endif /* __LP64__ */ #include <asm-generic/atomic.h>
include/asm-powerpc/atomic.h +40 -1
53 lines not on this page · raw message
include/asm-powerpc/system.h +129 -0
177 lines not on this page · raw message
include/asm-sparc64/atomic.h +23 -2
@@ -70,12 +70,13 @@ #define atomic64_dec(v) atomic64_sub(1,#define atomic_add_negative (i, v) (atomic_add_ret(i, v) < 0)#define atomic64_add_negative (i, v) (atomic64_add_ret(i, v) < 0) -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -89,6 +90,26 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + likely(c != (u)); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();
include/asm-x86_64/atomic.h +32 -2
@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_retu#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -402,7 +407,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -416,6 +421,31 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
include/asm-x86_64/system.h +36 -0
@@ -208,9 +208,45 @@ static inline unsigned long __cmpxchg(voreturn old; } +static inline unsigned long __cmpxchg_local (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; +switch (size) { +case 1 : + __asm__ __volatile__("cmpxchgb %b1,%2" + :"=a" (prev) + :"q" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 2 : + __asm__ __volatile__("cmpxchgw %w1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 4 : + __asm__ __volatile__("cmpxchgl %k1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 8 : + __asm__ __volatile__("cmpxchgq %1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; + } +return old; +} +#define cmpxchg (ptr,o,n)\ ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr)))) +#define cmpxchg_local (ptr,o,n)\ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ + (unsigned long)(n),sizeof(*(ptr))))#ifdef CONFIG_SMP #define smp_mb () mb()
---END---
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 1/2] atomic.h atomic64_t standardization
Quoted 6 lines · Mathieu Desnoyers
Hi,
I finalized the work for
atomic64_tcmpxchg andatomic64_add_unlesson all architectures.asm-generic/atomic.hatomic_long_tis also streamlined.Review is welcome.
Beautiful! Now I can do the rwsem consolidation. Thanks.
Nick
Signature · 2 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
Re: [PATCH 1/2] atomic.h atomic64_t standardization for 2.6.19
In reply to Mathieu DesnoyersHi,
Here is the 2.6.19-friendly diff of the atomic.h atomic64_t standardization. As it seems to be useful to at least one locking primitive (rwsem) in addition of LTTng (which is not in the mainline though), I think it should be considered for inclusion.
Regards,
Mathieu
---BEGIN---
include/asm-alpha/atomic.h +48 -3
@@ -175,19 +175,64 @@ static __inline__ long atomic64_sub_retureturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +/** +* atomic_add_unless - add unless the number is a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)
include/asm-alpha/system.h +227 -0
239 lines not on this page · raw message
include/asm-arm/atomic.h +1 -0
@@ -185,6 +185,7 @@ static inline int atomic_add_unless(atom c = old;return c != u; } +#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0)#define atomic_add (i, v) (void) atomic_add_return(i, v)
include/asm-generic/atomic.h +145 -0
156 lines not on this page · raw message
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-i386/system.h +58 -1
86 lines not on this page · raw message
include/asm-ia64/atomic.h +23 -2
@@ -88,12 +88,17 @@ ia64_atomic64_sub (__s64 i, atomic64_t *return new; } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__(v->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -107,6 +112,22 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__(v->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \
include/asm-mips/atomic.h +27 -3
@@ -292,8 +292,9 @@ static __inline__ int atomic_sub_if_posireturn result; } -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -306,7 +307,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -646,6 +647,29 @@ static __inline__ long atomic64_sub_if_preturn result; } +#define atomic64_cmpxchg (v, o, n) \ + (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v))
include/asm-parisc/atomic.h +28 -2
@@ -163,7 +163,8 @@ static __inline__ int atomic_read(const }/* exported interface */ -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -177,7 +178,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ c = old; \
@@ -270,6 +271,31 @@ #define atomic64_inc_and_test(v) (atomi#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0)#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i),(v)) == 0) +/* exported interface */ +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) + +#endif /* __LP64__ */ #include <asm-generic/atomic.h>
include/asm-powerpc/atomic.h +40 -1
53 lines not on this page · raw message
include/asm-powerpc/system.h +129 -0
177 lines not on this page · raw message
include/asm-sparc64/atomic.h +23 -2
@@ -70,12 +70,13 @@ #define atomic64_dec(v) atomic64_sub(1,#define atomic_add_negative (i, v) (atomic_add_ret(i, v) < 0)#define atomic64_add_negative (i, v) (atomic64_add_ret(i, v) < 0) -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -89,6 +90,26 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +#define atomic64_cmpxchg (v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + likely(c != (u)); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();
include/asm-x86_64/atomic.h +32 -2
@@ -388,7 +388,12 @@ static __inline__ long atomic64_sub_retu#define atomic64_inc_return (v) (atomic64_add_return(1,v))#define atomic64_dec_return (v) (atomic64_sub_return(1,v)) -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic64_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new)) +#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) + +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), old, new))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/**
@@ -402,7 +407,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
@@ -416,6 +421,31 @@ ({ \ })#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) +/** +* atomic64_add_unless - add unless the number is a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as it was not @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#define atomic64_add_unless (v, a, u) \ +({ \ + __typeof__((v)->counter) c, old; \ + c = atomic64_read(v); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic64_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +/* These are x86-specific, used by some header files */ #define atomic_clear_mask (mask, addr) \ __asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
include/asm-x86_64/system.h +36 -0
@@ -209,9 +209,45 @@ static inline unsigned long __cmpxchg(voreturn old; } +static inline unsigned long __cmpxchg_local (volatile void *ptr, +unsigned long old,unsigned long new,int size) +{ +unsigned long prev; +switch (size) { +case 1 : + __asm__ __volatile__("cmpxchgb %b1,%2" + :"=a" (prev) + :"q" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 2 : + __asm__ __volatile__("cmpxchgw %w1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 4 : + __asm__ __volatile__("cmpxchgl %k1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; +case 8 : + __asm__ __volatile__("cmpxchgq %1,%2" + :"=a" (prev) + :"r" (new),"m" (*__xg (ptr)),"0" (old) + :"memory" ); +return prev; + } +return old; +} +#define cmpxchg (ptr,o,n)\ ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr)))) +#define cmpxchg_local (ptr,o,n)\ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ + (unsigned long)(n),sizeof(*(ptr))))#ifdef CONFIG_SMP #define smp_mb () mb()
---END---
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 1/2] atomic.h atomic64_t standardization
In reply to Mathieu DesnoyersQuoted 2 lines · Mathieu Desnoyers
include/asm-generic/atomic.h +0
-0
[snip]
Quoted 7 lines
include/asm-generic/atomic.h +7
-0+#if 0 +/* Atomic add unless is only effective on atomic_t on powerpc (at least) */ +static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_add_unless (v, a, u);
include/asm-generic/atomic.h static inline void atomic_long_sub(long · line 141
Quoted 10 lines
include/asm-generic/atomic.h +10
-0+} + +static inline long atomic_long_inc_not_zero (atomic_long_t *l) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_inc_not_zero (v); +} +#endif //0 +
Why is this in the patch?
Re: [PATCH 1/2] atomic.h atomic64_t standardization
Quoted 15 lines · * Paul Mundt ([email protected]) wrote:
Quoted 2 lines · Mathieu Desnoyers
include/asm-generic/atomic.h +0
-0[snip]
Quoted 11 lines
include/asm-generic/atomic.h +11
-0+#if 0 +/* Atomic add unless is only effective on atomic_t on powerpc (at least) */ +static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) +{ +atomic_t *v = (atomic_t *)l; + +return atomic_add_unless (v, a, u); +} + +static inline long atomic_long_inc_not_zero (atomic_long_t *l) +{
Quoted 9 lines
include/asm-generic/atomic.h
Quoted 6 lines
include/asm-generic/atomic.h +6
-0+atomic_t *v = (atomic_t *)l; + + return atomic_inc_not_zero(v); +} +#endif //0 +Why is this in the patch?
Oops, I forgot to remove these comments after I fixed it in the powerpc code. Code for all other architectures will have to modified too : I just modified i386, x86_64, mips, arm and powerpc.
Thanks for reporting.
Mathieu
Here is the fix :
include/asm-generic/atomic.h +4 -7
@@ -122,22 +122,19 @@ static inline long atomic_long_dec_retur return atomic64_dec_return(v); } -#if 0 -/* Atomic add unless is only effective on atomic_t on powerpc (at least) */ static inline long atomic_long_add_unless (atomic_long_t *l,long a,long u) { -atomic_t *v = (atomic_t *)l; +atomic64_t *v = (atomic64_t *)l; -return atomic_add_unless (v, a, u); +return atomic64_add_unless (v, a, u); }static inline long atomic_long_inc_not_zero (atomic_long_t *l) { -atomic_t *v = (atomic_t *)l; +atomic64_t *v = (atomic64_t *)l; -return atomic_inc_not_zero (v); +return atomic64_inc_not_zero (v); } -#endif //0 #else
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
PATCH 2/2 local.h modifications
In reply to Christoph HellwigQuoted 5 lines · * Christoph Hellwig ([email protected]) wrote:
We already have
local_tinasm/local.hfor this purposed. Unfortunately several architecture implementations are rather suboptimal, but I'm sure the architecture maintainers would be interested in patches to optimize the various implementations.
Hi Christoph,
Here are the local.h modifications for i386, x86_64, powerpc, mips and arm (and asm-generic). It adds support for various per-cpu atomic operations. It applies on 2.6.18.
Mathieu
---BEGIN--
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-x86_64/local.h +134 -1
144 lines not on this page · raw message
include/asm-powerpc/local.h +340 -1
341 lines not on this page · raw message
include/asm-mips/local.h +1 -60
61 lines not on this page · raw message
include/asm-generic/local.h +13 -0
@@ -33,6 +33,19 @@ #define local_dec(l) atomic_long_dec(&(l#define local_add (i,l) atomic_long_add((i),(&(l)->a))#define local_sub (i,l) atomic_long_sub((i),(&(l)->a)) +#define local_sub_and_test (i, l) atomic_long_sub_and_test((i), (&(l)->a)) +#define local_dec_and_test (l) atomic_long_dec_and_test(&(l)->a) +#define local_inc_and_test (l) atomic_long_inc_and_test(&(l)->a) +#define local_add_negative (i, l) atomic_long_add_negative((i), (&(l)->a)) +#define local_add_return (i, l) atomic_long_add_return((i), (&(l)->a)) +#define local_sub_return (i, l) atomic_long_sub_return((i), (&(l)->a)) +#define local_inc_return (l) atomic_long_inc_return(&(l)->a) + +#define local_cmpxchg (l, old, new) atomic_long_cmpxchg((&(l)->a), (old), (new)) +#define local_xchg (l, new) atomic_long_xchg((&(l)->a), (new)) +#define local_add_unless (l, a, u) atomic_long_add_unless((&(l)->a), (a), (u)) +#define local_inc_not_zero (l) atomic_long_inc_not_zero(&(l)->a) +/* Non-atomic variants, ie. preemption disabled and won't be touched * in interrupt, etc. Some archs can optimize this case well. */ #define __local_inc (l) local_set((l), local_read(l) + 1)
---END---
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 2/2] local.h modifications
New version, fixes PowerPC typo (cut'n'pasted from the atomic.h typo).
Mathieu
include/asm-i386/atomic.h +4 -3
@@ -207,8 +207,9 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } -#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) +#define atomic_cmpxchg (v, old, new) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (old), (new))) +#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** * atomic_add_unless - add unless the number is a given value
@@ -221,7 +222,7 @@ #define atomic_xchg(v, new) (xchg(&((v)-*/ #define atomic_add_unless (v, a, u) \ ({ \ - int c, old; \ + __typeof__((v)->counter) c, old; \ c = atomic_read(v); \ for (;;) { \ if (unlikely(c == (u))) \
include/asm-x86_64/local.h +134 -1
144 lines not on this page · raw message
include/asm-powerpc/local.h +340 -1
341 lines not on this page · raw message
include/asm-mips/local.h +1 -60
61 lines not on this page · raw message
include/asm-generic/local.h +13 -0
@@ -33,6 +33,19 @@ #define local_dec(l) atomic_long_dec(&(l#define local_add (i,l) atomic_long_add((i),(&(l)->a))#define local_sub (i,l) atomic_long_sub((i),(&(l)->a)) +#define local_sub_and_test (i, l) atomic_long_sub_and_test((i), (&(l)->a)) +#define local_dec_and_test (l) atomic_long_dec_and_test(&(l)->a) +#define local_inc_and_test (l) atomic_long_inc_and_test(&(l)->a) +#define local_add_negative (i, l) atomic_long_add_negative((i), (&(l)->a)) +#define local_add_return (i, l) atomic_long_add_return((i), (&(l)->a)) +#define local_sub_return (i, l) atomic_long_sub_return((i), (&(l)->a)) +#define local_inc_return (l) atomic_long_inc_return(&(l)->a) + +#define local_cmpxchg (l, old, new) atomic_long_cmpxchg((&(l)->a), (old), (new)) +#define local_xchg (l, new) atomic_long_xchg((&(l)->a), (new)) +#define local_add_unless (l, a, u) atomic_long_add_unless((&(l)->a), (a), (u)) +#define local_inc_not_zero (l) atomic_long_inc_not_zero(&(l)->a) +/* Non-atomic variants, ie. preemption disabled and won't be touched * in interrupt, etc. Some archs can optimize this case well. */ #define __local_inc (l) local_set((l), local_read(l) + 1)
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 2/2] local.h modifications
Hi,
I also completed the support for all atomic operations in local.h for all architectures. The local_t type is now identical on each architectures : it contains an atomic_long_t field, just like the asm-generic implementation.
Please review.
Mathieu
---BEGIN---
include/asm-alpha/local.h +102 -24
@@ -4,37 +4,115 @@ #define _ALPHA_LOCAL_H#include <linux/percpu.h> #include <asm/atomic.h> -typedef atomic64_t local_t ; +typedef struct +{ +atomic_long_t a; +}local_t ; -#define LOCAL_INIT (i) ATOMIC64_INIT(i) -#define local_read (v) atomic64_read(v) -#define local_set (v,i) atomic64_set(v,i) +#define LOCAL_INIT (i) { ATOMIC_LONG_INIT(i) } +#define local_read (l) atomic_long_read(&(l)->a) +#define local_set (l,i) atomic_long_set(&(l)->a, (i)) +#define local_inc (l) atomic_long_inc(&(l)->a) +#define local_dec (l) atomic_long_dec(&(l)->a) +#define local_add (i,l) atomic_long_add((i),(&(l)->a)) +#define local_sub (i,l) atomic_long_sub((i),(&(l)->a)) -#define local_inc (v) atomic64_inc(v) -#define local_dec (v) atomic64_dec(v) -#define local_add (i, v) atomic64_add(i, v) -#define local_sub (i, v) atomic64_sub(i, v) +static __inline__long local_add_return (long i,local_t * l) +{ +long temp, result; + __asm__ __volatile__( +"1: ldq_l %0,%1\n" +" addq %0,%3,%2\n" +" addq %0,%3,%0\n" +" stq_c %0,%1\n" +" beq %0,2f\n" +".subsection 2\n" +"2: br 1b\n" +".previous" + :"=&r" (temp),"=m" (l->a.counter),"=&r" (result) + :"Ir" (i),"m" (l->a.counter) :"memory" ); +return result; +} -#define __local_inc (v) ((v)->counter++) -#define __local_dec (v) ((v)->counter++) -#define __local_add (i,v) ((v)->counter+=(i)) -#define __local_sub (i,v) ((v)->counter-=(i)) +static __inline__long local_sub_return (long i,local_t * v) +{ +long temp, result; + __asm__ __volatile__( +"1: ldq_l %0,%1\n" +" subq %0,%3,%2\n" +" subq %0,%3,%0\n" +" stq_c %0,%1\n" +" beq %0,2f\n" +".subsection 2\n" +"2: br 1b\n" +".previous" + :"=&r" (temp),"=m" (l->a.counter),"=&r" (result) + :"Ir" (i),"m" (l->a.counter) :"memory" ); +return result; +} + +#define local_cmpxchg (l, old, new) \ + ((long)cmpxchg_local(&((l)->a.counter), old, new)) +#define local_xchg (l, new) (xchg_local(&((l)->a.counter), new)) + +/** +* local_add_unless - add unless the number is a given value +* @l: pointer of type local_t +* @a: the amount to add to l... +* @u: ...unless l is equal to u. +* +* Atomically adds @a to @l, so long as it was not @u. +* Returns non-zero if @l was not @u, and zero otherwise. +*/ +#define local_add_unless (l, a, u) \ +({ \ + long c, old; \ + c = local_read(l); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = local_cmpxchg((l), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define local_inc_not_zero (l) local_add_unless((l), 1, 0) + +#define local_add_negative (a, l) (local_add_return((a), (l)) < 0) + +#define local_dec_return (l) local_sub_return(1,(l)) + +#define local_inc_return (l) local_add_return(1,(l)) + +#define local_sub_and_test (i,l) (local_sub_return((i), (l)) == 0) + +#define local_inc_and_test (l) (local_add_return(1, (l)) == 0) + +#define local_dec_and_test (l) (local_sub_return(1, (l)) == 0) + +/* Verify if faster than atomic ops */ +#define __local_inc (l) ((l)->a.counter++) +#define __local_dec (l) ((l)->a.counter++) +#define __local_add (i,l) ((l)->a.counter+=(i)) +#define __local_sub (i,l) ((l)->a.counter-=(i))/* Use these for per-cpu local_t variables: on some archs they are * much more efficient than these naive implementations. Note they take * a variable, not an address. */ -#define cpu_local_read (v) local_read(&__get_cpu_var(v)) -#define cpu_local_set (v, i) local_set(&__get_cpu_var(v), (i)) - -#define cpu_local_inc (v) local_inc(&__get_cpu_var(v)) -#define cpu_local_dec (v) local_dec(&__get_cpu_var(v)) -#define cpu_local_add (i, v) local_add((i), &__get_cpu_var(v)) -#define cpu_local_sub (i, v) local_sub((i), &__get_cpu_var(v)) - -#define __cpu_local_inc (v) __local_inc(&__get_cpu_var(v)) -#define __cpu_local_dec (v) __local_dec(&__get_cpu_var(v)) -#define __cpu_local_add (i, v) __local_add((i), &__get_cpu_var(v)) -#define __cpu_local_sub (i, v) __local_sub((i), &__get_cpu_var(v)) +#define cpu_local_read (l) local_read(&__get_cpu_var(l)) +#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) + +#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) +#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) +#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) +#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l)) + +#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) +#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) +#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) +#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l))#endif /* _ALPHA_LOCAL_H */
include/asm-generic/local.h +23 -10
@@ -33,6 +33,19 @@ #define local_dec(l) atomic_long_dec(&(l#define local_add (i,l) atomic_long_add((i),(&(l)->a))#define local_sub (i,l) atomic_long_sub((i),(&(l)->a)) +#define local_sub_and_test (i, l) atomic_long_sub_and_test((i), (&(l)->a)) +#define local_dec_and_test (l) atomic_long_dec_and_test(&(l)->a) +#define local_inc_and_test (l) atomic_long_inc_and_test(&(l)->a) +#define local_add_negative (i, l) atomic_long_add_negative((i), (&(l)->a)) +#define local_add_return (i, l) atomic_long_add_return((i), (&(l)->a)) +#define local_sub_return (i, l) atomic_long_sub_return((i), (&(l)->a)) +#define local_inc_return (l) atomic_long_inc_return(&(l)->a) + +#define local_cmpxchg (l, old, new) atomic_long_cmpxchg((&(l)->a), (old), (new)) +#define local_xchg (l, new) atomic_long_xchg((&(l)->a), (new)) +#define local_add_unless (l, a, u) atomic_long_add_unless((&(l)->a), (a), (u)) +#define local_inc_not_zero (l) atomic_long_inc_not_zero(&(l)->a) +/* Non-atomic variants, ie. preemption disabled and won't be touched * in interrupt, etc. Some archs can optimize this case well. */ #define __local_inc (l) local_set((l), local_read(l) + 1)
@@ -44,19 +57,19 @@ #define __local_sub(i,l) local_set((l),* much more efficient than these naive implementations. Note they take * a variable (eg. mystruct.foo), not an address. */ -#define cpu_local_read (v) local_read(&__get_cpu_var(v)) -#define cpu_local_set (v, i) local_set(&__get_cpu_var(v), (i)) -#define cpu_local_inc (v) local_inc(&__get_cpu_var(v)) -#define cpu_local_dec (v) local_dec(&__get_cpu_var(v)) -#define cpu_local_add (i, v) local_add((i), &__get_cpu_var(v)) -#define cpu_local_sub (i, v) local_sub((i), &__get_cpu_var(v)) +#define cpu_local_read (l) local_read(&__get_cpu_var(l)) +#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) +#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) +#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) +#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) +#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l))/* Non-atomic increments, ie. preemption disabled and won't be touched * in interrupt, etc. Some archs can optimize this case well. */ -#define __cpu_local_inc (v) __local_inc(&__get_cpu_var(v)) -#define __cpu_local_dec (v) __local_dec(&__get_cpu_var(v)) -#define __cpu_local_add (i, v) __local_add((i), &__get_cpu_var(v)) -#define __cpu_local_sub (i, v) __local_sub((i), &__get_cpu_var(v)) +#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) +#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) +#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) +#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l))#endif /* _ASM_GENERIC_LOCAL_H */
include/asm-i386/local.h +178 -27
252 lines not on this page · raw message
include/asm-ia64/local.h +1 -50
51 lines not on this page · raw message
include/asm-mips/local.h +1 -60
61 lines not on this page · raw message
include/asm-parisc/local.h +1 -40
41 lines not on this page · raw message
include/asm-powerpc/local.h +345 -1
346 lines not on this page · raw message
include/asm-s390/local.h +1 -58
59 lines not on this page · raw message
include/asm-sparc64/local.h +1 -40
41 lines not on this page · raw message
include/asm-x86_64/local.h +165 -31
241 lines not on this page · raw message
---END---
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
Re: [PATCH 2/2] local.h modifications for 2.6.19
Hi,
Here is the complete support for all atomic operations in local.h for all architectures. The local_t type is now identical on each architectures : it contains an atomic_long_t field, just like the asm-generic implementation.
This patch applies on 2.6.19. It is currently useful to my LTTng tracer, which is not in the mainline. I could foresee other users : any frequently used per-cpu atomic counter that is used by any code executing asynchronously on a processor. Therefore, I think it should be considered for inclusion for 2.6.20.
Regards,
Mathieu
---BEGIN---
include/asm-alpha/local.h +102 -24
@@ -4,37 +4,115 @@ #define _ALPHA_LOCAL_H#include <linux/percpu.h> #include <asm/atomic.h> -typedef atomic64_t local_t ; +typedef struct +{ +atomic_long_t a; +}local_t ; -#define LOCAL_INIT (i) ATOMIC64_INIT(i) -#define local_read (v) atomic64_read(v) -#define local_set (v,i) atomic64_set(v,i) +#define LOCAL_INIT (i) { ATOMIC_LONG_INIT(i) } +#define local_read (l) atomic_long_read(&(l)->a) +#define local_set (l,i) atomic_long_set(&(l)->a, (i)) +#define local_inc (l) atomic_long_inc(&(l)->a) +#define local_dec (l) atomic_long_dec(&(l)->a) +#define local_add (i,l) atomic_long_add((i),(&(l)->a)) +#define local_sub (i,l) atomic_long_sub((i),(&(l)->a)) -#define local_inc (v) atomic64_inc(v) -#define local_dec (v) atomic64_dec(v) -#define local_add (i, v) atomic64_add(i, v) -#define local_sub (i, v) atomic64_sub(i, v) +static __inline__long local_add_return (long i,local_t * l) +{ +long temp, result; + __asm__ __volatile__( +"1: ldq_l %0,%1\n" +" addq %0,%3,%2\n" +" addq %0,%3,%0\n" +" stq_c %0,%1\n" +" beq %0,2f\n" +".subsection 2\n" +"2: br 1b\n" +".previous" + :"=&r" (temp),"=m" (l->a.counter),"=&r" (result) + :"Ir" (i),"m" (l->a.counter) :"memory" ); +return result; +} -#define __local_inc (v) ((v)->counter++) -#define __local_dec (v) ((v)->counter++) -#define __local_add (i,v) ((v)->counter+=(i)) -#define __local_sub (i,v) ((v)->counter-=(i)) +static __inline__long local_sub_return (long i,local_t * v) +{ +long temp, result; + __asm__ __volatile__( +"1: ldq_l %0,%1\n" +" subq %0,%3,%2\n" +" subq %0,%3,%0\n" +" stq_c %0,%1\n" +" beq %0,2f\n" +".subsection 2\n" +"2: br 1b\n" +".previous" + :"=&r" (temp),"=m" (l->a.counter),"=&r" (result) + :"Ir" (i),"m" (l->a.counter) :"memory" ); +return result; +} + +#define local_cmpxchg (l, old, new) \ + ((long)cmpxchg_local(&((l)->a.counter), old, new)) +#define local_xchg (l, new) (xchg_local(&((l)->a.counter), new)) + +/** +* local_add_unless - add unless the number is a given value +* @l: pointer of type local_t +* @a: the amount to add to l... +* @u: ...unless l is equal to u. +* +* Atomically adds @a to @l, so long as it was not @u. +* Returns non-zero if @l was not @u, and zero otherwise. +*/ +#define local_add_unless (l, a, u) \ +({ \ + long c, old; \ + c = local_read(l); \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = local_cmpxchg((l), c, c + (a)); \ + if (likely(old == c)) \ + break; \ + c = old; \ + } \ + c != (u); \ +}) +#define local_inc_not_zero (l) local_add_unless((l), 1, 0) + +#define local_add_negative (a, l) (local_add_return((a), (l)) < 0) + +#define local_dec_return (l) local_sub_return(1,(l)) + +#define local_inc_return (l) local_add_return(1,(l)) + +#define local_sub_and_test (i,l) (local_sub_return((i), (l)) == 0) + +#define local_inc_and_test (l) (local_add_return(1, (l)) == 0) + +#define local_dec_and_test (l) (local_sub_return(1, (l)) == 0) + +/* Verify if faster than atomic ops */ +#define __local_inc (l) ((l)->a.counter++) +#define __local_dec (l) ((l)->a.counter++) +#define __local_add (i,l) ((l)->a.counter+=(i)) +#define __local_sub (i,l) ((l)->a.counter-=(i))/* Use these for per-cpu local_t variables: on some archs they are * much more efficient than these naive implementations. Note they take * a variable, not an address. */ -#define cpu_local_read (v) local_read(&__get_cpu_var(v)) -#define cpu_local_set (v, i) local_set(&__get_cpu_var(v), (i)) - -#define cpu_local_inc (v) local_inc(&__get_cpu_var(v)) -#define cpu_local_dec (v) local_dec(&__get_cpu_var(v)) -#define cpu_local_add (i, v) local_add((i), &__get_cpu_var(v)) -#define cpu_local_sub (i, v) local_sub((i), &__get_cpu_var(v)) - -#define __cpu_local_inc (v) __local_inc(&__get_cpu_var(v)) -#define __cpu_local_dec (v) __local_dec(&__get_cpu_var(v)) -#define __cpu_local_add (i, v) __local_add((i), &__get_cpu_var(v)) -#define __cpu_local_sub (i, v) __local_sub((i), &__get_cpu_var(v)) +#define cpu_local_read (l) local_read(&__get_cpu_var(l)) +#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) + +#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) +#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) +#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) +#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l)) + +#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) +#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) +#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) +#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l))#endif /* _ALPHA_LOCAL_H */
include/asm-generic/local.h +23 -10
@@ -33,6 +33,19 @@ #define local_dec(l) atomic_long_dec(&(l#define local_add (i,l) atomic_long_add((i),(&(l)->a))#define local_sub (i,l) atomic_long_sub((i),(&(l)->a)) +#define local_sub_and_test (i, l) atomic_long_sub_and_test((i), (&(l)->a)) +#define local_dec_and_test (l) atomic_long_dec_and_test(&(l)->a) +#define local_inc_and_test (l) atomic_long_inc_and_test(&(l)->a) +#define local_add_negative (i, l) atomic_long_add_negative((i), (&(l)->a)) +#define local_add_return (i, l) atomic_long_add_return((i), (&(l)->a)) +#define local_sub_return (i, l) atomic_long_sub_return((i), (&(l)->a)) +#define local_inc_return (l) atomic_long_inc_return(&(l)->a) + +#define local_cmpxchg (l, old, new) atomic_long_cmpxchg((&(l)->a), (old), (new)) +#define local_xchg (l, new) atomic_long_xchg((&(l)->a), (new)) +#define local_add_unless (l, a, u) atomic_long_add_unless((&(l)->a), (a), (u)) +#define local_inc_not_zero (l) atomic_long_inc_not_zero(&(l)->a) +/* Non-atomic variants, ie. preemption disabled and won't be touched * in interrupt, etc. Some archs can optimize this case well. */ #define __local_inc (l) local_set((l), local_read(l) + 1)
@@ -44,19 +57,19 @@ #define __local_sub(i,l) local_set((l),* much more efficient than these naive implementations. Note they take * a variable (eg. mystruct.foo), not an address. */ -#define cpu_local_read (v) local_read(&__get_cpu_var(v)) -#define cpu_local_set (v, i) local_set(&__get_cpu_var(v), (i)) -#define cpu_local_inc (v) local_inc(&__get_cpu_var(v)) -#define cpu_local_dec (v) local_dec(&__get_cpu_var(v)) -#define cpu_local_add (i, v) local_add((i), &__get_cpu_var(v)) -#define cpu_local_sub (i, v) local_sub((i), &__get_cpu_var(v)) +#define cpu_local_read (l) local_read(&__get_cpu_var(l)) +#define cpu_local_set (l, i) local_set(&__get_cpu_var(l), (i)) +#define cpu_local_inc (l) local_inc(&__get_cpu_var(l)) +#define cpu_local_dec (l) local_dec(&__get_cpu_var(l)) +#define cpu_local_add (i, l) local_add((i), &__get_cpu_var(l)) +#define cpu_local_sub (i, l) local_sub((i), &__get_cpu_var(l))/* Non-atomic increments, ie. preemption disabled and won't be touched * in interrupt, etc. Some archs can optimize this case well. */ -#define __cpu_local_inc (v) __local_inc(&__get_cpu_var(v)) -#define __cpu_local_dec (v) __local_dec(&__get_cpu_var(v)) -#define __cpu_local_add (i, v) __local_add((i), &__get_cpu_var(v)) -#define __cpu_local_sub (i, v) __local_sub((i), &__get_cpu_var(v)) +#define __cpu_local_inc (l) __local_inc(&__get_cpu_var(l)) +#define __cpu_local_dec (l) __local_dec(&__get_cpu_var(l)) +#define __cpu_local_add (i, l) __local_add((i), &__get_cpu_var(l)) +#define __cpu_local_sub (i, l) __local_sub((i), &__get_cpu_var(l))#endif /* _ASM_GENERIC_LOCAL_H */
include/asm-i386/local.h +178 -27
252 lines not on this page · raw message
include/asm-ia64/local.h +1 -50
51 lines not on this page · raw message
include/asm-mips/local.h +1 -60
61 lines not on this page · raw message
include/asm-parisc/local.h +1 -40
41 lines not on this page · raw message
include/asm-powerpc/local.h +345 -1
346 lines not on this page · raw message
include/asm-s390/local.h +1 -58
59 lines not on this page · raw message
include/asm-sparc64/local.h +1 -40
41 lines not on this page · raw message
include/asm-x86_64/local.h +165 -31
241 lines not on this page · raw message
---END---
OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68