ia64: Fix atomic ops vs memory barriers
Subsystems (3)
Peter Zijlstra
maintainerRFC PATCH 1/1 ia64: Fix atomic ops vs memory barriers
The IA64 SDM V1-4.4.7 describes the memory ordering rules of IA64.
It states that the cmpxchg.{rel,acq} really have release and acquire semantics and that xchg has acquire semantics.
Despite this the ia64 atomic_t code says that all atomic ops are fully serialized, and its smp_mb__{before,after}_atomic_{inc,dec} are barrier().
OTOH the atomic bitops, which are identically implemented (using cmpxchg.acq), state that they have acquire semantics and set smp_mb__before_clear_bit() to smp_mb().
Now either the SDM and the bitops are right, which means the atomic_t, cmpxchg() and xchg() implementations are wrong, or the SDM is wrong and cmpxchg.acq is actually fully serialized after all and the bitops implementation is wrong, not to mention there's a big fat comment missing someplace.
The below patch assumes the SDM is right (TM), and fixes the atomic_t, cmpxchg() and xchg() implementations by inserting a mf before the cmpxchg.acq (or xchg).
This way loads/stores before the mf stay there, loads/stores after the cmpxchg.acq also stay after and the entire thing acts as a fully serialized op the way it is supposed to.
For things like ia64_atomic_sub() which is a cmpxchg.acq loop with an unordered load in, it still works because there is a control dependency upon the cmpxchg op, therefore the load cannot escape.
--- arch/ia64/include/asm/atomic.h | 31 +++++++++++++++++++++++-------- arch/ia64/include/asm/bitops.h | 25 +++++++++++++++++++++++-- arch/ia64/include/uapi/asm/cmpxchg.h | 13 ++++++++----- 3 files changed, 54 insertions(+), 15 deletions(-)
arch/ia64/include/asm/atomic.h +23 -8
@@ -15,6 +15,7 @@#include <linux/types.h> #include <asm/intrinsics.h> +#include <asm/barrier.h> #define ATOMIC_INIT (i) { (i) }
@@ -125,6 +126,7 @@ static __inline__ long atomic64_add_unless(atomic64_t *v, long a, long u)#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \ + ia64_mf(); \ (__builtin_constant_p(i) \ && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \
@@ -137,6 +139,7 @@ static __inline__ long atomic64_add_unless(atomic64_t *v, long a, long u)#define atomic64_add_return (i,v) \ ({ \ long __ia64_aar_i = (i); \ + ia64_mf(); \ (__builtin_constant_p(i) \ && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \
@@ -162,7 +165,7 @@ atomic64_add_negative (__s64 i, atomic64_t *v)return atomic64_add_return (i, v) <0 ; } -#define atomic_sub_return (i,v) \ +#define __atomic_sub_return (i,v) \ ({ \ int __ia64_asr_i = (i); \ (__builtin_constant_p(i) \
@@ -174,7 +177,13 @@ atomic64_add_negative (__s64 i, atomic64_t *v) : ia64_atomic_sub(__ia64_asr_i, v); \ }) -#define atomic64_sub_return (i,v) \ +#define atomic_sub_return (i,v) \ +({ \ + ia64_mf(); \ + __atomic_sub_return((i), (v)); \ +}) + +#define __atomic64_sub_return (i,v) \ ({ \ long __ia64_asr_i = (i); \ (__builtin_constant_p(i) \
@@ -186,6 +195,12 @@ atomic64_add_negative (__s64 i, atomic64_t *v) : ia64_atomic64_sub(__ia64_asr_i, v); \ }) +#define atomic64_sub_return (i,v) \ +({ \ + ia64_mf(); \ + __atomic64_sub_return((i), (v)) \ +}) +#define atomic_dec_return (v) atomic_sub_return(1, (v))#define atomic_inc_return (v) atomic_add_return(1, (v))#define atomic64_dec_return (v) atomic64_sub_return(1, (v))
@@ -198,20 +213,20 @@ atomic64_add_negative (__s64 i, atomic64_t *v)#define atomic64_dec_and_test (v) (atomic64_sub_return(1, (v)) == 0)#define atomic64_inc_and_test (v) (atomic64_add_return(1, (v)) == 0) -#define atomic_add (i,v) atomic_add_return((i), (v)) -#define atomic_sub (i,v) atomic_sub_return((i), (v)) +#define atomic_add (i,v) (void)__atomic_add_return((i), (v)) +#define atomic_sub (i,v) (void)__atomic_sub_return((i), (v))#define atomic_inc (v) atomic_add(1, (v))#define atomic_dec (v) atomic_sub(1, (v)) -#define atomic64_add (i,v) atomic64_add_return((i), (v)) -#define atomic64_sub (i,v) atomic64_sub_return((i), (v)) +#define atomic64_add (i,v) (void)__atomic64_add_return((i), (v)) +#define atomic64_sub (i,v) (void)__atomic64_sub_return((i), (v))#define atomic64_inc (v) atomic64_add(1, (v))#define atomic64_dec (v) atomic64_sub(1, (v))/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec () barrier() +#define smp_mb__before_atomic_dec () smp_mb()#define smp_mb__after_atomic_dec () barrier() -#define smp_mb__before_atomic_inc () barrier() +#define smp_mb__before_atomic_inc () smp_mb()#define smp_mb__after_atomic_inc () barrier()#endif /* _ASM_IA64_ATOMIC_H */
arch/ia64/include/asm/bitops.h +23 -2
@@ -69,7 +69,7 @@ __set_bit (int nr, volatile void *addr)* clear_bit() has "acquire" semantics. */ #define smp_mb__before_clear_bit () smp_mb() -#define smp_mb__after_clear_bit () do {/* skip */ ; }while (0 ) +#define smp_mb__after_clear_bit () barrier()/** * clear_bit - Clears a bit in memory
@@ -208,6 +208,8 @@ test_and_set_bit (int nr, volatile void *addr)volatile __u32 *m;CMPXCHG_BUGCHECK_DECL +smp_mb (); + m = (volatile __u32 *) addr + (nr >>5 ); bit =1 << (nr &31 ); do {
@@ -225,7 +227,22 @@ test_and_set_bit (int nr, volatile void *addr)* * This is the same as test_and_set_bit on ia64 */ -#define test_and_set_bit_lock test_and_set_bit +static __inline__int +test_and_set_bit_lock (int nr,volatile void *addr) +{ +__u32 bit, old, new; +volatile __u32 *m; +CMPXCHG_BUGCHECK_DECL + + m = (volatile __u32 *) addr + (nr >>5 ); + bit =1 << (nr &31 ); +do { +CMPXCHG_BUGCHECK (m); + old = *m; + new = old | bit; + }while (cmpxchg_acq (m, old, new) != old); +return (old & bit) !=0 ; +}/** * __test_and_set_bit - Set a bit and return its old value
@@ -262,6 +279,8 @@ test_and_clear_bit (int nr, volatile void *addr)volatile __u32 *m;CMPXCHG_BUGCHECK_DECL +smp_mb (); + m = (volatile __u32 *) addr + (nr >>5 ); mask = ~(1 << (nr &31 )); do {
@@ -307,6 +326,8 @@ test_and_change_bit (int nr, volatile void *addr)volatile __u32 *m;CMPXCHG_BUGCHECK_DECL +smp_mb (); + m = (volatile __u32 *) addr + (nr >>5 ); bit = (1 << (nr &31 )); do {
arch/ia64/include/uapi/asm/cmpxchg.h +8 -5
@@ -53,7 +53,10 @@ extern void ia64_xchg_called_with_bad_pointer(void); })#define xchg (ptr, x) \ -((__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)))) +({ \ + ia64_mf(); \ + (__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)))) \ +)}/* * Atomic compare and exchange. Compare OLD with MEM, if identical,
@@ -119,11 +122,11 @@ extern long ia64_cmpxchg_called_with_bad_pointer(void);ia64_cmpxchg (rel, (ptr), (o), (n),sizeof (*(ptr))) /* for compatibility with other platforms: */ -#define cmpxchg (ptr, o, n) cmpxchg_acq((ptr), (o), (n)) -#define cmpxchg64 (ptr, o, n) cmpxchg_acq((ptr), (o), (n)) +#define cmpxchg (ptr, o, n) ({ ia64_mf(); cmpxchg_acq((ptr), (o), (n)); }) +#define cmpxchg64 (ptr, o, n) ({ ia64_mf(); cmpxchg_acq((ptr), (o), (n)); }) -#define cmpxchg_local cmpxchg -#define cmpxchg64_local cmpxchg64 +#define cmpxchg_local cmpxchg_acq +#define cmpxchg64_local cmpxchg_acq#ifdef CONFIG_IA64_DEBUG_CMPXCHG # define CMPXCHG_BUGCHECK_DECL int _cmpxchg_bugcheck_count = 128;
Peter Zijlstra
maintainerQuoted 25 lines · Peter Zijlstra
The IA64 SDM V1-4.4.7 describes the memory ordering rules of IA64.
It states that the cmpxchg.{rel,acq} really have release and acquire semantics and that xchg has acquire semantics.
Despite this the ia64
atomic_tcode says that all atomic ops are fully serialized, and its smp_mb__{before,after}_atomic_{inc,dec} arebarrier().OTOH the atomic bitops, which are identically implemented (using cmpxchg.acq), state that they have acquire semantics and set
smp_mb__before_clear_bit()tosmp_mb().Now either the SDM and the bitops are right, which means the
atomic_t,cmpxchg()andxchg()implementations are wrong, or the SDM is wrong and cmpxchg.acq is actually fully serialized after all and the bitops implementation is wrong, not to mention there's a big fat comment missing someplace.The below patch assumes the SDM is right (TM), and fixes the
atomic_t,cmpxchg()andxchg()implementations by inserting a mf before the cmpxchg.acq (or xchg).
Quoted 8 lines
This way loads/stores before the mf stay there, loads/stores after the cmpxchg.acq also stay after and the entire thing acts as a fully serialized op the way it is supposed to.
For things like
ia64_atomic_sub()which is a cmpxchg.acq loop with an unordered load in, it still works because there is a control dependency upon the cmpxchg op, therefore the load cannot escape.
And assuming all that is true; we also need the below on top.
Because when one writes:
atomic_dec(); smp_mb__after_atomic_dec();
One actually expects a full barrier, not an acquire.
---
arch/ia64/include/asm/atomic.h +2 -2
@@ -225,8 +225,8 @@ atomic64_add_negative (__s64 i, atomic64/* Atomic operations are already serializing */ #define smp_mb__before_atomic_dec () smp_mb() -#define smp_mb__after_atomic_dec () barrier() +#define smp_mb__after_atomic_dec () smp_mb()#define smp_mb__before_atomic_inc () smp_mb() -#define smp_mb__after_atomic_inc () barrier() +#define smp_mb__after_atomic_inc () smp_mb()#endif /* _ASM_IA64_ATOMIC_H */
arch/ia64/include/asm/bitops.h +1 -1
@@ -69,7 +69,7 @@ __set_bit (int nr, volatile void *addr)* clear_bit() has "acquire" semantics. */ #define smp_mb__before_clear_bit () smp_mb() -#define smp_mb__after_clear_bit () barrier() +#define smp_mb__after_clear_bit () smp_mb()/** * clear_bit - Clears a bit in memory
Quoted 4 lines · Peter Zijlstra
The below patch assumes the SDM is right (TM), and fixes the
atomic_t,cmpxchg()andxchg()implementations by inserting a mf before the cmpxchg.acq (or xchg).
You picked the wrong thing to be right. The SDM is wrong.
Last time this came up, Tony explained it thus:
Quoted 6 lines
Quoted 6 lines
Worse still - early processor implementations actually just ignored the acquire/release and did a full fence all the time. Unfortunately this meant a lot of badly written code that used .acq when they really wanted .rel became legacy out in the wild - so when we made a cpu that strictly did the .acq or .rel ... all that code started breaking - so we had to back-pedal and keep the "legacy" behavior of a full fence :-(
and since ia64 is basically on life support as an architecture, we can pretty much agree that the SDM is dead, and the only thing that matters is implementation.
The above quote was strictly in the context of just cmpxchg, though, so it's possible that the "fetchadd" instruction acts differently. I would personally expect it to have the same issues, but let's see what Tony says.. Tony?
Linus
Peter Zijlstra
maintainerQuoted 7 lines · Linus Torvalds
Quoted 4 lines · Peter Zijlstra
The below patch assumes the SDM is right (TM), and fixes the
atomic_t,cmpxchg()andxchg()implementations by inserting a mf before the cmpxchg.acq (or xchg).You picked the wrong thing to be right. The SDM is wrong.
Figured, just my luck :-)
Quoted 8 lines
Last time this came up, Tony explained it thus:
Quoted 6 lines
Quoted 6 lines
Worse still - early processor implementations actually just ignored the acquire/release and did a full fence all the time. Unfortunately this meant a lot of badly written code that used .acq when they really wanted .rel became legacy out in the wild - so when we made a cpu that strictly did the .acq or .rel ... all that code started breaking - so we had to back-pedal and keep the "legacy" behavior of a full fence :-(
That would make a lovely comment near ia64_cmpxchg().
Quoted 8 lines
and since ia64 is basically on life support as an architecture, we can pretty much agree that the SDM is dead, and the only thing that matters is implementation.
The above quote was strictly in the context of just cmpxchg, though, so it's possible that the "fetchadd" instruction acts differently. I would personally expect it to have the same issues, but let's see what Tony says.. Tony?
I would suspect it to be a full fence too, let me do the reverse patch.
---
arch/ia64/include/asm/bitops.h +2 -5
@@ -65,11 +65,8 @@ __set_bit (int nr, volatile void *addr) *((__u32 *) addr + (nr >>5 )) |= (1 << (nr &31 )); } -/* -* clear_bit() has "acquire" semantics. -*/ -#define smp_mb__before_clear_bit () smp_mb() -#define smp_mb__after_clear_bit () do {/* skip */ ; }while (0 ) +#define smp_mb__before_clear_bit () barrier(); +#define smp_mb__after_clear_bit () barrier();/** * clear_bit - Clears a bit in memory
arch/ia64/include/uapi/asm/cmpxchg.h +9 -0
@@ -118,6 +118,15 @@ extern long ia64_cmpxchg_called_with_bad#define cmpxchg_rel (ptr, o, n) \ ia64_cmpxchg(rel, (ptr), (o), (n), sizeof(*(ptr))) +/* +* Worse still - early processor implementations actually just ignored +* the acquire/release and did a full fence all the time. Unfortunately +* this meant a lot of badly written code that used .acq when they really +* wanted .rel became legacy out in the wild - so when we made a cpu +* that strictly did the .acq or .rel ... all that code started breaking - so +* we had to back-pedal and keep the "legacy" behavior of a full fence :-( +*/ +/* for compatibility with other platforms: */ #define cmpxchg (ptr, o, n) cmpxchg_acq((ptr), (o), (n))#define cmpxchg64 (ptr, o, n) cmpxchg_acq((ptr), (o), (n))