locking/atomic: correct (cmp)xhcg instrumentation
Mark Rutland
reviewerPATCH 1/1 locking/atomic: correct (cmp)xhcg instrumentation
All xchg() and cmpxchg() ops are atomic RMWs, but currently we instrument these with instrument_atomic_write() rather than instrument_atomic_read_write(), missing the read aspect.
Similarly, all try_cmpxchg() ops are non-atomic RMWs on *oldp, but we instrument these accesses with instrument_atomic_write() rather than instrument_read_write(), missing the read aspect and erroneously marking these as atomic.
Fix the instrumentation for both points.
--- include/linux/atomic/atomic-instrumented.h | 76 +++++++++++----------- scripts/atomic/gen-atomic-instrumented.sh | 6 +- 2 files changed, 41 insertions(+), 41 deletions(-)
Note: this is based on tip locking/core, with the head commit being:
561b081f19655a46 ("locking/x86: Define arch_try_cmpxchg_local")
Mark.
include/linux/atomic/atomic-instrumented.h +38 -38
@@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg (__ai_ptr, __VA_ARGS__); \ })#define xchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg_release (__ai_ptr, __VA_ARGS__); \ })#define xchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64 (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64 (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ })
@@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_sync_cmpxchg (__ai_ptr, __VA_ARGS__); \ })
@@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \arch_cmpxchg_double (__ai_ptr, __VA_ARGS__); \ })
@@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)#define cmpxchg_double_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ })#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 97fe4d79aa058d2164df824632cbc4f716d2a407 +// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
scripts/atomic/gen-atomic-instrumented.sh +3 -3
@@ -104,8 +104,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF
@@ -119,7 +119,7 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ }) EOF
Signature · 1 line
2.30.2
[tip: locking/core] locking/atomic: Correct (cmp)xhcg instrumentation
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 93eff603d6a2bc1895eeb7063dbd0661bb760b74 Gitweb: https://git.kernel.org/tip/93eff603d6a2bc1895eeb7063dbd0661bb760b74 Author: Mark Rutland <[email protected]> AuthorDate: Thu, 13 Apr 2023 17:06:44 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Apr 2023 16:08:24 +02:00
locking/atomic: Correct (cmp)xhcg instrumentation
All xchg() and cmpxchg() ops are atomic RMWs, but currently we instrument these with instrument_atomic_write() rather than instrument_atomic_read_write(), missing the read aspect.
Similarly, all try_cmpxchg() ops are non-atomic RMWs on *oldp, but we instrument these accesses with instrument_atomic_write() rather than instrument_read_write(), missing the read aspect and erroneously marking these as atomic.
Fix the instrumentation for both points.
--- include/linux/atomic/atomic-instrumented.h | 76 ++++++++++----------- scripts/atomic/gen-atomic-instrumented.sh | 6 +- 2 files changed, 41 insertions(+), 41 deletions(-)
include/linux/atomic/atomic-instrumented.h +38 -38
@@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg (__ai_ptr, __VA_ARGS__); \ })#define xchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg_release (__ai_ptr, __VA_ARGS__); \ })#define xchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64 (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64 (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ })
@@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_sync_cmpxchg (__ai_ptr, __VA_ARGS__); \ })
@@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \arch_cmpxchg_double (__ai_ptr, __VA_ARGS__); \ })
@@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)#define cmpxchg_double_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ })#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 97fe4d79aa058d2164df824632cbc4f716d2a407 +// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
scripts/atomic/gen-atomic-instrumented.sh +3 -3
@@ -104,8 +104,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF
@@ -119,7 +119,7 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ }) EOF
Quoted 10 lines · Mark Rutland
All
xchg()andcmpxchg()ops are atomic RMWs, but currently we instrument these withinstrument_atomic_write()rather thaninstrument_atomic_read_write(), missing the read aspect.Similarly, all
try_cmpxchg()ops are non-atomic RMWs on *oldp, but we instrument these accesses withinstrument_atomic_write()rather thaninstrument_read_write(), missing the read aspect and erroneously marking these as atomic.
Quoted 7 lines
Fix the instrumentation for both points.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Marco Elver <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Will Deacon <[email protected]>
Thanks!
Quoted 300 lines
--- include/linux/atomic/atomic-instrumented.h | 76 +++++++++++----------- scripts/atomic/gen-atomic-instrumented.sh | 6 +- 2 files changed, 41 insertions(+), 41 deletions(-)Note: this is based on tip locking/core, with the head commit being:
561b081f19655a46("locking/x86: Definearch_try_cmpxchg_local")Mark.
include/linux/atomic/atomic-instrumented.h +38
-38@@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg (__ai_ptr, __VA_ARGS__); \ })#define xchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ })@@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg_release (__ai_ptr, __VA_ARGS__); \ })#define xchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ })@@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ })@@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ })@@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64 (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ })@@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ })@@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64 (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ })@@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_sync_cmpxchg (__ai_ptr, __VA_ARGS__); \ })@@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })@@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \arch_cmpxchg_double (__ai_ptr, __VA_ARGS__); \ })@@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)#define cmpxchg_double_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ })#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 97fe4d79aa058d2164df824632cbc4f716d2a407 +// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e scripts/atomic/gen-atomic-instrumented.sh +3
-4@@ -104,8 +104,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF@@ -119,7 +119,7 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ }) EOF --2.30.2
[tip: locking/core] locking/atomic: Correct (cmp)xchg() instrumentation
In reply to Mark RutlandThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 3caea4fdfa78383297091792210eb8a2e2fb0b49 Gitweb: https://git.kernel.org/tip/3caea4fdfa78383297091792210eb8a2e2fb0b49 Author: Mark Rutland <[email protected]> AuthorDate: Thu, 13 Apr 2023 17:06:44 +01:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 29 Apr 2023 08:52:56 +02:00
locking/atomic: Correct (cmp)xchg() instrumentation
All xchg() and cmpxchg() ops are atomic RMWs, but currently we instrument these with instrument_atomic_write() rather than instrument_atomic_read_write(), missing the read aspect.
Similarly, all try_cmpxchg() ops are non-atomic RMWs on *oldp, but we instrument these accesses with instrument_atomic_write() rather than instrument_read_write(), missing the read aspect and erroneously marking these as atomic.
Fix the instrumentation for both points.
--- include/linux/atomic/atomic-instrumented.h | 76 ++++++++++----------- scripts/atomic/gen-atomic-instrumented.sh | 6 +- 2 files changed, 41 insertions(+), 41 deletions(-)
include/linux/atomic/atomic-instrumented.h +38 -38
@@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg (__ai_ptr, __VA_ARGS__); \ })#define xchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg_release (__ai_ptr, __VA_ARGS__); \ })#define xchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64 (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64 (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ })
@@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_sync_cmpxchg (__ai_ptr, __VA_ARGS__); \ })
@@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \arch_cmpxchg_double (__ai_ptr, __VA_ARGS__); \ })
@@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)#define cmpxchg_double_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ })#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 97fe4d79aa058d2164df824632cbc4f716d2a407 +// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
scripts/atomic/gen-atomic-instrumented.sh +3 -3
@@ -104,8 +104,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF
@@ -119,7 +119,7 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ }) EOF
[tip: locking/core] locking/atomic: Correct (cmp)xchg() instrumentation
In reply to Mark RutlandThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: ec570320b09f76d52819e60abdccf372658216b6 Gitweb: https://git.kernel.org/tip/ec570320b09f76d52819e60abdccf372658216b6 Author: Mark Rutland <[email protected]> AuthorDate: Thu, 13 Apr 2023 17:06:44 +01:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Sat, 29 Apr 2023 09:09:31 +02:00
locking/atomic: Correct (cmp)xchg() instrumentation
All xchg() and cmpxchg() ops are atomic RMWs, but currently we instrument these with instrument_atomic_write() rather than instrument_atomic_read_write(), missing the read aspect.
Similarly, all try_cmpxchg() ops are non-atomic RMWs on *oldp, but we instrument these accesses with instrument_atomic_write() rather than instrument_read_write(), missing the read aspect and erroneously marking these as atomic.
Fix the instrumentation for both points.
--- include/linux/atomic/atomic-instrumented.h | 76 ++++++++++----------- scripts/atomic/gen-atomic-instrumented.sh | 6 +- 2 files changed, 41 insertions(+), 41 deletions(-)
include/linux/atomic/atomic-instrumented.h +38 -38
@@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg (__ai_ptr, __VA_ARGS__); \ })#define xchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_xchg_release (__ai_ptr, __VA_ARGS__); \ })#define xchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64 (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_acquire (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ })
@@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_cmpxchg64_release (__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_relaxed (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ })
@@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64 (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_acquire (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v)typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \kcsan_release (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_release (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ })#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ })
@@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \arch_sync_cmpxchg (__ai_ptr, __VA_ARGS__); \ })
@@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \typeof (oldp) __ai_oldp = (oldp); \ -instrument_atomic_write (__ai_ptr,sizeof (*__ai_ptr)); \ -instrument_atomic_write (__ai_oldp,sizeof (*__ai_oldp)); \ +instrument_atomic_read_write (__ai_ptr,sizeof (*__ai_ptr)); \ +instrument_read_write (__ai_oldp,sizeof (*__ai_oldp)); \arch_try_cmpxchg64_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ })
@@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \typeof (ptr) __ai_ptr = (ptr); \kcsan_mb (); \ -instrument_atomic_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \ +instrument_atomic_read_write (__ai_ptr,2 *sizeof (*__ai_ptr)); \arch_cmpxchg_double (__ai_ptr, __VA_ARGS__); \ })
@@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v)#define cmpxchg_double_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ })#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 97fe4d79aa058d2164df824632cbc4f716d2a407 +// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
scripts/atomic/gen-atomic-instrumented.sh +3 -3
@@ -104,8 +104,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF
@@ -119,7 +119,7 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ }) EOF