locking/atomics: Fixup GENERIC_ATOMIC64 conflict with atomic-arch-fallback.h
Subsystems (5)
[email protected]
maintainerPATCH v3 1/2 locking/atomics: Fixup GENERIC_ATOMIC64 conflict with atomic-arch-fallback.h
From: Guo Ren <[email protected]>
Current GENERIC_ATOMIC64 in atomic-arch-fallback.h is broken. When a 32-bit arch use atomic-arch-fallback.h will cause compile error.
In file included from include/linux/atomic.h:81,
from include/linux/rcupdate.h:25,
from include/linux/rculist.h:11,
from include/linux/pid.h:5,
from include/linux/sched.h:14,
from arch/riscv/kernel/asm-offsets.c:10:
include/linux/atomic-arch-fallback.h: In function 'arch_atomic64_inc':
Quoted 1 line
Quoted 1 line
include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function 'arch_atomic64_add'; did you mean 'arch_atomic_add'? [-Werror=implicit-function-declaration]
1447 | arch_atomic64_add(1, v);
| ^~~~~~~~~~~~~~~~~
| arch_atomic_addThe atomic-arch-fallback.h & atomic-fallback.h & atomic-instrumented.h are generated by gen-atomic-fallback.sh & gen-atomic-instrumented.sh, so just take care the bash files.
Remove the dependency of atomic-*-fallback.h in atomic64.h.
Signed-off-by Guo Ren <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Arnd Bergmann <[email protected]>
--- include/asm-generic/atomic-instrumented.h | 264 +++++++++++++++--------------- include/asm-generic/atomic64.h | 89 ++++++++++ include/linux/atomic-arch-fallback.h | 5 +- include/linux/atomic-fallback.h | 5 +- scripts/atomic/gen-atomic-fallback.sh | 3 +- scripts/atomic/gen-atomic-instrumented.sh | 23 ++- 6 files changed, 251 insertions(+), 138 deletions(-)
include/asm-generic/atomic-instrumented.h +133 -131
@@ -831,6 +831,137 @@ atomic_dec_if_positive(atomic_t *v)#define atomic_dec_if_positive atomic_dec_if_positive#endif +#if !defined(arch_xchg_relaxed) || defined(arch_xchg) +#define xchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_acquire) +#define xchg_acquire (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_release) +#define xchg_release (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_relaxed) +#define xchg_relaxed (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) +#define cmpxchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_acquire) +#define cmpxchg_acquire (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_release) +#define cmpxchg_release (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_relaxed) +#define cmpxchg_relaxed (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) +#define try_cmpxchg (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_acquire) +#define try_cmpxchg_acquire (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_release) +#define try_cmpxchg_release (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_relaxed) +#define try_cmpxchg_relaxed (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#define cmpxchg_local (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ +}) + +#define sync_cmpxchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) + +#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline s64atomic64_read (const atomic64_t *v) {
@@ -1641,78 +1772,6 @@ atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec_if_positive atomic64_dec_if_positive#endif -#if !defined(arch_xchg_relaxed) || defined(arch_xchg) -#define xchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_acquire) -#define xchg_acquire (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_release) -#define xchg_release (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_release(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_relaxed) -#define xchg_relaxed (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) -#define cmpxchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_acquire) -#define cmpxchg_acquire (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_release) -#define cmpxchg_release (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_relaxed) -#define cmpxchg_relaxed (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ -}) -#endif -#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)#define cmpxchg64 (ptr, ...) \ ({ \
@@ -1749,57 +1808,6 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif -#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) -#define try_cmpxchg (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_acquire) -#define try_cmpxchg_acquire (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_release) -#define try_cmpxchg_release (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_relaxed) -#define try_cmpxchg_relaxed (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#define cmpxchg_local (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ -}) -#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \
@@ -1807,13 +1815,7 @@ atomic64_dec_if_positive(atomic64_t *v)arch_cmpxchg64_local (__ai_ptr, __VA_ARGS__); \ }) -#define sync_cmpxchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ -}) - +#endif #define cmpxchg_double (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \
@@ -1830,4 +1832,4 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ -// 4bec382e44520f4d8267e42620054db26a659ea3 +// 21c7f7e074cb2cb7e2f593fe7c8f6dec6ab9e7ea
include/asm-generic/atomic64.h +89 -0
103 lines not on this page · raw message
include/linux/atomic-arch-fallback.h +3 -2
@@ -1252,7 +1252,7 @@ arch_atomic_dec_if_positive(atomic_t *v)#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else #ifndef arch_atomic64_read_acquirestatic __always_inline s64
@@ -2357,5 +2357,6 @@ arch_atomic64_dec_if_positive(atomic64_t *v)#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive#endif +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// cca554917d7ea73d5e3e7397dd70c484cad9b2c4 +// ae31a21075855e67a9b2927f8241dedddafda046
include/linux/atomic-fallback.h +3 -2
@@ -1369,7 +1369,7 @@ atomic_dec_if_positive(atomic_t *v)#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else #define arch_atomic64_read atomic64_read#define arch_atomic64_read_acquire atomic64_read_acquire
@@ -2591,5 +2591,6 @@ atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec_if_positive atomic64_dec_if_positive#endif +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// d78e6c293c661c15188f0ec05bce45188c8d5892 +// b809c8e3c88910826f765bdba4a74f21c527029d
scripts/atomic/gen-atomic-fallback.sh +2 -1
@@ -247,7 +247,7 @@ done cat <<EOF #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else EOF
@@ -256,5 +256,6 @@ grep '^[a-z]' "$1" | while read name meta args; do done cat <<EOF +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ EOF
scripts/atomic/gen-atomic-instrumented.sh +21 -2
@@ -182,21 +182,40 @@ grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic" "int" ${args} done +for xchg in "xchg" "cmpxchg" "try_cmpxchg"; do + for order in "" "_acquire" "_release" "_relaxed"; do + gen_optional_xchg "${xchg}" "${order}" + done +done + +for xchg in "cmpxchg_local" "sync_cmpxchg"; do + gen_xchg "${xchg}" "" + printf "\n" +done + +cat <<EOF +#ifndef CONFIG_GENERIC_ATOMIC64 +EOF + grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do +for xchg in "cmpxchg64"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_optional_xchg "${xchg}" "${order}" done done -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do +for xchg in "cmpxchg64_local"; do gen_xchg "${xchg}" "" printf "\n" done +cat <<EOF +#endif +EOF + gen_xchg "cmpxchg_double" "2 * " printf "\n\n"
Signature · 1 line
2.7.4
[email protected]
maintainerPATCH v3 2/2 riscv: atomic: Using ARCH_ATOMIC in asm/atomic.h
From: Guo Ren <[email protected]>
The linux/atomic-arch-fallback.h has been there for a while, but only x86 & arm64 support it. Let's make riscv follow the linux/arch/* development trendy and make the codes more readable and maintainable.
This patch also cleanup some codes:
- Add atomic_andnot_* operation
- Using amoswap.w.rl & amoswap.w.aq instructions in xchg
- Remove cmpxchg_acquire/release unnecessary optimization
Change in v3:
- Revert to origin atomic(64)_read/set coding style
Change in v2:
- Fixup andnot bug by Peter Zijlstra
Signed-off-by Guo Ren <[email protected]>
Link https://lore.kernel.org/linux-riscv/CAK8P3a0FG3cpqBNUP7kXj3713cMUqV1WcEh-vcRnGKM00WXqxw@mail.gmail.com/ open here
Cc Arnd Bergmann <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Anup Patel <[email protected]>
Cc Palmer Dabbelt <[email protected]>
---
Signed-off-by Guo Ren <[email protected]>
--- arch/riscv/include/asm/atomic.h | 216 ++++++++++++++++----------------------- arch/riscv/include/asm/cmpxchg.h | 199 ++---------------------------------- 2 files changed, 98 insertions(+), 317 deletions(-)
arch/riscv/include/asm/atomic.h +89 -127
@@ -8,13 +8,8 @@#ifndef _ASM_RISCV_ATOMIC_H#define _ASM_RISCV_ATOMIC_H -#ifdef CONFIG_GENERIC_ATOMIC64 -# include <asm-generic/atomic64.h> -#else -# if (__riscv_xlen <64 ) -# error "64-bit atomics require XLEN to be at least 64" -# endif -#endif +#include <linux/compiler.h> +#include <linux/types.h> #include <asm/cmpxchg.h> #include <asm/barrier.h>
@@ -25,22 +20,22 @@#define __atomic_release_fence () \ __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); -static __always_inline int atomic_read(const atomic_t *v) +static __always_inline int arch_atomic_read(const atomic_t *v) {return READ_ONCE (v->counter); } -static __always_inline void atomic_set(atomic_t *v, int i) +static __always_inline void arch_atomic_set(atomic_t *v, int i) {WRITE_ONCE (v->counter, i); }#ifndef CONFIG_GENERIC_ATOMIC64 #define ATOMIC64_INIT (i) { (i) } -static __always_inline s64 atomic64_read(const atomic64_t *v) +static __always_inline s64 arch_atomic64_read(const atomic64_t *v) {return READ_ONCE (v->counter); } -static __always_inline void atomic64_set(atomic64_t *v, s64 i) +static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) {WRITE_ONCE (v->counter, i); }
@@ -53,7 +48,7 @@ static __always_inline void atomic64_set(atomic64_t *v, s64 i)*/ #define ATOMIC_OP (op, asm_op,I , asm_type, c_type, prefix) \ static __always_inline \ -void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ +void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ { \ __asm__ __volatile__ ( \ " amo" #asm_op "." #asm_type " zero, %1, %0" \
@@ -76,6 +71,12 @@ ATOMIC_OPS(sub, add, -i) ATOMIC_OPS(and, and, i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) +ATOMIC_OPS(andnot, and, ~i) + +#define arch_atomic_andnot arch_atomic_andnot +#ifndef CONFIG_GENERIC_ATOMIC64 +#define arch_atomic64_andnot arch_atomic64_andnot +#endif #undef ATOMIC_OP#undef ATOMIC_OPS
@@ -87,7 +88,7 @@ ATOMIC_OPS(xor, xor, i)*/ #define ATOMIC_FETCH_OP (op, asm_op,I , asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \ +c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ register c_type ret; \
@@ -99,7 +100,7 @@ c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \return ret; \ } \ static __always_inline \ -c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)\ { \ register c_type ret; \ __asm__ __volatile__ ( \
@@ -112,15 +113,16 @@ c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \#define ATOMIC_OP_RETURN (op, asm_op, c_op,I , asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_##op##_return_relaxed(c_type i, \ +c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ } \ static __always_inline \ -c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_##op##_return(c_type i, \ + atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ }#ifdef CONFIG_GENERIC_ATOMIC64
@@ -138,26 +140,26 @@ c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ ATOMIC_OPS(add, add, +, i) ATOMIC_OPS(sub, add, +, -i) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_add_return atomic_add_return -#define atomic_sub_return atomic_sub_return +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed -#define atomic_fetch_add atomic_fetch_add -#define atomic_fetch_sub atomic_fetch_sub +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#define atomic64_add_return atomic64_add_return -#define atomic64_sub_return atomic64_sub_return - -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed -#define atomic64_fetch_add atomic64_fetch_add -#define atomic64_fetch_sub atomic64_fetch_sub +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_sub_return arch_atomic64_sub_return + +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub#endif #undef ATOMIC_OPS
@@ -172,23 +174,28 @@ ATOMIC_OPS(sub, add, +, -i)#endif ATOMIC_OPS(and, and, i) +ATOMIC_OPS(andnot, and, ~i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed -#define atomic_fetch_and atomic_fetch_and -#define atomic_fetch_or atomic_fetch_or -#define atomic_fetch_xor atomic_fetch_xor +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed -#define atomic64_fetch_and atomic64_fetch_and -#define atomic64_fetch_or atomic64_fetch_or -#define atomic64_fetch_xor atomic64_fetch_xor +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor#endif #undef ATOMIC_OPS
@@ -197,7 +204,7 @@ ATOMIC_OPS(xor, xor, i)#undef ATOMIC_OP_RETURN/* This is required to provide a full barrier on success. */ -static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) {int prev, rc;
@@ -214,10 +221,10 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "memory");return prev; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) {s64 prev;long rc;
@@ -235,82 +242,10 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u : "memory");return prev; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless#endif -/* -* atomic_{cmp,}xchg is required to have exactly the same ordering semantics as -* {cmp,}xchg and the operations that return, so they need a full barrier. -*/ -#define ATOMIC_OP (c_t, prefix, size) \ -static __always_inline \ -c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_relaxed(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_acquire(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_release(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_relaxed(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_acquire(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_release(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ -{ \ - return __cmpxchg(&(v->counter), o, n, size); \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS () \ - ATOMIC_OP(int, , 4) -#else -#define ATOMIC_OPS () \ - ATOMIC_OP(int, , 4) \ - ATOMIC_OP(s64, 64, 8) -#endif - -ATOMIC_OPS() - -#define atomic_xchg_relaxed atomic_xchg_relaxed -#define atomic_xchg_acquire atomic_xchg_acquire -#define atomic_xchg_release atomic_xchg_release -#define atomic_xchg atomic_xchg -#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#define atomic_cmpxchg_release atomic_cmpxchg_release -#define atomic_cmpxchg atomic_cmpxchg - -#undef ATOMIC_OPS -#undef ATOMIC_OP - -static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) +static __always_inline int arch_atomic_sub_if_positive(atomic_t *v, int offset) {int prev, rc;
@@ -327,11 +262,11 @@ static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) : "memory");return prev - offset; } +#define arch_atomic_dec_if_positive (v) arch_atomic_sub_if_positive(v, 1) -#define atomic_dec_if_positive (v) atomic_sub_if_positive(v, 1)#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) +static __always_inline s64 arch_atomic64_sub_if_positive(atomic64_t *v, s64 offset) {s64 prev;long rc;
@@ -349,8 +284,35 @@ static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) : "memory");return prev - offset; } +#define arch_atomic64_dec_if_positive (v) arch_atomic64_sub_if_positive(v, 1) +#endif + +#define arch_atomic_xchg_relaxed (v, new) \ + arch_xchg_relaxed(&((v)->counter), (new)) +#define arch_atomic_xchg_acquire (v, new) \ + arch_xchg_acquire(&((v)->counter), (new)) +#define arch_atomic_xchg_release (v, new) \ + arch_xchg_release(&((v)->counter), (new)) +#define arch_atomic_xchg (v, new) \ + arch_xchg(&((v)->counter), (new)) + +#define arch_atomic_cmpxchg_relaxed (v, old, new) \ + arch_cmpxchg_relaxed(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg_acquire (v, old, new) \ + arch_cmpxchg_acquire(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg_release (v, old, new) \ + arch_cmpxchg_release(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg (v, old, new) \ + arch_cmpxchg(&((v)->counter), (old), (new)) -#define atomic64_dec_if_positive (v) atomic64_sub_if_positive(v, 1) +#ifndef CONFIG_GENERIC_ATOMIC64 +#define arch_atomic64_xchg_relaxed arch_atomic_xchg_relaxed +#define arch_atomic64_xchg arch_atomic_xchg + +#define arch_atomic64_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed +#define arch_atomic64_cmpxchg arch_atomic_cmpxchg#endif +#define ARCH_ATOMIC +#endif /* _ASM_RISCV_ATOMIC_H */
arch/riscv/include/asm/cmpxchg.h +9 -190
262 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [PATCH v3 2/2] riscv: atomic: Using ARCH_ATOMIC in asm/atomic.h
Quoted 11 lines · On Wed, 21 Apr 2021 05:53:46 PDT (-0700), [email protected] wrote:
From: Guo Ren <[email protected]>
The
linux/atomic-arch-fallback.hhas been there for a while, but only x86 & arm64 support it. Let's make riscv follow the linux/arch/* development trendy and make the codes more readable and maintainable.This patch also cleanup some codes:
- Add atomic_andnot_* operation
- Using amoswap.w.rl & amoswap.w.aq instructions in xchg
- Remove cmpxchg_acquire/release unnecessary optimization
Thanks. I haven't gotten the time to review this properly yet, but I don't really see any issues. Regardless, it was too late for this merge window anyway -- these sorts of things are a bit subtle and require a proper look, right at the end of the cycle things are just a bit too hectic.
I'll take a look once things calm down a bit, there's still some fixes out that I'd like to look at first.
I'm also only getting patch 2, and your lore link points to a different patch set.
Thanks!
Quoted 635 lines
Change in v3:
- Revert to origin atomic(64)_read/set coding styleChange in v2:
- Fixup andnot bug by Peter ZijlstraSigned-off-by Guo Ren <[email protected]>Link https://lore.kernel.org/linux-riscv/CAK8P3a0FG3cpqBNUP7kXj3713cMUqV1WcEh-vcRnGKM00WXqxw@mail.gmail.com/ open hereCc Arnd Bergmann <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Anup Patel <[email protected]>Cc Palmer Dabbelt <[email protected]>---
Signed-off-by Guo Ren <[email protected]>--- arch/riscv/include/asm/atomic.h | 216 ++++++++++++++++----------------------- arch/riscv/include/asm/cmpxchg.h | 199 ++---------------------------------- 2 files changed, 98 insertions(+), 317 deletions(-)arch/riscv/include/asm/atomic.h +89
-127@@ -8,13 +8,8 @@#ifndef _ASM_RISCV_ATOMIC_H#define _ASM_RISCV_ATOMIC_H -#ifdef CONFIG_GENERIC_ATOMIC64 -# include <asm-generic/atomic64.h> -#else -# if (__riscv_xlen <64 ) -# error "64-bit atomics require XLEN to be at least 64" -# endif -#endif +#include <linux/compiler.h> +#include <linux/types.h> #include <asm/cmpxchg.h> #include <asm/barrier.h> @@ -25,22 +20,22 @@#define __atomic_release_fence () \ __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); -static __always_inline int atomic_read(const atomic_t *v) +static __always_inline int arch_atomic_read(const atomic_t *v) {return READ_ONCE (v->counter); } -static __always_inline void atomic_set(atomic_t *v, int i) +static __always_inline void arch_atomic_set(atomic_t *v, int i) {WRITE_ONCE (v->counter, i); }#ifndef CONFIG_GENERIC_ATOMIC64 #define ATOMIC64_INIT (i) { (i) } -static __always_inline s64 atomic64_read(const atomic64_t *v) +static __always_inline s64 arch_atomic64_read(const atomic64_t *v) {return READ_ONCE (v->counter); } -static __always_inline void atomic64_set(atomic64_t *v, s64 i) +static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) {WRITE_ONCE (v->counter, i); }@@ -53,7 +48,7 @@ static __always_inline void atomic64_set(atomic64_t *v, s64 i)*/ #define ATOMIC_OP (op, asm_op,I , asm_type, c_type, prefix) \ static __always_inline \ -void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ +void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ { \ __asm__ __volatile__ ( \ " amo" #asm_op "." #asm_type " zero, %1, %0" \@@ -76,6 +71,12 @@ ATOMIC_OPS(sub, add, -i) ATOMIC_OPS(and, and, i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) +ATOMIC_OPS(andnot, and, ~i) + +#define arch_atomic_andnot arch_atomic_andnot +#ifndef CONFIG_GENERIC_ATOMIC64 +#define arch_atomic64_andnot arch_atomic64_andnot +#endif #undef ATOMIC_OP#undef ATOMIC_OPS@@ -87,7 +88,7 @@ ATOMIC_OPS(xor, xor, i)*/ #define ATOMIC_FETCH_OP (op, asm_op,I , asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \ +c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ register c_type ret; \@@ -99,7 +100,7 @@ c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \return ret; \ } \ static __always_inline \ -c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)\ { \ register c_type ret; \ __asm__ __volatile__ ( \@@ -112,15 +113,16 @@ c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \#define ATOMIC_OP_RETURN (op, asm_op, c_op,I , asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_##op##_return_relaxed(c_type i, \ +c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ } \ static __always_inline \ -c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_##op##_return(c_type i, \ + atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ }#ifdef CONFIG_GENERIC_ATOMIC64 @@ -138,26 +140,26 @@ c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ ATOMIC_OPS(add, add, +, i) ATOMIC_OPS(sub, add, +, -i) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_add_return atomic_add_return -#define atomic_sub_return atomic_sub_return +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed -#define atomic_fetch_add atomic_fetch_add -#define atomic_fetch_sub atomic_fetch_sub +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#define atomic64_add_return atomic64_add_return -#define atomic64_sub_return atomic64_sub_return - -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed -#define atomic64_fetch_add atomic64_fetch_add -#define atomic64_fetch_sub atomic64_fetch_sub +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_sub_return arch_atomic64_sub_return + +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub#endif #undef ATOMIC_OPS@@ -172,23 +174,28 @@ ATOMIC_OPS(sub, add, +, -i)#endif ATOMIC_OPS(and, and, i) +ATOMIC_OPS(andnot, and, ~i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed -#define atomic_fetch_and atomic_fetch_and -#define atomic_fetch_or atomic_fetch_or -#define atomic_fetch_xor atomic_fetch_xor +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed -#define atomic64_fetch_and atomic64_fetch_and -#define atomic64_fetch_or atomic64_fetch_or -#define atomic64_fetch_xor atomic64_fetch_xor +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor#endif #undef ATOMIC_OPS@@ -197,7 +204,7 @@ ATOMIC_OPS(xor, xor, i)#undef ATOMIC_OP_RETURN/* This is required to provide a full barrier on success. */ -static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) {int prev, rc;@@ -214,10 +221,10 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "memory");return prev; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) {s64 prev;long rc;@@ -235,82 +242,10 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u : "memory");return prev; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless#endif -/* -* atomic_{cmp,}xchg is required to have exactly the same ordering semantics as -* {cmp,}xchg and the operations that return, so they need a full barrier. -*/ -#define ATOMIC_OP (c_t, prefix, size) \ -static __always_inline \ -c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_relaxed(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_acquire(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_release(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_relaxed(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_acquire(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_release(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ -{ \ - return __cmpxchg(&(v->counter), o, n, size); \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS () \ - ATOMIC_OP(int, , 4) -#else -#define ATOMIC_OPS () \ - ATOMIC_OP(int, , 4) \ - ATOMIC_OP(s64, 64, 8) -#endif - -ATOMIC_OPS() - -#define atomic_xchg_relaxed atomic_xchg_relaxed -#define atomic_xchg_acquire atomic_xchg_acquire -#define atomic_xchg_release atomic_xchg_release -#define atomic_xchg atomic_xchg -#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#define atomic_cmpxchg_release atomic_cmpxchg_release -#define atomic_cmpxchg atomic_cmpxchg - -#undef ATOMIC_OPS -#undef ATOMIC_OP - -static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) +static __always_inline int arch_atomic_sub_if_positive(atomic_t *v, int offset) {int prev, rc;@@ -327,11 +262,11 @@ static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) : "memory");return prev - offset; } +#define arch_atomic_dec_if_positive (v) arch_atomic_sub_if_positive(v, 1) -#define atomic_dec_if_positive (v) atomic_sub_if_positive(v, 1)#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) +static __always_inline s64 arch_atomic64_sub_if_positive(atomic64_t *v, s64 offset) {s64 prev;long rc;@@ -349,8 +284,35 @@ static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) : "memory");return prev - offset; } +#define arch_atomic64_dec_if_positive (v) arch_atomic64_sub_if_positive(v, 1) +#endif + +#define arch_atomic_xchg_relaxed (v, new) \ + arch_xchg_relaxed(&((v)->counter), (new)) +#define arch_atomic_xchg_acquire (v, new) \ + arch_xchg_acquire(&((v)->counter), (new)) +#define arch_atomic_xchg_release (v, new) \ + arch_xchg_release(&((v)->counter), (new)) +#define arch_atomic_xchg (v, new) \ + arch_xchg(&((v)->counter), (new)) + +#define arch_atomic_cmpxchg_relaxed (v, old, new) \ + arch_cmpxchg_relaxed(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg_acquire (v, old, new) \ + arch_cmpxchg_acquire(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg_release (v, old, new) \ + arch_cmpxchg_release(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg (v, old, new) \ + arch_cmpxchg(&((v)->counter), (old), (new)) -#define atomic64_dec_if_positive (v) atomic64_sub_if_positive(v, 1) +#ifndef CONFIG_GENERIC_ATOMIC64 +#define arch_atomic64_xchg_relaxed arch_atomic_xchg_relaxed +#define arch_atomic64_xchg arch_atomic_xchg + +#define arch_atomic64_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed +#define arch_atomic64_cmpxchg arch_atomic_cmpxchg#endif +#define ARCH_ATOMIC +#endif /* _ASM_RISCV_ATOMIC_H */ arch/riscv/include/asm/cmpxchg.h +9
-181@@ -37,7 +37,7 @@ __ret; \ }) -#define xchg_relaxed (ptr, x) \ +#define arch_xchg_relaxed (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_relaxed((ptr), \@@ -52,16 +52,14 @@switch (size) { \case 4 : \ __asm__ __volatile__ ( \ -" amoswap.w %0, %2, %1\n" \ -RISCV_ACQUIRE_BARRIER \ +" amoswap.w.aq %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \break ; \case 8 : \ __asm__ __volatile__ ( \ -" amoswap.d %0, %2, %1\n" \ -RISCV_ACQUIRE_BARRIER \ +" amoswap.d.aq %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \@@ -72,7 +70,7 @@ __ret; \ }) -#define xchg_acquire (ptr, x) \ +#define arch_xchg_acquire (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_acquire((ptr), \@@ -87,16 +85,14 @@switch (size) { \case 4 : \ __asm__ __volatile__ ( \ -RISCV_RELEASE_BARRIER \ -" amoswap.w %0, %2, %1\n" \ +" amoswap.w.rl %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \break ; \case 8 : \ __asm__ __volatile__ ( \ -RISCV_RELEASE_BARRIER \ -" amoswap.d %0, %2, %1\n" \ +" amoswap.d.rl %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \@@ -107,7 +103,7 @@ __ret; \ }) -#define xchg_release (ptr, x) \ +#define arch_xchg_release (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_release((ptr), \@@ -140,24 +136,12 @@ __ret; \ }) -#define xchg (ptr, x) \ +#define arch_xchg (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \ }) -#define xchg32 (ptr, x) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - xchg((ptr), (x)); \ -}) - -#define xchg64 (ptr, x) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - xchg((ptr), (x)); \ -}) -/* * Atomic compare and exchange. Compare OLD with MEM, if identical, * store NEW in MEM. Return the initial value in MEM. Success is @@ -199,7 +183,7 @@ __ret; \ }) -#define cmpxchg_relaxed (ptr, o, n) \ +#define arch_cmpxchg_relaxed (ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \@@ -207,169 +191,4 @@ _o_, _n_, sizeof(*(ptr))); \ }) -#define __cmpxchg_acquire (ptr, old, new, size) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - register unsigned int __rc; \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - "0: lr.w %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.w %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - RISCV_ACQUIRE_BARRIER \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" ((long)__old), "rJ" (__new) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - "0: lr.d %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.d %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - RISCV_ACQUIRE_BARRIER \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ - : "memory"); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __ret; \ -}) - -#define cmpxchg_acquire (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ - _o_, _n_, sizeof(*(ptr))); \ -}) - -#define __cmpxchg_release (ptr, old, new, size) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - register unsigned int __rc; \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - RISCV_RELEASE_BARRIER \ - "0: lr.w %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.w %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" ((long)__old), "rJ" (__new) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - RISCV_RELEASE_BARRIER \ - "0: lr.d %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.d %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ - : "memory"); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __ret; \ -}) - -#define cmpxchg_release (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg_release((ptr), \ - _o_, _n_, sizeof(*(ptr))); \ -}) - -#define __cmpxchg (ptr, old, new, size) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - register unsigned int __rc; \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - "0: lr.w %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.w.rl %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - " fence rw, rw\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" ((long)__old), "rJ" (__new) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - "0: lr.d %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.d.rl %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - " fence rw, rw\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ - : "memory"); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __ret; \ -}) - -#define cmpxchg (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg((ptr), \ - _o_, _n_, sizeof(*(ptr))); \ -}) - -#define cmpxchg_local (ptr, o, n) \ - (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) - -#define cmpxchg32 (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - cmpxchg((ptr), (o), (n)); \ -}) - -#define cmpxchg32_local (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - cmpxchg_relaxed((ptr), (o), (n)) \ -}) - -#define cmpxchg64 (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
arch/riscv/include/asm/cmpxchg.h line 194
Quoted 10 lines
arch/riscv/include/asm/cmpxchg.h +0
-9-cmpxchg ((ptr), (o), (n)); \ -}) - -#define cmpxchg64_local (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_relaxed((ptr), (o), (n)); \ -}) -#endif /* _ASM_RISCV_CMPXCHG_H */
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv
Guo Ren
maintainerRe: [PATCH v3 2/2] riscv: atomic: Using ARCH_ATOMIC in asm/atomic.h
Hi Palmer,
Quoted 20 lines · Palmer Dabbelt
Quoted 11 lines · On Wed, 21 Apr 2021 05:53:46 PDT (-0700), [email protected] wrote:
From: Guo Ren <[email protected]>
The
linux/atomic-arch-fallback.hhas been there for a while, but only x86 & arm64 support it. Let's make riscv follow the linux/arch/* development trendy and make the codes more readable and maintainable.This patch also cleanup some codes:
- Add atomic_andnot_* operation
- Using amoswap.w.rl & amoswap.w.aq instructions in xchg
- Remove cmpxchg_acquire/release unnecessary optimizationThanks. I haven't gotten the time to review this properly yet, but I don't really see any issues. Regardless, it was too late for this merge window anyway -- these sorts of things are a bit subtle and require a proper look, right at the end of the cycle things are just a bit too hectic.
Quoted 5 lines
I'll take a look once things calm down a bit, there's still some fixes out that I'd like to look at first.
I'm also only getting patch 2, and your lore link points to a different patch set.
Mark Rutland is preparing a new patchset to let ARCH_ATOMIC become the default for all arch. I'm waiting for his patchset and atop my modification on it.
https://lore.kernel.org/linux-riscv/CAJF2gTS9y1QZx-8pu2NW22xs1Gky0y4Hs31Wrn7gZg3FiAq8NA@mail.gmail.com/
So, please abandon the patch, I'll send a new version patchset based on Mark's work.
Quoted 640 lines
Thanks!
Quoted 637 lines
Change in v3:
- Revert to origin atomic(64)_read/set coding styleChange in v2:
- Fixup andnot bug by Peter ZijlstraSigned-off-by Guo Ren <[email protected]>Link https://lore.kernel.org/linux-riscv/CAK8P3a0FG3cpqBNUP7kXj3713cMUqV1WcEh-vcRnGKM00WXqxw@mail.gmail.com/ open hereCc Arnd Bergmann <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Anup Patel <[email protected]>Cc Palmer Dabbelt <[email protected]>---
Signed-off-by Guo Ren <[email protected]>--- arch/riscv/include/asm/atomic.h | 216 ++++++++++++++++----------------------- arch/riscv/include/asm/cmpxchg.h | 199 ++---------------------------------- 2 files changed, 98 insertions(+), 317 deletions(-)arch/riscv/include/asm/atomic.h +89
-127@@ -8,13 +8,8 @@#ifndef _ASM_RISCV_ATOMIC_H#define _ASM_RISCV_ATOMIC_H -#ifdef CONFIG_GENERIC_ATOMIC64 -# include <asm-generic/atomic64.h> -#else -# if (__riscv_xlen <64 ) -# error "64-bit atomics require XLEN to be at least 64" -# endif -#endif +#include <linux/compiler.h> +#include <linux/types.h> #include <asm/cmpxchg.h> #include <asm/barrier.h> @@ -25,22 +20,22 @@#define __atomic_release_fence () \ __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); -static __always_inline int atomic_read(const atomic_t *v) +static __always_inline int arch_atomic_read(const atomic_t *v) {return READ_ONCE (v->counter); } -static __always_inline void atomic_set(atomic_t *v, int i) +static __always_inline void arch_atomic_set(atomic_t *v, int i) {WRITE_ONCE (v->counter, i); }#ifndef CONFIG_GENERIC_ATOMIC64 #define ATOMIC64_INIT (i) { (i) } -static __always_inline s64 atomic64_read(const atomic64_t *v) +static __always_inline s64 arch_atomic64_read(const atomic64_t *v) {return READ_ONCE (v->counter); } -static __always_inline void atomic64_set(atomic64_t *v, s64 i) +static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) {WRITE_ONCE (v->counter, i); }@@ -53,7 +48,7 @@ static __always_inline void atomic64_set(atomic64_t *v, s64 i)*/ #define ATOMIC_OP (op, asm_op,I , asm_type, c_type, prefix) \ static __always_inline \ -void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ +void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ { \ __asm__ __volatile__ ( \ " amo" #asm_op "." #asm_type " zero, %1, %0" \@@ -76,6 +71,12 @@ ATOMIC_OPS(sub, add, -i) ATOMIC_OPS(and, and, i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) +ATOMIC_OPS(andnot, and, ~i) + +#define arch_atomic_andnot arch_atomic_andnot +#ifndef CONFIG_GENERIC_ATOMIC64 +#define arch_atomic64_andnot arch_atomic64_andnot +#endif #undef ATOMIC_OP#undef ATOMIC_OPS@@ -87,7 +88,7 @@ ATOMIC_OPS(xor, xor, i)*/ #define ATOMIC_FETCH_OP (op, asm_op,I , asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \ +c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ register c_type ret; \@@ -99,7 +100,7 @@ c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \return ret; \ } \ static __always_inline \ -c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)\ { \ register c_type ret; \ __asm__ __volatile__ ( \@@ -112,15 +113,16 @@ c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \#define ATOMIC_OP_RETURN (op, asm_op, c_op,I , asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_##op##_return_relaxed(c_type i, \ +c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ } \ static __always_inline \ -c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_##op##_return(c_type i, \ + atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ }#ifdef CONFIG_GENERIC_ATOMIC64 @@ -138,26 +140,26 @@ c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ ATOMIC_OPS(add, add, +, i) ATOMIC_OPS(sub, add, +, -i) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_add_return atomic_add_return -#define atomic_sub_return atomic_sub_return +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed -#define atomic_fetch_add atomic_fetch_add -#define atomic_fetch_sub atomic_fetch_sub +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#define atomic64_add_return atomic64_add_return -#define atomic64_sub_return atomic64_sub_return - -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed -#define atomic64_fetch_add atomic64_fetch_add -#define atomic64_fetch_sub atomic64_fetch_sub +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_sub_return arch_atomic64_sub_return + +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub#endif #undef ATOMIC_OPS@@ -172,23 +174,28 @@ ATOMIC_OPS(sub, add, +, -i)#endif ATOMIC_OPS(and, and, i) +ATOMIC_OPS(andnot, and, ~i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed -#define atomic_fetch_and atomic_fetch_and -#define atomic_fetch_or atomic_fetch_or -#define atomic_fetch_xor atomic_fetch_xor +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed -#define atomic64_fetch_and atomic64_fetch_and -#define atomic64_fetch_or atomic64_fetch_or -#define atomic64_fetch_xor atomic64_fetch_xor +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor#endif #undef ATOMIC_OPS@@ -197,7 +204,7 @@ ATOMIC_OPS(xor, xor, i)#undef ATOMIC_OP_RETURN/* This is required to provide a full barrier on success. */ -static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) {int prev, rc;@@ -214,10 +221,10 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "memory");return prev; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) {s64 prev;long rc;@@ -235,82 +242,10 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u : "memory");return prev; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless#endif -/* -* atomic_{cmp,}xchg is required to have exactly the same ordering semantics as -* {cmp,}xchg and the operations that return, so they need a full barrier. -*/ -#define ATOMIC_OP (c_t, prefix, size) \ -static __always_inline \ -c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_relaxed(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_acquire(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg_release(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ -{ \ - return __xchg(&(v->counter), n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_relaxed(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_acquire(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ - c_t o, c_t n) \ -{ \ - return __cmpxchg_release(&(v->counter), o, n, size); \ -} \ -static __always_inline \ -c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ -{ \ - return __cmpxchg(&(v->counter), o, n, size); \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS () \ - ATOMIC_OP(int, , 4) -#else -#define ATOMIC_OPS () \ - ATOMIC_OP(int, , 4) \ - ATOMIC_OP(s64, 64, 8) -#endif - -ATOMIC_OPS() - -#define atomic_xchg_relaxed atomic_xchg_relaxed -#define atomic_xchg_acquire atomic_xchg_acquire -#define atomic_xchg_release atomic_xchg_release -#define atomic_xchg atomic_xchg -#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#define atomic_cmpxchg_release atomic_cmpxchg_release -#define atomic_cmpxchg atomic_cmpxchg - -#undef ATOMIC_OPS -#undef ATOMIC_OP - -static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) +static __always_inline int arch_atomic_sub_if_positive(atomic_t *v, int offset) {int prev, rc;@@ -327,11 +262,11 @@ static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) : "memory");return prev - offset; } +#define arch_atomic_dec_if_positive (v) arch_atomic_sub_if_positive(v, 1) -#define atomic_dec_if_positive (v) atomic_sub_if_positive(v, 1)#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) +static __always_inline s64 arch_atomic64_sub_if_positive(atomic64_t *v, s64 offset) {s64 prev;long rc;@@ -349,8 +284,35 @@ static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) : "memory");return prev - offset; } +#define arch_atomic64_dec_if_positive (v) arch_atomic64_sub_if_positive(v, 1) +#endif + +#define arch_atomic_xchg_relaxed (v, new) \ + arch_xchg_relaxed(&((v)->counter), (new)) +#define arch_atomic_xchg_acquire (v, new) \ + arch_xchg_acquire(&((v)->counter), (new)) +#define arch_atomic_xchg_release (v, new) \ + arch_xchg_release(&((v)->counter), (new)) +#define arch_atomic_xchg (v, new) \ + arch_xchg(&((v)->counter), (new)) + +#define arch_atomic_cmpxchg_relaxed (v, old, new) \ + arch_cmpxchg_relaxed(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg_acquire (v, old, new) \ + arch_cmpxchg_acquire(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg_release (v, old, new) \ + arch_cmpxchg_release(&((v)->counter), (old), (new)) +#define arch_atomic_cmpxchg (v, old, new) \ + arch_cmpxchg(&((v)->counter), (old), (new)) -#define atomic64_dec_if_positive (v) atomic64_sub_if_positive(v, 1) +#ifndef CONFIG_GENERIC_ATOMIC64 +#define arch_atomic64_xchg_relaxed arch_atomic_xchg_relaxed +#define arch_atomic64_xchg arch_atomic_xchg + +#define arch_atomic64_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed +#define arch_atomic64_cmpxchg arch_atomic_cmpxchg#endif +#define ARCH_ATOMIC +#endif /* _ASM_RISCV_ATOMIC_H */ arch/riscv/include/asm/cmpxchg.h +9
-183@@ -37,7 +37,7 @@ __ret; \ }) -#define xchg_relaxed (ptr, x) \ +#define arch_xchg_relaxed (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_relaxed((ptr), \@@ -52,16 +52,14 @@switch (size) { \case 4 : \ __asm__ __volatile__ ( \ -" amoswap.w %0, %2, %1\n" \ -RISCV_ACQUIRE_BARRIER \ +" amoswap.w.aq %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \break ; \case 8 : \ __asm__ __volatile__ ( \ -" amoswap.d %0, %2, %1\n" \ -RISCV_ACQUIRE_BARRIER \ +" amoswap.d.aq %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \@@ -72,7 +70,7 @@ __ret; \ }) -#define xchg_acquire (ptr, x) \ +#define arch_xchg_acquire (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_acquire((ptr), \@@ -87,16 +85,14 @@switch (size) { \case 4 : \ __asm__ __volatile__ ( \ -RISCV_RELEASE_BARRIER \ -" amoswap.w %0, %2, %1\n" \ +" amoswap.w.rl %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \break ; \case 8 : \ __asm__ __volatile__ ( \ -RISCV_RELEASE_BARRIER \ -" amoswap.d %0, %2, %1\n" \ +" amoswap.d.rl %0, %2, %1\n" \ :"=r" (__ret),"+A" (*__ptr) \ :"r" (__new) \ :"memory" ); \@@ -107,7 +103,7 @@ __ret; \ }) -#define xchg_release (ptr, x) \ +#define arch_xchg_release (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_release((ptr), \@@ -140,24 +136,12 @@ __ret; \ }) -#define xchg (ptr, x) \ +#define arch_xchg (ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \ }) -#define xchg32 (ptr, x) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - xchg((ptr), (x)); \ -}) - -#define xchg64 (ptr, x) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - xchg((ptr), (x)); \ -}) -/* * Atomic compare and exchange. Compare OLD with MEM, if identical, * store NEW in MEM. Return the initial value in MEM. Success is @@ -199,7 +183,7 @@ __ret; \ }) -#define cmpxchg_relaxed (ptr, o, n) \ +#define arch_cmpxchg_relaxed (ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \@@ -207,169 +191,4 @@ _o_, _n_, sizeof(*(ptr))); \ }) -#define __cmpxchg_acquire (ptr, old, new, size) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - register unsigned int __rc; \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - "0: lr.w %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.w %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - RISCV_ACQUIRE_BARRIER \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" ((long)__old), "rJ" (__new) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - "0: lr.d %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.d %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - RISCV_ACQUIRE_BARRIER \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ - : "memory"); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __ret; \ -}) - -#define cmpxchg_acquire (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ - _o_, _n_, sizeof(*(ptr))); \ -}) - -#define __cmpxchg_release (ptr, old, new, size) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - register unsigned int __rc; \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - RISCV_RELEASE_BARRIER \ - "0: lr.w %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.w %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" ((long)__old), "rJ" (__new) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - RISCV_RELEASE_BARRIER \ - "0: lr.d %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.d %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ - : "memory"); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __ret; \ -}) - -#define cmpxchg_release (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg_release((ptr), \ - _o_, _n_, sizeof(*(ptr))); \ -}) - -#define __cmpxchg (ptr, old, new, size) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - register unsigned int __rc; \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - "0: lr.w %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.w.rl %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - " fence rw, rw\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" ((long)__old), "rJ" (__new) \ - : "memory"); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - "0: lr.d %0, %2\n" \ - " bne %0, %z3, 1f\n" \ - " sc.d.rl %1, %z4, %2\n" \ - " bnez %1, 0b\n" \ - " fence rw, rw\n" \ - "1:\n" \ - : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ - : "rJ" (__old), "rJ" (__new) \ - : "memory"); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __ret; \ -}) - -#define cmpxchg (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg((ptr), \ - _o_, _n_, sizeof(*(ptr))); \ -}) - -#define cmpxchg_local (ptr, o, n) \ - (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) - -#define cmpxchg32 (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - cmpxchg((ptr), (o), (n)); \ -}) - -#define cmpxchg32_local (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - cmpxchg_relaxed((ptr), (o), (n)) \ -}) - -#define cmpxchg64 (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ -})
Quoted 8 lines
arch/riscv/include/asm/cmpxchg.h line 194
Quoted 8 lines
arch/riscv/include/asm/cmpxchg.h +0
-7- -#define cmpxchg64_local (ptr, o, n) \ -({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_relaxed((ptr), (o), (n)); \ -}) -#endif /* _ASM_RISCV_CMPXCHG_H */
--
Best Regards
Guo Ren
Mark Rutland
reviewerRe: [PATCH v3 2/2] riscv: atomic: Using ARCH_ATOMIC in asm/atomic.h
Quoted 28 lines · Guo Ren
Hi Palmer,
Quoted 20 lines · Palmer Dabbelt
Quoted 11 lines · On Wed, 21 Apr 2021 05:53:46 PDT (-0700), [email protected] wrote:
From: Guo Ren <[email protected]>
The
linux/atomic-arch-fallback.hhas been there for a while, but only x86 & arm64 support it. Let's make riscv follow the linux/arch/* development trendy and make the codes more readable and maintainable.This patch also cleanup some codes:
- Add atomic_andnot_* operation
- Using amoswap.w.rl & amoswap.w.aq instructions in xchg
- Remove cmpxchg_acquire/release unnecessary optimizationThanks. I haven't gotten the time to review this properly yet, but I don't really see any issues. Regardless, it was too late for this merge window anyway -- these sorts of things are a bit subtle and require a proper look, right at the end of the cycle things are just a bit too hectic.
Quoted 5 lines
I'll take a look once things calm down a bit, there's still some fixes out that I'd like to look at first.
I'm also only getting patch 2, and your lore link points to a different patch set.
Quoted 7 lines
Mark Rutland is preparing a new patchset to let
ARCH_ATOMICbecome the default for all arch. I'm waiting for his patchset and atop my modification on it.
https://lore.kernel.org/linux-riscv/CAJF2gTS9y1QZx-8pu2NW22xs1Gky0y4Hs31Wrn7gZg3FiAq8NA@mail.gmail.com/So, please abandon the patch, I'll send a new version patchset based on Mark's work.
For context, the series is basically ready now, but I'm intending to rebase it atop v5.13-rc1 before posting (e.g. since s390 just moved to ARCH_ATOMIC), and plan to post that early next week.
If you want an early look, my atomics/arch-atomic branch on kernel.org has the WIP (atop v5.12):
https://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.git/log/?h=atomics/arch-atomic
git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.git atomics/arch-atomic
.. and the riscv-specific changes are just renames:
Thanks,
Mark.
Mark Rutland
reviewerPATCH 1/1 asm-generic: atomic64: handle ARCH_ATOMIC builds (was "Re: [PATCH v3 1/2] locking/atomics: Fixup GENERIC_ATOMIC64 conflict") with atomic-arch-fallback.h
In reply to [email protected]Hi Guo,
Quoted 12 lines · On Wed, Apr 21, 2021 at 12:53:45PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Current
GENERIC_ATOMIC64inatomic-arch-fallback.his broken. When a 32-bit arch useatomic-arch-fallback.hwill cause compile error.In file included from
include/linux/atomic.h:81,
frominclude/linux/rcupdate.h:25,
frominclude/linux/rculist.h:11,
frominclude/linux/pid.h:5,
frominclude/linux/sched.h:14,
fromarch/riscv/kernel/asm-offsets.c:10:
include/linux/atomic-arch-fallback.h: In function 'arch_atomic64_inc':
Quoted 4 lines
Quoted 1 line
Quoted 1 line
include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function 'arch_atomic64_add'; did you mean 'arch_atomic_add'? [-Werror=implicit-function-declaration]1447 | arch_atomic64_add(1, v); | ^~~~~~~~~~~~~~~~~ | arch_atomic_add
This is expected; GENERIC_ATOMIC64 doesn't implement arch_atomic64_*(), and thus violates the expectations of the fallback code.
To make GENERIC_ATOMIC64 compatible with ARCH_ATOMIC, the GENERIC_ATOMIC64 implementation *must* provide arch_atomic64_*() functions.
Quoted 5 lines
The
atomic-arch-fallback.h&atomic-fallback.h&atomic-instrumented.hare generated bygen-atomic-fallback.sh&gen-atomic-instrumented.sh, so just take care the bash files.Remove the dependency of atomic-*-fallback.h in
atomic64.h.
Please don't duplicate the fallbacks; this'll make it harder to move other over and eventually remove the non-ARCH_ATOMIC implementations.
Does the patch below make things work for you, or have I missed something?
I've given this a basic build test on an arm config using GENERIC_ATOMIC64 (but not ARCH_ATOMIC).
Thanks,
Mark.
---->8----
From 7f0389c8a1f41ecb5b2700f6ba38ff2ba093eb33 Mon Sep 17 00:00:00 2001
From: Mark Rutland <[email protected]>
Date: Thu, 22 Apr 2021 11:26:04 +0100
Subject: [PATCH] asm-generic: atomic64: handle ARCH_ATOMIC builds
We'd like all architectures to convert to ARCH_ATOMIC, as this will enable functionality, and once all architectures are converted it will be possible to make significant cleanups to the atomic headers.
A number of architectures use GENERIC_ATOMIC64, and it's impractical to convert them all in one go. To make it possible to convert them one-by-one, let's make the GENERIC_ATOMIC64 implementation function as either atomic64_*() or arch_atomic64_*() depending on whether ARCH_ATOMIC is selected. To do this, the C implementations are prefixed as generic_atomic64_*(), and the asm-generic/atomic64.h header maps atomic64_*()/arch_atomic64_*() onto these as appropriate via teh preprocessor.
Once all users are moved over to ARCH_ATOMIC the ifdeffery in the header can be simplified and/or removed entirely.
For existing users (none of which select ARCH_ATOMIC), there should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>
Cc Arnd Bergmann <[email protected]>
Cc Guo Ren <[email protected]>
Cc Peter Zijlstra <[email protected]>
--- include/asm-generic/atomic64.h | 74 ++++++++++++++++++++++++++++++++++-------- lib/atomic64.c | 36 ++++++++++---------- 2 files changed, 79 insertions(+), 31 deletions(-)
include/asm-generic/atomic64.h +61 -13
@@ -15,19 +15,17 @@ typedef struct {#define ATOMIC64_INIT (i) { (i) } -extern s64 atomic64_read (const atomic64_t *v); -extern void atomic64_set (atomic64_t *v,s64 i); - -#define atomic64_set_release (v, i) atomic64_set((v), (i)) +extern s64 generic_atomic64_read (const atomic64_t *v); +extern void generic_atomic64_set (atomic64_t *v,s64 i);#define ATOMIC64_OP (op) \ -extern void atomic64_##op(s64 a, atomic64_t *v); +extern void generic_atomic64_##op(s64 a, atomic64_t *v);#define ATOMIC64_OP_RETURN (op) \ -extern s64 atomic64_##op##_return(s64 a, atomic64_t *v); +extern s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v);#define ATOMIC64_FETCH_OP (op) \ -extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); +extern s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v);#define ATOMIC64_OPS (op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op)
@@ -46,11 +44,61 @@ ATOMIC64_OPS(xor)#undef ATOMIC64_OP_RETURN#undef ATOMIC64_OP -extern s64 atomic64_dec_if_positive (atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive -extern s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); -extern s64 atomic64_xchg (atomic64_t *v,s64 new); -extern s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u); -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +extern s64 generic_atomic64_dec_if_positive (atomic64_t *v); +extern s64 generic_atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); +extern s64 generic_atomic64_xchg (atomic64_t *v,s64 new); +extern s64 generic_atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u); + +#ifdef ARCH_ATOMIC + +#define arch_atomic64_read generic_atomic64_read +#define arch_atomic64_set generic_atomic64_set +#define arch_atomic64_set_release generic_atomic64_set + +#define arch_atomic64_add generic_atomic64_add +#define arch_atomic64_add_return generic_atomic64_add_return +#define arch_atomic64_fetch_add generic_atomic64_fetch_add +#define arch_atomic64_sub generic_atomic64_sub +#define arch_atomic64_sub_return generic_atomic64_sub_return +#define arch_atomic64_fetch_sub generic_atomic64_fetch_sub + +#define arch_atomic64_and generic_atomic64_and +#define arch_atomic64_fetch_and generic_atomic64_fetch_and +#define arch_atomic64_or generic_atomic64_or +#define arch_atomic64_fetch_or generic_atomic64_fetch_or +#define arch_atomic64_xor generic_atomic64_xor +#define arch_atomic64_fetch_xor generic_atomic64_fetch_xor + +#define arch_atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define arch_atomic64_cmpxchg generic_atomic64_cmpxchg +#define arch_atomic64_xchg generic_atomic64_xchg +#define arch_atomic64_fetch_add_unless generic_atomic64_fetch_add_unless + +#else /* ARCH_ATOMIC */ + +#define atomic64_read generic_atomic64_read +#define atomic64_set generic_atomic64_set +#define atomic64_set_release generic_atomic64_set + +#define atomic64_add generic_atomic64_add +#define atomic64_add_return generic_atomic64_add_return +#define atomic64_fetch_add generic_atomic64_fetch_add +#define atomic64_sub generic_atomic64_sub +#define atomic64_sub_return generic_atomic64_sub_return +#define atomic64_fetch_sub generic_atomic64_fetch_sub + +#define atomic64_and generic_atomic64_and +#define atomic64_fetch_and generic_atomic64_fetch_and +#define atomic64_or generic_atomic64_or +#define atomic64_fetch_or generic_atomic64_fetch_or +#define atomic64_xor generic_atomic64_xor +#define atomic64_fetch_xor generic_atomic64_fetch_xor + +#define atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define atomic64_cmpxchg generic_atomic64_cmpxchg +#define atomic64_xchg generic_atomic64_xchg +#define atomic64_fetch_add_unless generic_atomic64_fetch_add_unless + +#endif /* ARCH_ATOMIC */ #endif /* _ASM_GENERIC_ATOMIC64_H */
lib/atomic64.c +18 -18
@@ -42,7 +42,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v)return &atomic64_lock[addr & (NR_LOCKS -1 )].lock; } -s64 atomic64_read (const atomic64_t *v) +s64 generic_atomic64_read (const atomic64_t *v) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);
@@ -53,9 +53,9 @@ s64 atomic64_read(const atomic64_t *v)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_read); +EXPORT_SYMBOL (generic_atomic64_read); -void atomic64_set (atomic64_t *v,s64 i) +void generic_atomic64_set (atomic64_t *v,s64 i) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);
@@ -64,10 +64,10 @@ void atomic64_set(atomic64_t *v, s64 i) v->counter = i;raw_spin_unlock_irqrestore (lock, flags); } -EXPORT_SYMBOL (atomic64_set); +EXPORT_SYMBOL (generic_atomic64_set);#define ATOMIC64_OP (op, c_op) \ -void atomic64_##op(s64 a, atomic64_t *v) \ +void generic_atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \
@@ -76,10 +76,10 @@ void atomic64_##op(s64 a, atomic64_t *v) \ v->counter c_op a; \raw_spin_unlock_irqrestore (lock, flags); \ } \ -EXPORT_SYMBOL(atomic64_##op); +EXPORT_SYMBOL(generic_atomic64_##op);#define ATOMIC64_OP_RETURN (op, c_op) \ -s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \
@@ -90,10 +90,10 @@ s64 atomic64_##op##_return(s64 a, atomic64_t *v) \raw_spin_unlock_irqrestore (lock, flags); \return val; \ } \ -EXPORT_SYMBOL(atomic64_##op##_return); +EXPORT_SYMBOL(generic_atomic64_##op##_return);#define ATOMIC64_FETCH_OP (op, c_op) \ -s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \
@@ -105,7 +105,7 @@ s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \raw_spin_unlock_irqrestore (lock, flags); \return val; \ } \ -EXPORT_SYMBOL(atomic64_fetch_##op); +EXPORT_SYMBOL(generic_atomic64_fetch_##op);#define ATOMIC64_OPS (op, c_op) \ ATOMIC64_OP(op, c_op) \
@@ -130,7 +130,7 @@ ATOMIC64_OPS(xor, ^=)#undef ATOMIC64_OP_RETURN#undef ATOMIC64_OP -s64 atomic64_dec_if_positive (atomic64_t *v) +s64 generic_atomic64_dec_if_positive (atomic64_t *v) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);
@@ -143,9 +143,9 @@ s64 atomic64_dec_if_positive(atomic64_t *v)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_dec_if_positive); +EXPORT_SYMBOL (generic_atomic64_dec_if_positive); -s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n) +s64 generic_atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);
@@ -158,9 +158,9 @@ s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_cmpxchg); +EXPORT_SYMBOL (generic_atomic64_cmpxchg); -s64 atomic64_xchg (atomic64_t *v,s64 new) +s64 generic_atomic64_xchg (atomic64_t *v,s64 new) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);
@@ -172,9 +172,9 @@ s64 atomic64_xchg(atomic64_t *v, s64 new)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_xchg); +EXPORT_SYMBOL (generic_atomic64_xchg); -s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u) +s64 generic_atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);
@@ -188,4 +188,4 @@ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)return val; } -EXPORT_SYMBOL (atomic64_fetch_add_unless); +EXPORT_SYMBOL (generic_atomic64_fetch_add_unless);
Signature · 1 line
2.11.0
Guo Ren
maintainerRe: [PATCH] asm-generic: atomic64: handle ARCH_ATOMIC builds (was "Re: [PATCH v3 1/2] locking/atomics: Fixup GENERIC_ATOMIC64 conflict") with atomic-arch-fallback.h
Hi Mark,
Quoted 21 lines · Mark Rutland
Hi Guo,
Quoted 12 lines · On Wed, Apr 21, 2021 at 12:53:45PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Current
GENERIC_ATOMIC64inatomic-arch-fallback.his broken. When a 32-bit arch useatomic-arch-fallback.hwill cause compile error.In file included from
include/linux/atomic.h:81,
frominclude/linux/rcupdate.h:25,
frominclude/linux/rculist.h:11,
frominclude/linux/pid.h:5,
frominclude/linux/sched.h:14,
fromarch/riscv/kernel/asm-offsets.c:10:
include/linux/atomic-arch-fallback.h: In function 'arch_atomic64_inc':Quoted 4 lines
Quoted 1 line
Quoted 1 line
include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function 'arch_atomic64_add'; did you mean 'arch_atomic_add'? [-Werror=implicit-function-declaration]1447 | arch_atomic64_add(1, v); | ^~~~~~~~~~~~~~~~~ | arch_atomic_add
Quoted 6 lines
This is expected;
GENERIC_ATOMIC64doesn't implement arch_atomic64_*(), and thus violates the expectations of the fallback code.To make
GENERIC_ATOMIC64compatible withARCH_ATOMIC, theGENERIC_ATOMIC64implementation *must* provide arch_atomic64_*() functions.
How do you let a "static __always_inline" of "asm-generic/atomic-instrumented.h" call a real function? See lib/atomic64.c.
Quoted 5 lines
Quoted 5 lines
The
atomic-arch-fallback.h&atomic-fallback.h&atomic-instrumented.hare generated bygen-atomic-fallback.sh&gen-atomic-instrumented.sh, so just take care the bash files.
Quoted 7 lines
Quoted 1 line
Remove the dependency of atomic-*-fallback.h in
atomic64.h.Please don't duplicate the fallbacks; this'll make it harder to move other over and eventually remove the non-ARCH_ATOMIC implementations.
Does the patch below make things work for you, or have I missed something?
RISC-V combines 32bit & 64bit together just like x86. Current ARCH_ATOMIC could work perfectly with RV64, but not RV32.
RV32 still could use ARCH_ATOMIC to improve kasan check.
Quoted 264 lines
I've given this a basic build test on an arm config using
GENERIC_ATOMIC64(but notARCH_ATOMIC).Thanks,
Mark.
---->8----
From7f0389c8a1f41ecb5b2700f6ba38ff2ba093eb33Mon Sep 17 00:00:00 2001
From: Mark Rutland <[email protected]>
Date: Thu, 22 Apr 2021 11:26:04 +0100
Subject: [PATCH] asm-generic: atomic64: handleARCH_ATOMICbuildsWe'd like all architectures to convert to
ARCH_ATOMIC, as this will enable functionality, and once all architectures are converted it will be possible to make significant cleanups to the atomic headers.A number of architectures use
GENERIC_ATOMIC64, and it's impractical to convert them all in one go. To make it possible to convert them one-by-one, let's make theGENERIC_ATOMIC64implementation function as either atomic64_*() or arch_atomic64_*() depending on whetherARCH_ATOMICis selected. To do this, the C implementations are prefixed as generic_atomic64_*(), and theasm-generic/atomic64.hheader maps atomic64_*()/arch_atomic64_*() onto these as appropriate via teh preprocessor.Once all users are moved over to
ARCH_ATOMICthe ifdeffery in the header can be simplified and/or removed entirely.For existing users (none of which select
ARCH_ATOMIC), there should be no functional change as a result of this patch.Signed-off-by Mark Rutland <[email protected]>Cc Arnd Bergmann <[email protected]>Cc Guo Ren <[email protected]>Cc Peter Zijlstra <[email protected]>--- include/asm-generic/atomic64.h | 74 ++++++++++++++++++++++++++++++++++-------- lib/atomic64.c | 36 ++++++++++---------- 2 files changed, 79 insertions(+), 31 deletions(-)include/asm-generic/atomic64.h +61
-13@@ -15,19 +15,17 @@ typedef struct {#define ATOMIC64_INIT (i) { (i) } -extern s64 atomic64_read (const atomic64_t *v); -extern void atomic64_set (atomic64_t *v,s64 i); - -#define atomic64_set_release (v, i) atomic64_set((v), (i)) +extern s64 generic_atomic64_read (const atomic64_t *v); +extern void generic_atomic64_set (atomic64_t *v,s64 i);#define ATOMIC64_OP (op) \ -extern void atomic64_##op(s64 a, atomic64_t *v); +extern void generic_atomic64_##op(s64 a, atomic64_t *v);#define ATOMIC64_OP_RETURN (op) \ -extern s64 atomic64_##op##_return(s64 a, atomic64_t *v); +extern s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v);#define ATOMIC64_FETCH_OP (op) \ -extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); +extern s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v);#define ATOMIC64_OPS (op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op)@@ -46,11 +44,61 @@ ATOMIC64_OPS(xor)#undef ATOMIC64_OP_RETURN#undef ATOMIC64_OP -extern s64 atomic64_dec_if_positive (atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive -extern s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); -extern s64 atomic64_xchg (atomic64_t *v,s64 new); -extern s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u); -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +extern s64 generic_atomic64_dec_if_positive (atomic64_t *v); +extern s64 generic_atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); +extern s64 generic_atomic64_xchg (atomic64_t *v,s64 new); +extern s64 generic_atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u); + +#ifdef ARCH_ATOMIC + +#define arch_atomic64_read generic_atomic64_read +#define arch_atomic64_set generic_atomic64_set +#define arch_atomic64_set_release generic_atomic64_set + +#define arch_atomic64_add generic_atomic64_add +#define arch_atomic64_add_return generic_atomic64_add_return +#define arch_atomic64_fetch_add generic_atomic64_fetch_add +#define arch_atomic64_sub generic_atomic64_sub +#define arch_atomic64_sub_return generic_atomic64_sub_return +#define arch_atomic64_fetch_sub generic_atomic64_fetch_sub + +#define arch_atomic64_and generic_atomic64_and +#define arch_atomic64_fetch_and generic_atomic64_fetch_and +#define arch_atomic64_or generic_atomic64_or +#define arch_atomic64_fetch_or generic_atomic64_fetch_or +#define arch_atomic64_xor generic_atomic64_xor +#define arch_atomic64_fetch_xor generic_atomic64_fetch_xor + +#define arch_atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define arch_atomic64_cmpxchg generic_atomic64_cmpxchg +#define arch_atomic64_xchg generic_atomic64_xchg +#define arch_atomic64_fetch_add_unless generic_atomic64_fetch_add_unless + +#else /* ARCH_ATOMIC */ + +#define atomic64_read generic_atomic64_read +#define atomic64_set generic_atomic64_set +#define atomic64_set_release generic_atomic64_set + +#define atomic64_add generic_atomic64_add +#define atomic64_add_return generic_atomic64_add_return +#define atomic64_fetch_add generic_atomic64_fetch_add +#define atomic64_sub generic_atomic64_sub +#define atomic64_sub_return generic_atomic64_sub_return +#define atomic64_fetch_sub generic_atomic64_fetch_sub + +#define atomic64_and generic_atomic64_and +#define atomic64_fetch_and generic_atomic64_fetch_and +#define atomic64_or generic_atomic64_or +#define atomic64_fetch_or generic_atomic64_fetch_or +#define atomic64_xor generic_atomic64_xor +#define atomic64_fetch_xor generic_atomic64_fetch_xor + +#define atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define atomic64_cmpxchg generic_atomic64_cmpxchg +#define atomic64_xchg generic_atomic64_xchg +#define atomic64_fetch_add_unless generic_atomic64_fetch_add_unless + +#endif /* ARCH_ATOMIC */ #endif /* _ASM_GENERIC_ATOMIC64_H */ lib/atomic64.c +18
-19@@ -42,7 +42,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v)return &atomic64_lock[addr & (NR_LOCKS -1 )].lock; } -s64 atomic64_read (const atomic64_t *v) +s64 generic_atomic64_read (const atomic64_t *v) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -53,9 +53,9 @@ s64 atomic64_read(const atomic64_t *v)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_read); +EXPORT_SYMBOL (generic_atomic64_read); -void atomic64_set (atomic64_t *v,s64 i) +void generic_atomic64_set (atomic64_t *v,s64 i) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -64,10 +64,10 @@ void atomic64_set(atomic64_t *v, s64 i) v->counter = i;raw_spin_unlock_irqrestore (lock, flags); } -EXPORT_SYMBOL (atomic64_set); +EXPORT_SYMBOL (generic_atomic64_set);#define ATOMIC64_OP (op, c_op) \ -void atomic64_##op(s64 a, atomic64_t *v) \ +void generic_atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \@@ -76,10 +76,10 @@ void atomic64_##op(s64 a, atomic64_t *v) \ v->counter c_op a; \raw_spin_unlock_irqrestore (lock, flags); \ } \ -EXPORT_SYMBOL(atomic64_##op); +EXPORT_SYMBOL(generic_atomic64_##op);#define ATOMIC64_OP_RETURN (op, c_op) \ -s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \@@ -90,10 +90,10 @@ s64 atomic64_##op##_return(s64 a, atomic64_t *v) \raw_spin_unlock_irqrestore (lock, flags); \return val; \ } \ -EXPORT_SYMBOL(atomic64_##op##_return); +EXPORT_SYMBOL(generic_atomic64_##op##_return);#define ATOMIC64_FETCH_OP (op, c_op) \ -s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \@@ -105,7 +105,7 @@ s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \raw_spin_unlock_irqrestore (lock, flags); \return val; \ } \ -EXPORT_SYMBOL(atomic64_fetch_##op); +EXPORT_SYMBOL(generic_atomic64_fetch_##op);#define ATOMIC64_OPS (op, c_op) \ ATOMIC64_OP(op, c_op) \@@ -130,7 +130,7 @@ ATOMIC64_OPS(xor, ^=)#undef ATOMIC64_OP_RETURN#undef ATOMIC64_OP -s64 atomic64_dec_if_positive (atomic64_t *v) +s64 generic_atomic64_dec_if_positive (atomic64_t *v) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -143,9 +143,9 @@ s64 atomic64_dec_if_positive(atomic64_t *v)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_dec_if_positive); +EXPORT_SYMBOL (generic_atomic64_dec_if_positive); -s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n) +s64 generic_atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -158,9 +158,9 @@ s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_cmpxchg); +EXPORT_SYMBOL (generic_atomic64_cmpxchg); -s64 atomic64_xchg (atomic64_t *v,s64 new) +s64 generic_atomic64_xchg (atomic64_t *v,s64 new) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -172,9 +172,9 @@ s64 atomic64_xchg(atomic64_t *v, s64 new)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_xchg); +EXPORT_SYMBOL (generic_atomic64_xchg); -s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u) +s64 generic_atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -188,4 +188,4 @@ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)return val; } -EXPORT_SYMBOL (atomic64_fetch_add_unless); +EXPORT_SYMBOL (generic_atomic64_fetch_add_unless); --2.11.0
Signature · 4 lines
Best Regards Guo Ren ML: https://lore.kernel.org/linux-csky/
Mark Rutland
reviewerRe: [PATCH] asm-generic: atomic64: handle ARCH_ATOMIC builds (was "Re: [PATCH v3 1/2] locking/atomics: Fixup GENERIC_ATOMIC64 conflict") with atomic-arch-fallback.h
Quoted 25 lines · Guo Ren
Quoted 24 lines · Mark Rutland
Hi Guo,
Quoted 12 lines · On Wed, Apr 21, 2021 at 12:53:45PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Current
GENERIC_ATOMIC64inatomic-arch-fallback.his broken. When a 32-bit arch useatomic-arch-fallback.hwill cause compile error.In file included from
include/linux/atomic.h:81,
frominclude/linux/rcupdate.h:25,
frominclude/linux/rculist.h:11,
frominclude/linux/pid.h:5,
frominclude/linux/sched.h:14,
fromarch/riscv/kernel/asm-offsets.c:10:
include/linux/atomic-arch-fallback.h: In function 'arch_atomic64_inc':Quoted 4 lines
Quoted 1 line
Quoted 1 line
include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function 'arch_atomic64_add'; did you mean 'arch_atomic_add'? [-Werror=implicit-function-declaration]1447 | arch_atomic64_add(1, v); | ^~~~~~~~~~~~~~~~~ | arch_atomic_addThis is expected;
GENERIC_ATOMIC64doesn't implement arch_atomic64_*(), and thus violates the expectations of the fallback code.
Quoted 6 lines
Quoted 3 lines
To make
GENERIC_ATOMIC64compatible withARCH_ATOMIC, theGENERIC_ATOMIC64implementation *must* provide arch_atomic64_*() functions.How do you let a "static
__always_inline" of "asm-generic/atomic-instrumented.h" call a real function? Seelib/atomic64.c.
Can you rephrase the question? I don't understand what you're asking here.
If you're asking about how the calls are directed to generic_atomic64_*(), the atomic-instrumented atomic64_<foo>() function will try to call arch_atomic64_<foo>(), and the pre-processor definitions in asm-generic/atomic64.h will direct that to generic_atomic64_<foo>().
If you're asking about inlining specifically, I'm afraid I don't understand. It's legitimate for a static __always_inline function A to call a non-inlined function B, and this works just fine. In that case, A will get inlined into its caller, and B will not, but nothing stops A from calling B.
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
The
atomic-arch-fallback.h&atomic-fallback.h&atomic-instrumented.hare generated bygen-atomic-fallback.sh&gen-atomic-instrumented.sh, so just take care the bash files.Remove the dependency of atomic-*-fallback.h in
atomic64.h.Please don't duplicate the fallbacks; this'll make it harder to move other over and eventually remove the non-ARCH_ATOMIC implementations.
Quoted 6 lines
Quoted 2 lines
Does the patch below make things work for you, or have I missed something?
RISC-V combines 32bit & 64bit together just like x86. Current
ARCH_ATOMICcould work perfectly with RV64, but not RV32.RV32 still could use
ARCH_ATOMICto improve kasan check.
I understand that (and I want riscv to use ARCH_ATOMIC), but that doesn't answer my question.
I went and built this locally, starting with v5.12-rc8, applying my patch, then applying your second patch atop. Both defconfig and rv32_defconfig build just fine, though I wasn't able to check KASAN with GCC 10.1.0.
Is there a problem that I've missed, or does my patch work?
Thanks,
Mark.
Quoted 270 lines
Quoted 263 lines
I've given this a basic build test on an arm config using
GENERIC_ATOMIC64(but notARCH_ATOMIC).Thanks,
Mark.
---->8----
From7f0389c8a1f41ecb5b2700f6ba38ff2ba093eb33Mon Sep 17 00:00:00 2001
From: Mark Rutland <[email protected]>
Date: Thu, 22 Apr 2021 11:26:04 +0100
Subject: [PATCH] asm-generic: atomic64: handleARCH_ATOMICbuildsWe'd like all architectures to convert to
ARCH_ATOMIC, as this will enable functionality, and once all architectures are converted it will be possible to make significant cleanups to the atomic headers.A number of architectures use
GENERIC_ATOMIC64, and it's impractical to convert them all in one go. To make it possible to convert them one-by-one, let's make theGENERIC_ATOMIC64implementation function as either atomic64_*() or arch_atomic64_*() depending on whetherARCH_ATOMICis selected. To do this, the C implementations are prefixed as generic_atomic64_*(), and theasm-generic/atomic64.hheader maps atomic64_*()/arch_atomic64_*() onto these as appropriate via teh preprocessor.Once all users are moved over to
ARCH_ATOMICthe ifdeffery in the header can be simplified and/or removed entirely.For existing users (none of which select
ARCH_ATOMIC), there should be no functional change as a result of this patch.Signed-off-by Mark Rutland <[email protected]>Cc Arnd Bergmann <[email protected]>Cc Guo Ren <[email protected]>Cc Peter Zijlstra <[email protected]>--- include/asm-generic/atomic64.h | 74 ++++++++++++++++++++++++++++++++++-------- lib/atomic64.c | 36 ++++++++++---------- 2 files changed, 79 insertions(+), 31 deletions(-)include/asm-generic/atomic64.h +61
-13@@ -15,19 +15,17 @@ typedef struct {#define ATOMIC64_INIT (i) { (i) } -extern s64 atomic64_read (const atomic64_t *v); -extern void atomic64_set (atomic64_t *v,s64 i); - -#define atomic64_set_release (v, i) atomic64_set((v), (i)) +extern s64 generic_atomic64_read (const atomic64_t *v); +extern void generic_atomic64_set (atomic64_t *v,s64 i);#define ATOMIC64_OP (op) \ -extern void atomic64_##op(s64 a, atomic64_t *v); +extern void generic_atomic64_##op(s64 a, atomic64_t *v);#define ATOMIC64_OP_RETURN (op) \ -extern s64 atomic64_##op##_return(s64 a, atomic64_t *v); +extern s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v);#define ATOMIC64_FETCH_OP (op) \ -extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); +extern s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v);#define ATOMIC64_OPS (op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op)@@ -46,11 +44,61 @@ ATOMIC64_OPS(xor)#undef ATOMIC64_OP_RETURN#undef ATOMIC64_OP -extern s64 atomic64_dec_if_positive (atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive -extern s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); -extern s64 atomic64_xchg (atomic64_t *v,s64 new); -extern s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u); -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +extern s64 generic_atomic64_dec_if_positive (atomic64_t *v); +extern s64 generic_atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); +extern s64 generic_atomic64_xchg (atomic64_t *v,s64 new); +extern s64 generic_atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u); + +#ifdef ARCH_ATOMIC + +#define arch_atomic64_read generic_atomic64_read +#define arch_atomic64_set generic_atomic64_set +#define arch_atomic64_set_release generic_atomic64_set + +#define arch_atomic64_add generic_atomic64_add +#define arch_atomic64_add_return generic_atomic64_add_return +#define arch_atomic64_fetch_add generic_atomic64_fetch_add +#define arch_atomic64_sub generic_atomic64_sub +#define arch_atomic64_sub_return generic_atomic64_sub_return +#define arch_atomic64_fetch_sub generic_atomic64_fetch_sub + +#define arch_atomic64_and generic_atomic64_and +#define arch_atomic64_fetch_and generic_atomic64_fetch_and +#define arch_atomic64_or generic_atomic64_or +#define arch_atomic64_fetch_or generic_atomic64_fetch_or +#define arch_atomic64_xor generic_atomic64_xor +#define arch_atomic64_fetch_xor generic_atomic64_fetch_xor + +#define arch_atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define arch_atomic64_cmpxchg generic_atomic64_cmpxchg +#define arch_atomic64_xchg generic_atomic64_xchg +#define arch_atomic64_fetch_add_unless generic_atomic64_fetch_add_unless + +#else /* ARCH_ATOMIC */ + +#define atomic64_read generic_atomic64_read +#define atomic64_set generic_atomic64_set +#define atomic64_set_release generic_atomic64_set + +#define atomic64_add generic_atomic64_add +#define atomic64_add_return generic_atomic64_add_return +#define atomic64_fetch_add generic_atomic64_fetch_add +#define atomic64_sub generic_atomic64_sub +#define atomic64_sub_return generic_atomic64_sub_return +#define atomic64_fetch_sub generic_atomic64_fetch_sub + +#define atomic64_and generic_atomic64_and +#define atomic64_fetch_and generic_atomic64_fetch_and +#define atomic64_or generic_atomic64_or +#define atomic64_fetch_or generic_atomic64_fetch_or +#define atomic64_xor generic_atomic64_xor +#define atomic64_fetch_xor generic_atomic64_fetch_xor + +#define atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define atomic64_cmpxchg generic_atomic64_cmpxchg +#define atomic64_xchg generic_atomic64_xchg +#define atomic64_fetch_add_unless generic_atomic64_fetch_add_unless + +#endif /* ARCH_ATOMIC */ #endif /* _ASM_GENERIC_ATOMIC64_H */ lib/atomic64.c +18
-19@@ -42,7 +42,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v)return &atomic64_lock[addr & (NR_LOCKS -1 )].lock; } -s64 atomic64_read (const atomic64_t *v) +s64 generic_atomic64_read (const atomic64_t *v) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -53,9 +53,9 @@ s64 atomic64_read(const atomic64_t *v)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_read); +EXPORT_SYMBOL (generic_atomic64_read); -void atomic64_set (atomic64_t *v,s64 i) +void generic_atomic64_set (atomic64_t *v,s64 i) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -64,10 +64,10 @@ void atomic64_set(atomic64_t *v, s64 i) v->counter = i;raw_spin_unlock_irqrestore (lock, flags); } -EXPORT_SYMBOL (atomic64_set); +EXPORT_SYMBOL (generic_atomic64_set);#define ATOMIC64_OP (op, c_op) \ -void atomic64_##op(s64 a, atomic64_t *v) \ +void generic_atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \@@ -76,10 +76,10 @@ void atomic64_##op(s64 a, atomic64_t *v) \ v->counter c_op a; \raw_spin_unlock_irqrestore (lock, flags); \ } \ -EXPORT_SYMBOL(atomic64_##op); +EXPORT_SYMBOL(generic_atomic64_##op);#define ATOMIC64_OP_RETURN (op, c_op) \ -s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \@@ -90,10 +90,10 @@ s64 atomic64_##op##_return(s64 a, atomic64_t *v) \raw_spin_unlock_irqrestore (lock, flags); \return val; \ } \ -EXPORT_SYMBOL(atomic64_##op##_return); +EXPORT_SYMBOL(generic_atomic64_##op##_return);#define ATOMIC64_FETCH_OP (op, c_op) \ -s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \@@ -105,7 +105,7 @@ s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \raw_spin_unlock_irqrestore (lock, flags); \return val; \ } \ -EXPORT_SYMBOL(atomic64_fetch_##op); +EXPORT_SYMBOL(generic_atomic64_fetch_##op);#define ATOMIC64_OPS (op, c_op) \ ATOMIC64_OP(op, c_op) \@@ -130,7 +130,7 @@ ATOMIC64_OPS(xor, ^=)#undef ATOMIC64_OP_RETURN#undef ATOMIC64_OP -s64 atomic64_dec_if_positive (atomic64_t *v) +s64 generic_atomic64_dec_if_positive (atomic64_t *v) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -143,9 +143,9 @@ s64 atomic64_dec_if_positive(atomic64_t *v)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_dec_if_positive); +EXPORT_SYMBOL (generic_atomic64_dec_if_positive); -s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n) +s64 generic_atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -158,9 +158,9 @@ s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_cmpxchg); +EXPORT_SYMBOL (generic_atomic64_cmpxchg); -s64 atomic64_xchg (atomic64_t *v,s64 new) +s64 generic_atomic64_xchg (atomic64_t *v,s64 new) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -172,9 +172,9 @@ s64 atomic64_xchg(atomic64_t *v, s64 new)raw_spin_unlock_irqrestore (lock, flags);return val; } -EXPORT_SYMBOL (atomic64_xchg); +EXPORT_SYMBOL (generic_atomic64_xchg); -s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u) +s64 generic_atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v);@@ -188,4 +188,4 @@ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)return val; } -EXPORT_SYMBOL (atomic64_fetch_add_unless); +EXPORT_SYMBOL (generic_atomic64_fetch_add_unless); --2.11.0
Signature · 4 lines
Best Regards Guo Ren ML: https://lore.kernel.org/linux-csky/
Mark Rutland
reviewerHi Guo,
I had a look at completing the treewide conversion to ARCH_ATOMIC, and I've pushed the results to my atomics/arch-atomic branch on kernel.org:
https://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.git/log/?h=atomics/arch-atomic
git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.git atomics/arch-atomic
Once v5.13-rc1 is out I intend to rebase and post the series to LKML. The Kbuild test robot has been attacked the series for the last few days, and I think it's fairly robust now, but if you wanted to take an early look at the riscv or csky bits that'd be much appreciated.
I've omitted the structural changes and optimizations you had prepared for riscv, but I think those can be applied atop (or applied as a preparatory step for the ARCH_ATOMIC conversion). I'd like to keep the two steps separate to make bisection easier.
Thanks,
Mark.
Quoted 556 lines · On Wed, Apr 21, 2021 at 12:53:45PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Current
GENERIC_ATOMIC64inatomic-arch-fallback.his broken. When a 32-bit arch useatomic-arch-fallback.hwill cause compile error.In file included from
include/linux/atomic.h:81,
frominclude/linux/rcupdate.h:25,
frominclude/linux/rculist.h:11,
frominclude/linux/pid.h:5,
frominclude/linux/sched.h:14,
fromarch/riscv/kernel/asm-offsets.c:10:
include/linux/atomic-arch-fallback.h: In function 'arch_atomic64_inc':Quoted 1 line
Quoted 1 line
include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function 'arch_atomic64_add'; did you mean 'arch_atomic_add'? [-Werror=implicit-function-declaration]1447 | arch_atomic64_add(1, v); | ^~~~~~~~~~~~~~~~~ | arch_atomic_addThe
atomic-arch-fallback.h&atomic-fallback.h&atomic-instrumented.hare generated bygen-atomic-fallback.sh&gen-atomic-instrumented.sh, so just take care the bash files.Remove the dependency of atomic-*-fallback.h in
atomic64.h.Signed-off-by Guo Ren <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Arnd Bergmann <[email protected]>--- include/asm-generic/atomic-instrumented.h | 264 +++++++++++++++--------------- include/asm-generic/atomic64.h | 89 ++++++++++ include/linux/atomic-arch-fallback.h | 5 +- include/linux/atomic-fallback.h | 5 +- scripts/atomic/gen-atomic-fallback.sh | 3 +- scripts/atomic/gen-atomic-instrumented.sh | 23 ++- 6 files changed, 251 insertions(+), 138 deletions(-)include/asm-generic/atomic-instrumented.h +133
-131@@ -831,6 +831,137 @@ atomic_dec_if_positive(atomic_t *v)#define atomic_dec_if_positive atomic_dec_if_positive#endif +#if !defined(arch_xchg_relaxed) || defined(arch_xchg) +#define xchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_acquire) +#define xchg_acquire (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_release) +#define xchg_release (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_relaxed) +#define xchg_relaxed (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) +#define cmpxchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_acquire) +#define cmpxchg_acquire (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_release) +#define cmpxchg_release (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_relaxed) +#define cmpxchg_relaxed (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) +#define try_cmpxchg (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_acquire) +#define try_cmpxchg_acquire (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_release) +#define try_cmpxchg_release (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_relaxed) +#define try_cmpxchg_relaxed (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#define cmpxchg_local (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ +}) + +#define sync_cmpxchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) + +#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline s64atomic64_read (const atomic64_t *v) {@@ -1641,78 +1772,6 @@ atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec_if_positive atomic64_dec_if_positive#endif -#if !defined(arch_xchg_relaxed) || defined(arch_xchg) -#define xchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_acquire) -#define xchg_acquire (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_release) -#define xchg_release (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_release(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_relaxed) -#define xchg_relaxed (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) -#define cmpxchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_acquire) -#define cmpxchg_acquire (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_release) -#define cmpxchg_release (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_relaxed) -#define cmpxchg_relaxed (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ -}) -#endif -#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)#define cmpxchg64 (ptr, ...) \ ({ \@@ -1749,57 +1808,6 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif -#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) -#define try_cmpxchg (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_acquire) -#define try_cmpxchg_acquire (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_release) -#define try_cmpxchg_release (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_relaxed) -#define try_cmpxchg_relaxed (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#define cmpxchg_local (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ -}) -#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \@@ -1807,13 +1815,7 @@ atomic64_dec_if_positive(atomic64_t *v)arch_cmpxchg64_local (__ai_ptr, __VA_ARGS__); \ }) -#define sync_cmpxchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ -}) - +#endif #define cmpxchg_double (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \@@ -1830,4 +1832,4 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ -// 4bec382e44520f4d8267e42620054db26a659ea3 +// 21c7f7e074cb2cb7e2f593fe7c8f6dec6ab9e7ea include/asm-generic/atomic64.h +89
-0@@ -34,6 +34,18 @@ extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); ATOMIC64_OPS(add) ATOMIC64_OPS(sub) +#define atomic64_add_relaxed atomic64_add +#define atomic64_add_acquire atomic64_add +#define atomic64_add_release atomic64_add + +#define atomic64_add_return_relaxed atomic64_add_return +#define atomic64_add_return_acquire atomic64_add_return +#define atomic64_add_return_release atomic64_add_return + +#define atomic64_fetch_add_relaxed atomic64_fetch_add +#define atomic64_fetch_add_acquire atomic64_fetch_add +#define atomic64_fetch_add_release atomic64_fetch_add +#undef ATOMIC64_OPS#define ATOMIC64_OPS (op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op)@@ -49,8 +61,85 @@ ATOMIC64_OPS(xor)extern s64 atomic64_dec_if_positive (atomic64_t *v);#define atomic64_dec_if_positive atomic64_dec_if_positiveextern s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); +#define atomic64_cmpxchg_relaxed atomic64_cmpxchg +#define atomic64_cmpxchg_acquire atomic64_cmpxchg +#define atomic64_cmpxchg_release atomic64_cmpxchgextern s64 atomic64_xchg (atomic64_t *v,s64 new); +#define atomic64_xchg_relaxed atomic64_xchg +#define atomic64_xchg_acquire atomic64_xchg +#define atomic64_xchg_release atomic64_xchgextern s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u);#define atomic64_fetch_add_unless atomic64_fetch_add_unless +static __always_inline void +atomic64_inc (atomic64_t *v) +{ +atomic64_add (1 , v); +} + +static __always_inline s64 +atomic64_inc_return (atomic64_t *v) +{ +return atomic64_add_return (1 , v); +} + +static __always_inline s64 +atomic64_fetch_inc (atomic64_t *v) +{ +return atomic64_fetch_add (1 , v); +} + +static __always_inline void +atomic64_dec (atomic64_t *v) +{ +atomic64_sub (1 , v); +} + +static __always_inline s64 +atomic64_dec_return (atomic64_t *v) +{ +return atomic64_sub_return (1 , v); +} + +static __always_inline s64 +atomic64_fetch_dec (atomic64_t *v) +{ +return atomic64_fetch_sub (1 , v); +} + +static __always_inline void +atomic64_andnot (s64 i,atomic64_t *v) +{ +atomic64_and (~i, v); +} + +static __always_inline s64 +atomic64_fetch_andnot (s64 i,atomic64_t *v) +{ +return atomic64_fetch_and (~i, v); +} + +static __always_inline bool +atomic64_sub_and_test (int i,atomic64_t *v) +{ +return atomic64_sub_return (i, v) ==0 ; +} + +static __always_inline bool +atomic64_dec_and_test (atomic64_t *v) +{ +return atomic64_dec_return (v) ==0 ; +} + +static __always_inline bool +atomic64_inc_and_test (atomic64_t *v) +{ +return atomic64_inc_return (v) ==0 ; +} + +static __always_inline bool +atomic64_add_negative (s64 i,atomic64_t *v) +{ +return atomic64_add_return (i, v) <0 ; +}#endif /* _ASM_GENERIC_ATOMIC64_H */ include/linux/atomic-arch-fallback.h +3
-2@@ -1252,7 +1252,7 @@ arch_atomic_dec_if_positive(atomic_t *v)#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else #ifndef arch_atomic64_read_acquirestatic __always_inline s64@@ -2357,5 +2357,6 @@ arch_atomic64_dec_if_positive(atomic64_t *v)#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive#endif +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// cca554917d7ea73d5e3e7397dd70c484cad9b2c4 +// ae31a21075855e67a9b2927f8241dedddafda046 include/linux/atomic-fallback.h +3
-2@@ -1369,7 +1369,7 @@ atomic_dec_if_positive(atomic_t *v)#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else #define arch_atomic64_read atomic64_read#define arch_atomic64_read_acquire atomic64_read_acquire@@ -2591,5 +2591,6 @@ atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec_if_positive atomic64_dec_if_positive#endif +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// d78e6c293c661c15188f0ec05bce45188c8d5892 +// b809c8e3c88910826f765bdba4a74f21c527029d scripts/atomic/gen-atomic-fallback.sh +2
-1@@ -247,7 +247,7 @@ done cat <<EOF #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else EOF@@ -256,5 +256,6 @@ grep '^[a-z]' "$1" | while read name meta args; do done cat <<EOF +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ EOFscripts/atomic/gen-atomic-instrumented.sh +21
-2@@ -182,21 +182,40 @@ grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic" "int" ${args} done +for xchg in "xchg" "cmpxchg" "try_cmpxchg"; do + for order in "" "_acquire" "_release" "_relaxed"; do + gen_optional_xchg "${xchg}" "${order}" + done +done + +for xchg in "cmpxchg_local" "sync_cmpxchg"; do + gen_xchg "${xchg}" "" + printf "\n" +done + +cat <<EOF +#ifndef CONFIG_GENERIC_ATOMIC64 +EOF + grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do +for xchg in "cmpxchg64"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_optional_xchg "${xchg}" "${order}" done done -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do +for xchg in "cmpxchg64_local"; do gen_xchg "${xchg}" "" printf "\n" done +cat <<EOF +#endif +EOF + gen_xchg "cmpxchg_double" "2 * " printf "\n\n"Signature · 1 line
2.7.4
Guo Ren
maintainerQuoted 6 lines · Mark Rutland
Hi Guo,
I had a look at completing the treewide conversion to
ARCH_ATOMIC, and I've pushed the results to my atomics/arch-atomic branch on kernel.org:
Quoted 7 lines
https://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.git/log/?h=atomics/arch-atomic
git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.gitatomics/arch-atomicOnce v5.13-rc1 is out I intend to rebase and post the series to LKML. The
Kbuildtest robot has been attacked the series for the last few days, and I think it's fairly robust now, but if you wanted to take an early look at the riscv or csky bits that'd be much appreciated.
riscv is ok, but csky:
UPD include/generated/timeconst.h
In file included from ./include/asm-generic/atomic.h:12:0,
from ./arch/csky/include/generated/asm/atomic.h:1,
from ./include/linux/atomic.h:7,
from ./include/asm-generic/bitops/lock.h:5,
from ./arch/csky/include/asm/bitops.h:70,
from ./include/linux/bitops.h:32,
from ./include/linux/kernel.h:11,
from ./include/asm-generic/bug.h:20,
from ./arch/csky/include/asm/bug.h:18,
from ./include/linux/bug.h:5,
from ./include/linux/page-flags.h:10,
from kernel/bounds.c:10:
./include/asm-generic/atomic.h: In function ‘generic_atomic_add_return’:
./arch/csky/include/asm/cmpxchg.h:71:10: error: implicit declaration
of function ‘cmpxchg_relaxed’ [-Werror=implicit-function-declaration]
__ret = cmpxchg_relaxed(ptr, o, n); \
^
./include/asm-generic/atomic.h:35:16: note: in expansion of macro ‘arch_cmpxchg’
while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \
^~~~~~~~~~~~./include/asm-generic/atomic.h:96:1: note: in expansion of macro ‘ATOMIC_OP_RETURN’
ATOMIC_OP_RETURN(add, +)
^~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors scripts/Makefile.build:116: recipe for target 'kernel/bounds.s' failed make[1]: *** [kernel/bounds.s] Error 1
Makefile:1234: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2
you can fixup with:
arch/csky/include/asm/cmpxchg.h +0 -9
@@ -64,15 +64,6 @@ extern void __bad_xchg(void);#define arch_cmpxchg_relaxed (ptr, o, n) \ (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) -#define arch_cmpxchg (ptr, o, n) \ -({ \ - __typeof__(*(ptr)) __ret; \ - __smp_release_fence(); \ - __ret = cmpxchg_relaxed(ptr, o, n); \ - __smp_acquire_fence(); \ - __ret; \ -}) -#else #include <asm-generic/cmpxchg.h> #endif
include/asm-generic/atomic.h +11 -0
@@ -14,6 +14,17 @@#ifdef CONFIG_SMP +#ifndef arch_cmpxchg +#define arch_cmpxchg (ptr, o, n) \ +({ \ + __typeof__(*(ptr)) __ret; \ + __smp_release_fence(); \ + __ret = arch_cmpxchg_relaxed(ptr, o, n); \ + __smp_acquire_fence(); \ + __ret; \ +}) +#endif +/* we can build all atomic primitives from cmpxchg */ #define ATOMIC_OP (op, c_op) \
I think we could optimize asm-generic/atomic.h with acquire | release | full_fence for the arch which only supports "cas" or "lrsc" (no atomic instructions) could use "asm-generic/atomic.h" as the default implementation.
Quoted 562 lines
I've omitted the structural changes and optimizations you had prepared for riscv, but I think those can be applied atop (or applied as a preparatory step for the
ARCH_ATOMICconversion). I'd like to keep the two steps separate to make bisection easier.Thanks,
Mark.Quoted 552 lines · On Wed, Apr 21, 2021 at 12:53:45PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Current
GENERIC_ATOMIC64inatomic-arch-fallback.his broken. When a 32-bit arch useatomic-arch-fallback.hwill cause compile error.In file included from
include/linux/atomic.h:81,
frominclude/linux/rcupdate.h:25,
frominclude/linux/rculist.h:11,
frominclude/linux/pid.h:5,
frominclude/linux/sched.h:14,
fromarch/riscv/kernel/asm-offsets.c:10:
include/linux/atomic-arch-fallback.h: In function 'arch_atomic64_inc':Quoted 1 line
Quoted 1 line
include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function 'arch_atomic64_add'; did you mean 'arch_atomic_add'? [-Werror=implicit-function-declaration]1447 | arch_atomic64_add(1, v); | ^~~~~~~~~~~~~~~~~ | arch_atomic_addThe
atomic-arch-fallback.h&atomic-fallback.h&atomic-instrumented.hare generated bygen-atomic-fallback.sh&gen-atomic-instrumented.sh, so just take care the bash files.Remove the dependency of atomic-*-fallback.h in
atomic64.h.Signed-off-by Guo Ren <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Arnd Bergmann <[email protected]>--- include/asm-generic/atomic-instrumented.h | 264 +++++++++++++++--------------- include/asm-generic/atomic64.h | 89 ++++++++++ include/linux/atomic-arch-fallback.h | 5 +- include/linux/atomic-fallback.h | 5 +- scripts/atomic/gen-atomic-fallback.sh | 3 +- scripts/atomic/gen-atomic-instrumented.sh | 23 ++- 6 files changed, 251 insertions(+), 138 deletions(-)include/asm-generic/atomic-instrumented.h +133
-131@@ -831,6 +831,137 @@ atomic_dec_if_positive(atomic_t *v)#define atomic_dec_if_positive atomic_dec_if_positive#endif +#if !defined(arch_xchg_relaxed) || defined(arch_xchg) +#define xchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_acquire) +#define xchg_acquire (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_release) +#define xchg_release (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_xchg_relaxed) +#define xchg_relaxed (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) +#define cmpxchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_acquire) +#define cmpxchg_acquire (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_release) +#define cmpxchg_release (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_cmpxchg_relaxed) +#define cmpxchg_relaxed (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ +}) +#endif + +#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) +#define try_cmpxchg (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_acquire) +#define try_cmpxchg_acquire (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_release) +#define try_cmpxchg_release (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg_relaxed) +#define try_cmpxchg_relaxed (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#define cmpxchg_local (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ +}) + +#define sync_cmpxchg (ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ +}) + +#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline s64atomic64_read (const atomic64_t *v) {@@ -1641,78 +1772,6 @@ atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec_if_positive atomic64_dec_if_positive#endif -#if !defined(arch_xchg_relaxed) || defined(arch_xchg) -#define xchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_acquire) -#define xchg_acquire (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_release) -#define xchg_release (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_release(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_xchg_relaxed) -#define xchg_relaxed (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) -#define cmpxchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_acquire) -#define cmpxchg_acquire (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_release) -#define cmpxchg_release (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_cmpxchg_relaxed) -#define cmpxchg_relaxed (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ -}) -#endif -#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)#define cmpxchg64 (ptr, ...) \ ({ \@@ -1749,57 +1808,6 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif -#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) -#define try_cmpxchg (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_acquire) -#define try_cmpxchg_acquire (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_release) -#define try_cmpxchg_release (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#if defined(arch_try_cmpxchg_relaxed) -#define try_cmpxchg_relaxed (ptr, oldp, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ -}) -#endif - -#define cmpxchg_local (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ -}) -#define cmpxchg64_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \@@ -1807,13 +1815,7 @@ atomic64_dec_if_positive(atomic64_t *v)arch_cmpxchg64_local (__ai_ptr, __VA_ARGS__); \ }) -#define sync_cmpxchg (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ -}) - +#endif #define cmpxchg_double (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \@@ -1830,4 +1832,4 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ -// 4bec382e44520f4d8267e42620054db26a659ea3 +// 21c7f7e074cb2cb7e2f593fe7c8f6dec6ab9e7ea include/asm-generic/atomic64.h +89
-0@@ -34,6 +34,18 @@ extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); ATOMIC64_OPS(add) ATOMIC64_OPS(sub) +#define atomic64_add_relaxed atomic64_add +#define atomic64_add_acquire atomic64_add +#define atomic64_add_release atomic64_add + +#define atomic64_add_return_relaxed atomic64_add_return +#define atomic64_add_return_acquire atomic64_add_return +#define atomic64_add_return_release atomic64_add_return + +#define atomic64_fetch_add_relaxed atomic64_fetch_add +#define atomic64_fetch_add_acquire atomic64_fetch_add +#define atomic64_fetch_add_release atomic64_fetch_add +#undef ATOMIC64_OPS#define ATOMIC64_OPS (op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op)@@ -49,8 +61,85 @@ ATOMIC64_OPS(xor)extern s64 atomic64_dec_if_positive (atomic64_t *v);#define atomic64_dec_if_positive atomic64_dec_if_positiveextern s64 atomic64_cmpxchg (atomic64_t *v,s64 o,s64 n); +#define atomic64_cmpxchg_relaxed atomic64_cmpxchg +#define atomic64_cmpxchg_acquire atomic64_cmpxchg +#define atomic64_cmpxchg_release atomic64_cmpxchgextern s64 atomic64_xchg (atomic64_t *v,s64 new); +#define atomic64_xchg_relaxed atomic64_xchg +#define atomic64_xchg_acquire atomic64_xchg +#define atomic64_xchg_release atomic64_xchgextern s64 atomic64_fetch_add_unless (atomic64_t *v,s64 a,s64 u);#define atomic64_fetch_add_unless atomic64_fetch_add_unless +static __always_inline void +atomic64_inc (atomic64_t *v) +{ +atomic64_add (1 , v); +} + +static __always_inline s64 +atomic64_inc_return (atomic64_t *v) +{ +return atomic64_add_return (1 , v); +} + +static __always_inline s64 +atomic64_fetch_inc (atomic64_t *v) +{ +return atomic64_fetch_add (1 , v); +} + +static __always_inline void +atomic64_dec (atomic64_t *v) +{ +atomic64_sub (1 , v); +} + +static __always_inline s64 +atomic64_dec_return (atomic64_t *v) +{ +return atomic64_sub_return (1 , v); +} + +static __always_inline s64 +atomic64_fetch_dec (atomic64_t *v) +{ +return atomic64_fetch_sub (1 , v); +} + +static __always_inline void +atomic64_andnot (s64 i,atomic64_t *v) +{ +atomic64_and (~i, v); +} + +static __always_inline s64 +atomic64_fetch_andnot (s64 i,atomic64_t *v) +{ +return atomic64_fetch_and (~i, v); +} + +static __always_inline bool +atomic64_sub_and_test (int i,atomic64_t *v) +{ +return atomic64_sub_return (i, v) ==0 ; +} + +static __always_inline bool +atomic64_dec_and_test (atomic64_t *v) +{ +return atomic64_dec_return (v) ==0 ; +} + +static __always_inline bool +atomic64_inc_and_test (atomic64_t *v) +{ +return atomic64_inc_return (v) ==0 ; +} + +static __always_inline bool +atomic64_add_negative (s64 i,atomic64_t *v) +{ +return atomic64_add_return (i, v) <0 ; +}#endif /* _ASM_GENERIC_ATOMIC64_H */ include/linux/atomic-arch-fallback.h +3
-2@@ -1252,7 +1252,7 @@ arch_atomic_dec_if_positive(atomic_t *v)#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else #ifndef arch_atomic64_read_acquirestatic __always_inline s64@@ -2357,5 +2357,6 @@ arch_atomic64_dec_if_positive(atomic64_t *v)#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive#endif +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// cca554917d7ea73d5e3e7397dd70c484cad9b2c4 +// ae31a21075855e67a9b2927f8241dedddafda046 include/linux/atomic-fallback.h +3
-2@@ -1369,7 +1369,7 @@ atomic_dec_if_positive(atomic_t *v)#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else #define arch_atomic64_read atomic64_read#define arch_atomic64_read_acquire atomic64_read_acquire@@ -2591,5 +2591,6 @@ atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec_if_positive atomic64_dec_if_positive#endif +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// d78e6c293c661c15188f0ec05bce45188c8d5892 +// b809c8e3c88910826f765bdba4a74f21c527029d scripts/atomic/gen-atomic-fallback.sh +2
-1@@ -247,7 +247,7 @@ done cat <<EOF #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> -#endif +#else EOF@@ -256,5 +256,6 @@ grep '^[a-z]' "$1" | while read name meta args; do done cat <<EOF +#endif /* CONFIG_GENERIC_ATOMIC64 */ #endif /* _LINUX_ATOMIC_FALLBACK_H */ EOFscripts/atomic/gen-atomic-instrumented.sh +21
-2@@ -182,21 +182,40 @@ grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic" "int" ${args} done +for xchg in "xchg" "cmpxchg" "try_cmpxchg"; do + for order in "" "_acquire" "_release" "_relaxed"; do + gen_optional_xchg "${xchg}" "${order}" + done +done + +for xchg in "cmpxchg_local" "sync_cmpxchg"; do + gen_xchg "${xchg}" "" + printf "\n" +done + +cat <<EOF +#ifndef CONFIG_GENERIC_ATOMIC64 +EOF + grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do +for xchg in "cmpxchg64"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_optional_xchg "${xchg}" "${order}" done done -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do +for xchg in "cmpxchg64_local"; do gen_xchg "${xchg}" "" printf "\n" done +cat <<EOF +#endif +EOF + gen_xchg "cmpxchg_double" "2 * "
Quoted 4 lines
Quoted 4 lines
scripts/atomic/gen-atomic-instrumented.sh +0
-1printf "\n\n" --2.7.4
--
Best Regards
Guo Ren