atomics: API cleanups
Subsystems (29)
Mark Rutland
reviewerv2 cover atomics: API cleanups
This series contains a few cleanups of the atomic API, fixing inconsistencies between atomic_* and atomic64_*, and minimizing repetition in arch code. This is nicer for arch code, and the improved regularity will help when generating the atomic headers in future.
Since v1 [1]:
* Add missing inc_not_zero #define for x86_32
* Remove newly redundant test op definitions for riscv
* Remove atomic_inc_not_zero_hint()
* Make inc/dec ops optional
The bulk of the patches reorganise things so architectures consistently provide <atomic>_fetch_add_unless(), with atomic_fetch_add_unless() provided as a wrapper by core code. A generic fallback is provided for <atomic>_fetch_add_unless(), based on <atomic>_read() and <atomic>_try_cmpxchg().
Other patches in the series add common fallbacks for:
* atomic64_inc_not_zero()
* <atomic>_inc_and_test()
* <atomic>_dec_and_test()
* <atomic>_sub_and_test()
* <atomic>add_negative()
* <atomic>_*inc*()
* <atomic>_*dec*()
... as almost all architectures provide (near-)identical implementation of these today. Where an architecture provides a non-trivial definition, it is updated to provide a matching preprocessor symbol, and the instrumented atomics are updated correspondingly.
The end result is a strongly negative diffstat, though <linux/atomic.h> grows by a reasonable amount. We could halve this in future by templating the various fallbacks for atomic{,64}_t with code generation.
I've pushed this out to my atomics/api-cleanup branch [2] on kernel.org.
Thanks,
Mark.
[1] https://lkml.kernel.org/r/[email protected]
[2] git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux.git atomics/api-cleanup
Mark Rutland (16):
atomics/treewide: s/__atomic_add_unless/atomic_fetch_add_unless/ atomics/treewide: remove redundant atomic_inc_not_zero() definitions atomics/treewide: make atomic64_inc_not_zero() optional atomics/treewide: make atomic_fetch_add_unless() optional
atomics: prepare for atomic64_fetch_add_unless() atomics/generic: define atomic64_fetch_add_unless() atomics/alpha: define atomic64_fetch_add_unless() atomics/arc: define atomic64_fetch_add_unless() atomics/arm: define atomic64_fetch_add_unless() atomics/powerpc: define atomic64_fetch_add_unless() atomics/riscv: define atomic64_fetch_add_unless() atomics/treewide: make atomic64_fetch_add_unless() optional atomics/treewide: make test ops optional atomics/treewide: remove atomic_inc_not_zero_hint() atomics/treewide: make unconditional inc/dec ops optional atomics/treewide: make conditional inc/dec ops optional
arch/alpha/include/asm/atomic.h | 56 ++--- arch/arc/include/asm/atomic.h | 78 ++----- arch/arm/include/asm/atomic.h | 55 ++--- arch/arm64/include/asm/atomic.h | 47 +--- arch/h8300/include/asm/atomic.h | 15 +- arch/hexagon/include/asm/atomic.h | 18 +- arch/ia64/include/asm/atomic.h | 81 ------- arch/m68k/include/asm/atomic.h | 24 +- arch/mips/include/asm/atomic.h | 172 -------------- arch/openrisc/include/asm/atomic.h | 4 +- arch/parisc/include/asm/atomic.h | 107 --------- arch/powerpc/include/asm/atomic.h | 52 ++--- arch/riscv/include/asm/atomic.h | 149 +----------- arch/s390/include/asm/atomic.h | 65 ------ arch/sh/include/asm/atomic.h | 35 --- arch/sparc/include/asm/atomic_32.h | 24 +- arch/sparc/include/asm/atomic_64.h | 65 +----- arch/sparc/lib/atomic32.c | 4 +- arch/x86/include/asm/atomic.h | 30 +-- arch/x86/include/asm/atomic64_32.h | 61 +---- arch/x86/include/asm/atomic64_64.h | 48 +--- arch/xtensa/include/asm/atomic.h | 98 -------- drivers/block/rbd.c | 2 +- drivers/infiniband/core/rdma_core.c | 2 +- fs/afs/rxrpc.c | 2 +- include/asm-generic/atomic-instrumented.h | 69 +++++- include/asm-generic/atomic.h | 33 --- include/asm-generic/atomic64.h | 14 +- include/linux/atomic.h | 371 +++++++++++++++++++++++++----- kernel/bpf/syscall.c | 4 +- lib/atomic64.c | 13 +- net/atm/pppoatm.c | 2 +- net/rxrpc/call_object.c | 2 +- net/rxrpc/conn_object.c | 4 +- net/rxrpc/local_object.c | 2 +- net/rxrpc/peer_object.c | 2 +- 36 files changed, 499 insertions(+), 1311 deletions(-)
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 1/16 atomics/treewide: s/__atomic_add_unless/atomic_fetch_add_unless/
While __atomic_add_unless was originally intended as a building-block for atomic_add_unless, it's now used in a number of places around the kernel. It's the only common atomic operation named __atomic*(), rather than atomic_*(), and for consistency it would be better named atomic_fetch_add_unless().
This lack of consistency is slightly confusing, and gets in the way of scripting atomics. Given that, let's clean things up and promote it to an offical part of the atomics API, in the form of atomic_fetch_add_unless().
This patch converts definitions and uses over to the new name, including the instrumented version, using the following script:
----
git grep -w __atomic_add_unless | while read line; do
sed -i '{s/\<__atomic_add_unless\>/atomic_fetch_add_unless/}' "${line%%:*}"; done
git grep -w __arch_atomic_add_unless | while read line; do
sed -i '{s/\<__arch_atomic_add_unless\>/arch_atomic_fetch_add_unless/}' "${line%%:*}"; done
----
Note that we do not have atomic{64,_long}_fetch_add_unless(), which will be introduced by later patches.
There should be no functional change as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 4 ++-- arch/arc/include/asm/atomic.h | 4 ++-- arch/arm/include/asm/atomic.h | 4 ++-- arch/arm64/include/asm/atomic.h | 2 +- arch/h8300/include/asm/atomic.h | 2 +- arch/hexagon/include/asm/atomic.h | 4 ++-- arch/ia64/include/asm/atomic.h | 2 +- arch/m68k/include/asm/atomic.h | 2 +- arch/mips/include/asm/atomic.h | 4 ++-- arch/openrisc/include/asm/atomic.h | 4 ++-- arch/parisc/include/asm/atomic.h | 4 ++-- arch/powerpc/include/asm/atomic.h | 8 ++++---- arch/riscv/include/asm/atomic.h | 4 ++-- arch/s390/include/asm/atomic.h | 2 +- arch/sh/include/asm/atomic.h | 4 ++-- arch/sparc/include/asm/atomic_32.h | 2 +- arch/sparc/include/asm/atomic_64.h | 2 +- arch/sparc/lib/atomic32.c | 4 ++-- arch/x86/include/asm/atomic.h | 4 ++-- arch/xtensa/include/asm/atomic.h | 4 ++-- drivers/block/rbd.c | 2 +- drivers/infiniband/core/rdma_core.c | 2 +- fs/afs/rxrpc.c | 2 +- include/asm-generic/atomic-instrumented.h | 4 ++-- include/asm-generic/atomic.h | 4 ++-- include/linux/atomic.h | 2 +- kernel/bpf/syscall.c | 4 ++-- net/rxrpc/call_object.c | 2 +- net/rxrpc/conn_object.c | 4 ++-- net/rxrpc/local_object.c | 2 +- net/rxrpc/peer_object.c | 2 +- 31 files changed, 50 insertions(+), 50 deletions(-)
arch/alpha/include/asm/atomic.h +2 -2
@@ -206,7 +206,7 @@ ATOMIC_OPS(xor, xor)#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -214,7 +214,7 @@ ATOMIC_OPS(xor, xor)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, new, old;smp_mb ();
arch/arc/include/asm/atomic.h +2 -2
@@ -309,7 +309,7 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)#undef ATOMIC_OP/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -317,7 +317,7 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v */ -#define __atomic_add_unless (v, a, u) \ +#define atomic_fetch_add_unless (v, a, u) \ ({ \ int c, old; \ \
arch/arm/include/asm/atomic.h +2 -2
@@ -130,7 +130,7 @@ static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) }#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int oldval, newval;unsigned long tmp;
@@ -215,7 +215,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)return ret; } -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old;
arch/arm64/include/asm/atomic.h +1 -1
@@ -125,7 +125,7 @@#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0)#define atomic_sub_and_test (i, v) (atomic_sub_return((i), (v)) == 0)#define atomic_add_negative (i, v) (atomic_add_return((i), (v)) < 0) -#define __atomic_add_unless (v, a, u) ___atomic_add_unless(v, a, u,) +#define atomic_fetch_add_unless (v, a, u) ___atomic_add_unless(v, a, u,)#define atomic_andnot atomic_andnot/*
arch/h8300/include/asm/atomic.h +1 -1
@@ -94,7 +94,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)return ret; } -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int ret;h8300flags flags;
arch/hexagon/include/asm/atomic.h +2 -2
@@ -164,7 +164,7 @@ ATOMIC_OPS(xor)#undef ATOMIC_OP/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer to value * @a: amount to add * @u: unless value is equal to u
@@ -173,7 +173,7 @@ ATOMIC_OPS(xor)* */ -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int __oldval; registerint tmp;
arch/ia64/include/asm/atomic.h +1 -1
@@ -215,7 +215,7 @@ ATOMIC64_FETCH_OP(xor, ^) (cmpxchg(&((v)->counter), old, new))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/m68k/include/asm/atomic.h +1 -1
@@ -211,7 +211,7 @@ static inline int atomic_add_negative(int i, atomic_t *v)return c !=0 ; } -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/mips/include/asm/atomic.h +2 -2
@@ -275,7 +275,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)#define atomic_xchg (v, new) (xchg(&((v)->counter), (new)))/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -283,7 +283,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/openrisc/include/asm/atomic.h +2 -2
@@ -100,7 +100,7 @@ ATOMIC_OP(xor)* * This is often used through atomic_inc_not_zero() */ -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int old, tmp;
@@ -119,7 +119,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)return old; } -#define __atomic_add_unless __atomic_add_unless +#define atomic_fetch_add_unless atomic_fetch_add_unless#include <asm-generic/atomic.h>
arch/parisc/include/asm/atomic.h +2 -2
@@ -78,7 +78,7 @@ static __inline__ int atomic_read(const atomic_t *v)#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -86,7 +86,7 @@ static __inline__ int atomic_read(const atomic_t *v)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/powerpc/include/asm/atomic.h +4 -4
@@ -218,7 +218,7 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v)#define atomic_xchg_relaxed (v, new) xchg_relaxed(&((v)->counter), (new))/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -226,13 +226,13 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int t; __asm__ __volatile__ (PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # __atomic_add_unless\n\ +"1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ cmpw 0,%0,%3 \n\ beq 2f \n\ add %0,%2,%0 \n"
@@ -538,7 +538,7 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) __asm__ __volatile__ (PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # __atomic_add_unless\n\ +"1: ldarx %0,0,%1 # atomic_fetch_add_unless\n\ cmpd 0,%0,%3 \n\ beq 2f \n\ add %0,%2,%0 \n"
arch/riscv/include/asm/atomic.h +2 -2
@@ -332,7 +332,7 @@ ATOMIC_OP(dec_and_test, dec, ==, 0, 64)#undef ATOMIC_OP/* This is required to provide a full barrier on success. */ -static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) {int prev, rc;
@@ -381,7 +381,7 @@ static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)*/ static __always_inline int atomic_inc_not_zero(atomic_t *v) { -return __atomic_add_unless (v,1 ,0 ); +return atomic_fetch_add_unless (v,1 ,0 ); }#ifndef CONFIG_GENERIC_ATOMIC64
arch/s390/include/asm/atomic.h +1 -1
@@ -90,7 +90,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)return __atomic_cmpxchg (&v->counter, old, new); } -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/sh/include/asm/atomic.h +2 -2
@@ -46,7 +46,7 @@#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -54,7 +54,7 @@* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/sparc/include/asm/atomic_32.h +1 -1
@@ -27,7 +27,7 @@ int atomic_fetch_or(int, atomic_t *);int atomic_fetch_xor (int ,atomic_t *);int atomic_cmpxchg (atomic_t *,int ,int );int atomic_xchg (atomic_t *,int ); -int __atomic_add_unless (atomic_t *,int ,int ); +int atomic_fetch_add_unless (atomic_t *,int ,int );void atomic_set (atomic_t *,int );#define atomic_set_release (v, i) atomic_set((v), (i))
arch/sparc/include/asm/atomic_64.h +1 -1
@@ -89,7 +89,7 @@ static inline int atomic_xchg(atomic_t *v, int new)return xchg (&v->counter, new); } -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
arch/sparc/lib/atomic32.c +2 -2
@@ -95,7 +95,7 @@ int atomic_cmpxchg(atomic_t *v, int old, int new) }EXPORT_SYMBOL (atomic_cmpxchg); -int __atomic_add_unless (atomic_t *v,int a,int u) +int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int ret;unsigned long flags;
@@ -107,7 +107,7 @@ int __atomic_add_unless(atomic_t *v, int a, int u)spin_unlock_irqrestore (ATOMIC_HASH (v), flags);return ret; } -EXPORT_SYMBOL (__atomic_add_unless); +EXPORT_SYMBOL (atomic_fetch_add_unless);/* Atomic operations are already serializing */ void atomic_set (atomic_t *v,int i)
arch/x86/include/asm/atomic.h +2 -2
@@ -254,7 +254,7 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) }/** -* __arch_atomic_add_unless - add unless the number is already a given value +* arch_atomic_fetch_add_unless - add unless the number is already a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -262,7 +262,7 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v)* Atomically adds @a to @v, so long as @v was not already @u. * Returns the old value of @v. */ -static __always_inline int __arch_atomic_add_unless(atomic_t *v, int a, int u) +static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) {int c =arch_atomic_read (v);
arch/xtensa/include/asm/atomic.h +2 -2
@@ -275,7 +275,7 @@ ATOMIC_OPS(xor)#define atomic_xchg (v, new) (xchg(&((v)->counter), new))/** -* __atomic_add_unless - add unless the number is a given value +* atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -283,7 +283,7 @@ ATOMIC_OPS(xor)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__int __atomic_add_unless (atomic_t *v,int a,int u) +static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
drivers/block/rbd.c +1 -1
@@ -61,7 +61,7 @@ static int atomic_inc_return_safe(atomic_t *v) {unsigned int counter; - counter = (unsigned int )__atomic_add_unless (v,1 ,0 ); + counter = (unsigned int )atomic_fetch_add_unless (v,1 ,0 );if (counter <= (unsigned int )INT_MAX )return (int )counter;
drivers/infiniband/core/rdma_core.c +1 -1
@@ -121,7 +121,7 @@ static int uverbs_try_lock_object(struct ib_uobject *uobj, bool exclusive)* this lock. */ if (!exclusive) -return __atomic_add_unless (&uobj->usecnt,1 ,-1 ) ==-1 ? +return atomic_fetch_add_unless (&uobj->usecnt,1 ,-1 ) ==-1 ? -EBUSY :0 ;/* lock is either WRITE or DESTROY - should be exclusive */
fs/afs/rxrpc.c +1 -1
@@ -648,7 +648,7 @@ static void afs_wake_up_async_call(struct sock *sk, struct rxrpc_call *rxcall,trace_afs_notify_call (rxcall, call); call->need_attention = true; - u =__atomic_add_unless (&call->usage,1 ,0 ); + u =atomic_fetch_add_unless (&call->usage,1 ,0 );if (u !=0 ) { trace_afs_call(call, afs_call_trace_wake, u, atomic_read(&call->net->nr_outstanding_calls),
include/asm-generic/atomic-instrumented.h +2 -2
@@ -84,10 +84,10 @@ static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 ne }#endif -static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) {kasan_check_write (v,sizeof (*v)); -return __arch_atomic_add_unless (v, a, u); +return arch_atomic_fetch_add_unless (v, a, u); }
include/asm-generic/atomic.h +2 -2
@@ -221,8 +221,8 @@ static inline void atomic_dec(atomic_t *v)#define atomic_xchg (ptr, v) (xchg(&(ptr)->counter, (v)))#define atomic_cmpxchg (v, old, new) (cmpxchg(&((v)->counter), (old), (new))) -#ifndef __atomic_add_unless -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +#ifndef atomic_fetch_add_unless +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
include/linux/atomic.h +1 -1
@@ -530,7 +530,7 @@*/ static inline int atomic_add_unless (atomic_t *v,int a,int u) { -return __atomic_add_unless (v, a, u) != u; +return atomic_fetch_add_unless (v, a, u) != u; }/**
kernel/bpf/syscall.c +2 -2
@@ -538,7 +538,7 @@ static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, {int refold; - refold =__atomic_add_unless (&map->refcnt,1 ,0 ); + refold =atomic_fetch_add_unless (&map->refcnt,1 ,0 );if (refold >=BPF_MAX_REFCNT ) {__bpf_map_put (map, false);
@@ -1115,7 +1115,7 @@ struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) {int refold; - refold =__atomic_add_unless (&prog->aux->refcnt,1 ,0 ); + refold =atomic_fetch_add_unless (&prog->aux->refcnt,1 ,0 );if (refold >=BPF_MAX_REFCNT ) {__bpf_prog_put (prog, false);
net/rxrpc/call_object.c +1 -1
@@ -415,7 +415,7 @@ void rxrpc_incoming_call(struct rxrpc_sock *rx,bool rxrpc_queue_call (struct rxrpc_call *call) {const void *here =__builtin_return_address (0 ); -int n =__atomic_add_unless (&call->usage,1 ,0 ); +int n =atomic_fetch_add_unless (&call->usage,1 ,0 );if (n ==0 )return false;if (rxrpc_queue_work (&call->processor))
net/rxrpc/conn_object.c +2 -2
@@ -266,7 +266,7 @@ void rxrpc_kill_connection(struct rxrpc_connection *conn)bool rxrpc_queue_conn (struct rxrpc_connection *conn) {const void *here =__builtin_return_address (0 ); -int n =__atomic_add_unless (&conn->usage,1 ,0 ); +int n =atomic_fetch_add_unless (&conn->usage,1 ,0 );if (n ==0 )return false;if (rxrpc_queue_work (&conn->processor))
@@ -309,7 +309,7 @@ rxrpc_get_connection_maybe(struct rxrpc_connection *conn)const void *here =__builtin_return_address (0 );if (conn) { -int n =__atomic_add_unless (&conn->usage,1 ,0 ); +int n =atomic_fetch_add_unless (&conn->usage,1 ,0 );if (n >0 )trace_rxrpc_conn (conn, rxrpc_conn_got, n +1 , here);else
net/rxrpc/local_object.c +1 -1
@@ -305,7 +305,7 @@ struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *local)const void *here =__builtin_return_address (0 );if (local) { -int n =__atomic_add_unless (&local->usage,1 ,0 ); +int n =atomic_fetch_add_unless (&local->usage,1 ,0 );if (n >0 )trace_rxrpc_local (local, rxrpc_local_got, n +1 , here);else
net/rxrpc/peer_object.c +1 -1
@@ -406,7 +406,7 @@ struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *peer)const void *here =__builtin_return_address (0 );if (peer) { -int n =__atomic_add_unless (&peer->usage,1 ,0 ); +int n =atomic_fetch_add_unless (&peer->usage,1 ,0 );if (n >0 )trace_rxrpc_peer (peer, rxrpc_peer_got, n +1 , here);else
Signature · 1 line
2.11.0
Re: [PATCHv2 01/16] atomics/treewide: s/__atomic_add_unless/atomic_fetch_add_unless/
Quoted 94 lines · On Tue, 29 May 2018 08:43:31 PDT (-0700), [email protected] wrote:
While
__atomic_add_unlesswas originally intended as a building-block foratomic_add_unless, it's now used in a number of places around the kernel. It's the only common atomic operation named __atomic*(), rather than atomic_*(), and for consistency it would be better namedatomic_fetch_add_unless().This lack of consistency is slightly confusing, and gets in the way of scripting atomics. Given that, let's clean things up and promote it to an offical part of the atomics API, in the form of
atomic_fetch_add_unless().This patch converts definitions and uses over to the new name, including the instrumented version, using the following script:
----
git grep -w__atomic_add_unless| while read line; do
sed -i '{s/\<__atomic_add_unless\>/atomic_fetch_add_unless/}' "${line%%:*}"; done
git grep -w__arch_atomic_add_unless| while read line; do
sed -i '{s/\<__arch_atomic_add_unless\>/arch_atomic_fetch_add_unless/}' "${line%%:*}"; done
----Note that we do not have atomic{64,_long}_fetch_add_unless(), which will be introduced by later patches.
There should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Will Deacon <[email protected]>--- arch/alpha/include/asm/atomic.h | 4 ++-- arch/arc/include/asm/atomic.h | 4 ++-- arch/arm/include/asm/atomic.h | 4 ++-- arch/arm64/include/asm/atomic.h | 2 +- arch/h8300/include/asm/atomic.h | 2 +- arch/hexagon/include/asm/atomic.h | 4 ++-- arch/ia64/include/asm/atomic.h | 2 +- arch/m68k/include/asm/atomic.h | 2 +- arch/mips/include/asm/atomic.h | 4 ++-- arch/openrisc/include/asm/atomic.h | 4 ++-- arch/parisc/include/asm/atomic.h | 4 ++-- arch/powerpc/include/asm/atomic.h | 8 ++++---- arch/riscv/include/asm/atomic.h | 4 ++-- arch/s390/include/asm/atomic.h | 2 +- arch/sh/include/asm/atomic.h | 4 ++-- arch/sparc/include/asm/atomic_32.h | 2 +- arch/sparc/include/asm/atomic_64.h | 2 +- arch/sparc/lib/atomic32.c | 4 ++-- arch/x86/include/asm/atomic.h | 4 ++-- arch/xtensa/include/asm/atomic.h | 4 ++-- drivers/block/rbd.c | 2 +- drivers/infiniband/core/rdma_core.c | 2 +- fs/afs/rxrpc.c | 2 +- include/asm-generic/atomic-instrumented.h | 4 ++-- include/asm-generic/atomic.h | 4 ++-- include/linux/atomic.h | 2 +- kernel/bpf/syscall.c | 4 ++-- net/rxrpc/call_object.c | 2 +- net/rxrpc/conn_object.c | 4 ++-- net/rxrpc/local_object.c | 2 +- net/rxrpc/peer_object.c | 2 +- 31 files changed, 50 insertions(+), 50 deletions(-)[...]
arch/riscv/include/asm/atomic.h +2
-2@@ -332,7 +332,7 @@ ATOMIC_OP(dec_and_test, dec, ==, 0, 64)#undef ATOMIC_OP/* This is required to provide a full barrier on success. */ -static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) {int prev, rc;@@ -381,7 +381,7 @@ static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)*/ static __always_inline int atomic_inc_not_zero(atomic_t *v) { -return __atomic_add_unless (v,1 ,0 ); +return atomic_fetch_add_unless (v,1 ,0 ); }#ifndef CONFIG_GENERIC_ATOMIC64 arch/s390/include/asm/atomic.h +0
-0
arch/s390/include/asm/atomic.h static inline int atomic_cmpxchg(atomic_t *v, int old, int new) · line 96
Quoted 8 lines
arch/s390/include/asm/atomic.h +1
-1@@ -90,7 +90,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)return __atomic_cmpxchg (&v->counter, old, new); } -static inline int __atomic_add_unless (atomic_t *v,int a,int u) +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) {int c, old; c =atomic_read (v);
Thanks!
Mark Rutland
reviewerPATCH v2 2/16 atomics/treewide: remove redundant atomic_inc_not_zero() definitions
In reply to Mark RutlandWhen atomic_inc_not_zero(v) isn't defined, <linux/atomic.h> will define it as falling back to atomic_add_unless((v), 1, 0), so there's no need for arch code to do so.
There should be no functional change as a result of this patch.
--- arch/arc/include/asm/atomic.h | 2 -- arch/hexagon/include/asm/atomic.h | 2 -- arch/riscv/include/asm/atomic.h | 9 --------- 3 files changed, 13 deletions(-)
arch/arc/include/asm/atomic.h +0 -2
@@ -336,8 +336,6 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) c; \ }) -#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) -#define atomic_inc (v) atomic_add(1, v)#define atomic_dec (v) atomic_sub(1, v)
arch/hexagon/include/asm/atomic.h +0 -2
@@ -197,8 +197,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)return __oldval; } -#define atomic_inc_not_zero (v) atomic_add_unless((v), 1, 0) -#define atomic_inc (v) atomic_add(1, (v))#define atomic_dec (v) atomic_sub(1, (v))
arch/riscv/include/asm/atomic.h +0 -9
@@ -375,15 +375,6 @@ static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) }#endif -/* -* The extra atomic operations that are constructed from one of the core -* LR/SC-based operations above. -*/ -static __always_inline int atomic_inc_not_zero(atomic_t *v) -{ -return atomic_fetch_add_unless (v,1 ,0 ); -} -#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline long atomic64_inc_not_zero (atomic64_t *v) {
Signature · 1 line
2.11.0
Re: [PATCHv2 02/16] atomics/treewide: remove redundant atomic_inc_not_zero() definitions
Quoted 26 lines · On Tue, 29 May 2018 08:43:32 PDT (-0700), [email protected] wrote:
When
atomic_inc_not_zero(v)isn't defined, <linux/atomic.h> will define it as falling back toatomic_add_unless((v), 1, 0), so there's no need for arch code to do so.There should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Will Deacon <[email protected]>--- arch/arc/include/asm/atomic.h | 2 -- arch/hexagon/include/asm/atomic.h | 2 -- arch/riscv/include/asm/atomic.h | 9 --------- 3 files changed, 13 deletions(-)[...]
arch/riscv/include/asm/atomic.h +0
-2@@ -375,15 +375,6 @@ static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) }#endif -/* -* The extra atomic operations that are constructed from one of the core
arch/riscv/include/asm/atomic.h static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) · line 380
Quoted 10 lines
arch/riscv/include/asm/atomic.h +0
-7-* LR/SC-based operations above. -*/ -static __always_inline int atomic_inc_not_zero(atomic_t *v) -{ -return atomic_fetch_add_unless (v,1 ,0 ); -} -#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline long atomic64_inc_not_zero (atomic64_t *v) {
Thanks!
Mark Rutland
reviewerPATCH v2 3/16 atomics/treewide: make atomic64_inc_not_zero() optional
In reply to Mark RutlandWe define a trivial fallback for atomic_inc_not_zero(), but don't do the same for atmic64_inc_not_zero(), leading most architectures to define the same boilerplate.
Let's add a fallback in <linux/atomic.h>, and remove the redundant implementations. Note that atomic64_add_unless() is always defined in <linux/atomic.h>, and promotes its arguments to the requisite types, so we need not do this explicitly.
There should be no functional change as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 2 -- arch/arc/include/asm/atomic.h | 1 - arch/arm/include/asm/atomic.h | 1 - arch/arm64/include/asm/atomic.h | 2 -- arch/ia64/include/asm/atomic.h | 2 -- arch/mips/include/asm/atomic.h | 2 -- arch/parisc/include/asm/atomic.h | 2 -- arch/powerpc/include/asm/atomic.h | 1 + arch/riscv/include/asm/atomic.h | 7 ------- arch/s390/include/asm/atomic.h | 1 - arch/sparc/include/asm/atomic_64.h | 2 -- arch/x86/include/asm/atomic64_32.h | 2 +- arch/x86/include/asm/atomic64_64.h | 2 -- include/asm-generic/atomic-instrumented.h | 3 +++ include/asm-generic/atomic64.h | 1 - include/linux/atomic.h | 11 +++++++++++ 16 files changed, 16 insertions(+), 26 deletions(-)
arch/alpha/include/asm/atomic.h +0 -2
@@ -296,8 +296,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)return old -1 ; } -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)
arch/arc/include/asm/atomic.h +0 -1
@@ -603,7 +603,6 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)#define atomic64_dec (v) atomic64_sub(1LL, (v))#define atomic64_dec_return (v) atomic64_sub_return(1LL, (v))#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0) -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1LL, 0LL)#endif /* !CONFIG_GENERIC_ATOMIC64 */
arch/arm/include/asm/atomic.h +0 -1
@@ -534,7 +534,6 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)#define atomic64_dec (v) atomic64_sub(1LL, (v))#define atomic64_dec_return_relaxed (v) atomic64_sub_return_relaxed(1LL, (v))#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0) -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1LL, 0LL)#endif /* !CONFIG_GENERIC_ATOMIC64 */ #endif
arch/arm64/include/asm/atomic.h +0 -2
@@ -204,7 +204,5 @@#define atomic64_add_unless (v, a, u) (___atomic_add_unless(v, a, u, 64) != u)#define atomic64_andnot atomic64_andnot -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) -#endif #endif
arch/ia64/include/asm/atomic.h +0 -2
@@ -246,8 +246,6 @@ static __inline__ long atomic64_add_unless(atomic64_t *v, long a, long u)return c != (u); } -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) -static __inline__long atomic64_dec_if_positive (atomic64_t *v) {long c, old, dec;
arch/mips/include/asm/atomic.h +0 -2
@@ -644,8 +644,6 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)return c != (u); } -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) -#define atomic64_dec_return (v) atomic64_sub_return(1, (v))#define atomic64_inc_return (v) atomic64_add_return(1, (v))
arch/parisc/include/asm/atomic.h +0 -2
@@ -305,8 +305,6 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)return c != (u); } -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) -/* * atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t
arch/powerpc/include/asm/atomic.h +1 -0
@@ -582,6 +582,7 @@ static __inline__ int atomic64_inc_not_zero(atomic64_t *v) return t1 != 0; } +#define atomic64_inc_not_zero (v) atomic64_inc_not_zero((v))#endif /* __powerpc64__ */
arch/riscv/include/asm/atomic.h +0 -7
@@ -375,13 +375,6 @@ static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) }#endif -#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline long atomic64_inc_not_zero (atomic64_t *v) -{ -return atomic64_add_unless (v,1 ,0 ); -} -#endif -/* * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as * {cmp,}xchg and the operations that return, so they need a full barrier.
arch/s390/include/asm/atomic.h +0 -1
@@ -212,6 +212,5 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)#define atomic64_dec (_v) atomic64_sub(1, _v)#define atomic64_dec_return (_v) atomic64_sub_return(1, _v)#define atomic64_dec_and_test (_v) (atomic64_sub_return(1, _v) == 0) -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0)#endif /* __ARCH_S390_ATOMIC__ */
arch/sparc/include/asm/atomic_64.h +0 -2
@@ -123,8 +123,6 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u) return c != (u); } -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) -long atomic64_dec_if_positive (atomic64_t *v);#endif /* !(__ARCH_SPARC64_ATOMIC__) */
arch/x86/include/asm/atomic64_32.h +1 -1
@@ -295,7 +295,7 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, long long a,return (int )a; } - +#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zerostatic inline int arch_atomic64_inc_not_zero (atomic64_t *v) {int r;
arch/x86/include/asm/atomic64_64.h +0 -2
@@ -207,8 +207,6 @@ static inline bool arch_atomic64_add_unless(atomic64_t *v, long a, long u)return true; } -#define arch_atomic64_inc_not_zero (v) arch_atomic64_add_unless((v), 1, 0) -/* * arch_atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t
include/asm-generic/atomic-instrumented.h +3 -0
@@ -205,11 +205,14 @@ static __always_inline s64 atomic64_dec_return(atomic64_t *v)return arch_atomic64_dec_return (v); } +#ifdef arch_atomic64_inc_not_zero +#define atomic64_inc_not_zero atomic64_inc_not_zero static __always_inline s64 atomic64_inc_not_zero(atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_inc_not_zero (v); } +#endif static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) {
include/asm-generic/atomic64.h +0 -1
@@ -62,6 +62,5 @@ extern int atomic64_add_unless(atomic64_t *v, long long a, long long u);#define atomic64_dec (v) atomic64_sub(1LL, (v))#define atomic64_dec_return (v) atomic64_sub_return(1LL, (v))#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0) -#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1LL, 0LL)#endif /* _ASM_GENERIC_ATOMIC64_H */
include/linux/atomic.h +11 -0
@@ -1048,6 +1048,17 @@ static inline int atomic_dec_if_positive(atomic_t *v)#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg#endif /* atomic64_try_cmpxchg */ +/** +* atomic64_inc_not_zero - increment unless the number is zero +* @v: pointer of type atomic64_t +* +* Atomically increments @v by 1, so long as @v is non-zero. +* Returns non-zero if @v was non-zero, and zero otherwise. +*/ +#ifndef atomic64_inc_not_zero +#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0) +#endif +#ifndef atomic64_andnotstatic inline void atomic64_andnot (long long i,atomic64_t *v) {
Signature · 1 line
2.11.0
Re: [PATCHv2 03/16] atomics/treewide: make atomic64_inc_not_zero() optional
Quoted 3 lines · On Tue, 29 May 2018 08:43:33 PDT (-0700), [email protected] wrote:
We define a trivial fallback for
atomic_inc_not_zero(), but don't do the same foratmic64_inc_not_zero(), leading most architectures to define the same boilerplate.
atmic64
Quoted 38 lines
Let's add a fallback in <
linux/atomic.h>, and remove the redundant implementations. Note thatatomic64_add_unless()is always defined in <linux/atomic.h>, and promotes its arguments to the requisite types, so we need not do this explicitly.There should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Will Deacon <[email protected]>--- arch/alpha/include/asm/atomic.h | 2 -- arch/arc/include/asm/atomic.h | 1 - arch/arm/include/asm/atomic.h | 1 - arch/arm64/include/asm/atomic.h | 2 -- arch/ia64/include/asm/atomic.h | 2 -- arch/mips/include/asm/atomic.h | 2 -- arch/parisc/include/asm/atomic.h | 2 -- arch/powerpc/include/asm/atomic.h | 1 + arch/riscv/include/asm/atomic.h | 7 ------- arch/s390/include/asm/atomic.h | 1 - arch/sparc/include/asm/atomic_64.h | 2 -- arch/x86/include/asm/atomic64_32.h | 2 +- arch/x86/include/asm/atomic64_64.h | 2 -- include/asm-generic/atomic-instrumented.h | 3 +++ include/asm-generic/atomic64.h | 1 - include/linux/atomic.h | 11 +++++++++++ 16 files changed, 16 insertions(+), 26 deletions(-)[...]
arch/riscv/include/asm/atomic.h +0
-0@@ -375,13 +375,6 @@ static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) }#endif
arch/riscv/include/asm/atomic.h static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) · line 380
Quoted 10 lines
arch/riscv/include/asm/atomic.h +0
-7-#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline long atomic64_inc_not_zero (atomic64_t *v) -{ -return atomic64_add_unless (v,1 ,0 ); -} -#endif -/* * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as * {cmp,}xchg and the operations that return, so they need a full barrier.
Thanks!
Mark Rutland
reviewerRe: [PATCHv2 03/16] atomics/treewide: make atomic64_inc_not_zero() optional
Quoted 6 lines · Palmer Dabbelt
Quoted 3 lines · On Tue, 29 May 2018 08:43:33 PDT (-0700), [email protected] wrote:
We define a trivial fallback for
atomic_inc_not_zero(), but don't do the same foratmic64_inc_not_zero(), leading most architectures to define the same boilerplate.atmic64
Cheers for the spot; fixed now!
Mark.
Mark Rutland
reviewerPATCH v2 4/16 atomics/treewide: make atomic_fetch_add_unless() optional
In reply to Mark RutlandSeveral architectures these have a near-identical implementation based on atomic_read() and atomic_cmpxchg() that we can instead define in <linux/atomic.h>, so let's do so, using something close to the existing x86 implementation with try_cmpxchg().
Where an architecture provides its own atomic_fetch_add_unless(), it must define a preprocessor symbol for it. The instrumented atomics are updated accordingly.
Note that arch/arc's existing atomic_fetch_add_unless() had redundant barriers, as these are already present in its atomic_cmpxchg() implementation.
There should be no functional change as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 2 +- arch/arc/include/asm/atomic.h | 28 ---------------------------- arch/arm/include/asm/atomic.h | 11 +---------- arch/arm64/include/asm/atomic.h | 1 - arch/h8300/include/asm/atomic.h | 1 + arch/hexagon/include/asm/atomic.h | 1 + arch/ia64/include/asm/atomic.h | 16 ---------------- arch/m68k/include/asm/atomic.h | 15 --------------- arch/mips/include/asm/atomic.h | 24 ------------------------ arch/parisc/include/asm/atomic.h | 24 ------------------------ arch/powerpc/include/asm/atomic.h | 1 + arch/riscv/include/asm/atomic.h | 1 + arch/s390/include/asm/atomic.h | 15 --------------- arch/sh/include/asm/atomic.h | 25 ------------------------- arch/sparc/include/asm/atomic_32.h | 2 ++ arch/sparc/include/asm/atomic_64.h | 15 --------------- arch/x86/include/asm/atomic.h | 21 --------------------- arch/xtensa/include/asm/atomic.h | 24 ------------------------ include/asm-generic/atomic-instrumented.h | 4 +++- include/asm-generic/atomic.h | 11 ----------- include/linux/atomic.h | 23 +++++++++++++++++++++++ 21 files changed, 34 insertions(+), 231 deletions(-)
arch/alpha/include/asm/atomic.h +1 -1
@@ -235,7 +235,7 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u)smp_mb ();return old; } - +#define atomic_fetch_add_unless atomic_fetch_add_unless/** * atomic64_add_unless - add unless the number is a given value
arch/arc/include/asm/atomic.h +0 -28
@@ -308,34 +308,6 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -/** -* atomic_fetch_add_unless - add unless the number is a given value -* @v: pointer of type atomic_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v -*/ -#define atomic_fetch_add_unless (v, a, u) \ -({ \ - int c, old; \ - \ -/* \ -* Explicit full memory barrier needed before/after as \ -* LLOCK/SCOND thmeselves don't provide any such semantics \ -*/ \ -smp_mb (); \ - \ - c =atomic_read (v); \ -while (c != (u ) && (old =atomic_cmpxchg ((v), c, c + (a))) != c)\ - c = old; \ - \ -smp_mb (); \ - \ - c; \ -}) -#define atomic_inc (v) atomic_add(1, v)#define atomic_dec (v) atomic_sub(1, v)
arch/arm/include/asm/atomic.h +1 -10
@@ -156,6 +156,7 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) return oldval; } +#define atomic_fetch_add_unless atomic_fetch_add_unless#else /* ARM_ARCH_6 */
@@ -215,16 +216,6 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } -static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - - c =atomic_read (v); -while (c != u && (old =atomic_cmpxchg ((v), c, c + a)) != c) - c = old; -return c; -} -#endif /* __LINUX_ARM_ARCH__ */ #define ATOMIC_OPS (op, c_op, asm_op) \
arch/arm64/include/asm/atomic.h +0 -1
@@ -125,7 +125,6 @@#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0)#define atomic_sub_and_test (i, v) (atomic_sub_return((i), (v)) == 0)#define atomic_add_negative (i, v) (atomic_add_return((i), (v)) < 0) -#define atomic_fetch_add_unless (v, a, u) ___atomic_add_unless(v, a, u,)#define atomic_andnot atomic_andnot/*
arch/h8300/include/asm/atomic.h +1 -0
@@ -106,5 +106,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)arch_local_irq_restore (flags); return ret; } +#define atomic_fetch_add_unless atomic_fetch_add_unless#endif /* __ARCH_H8300_ATOMIC __ */
arch/hexagon/include/asm/atomic.h +1 -0
@@ -196,6 +196,7 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) );return __oldval; } +#define atomic_fetch_add_unless atomic_fetch_add_unless#define atomic_inc (v) atomic_add(1, (v))#define atomic_dec (v) atomic_sub(1, (v))
arch/ia64/include/asm/atomic.h +0 -16
@@ -215,22 +215,6 @@ ATOMIC64_FETCH_OP(xor, ^) (cmpxchg(&((v)->counter), old, new))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c; -} - -static __inline__long atomic64_add_unless (atomic64_t *v,long a,long u) {long c, old;
arch/m68k/include/asm/atomic.h +0 -15
@@ -211,19 +211,4 @@ static inline int atomic_add_negative(int i, atomic_t *v) return c != 0; } -static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c; -} -#endif /* __ARCH_M68K_ATOMIC __ */
arch/mips/include/asm/atomic.h +0 -24
@@ -274,30 +274,6 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), (new))) -/** -* atomic_fetch_add_unless - add unless the number is a given value -* @v: pointer of type atomic_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v. -*/ -static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c; -} -#define atomic_dec_return (v) atomic_sub_return(1, (v))#define atomic_inc_return (v) atomic_add_return(1, (v))
arch/parisc/include/asm/atomic.h +0 -24
@@ -77,30 +77,6 @@ static __inline__ int atomic_read(const atomic_t *v)#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) -/** -* atomic_fetch_add_unless - add unless the number is a given value -* @v: pointer of type atomic_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v. -*/ -static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c; -} -#define ATOMIC_OP (op, c_op) \ static __inline__ void atomic_##op(int i, atomic_t *v) \ { \
arch/powerpc/include/asm/atomic.h +1 -0
@@ -248,6 +248,7 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u)return t; } +#define atomic_fetch_add_unless atomic_fetch_add_unless/** * atomic_inc_not_zero - increment unless the number is zero
arch/riscv/include/asm/atomic.h +1 -0
@@ -349,6 +349,7 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "memory");return prev; } +#define atomic_fetch_add_unless atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline long __atomic64_add_unless (atomic64_t *v,long a,long u)
arch/s390/include/asm/atomic.h +0 -15
@@ -90,21 +90,6 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)return __atomic_cmpxchg (&v->counter, old, new); } -static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == u)) -break ; - old =atomic_cmpxchg (v, c, c + a); -if (likely (old == c)) -break ; - c = old; - } -return c; -} -#define ATOMIC64_INIT (i) { (i) }static inline long atomic64_read (const atomic64_t *v)
arch/sh/include/asm/atomic.h +0 -25
@@ -45,31 +45,6 @@#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -/** -* atomic_fetch_add_unless - add unless the number is a given value -* @v: pointer of type atomic_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v. -*/ -static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } - -return c; -} -#endif /* CONFIG_CPU_J2 */ #endif /* __ASM_SH_ATOMIC_H */
arch/sparc/include/asm/atomic_32.h +2 -0
@@ -30,6 +30,8 @@ int atomic_xchg(atomic_t *, int);int atomic_fetch_add_unless (atomic_t *,int ,int );void atomic_set (atomic_t *,int ); +#define atomic_fetch_add_unless atomic_fetch_add_unless +#define atomic_set_release (v, i) atomic_set((v), (i))#define atomic_read (v) READ_ONCE((v)->counter)
arch/sparc/include/asm/atomic_64.h +0 -15
@@ -89,21 +89,6 @@ static inline int atomic_xchg(atomic_t *v, int new)return xchg (&v->counter, new); } -static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c; -} -#define atomic64_cmpxchg (v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new))
arch/x86/include/asm/atomic.h +0 -21
@@ -253,27 +253,6 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v)return val; } -/** -* arch_atomic_fetch_add_unless - add unless the number is already a given value -* @v: pointer of type atomic_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as @v was not already @u. -* Returns the old value of @v. -*/ -static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) -{ -int c =arch_atomic_read (v); - -do { -if (unlikely (c == u)) -break ; - }while (!arch_atomic_try_cmpxchg (v, &c, c + a)); - -return c; -} -#ifdef CONFIG_X86_32 # include <asm/atomic64_32.h> #else
arch/xtensa/include/asm/atomic.h +0 -24
@@ -274,30 +274,6 @@ ATOMIC_OPS(xor)#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) -/** -* atomic_fetch_add_unless - add unless the number is a given value -* @v: pointer of type atomic_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v. -*/ -static __inline__int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c; -} -#endif /* __KERNEL__ */ #endif /* _XTENSA_ATOMIC_H */
include/asm-generic/atomic-instrumented.h +3 -1
@@ -84,12 +84,14 @@ static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 ne }#endif +#ifdef arch_atomic_fetch_add_unless +#define atomic_fetch_add_unless atomic_fetch_add_unless static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) {kasan_check_write (v,sizeof (*v));return arch_atomic_fetch_add_unless (v, a, u); } - +#endif static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) {
include/asm-generic/atomic.h +0 -11
@@ -221,15 +221,4 @@ static inline void atomic_dec(atomic_t *v)#define atomic_xchg (ptr, v) (xchg(&(ptr)->counter, (v)))#define atomic_cmpxchg (v, old, new) (cmpxchg(&((v)->counter), (old), (new))) -#ifndef atomic_fetch_add_unless -static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) -{ -int c, old; - c =atomic_read (v); -while (c != u && (old =atomic_cmpxchg (v, c, c + a)) != c) - c = old; -return c; -} -#endif -#endif /* __ASM_GENERIC_ATOMIC_H */
include/linux/atomic.h +23 -0
@@ -520,6 +520,29 @@#endif /* xchg_relaxed */ /** +* atomic_fetch_add_unless - add unless the number is already a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as @v was not already @u. +* Returns original value of @v +*/ +#ifndef atomic_fetch_add_unless +static inline int atomic_fetch_add_unless (atomic_t *v,int a,int u) +{ +int c =atomic_read (v); + +do { +if (unlikely (c == u)) +break ; + }while (!atomic_try_cmpxchg (v, &c, c + a)); + +return c; +} +#endif + +/** * atomic_add_unless - add unless the number is already a given value * @v: pointer of type atomic_t * @a: the amount to add to v...
Signature · 1 line
2.11.0
Re: [PATCHv2 04/16] atomics/treewide: make atomic_fetch_add_unless() optional
Quoted 51 lines · On Tue, 29 May 2018 08:43:34 PDT (-0700), [email protected] wrote:
Several architectures these have a near-identical implementation based on
atomic_read()andatomic_cmpxchg()that we can instead define in <linux/atomic.h>, so let's do so, using something close to the existing x86 implementation withtry_cmpxchg().Where an architecture provides its own
atomic_fetch_add_unless(), it must define a preprocessor symbol for it. The instrumented atomics are updated accordingly.Note that arch/arc's existing
atomic_fetch_add_unless()had redundant barriers, as these are already present in itsatomic_cmpxchg()implementation.There should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Reviewed-by Geert Uytterhoeven <[email protected]>Cc Boqun Feng <[email protected]>Cc Will Deacon <[email protected]>Cc Vineet Gupta <[email protected]>--- arch/alpha/include/asm/atomic.h | 2 +- arch/arc/include/asm/atomic.h | 28 ---------------------------- arch/arm/include/asm/atomic.h | 11 +---------- arch/arm64/include/asm/atomic.h | 1 - arch/h8300/include/asm/atomic.h | 1 + arch/hexagon/include/asm/atomic.h | 1 + arch/ia64/include/asm/atomic.h | 16 ---------------- arch/m68k/include/asm/atomic.h | 15 --------------- arch/mips/include/asm/atomic.h | 24 ------------------------ arch/parisc/include/asm/atomic.h | 24 ------------------------ arch/powerpc/include/asm/atomic.h | 1 + arch/riscv/include/asm/atomic.h | 1 + arch/s390/include/asm/atomic.h | 15 --------------- arch/sh/include/asm/atomic.h | 25 ------------------------- arch/sparc/include/asm/atomic_32.h | 2 ++ arch/sparc/include/asm/atomic_64.h | 15 --------------- arch/x86/include/asm/atomic.h | 21 --------------------- arch/xtensa/include/asm/atomic.h | 24 ------------------------ include/asm-generic/atomic-instrumented.h | 4 +++- include/asm-generic/atomic.h | 11 ----------- include/linux/atomic.h | 23 +++++++++++++++++++++++ 21 files changed, 34 insertions(+), 231 deletions(-)[...]
arch/riscv/include/asm/atomic.h +0
-0
arch/riscv/include/asm/atomic.h static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) · line 355
Quoted 7 lines
arch/riscv/include/asm/atomic.h +1
-0@@ -349,6 +349,7 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "memory");return prev; } +#define atomic_fetch_add_unless atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 static __always_inline long __atomic64_add_unless (atomic64_t *v,long a,long u)
Thanks!
Mark Rutland
reviewerPATCH v2 5/16 atomics: prepare for atomic64_fetch_add_unless()
In reply to Mark RutlandCurrently architecture must implement atomic_fetch_add_unless(), with common code providing atomic_add_unless(). Architectures must also implement atmic64_add_unless() directly, with no corresponding atomic64_fetch_add_unless().
This divergenece is unfortunate, and means that the APIs for atomic_t, atomic64_t, and atomic_long_t differ.
In preparation for unifying things, with architectures providing atomic64_fetch_add_unless, this patch adds a generic atomic64_add_unless() which will use atomic64_fetch_add_unless(). The instrumented atomics are updated to take this case into account.
There should be no functional change as a result of this patch.
--- include/asm-generic/atomic-instrumented.h | 9 +++++++++ include/linux/atomic.h | 16 ++++++++++++++++ 2 files changed, 25 insertions(+)
include/asm-generic/atomic-instrumented.h +9 -0
@@ -93,11 +93,20 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) }#endif +#ifdef arch_atomic64_fetch_add_unless +#define atomic64_fetch_add_unless atomic64_fetch_add_unless +static __always_inline int atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ +kasan_check_write (v,sizeof (*v)); +return arch_atomic64_fetch_add_unless (v, a, u); +} +#else static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) {kasan_check_write (v,sizeof (*v));return arch_atomic64_add_unless (v, a, u); } +#endif static __always_inline void atomic_inc(atomic_t *v) {
include/linux/atomic.h +16 -0
@@ -1072,6 +1072,22 @@ static inline int atomic_dec_if_positive(atomic_t *v)#endif /* atomic64_try_cmpxchg */ /** +* atomic64_add_unless - add unless the number is already a given value +* @v: pointer of type atomic_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as @v was not already @u. +* Returns non-zero if @v was not @u, and zero otherwise. +*/ +#ifdef atomic64_fetch_add_unless +static inline int atomic64_add_unless (atomic64_t *v,long long a,long long u) +{ +return atomic64_fetch_add_unless (v, a, u) != u; +} +#endif + +/** * atomic64_inc_not_zero - increment unless the number is zero * @v: pointer of type atomic64_t *
Signature · 1 line
2.11.0
Peter Zijlstra
maintainerRe: [PATCHv2 05/16] atomics: prepare for atomic64_fetch_add_unless()
Quoted 8 lines · Mark Rutland
/**
+ *atomic64_add_unless- add unless the number is already a given value
+ * @v: pointer of typeatomic_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as @v was not already @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.
I always get confused by that wording; would something like: "Returns true if the addition was done" not be more clear?
include/linux/atomic.h static inline int atomic_dec_if_positive(atomic_t *v) · line 1084
Quoted 3 lines
+ */
include/linux/atomic.h +2
-0+#ifdef atomic64_fetch_add_unless +static inline int atomic64_add_unless (atomic64_t *v,long long a,long long u)
Do we want to make that a "bool' return?
Quoted 4 lines
include/linux/atomic.h +4
-0+{ +return atomic64_fetch_add_unless (v, a, u) != u; +} +#endif
Mark Rutland
reviewerRe: [PATCHv2 05/16] atomics: prepare for atomic64_fetch_add_unless()
Quoted 10 lines · Peter Zijlstra
Quoted 8 lines · Mark Rutland
/**
+ *atomic64_add_unless- add unless the number is already a given value
+ * @v: pointer of typeatomic_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as @v was not already @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.
Quoted 2 lines
I always get confused by that wording; would something like: "Returns true if the addition was done" not be more clear?
Sounds clearer to me; I just stole the wording from the existing atomic_add_unless().
I guess you'll want similar for the conditional inc/dec ops, e.g.
/**
* atomic_inc_not_zero - increment unless the number is zero
* @v: pointer of type atomic_t
*
* Atomically increments @v by 1, so long as @v is non-zero.
* Returns non-zero if @v was non-zero, and zero otherwise. */
Quoted 5 lines
include/linux/atomic.h static inline int atomic_dec_if_positive(atomic_t *v) · line 1084
Quoted 3 lines
+ */
include/linux/atomic.h +2
-0+#ifdef atomic64_fetch_add_unless +static inline int atomic64_add_unless (atomic64_t *v,long long a,long long u)Do we want to make that a "bool' return?
I think so -- that's what the instrumented wrappers (and x86) do today anyhow, and what I ended up using for the generated headers.
I'll spin a prep patch cleaning up the existing fallbacks in <linux/atomic.h>, along with the comment fixup above, then rework the additions likewise.
Thanks,
Mark.
Michael Ellerman
reviewerRe: [PATCHv2 05/16] atomics: prepare for atomic64_fetch_add_unless()
Quoted 19 lines · Mark Rutland
Quoted 10 lines · Peter Zijlstra
Quoted 8 lines · Mark Rutland
/**
+ *atomic64_add_unless- add unless the number is already a given value
+ * @v: pointer of typeatomic_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as @v was not already @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.Quoted 2 lines
I always get confused by that wording; would something like: "Returns true if the addition was done" not be more clear?
Sounds clearer to me; I just stole the wording from the existing
atomic_add_unless().I guess you'll want similar for the conditional inc/dec ops, e.g.
Quoted 7 lines
/**
*atomic_inc_not_zero- increment unless the number is zero
* @v: pointer of typeatomic_t
*
* Atomically increments @v by 1, so long as @v is non-zero.
* Returns non-zero if @v was non-zero, and zero otherwise. */
If we're bike-shedding .. :)
I think "so long as" is overly wordy and not helpful. It'd be clearer just as:
* Atomically increments @v by 1, if @v is non-zero.
cheers
Mark Rutland
reviewerRe: [PATCHv2 05/16] atomics: prepare for atomic64_fetch_add_unless()
Quoted 28 lines · Michael Ellerman
Quoted 19 lines · Mark Rutland
Quoted 10 lines · Peter Zijlstra
Quoted 8 lines · Mark Rutland
/**
+ *atomic64_add_unless- add unless the number is already a given value
+ * @v: pointer of typeatomic_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as @v was not already @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.Quoted 2 lines
I always get confused by that wording; would something like: "Returns true if the addition was done" not be more clear?
Sounds clearer to me; I just stole the wording from the existing
atomic_add_unless().I guess you'll want similar for the conditional inc/dec ops, e.g.
Quoted 7 lines
/**
*atomic_inc_not_zero- increment unless the number is zero
* @v: pointer of typeatomic_t
*
* Atomically increments @v by 1, so long as @v is non-zero.
* Returns non-zero if @v was non-zero, and zero otherwise. */
Quoted 6 lines
If we're bike-shedding .. :)
I think "so long as" is overly wordy and not helpful. It'd be clearer just as:
* Atomically increments @v by 1, if @v is non-zero.
I agree; done.
Mark.
Michael Ellerman
reviewerRe: [PATCHv2 05/16] atomics: prepare for atomic64_fetch_add_unless()
Quoted 29 lines · Mark Rutland
Quoted 28 lines · Michael Ellerman
Quoted 26 lines · Mark Rutland
Quoted 10 lines · Peter Zijlstra
Quoted 8 lines · Mark Rutland
/**
+ *atomic64_add_unless- add unless the number is already a given value
+ * @v: pointer of typeatomic_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as @v was not already @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.Quoted 2 lines
I always get confused by that wording; would something like: "Returns true if the addition was done" not be more clear?
Sounds clearer to me; I just stole the wording from the existing
atomic_add_unless().I guess you'll want similar for the conditional inc/dec ops, e.g.
/**
*atomic_inc_not_zero- increment unless the number is zero
* @v: pointer of typeatomic_t
*
* Atomically increments @v by 1, so long as @v is non-zero.
* Returns non-zero if @v was non-zero, and zero otherwise. */
Quoted 8 lines
Quoted 6 lines
If we're bike-shedding .. :)
I think "so long as" is overly wordy and not helpful. It'd be clearer just as:
* Atomically increments @v by 1, if @v is non-zero.
I agree; done.
Thanks.
cheers
Mark Rutland
reviewerPATCH v2 6/16 atomics/generic: define atomic64_fetch_add_unless()
In reply to Mark RutlandAs a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the generic implementation of atomic64_add_unless() into a generic implementation of atomic64_fetch_add_unless().
A wrapper in <linux/atomic.h> will build atomic_add_unless() atop of this, provided it is given a preprocessor definition.
No functional change is intended as a result of this patch.
--- include/asm-generic/atomic64.h | 3 ++- lib/atomic64.c | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)
include/asm-generic/atomic64.h +2 -1
@@ -52,7 +52,8 @@ ATOMIC64_OPS(xor)extern long long atomic64_dec_if_positive (atomic64_t *v);extern long long atomic64_cmpxchg (atomic64_t *v,long long o,long long n);extern long long atomic64_xchg (atomic64_t *v,long long new); -extern int atomic64_add_unless (atomic64_t *v,long long a,long long u); +extern long long atomic64_fetch_add_unless (atomic64_t *v,long long a,long long u); +#define atomic64_fetch_add_unless atomic64_fetch_add_unless#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)#define atomic64_inc (v) atomic64_add(1LL, (v))
lib/atomic64.c +6 -7
@@ -178,18 +178,17 @@ long long atomic64_xchg(atomic64_t *v, long long new) }EXPORT_SYMBOL (atomic64_xchg); -int atomic64_add_unless (atomic64_t *v,long long a,long long u) +long long atomic64_fetch_add_unless (atomic64_t *v,long long a,long long u) {unsigned long flags;raw_spinlock_t *lock =lock_addr (v); -int ret =0 ; +long long val;raw_spin_lock_irqsave (lock, flags); -if (v->counter != u) { + val = v->counter; +if (val != u) v->counter += a; - ret =1 ; - }raw_spin_unlock_irqrestore (lock, flags); -return ret; +return val; } -EXPORT_SYMBOL (atomic64_add_unless); +EXPORT_SYMBOL (atomic64_fetch_add_unless);
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 7/16 atomics/alpha: define atomic64_fetch_add_unless()
In reply to Mark RutlandAs a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the arch/alpha implementation of atomic64_add_unless() into an implementation of atomic64_fetch_add_unless().
A wrapper in <linux/atomic.h> will build atomic_add_unless() atop of this, provided it is given a preprocessor definition.
No functional change is intended as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-)
arch/alpha/include/asm/atomic.h +12 -11
@@ -238,35 +238,36 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u)#define atomic_fetch_add_unless atomic_fetch_add_unless/** -* atomic64_add_unless - add unless the number is a given value +* atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. * * Atomically adds @a to @v, so long as it was not @u. -* Returns true iff @v was not @u. +* Returns the old value of @v. */ -static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) +static __inline__int atomic64_fetch_add_unless (atomic64_t *v,long a,long u) { -long c, tmp; +long c, new, old;smp_mb (); __asm__ __volatile__( -"1: ldq_l %[tmp],%[mem]\n" -" cmpeq %[tmp],%[u],%[c]\n" -" addq %[tmp],%[a],%[tmp]\n" +"1: ldq_l %[old],%[mem]\n" +" cmpeq %[old],%[u],%[c]\n" +" addq %[old],%[a],%[new]\n" " bne %[c],2f\n" -" stq_c %[tmp],%[mem]\n" -" beq %[tmp],3f\n" +" stq_c %[new],%[mem]\n" +" beq %[new],3f\n" "2:\n" ".subsection 2\n" "3: br 1b\n" ".previous" - : [tmp]"=&r" (tmp), [c]"=&r" (c) + : [old]"=&r" (old), [new]"=&r" (new), [c]"=&r" (c) : [mem]"m" (*v), [a]"rI" (a), [u]"rI" (u) :"memory" );smp_mb (); -return !c; +return old; } +#define atomic64_fetch_add_unless atomic64_fetch_add_unless/* * atomic64_dec_if_positive - decrement by 1 if old value positive
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 8/16 atomics/arc: define atomic64_fetch_add_unless()
In reply to Mark RutlandAs a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the arch/arc implementation of atomic64_add_unless() into an implementation of atomic64_fetch_add_unless().
A wrapper in <linux/atomic.h> will build atomic_add_unless() atop of this, provided it is given a preprocessor definition.
No functional change is intended as a result of this patch.
--- arch/arc/include/asm/atomic.h | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)
arch/arc/include/asm/atomic.h +12 -13
@@ -531,41 +531,40 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) }/** -* atomic64_add_unless - add unless the number is a given value +* atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. * -* if (v != u) { v += a; ret = 1} else {ret = 0} -* Returns 1 iff @v was not @u (i.e. if add actually happened) +* Atomically adds @a to @v, so long as it was not @u. +* Returns the old value of @v */ -static inline int atomic64_add_unless (atomic64_t *v,long long a,long long u) +static inline long long atomic64_fetch_add_unless (atomic64_t *v,long long a, +long long u) { -long long val; -int op_done; +long long old, temp;smp_mb (); __asm__ __volatile__("1: llockd %0, [%2] \n" -" mov %1, 1 \n" " brne %L0, %L4, 2f # continue to add since v != u \n" " breq.d %H0, %H4, 3f # return since v == u \n" -" mov %1, 0 \n" "2: \n" -" add.f %L0, %L0, %L3 \n" -" adc %H0, %H0, %H3 \n" -" scondd %0, [%2] \n" +" add.f %L1, %L0, %L3 \n" +" adc %H1, %H0, %H3 \n" +" scondd %1, [%2] \n" " bnz 1b \n" "3: \n" - :"=&r" (val),"=&r" (op_done) + :"=&r" (old),"=&r" (temp) :"r" (&v->counter),"r" (a),"r" (u) :"cc" );/* memory clobber comes from smp_mb() */ smp_mb (); -return op_done; +return old; } +#define atomic64_fetch_add_unless atomic64_fetch_add_unless#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)#define atomic64_inc (v) atomic64_add(1LL, (v))
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 9/16 atomics/arm: define atomic64_fetch_add_unless()
In reply to Mark RutlandAs a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the arch/arm implementation of atomic64_add_unless() into an implementation of atomic64_fetch_add_unless().
A wrapper in <linux/atomic.h> will build atomic_add_unless() atop of this, provided it is given a preprocessor definition.
No functional change is intended as a result of this patch.
--- arch/arm/include/asm/atomic.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
arch/arm/include/asm/atomic.h +10 -10
@@ -486,11 +486,11 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v)return result; } -static inline int atomic64_add_unless (atomic64_t *v,long long a,long long u) +static inline long long atomic64_fetch_add_unless (atomic64_t *v,long long a, +long long u) { -long long val; +long long oldval, newval;unsigned long tmp; -int ret =1 ;smp_mb ();prefetchw (&v->counter);
@@ -499,23 +499,23 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)"1: ldrexd %0, %H0, [%4]\n" " teq %0, %5\n" " teqeq %H0, %H5\n" -" moveq %1, #0\n" " beq 2f\n" -" adds %Q0, %Q0, %Q6\n" -" adc %R0, %R0, %R6\n" -" strexd %2, %0, %H0, [%4]\n" +" adds %Q1, %Q0, %Q6\n" +" adc %R1, %R0, %R6\n" +" strexd %2, %1, %H1, [%4]\n" " teq %2, #0\n" " bne 1b\n" "2:" - : "=&r" (val), "+r" (ret), "=&r" (tmp), "+Qo" (v->counter) + : "=&r" (oldval), "=&r" (newval), "=&r" (tmp), "+Qo" (v->counter) : "r" (&v->counter), "r" (u), "r" (a) : "cc"); - if (ret) + if (oldval != u)smp_mb (); -return ret; +return oldval; } +#define atomic64_fetch_add_unless atomic64_fetch_add_unless#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)#define atomic64_inc (v) atomic64_add(1LL, (v))
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 10/16 atomics/powerpc: define atomic64_fetch_add_unless()
In reply to Mark RutlandAs a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the arch/powerpc implementation of atomic64_add_unless() into an implementation of atomic64_fetch_add_unless().
A wrapper in <linux/atomic.h> will build atomic_add_unless() atop of this, provided it is given a preprocessor definition.
No functional change is intended as a result of this patch.
--- arch/powerpc/include/asm/atomic.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
arch/powerpc/include/asm/atomic.h +5 -4
@@ -525,7 +525,7 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v)#define atomic64_xchg_relaxed (v, new) xchg_relaxed(&((v)->counter), (new))/** -* atomic64_add_unless - add unless the number is a given value +* atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u.
@@ -533,13 +533,13 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v)* Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) +static __inline__long atomic64_fetch_add_unless (atomic64_t *v,long a,long u) {long t; __asm__ __volatile__ (PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # atomic_fetch_add_unless\n\ +"1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ cmpd 0,%0,%3 \n\ beq 2f \n\ add %0,%2,%0 \n"
@@ -552,8 +552,9 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) : "r" (&v->counter), "r" (a), "r" (u) : "cc", "memory"); -return t != u; +return t; } +#define atomic64_fetch_add_unless atomic64_fetch_add_unless/** * atomic_inc64_not_zero - increment unless the number is zero
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 11/16 atomics/riscv: define atomic64_fetch_add_unless()
In reply to Mark RutlandAs a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the arch/riscv implementation of atomic64_add_unless() into an implementation of atomic64_fetch_add_unless().
A wrapper in <linux/atomic.h> will build atomic_add_unless() atop of this, provided it is given a preprocessor definition.
No functional change is intended as a result of this patch.
--- arch/riscv/include/asm/atomic.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
arch/riscv/include/asm/atomic.h +2 -6
@@ -352,7 +352,7 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)#define atomic_fetch_add_unless atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline long __atomic64_add_unless (atomic64_t *v,long a,long u) +static __always_inline long atomic64_fetch_add_unless (atomic64_t *v,long a,long u) {long prev, rc;
@@ -369,11 +369,7 @@ static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u) : "memory"); return prev; } - -static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) -{ -return __atomic64_add_unless (v, a, u) != u; -} +#define atomic64_fetch_add_unless atomic64_fetch_add_unless#endif /*
Signature · 1 line
2.11.0
Re: [PATCHv2 11/16] atomics/riscv: define atomic64_fetch_add_unless()
Quoted 35 lines · On Tue, 29 May 2018 08:43:41 PDT (-0700), [email protected] wrote:
As a step towards unifying the atomic/atomic64/atomic_long APIs, this patch converts the
arch/riscvimplementation ofatomic64_add_unless()into an implementation ofatomic64_fetch_add_unless().A wrapper in <
linux/atomic.h> will buildatomic_add_unless()atop of this, provided it is given a preprocessor definition.No functional change is intended as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Will Deacon <[email protected]>Cc Palmer Dabbelt <[email protected]>Cc Albert Ou <[email protected]>--- arch/riscv/include/asm/atomic.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)arch/riscv/include/asm/atomic.h +1
-1@@ -352,7 +352,7 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)#define atomic_fetch_add_unless atomic_fetch_add_unless#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline long __atomic64_add_unless (atomic64_t *v,long a,long u) +static __always_inline long atomic64_fetch_add_unless (atomic64_t *v,long a,long u) {long prev, rc;@@ -369,11 +369,7 @@ static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u) : "memory");return prev;
arch/riscv/include/asm/atomic.h static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u) · line 375
Quoted 10 lines
arch/riscv/include/asm/atomic.h +1
-5} - -static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) -{ -return __atomic64_add_unless (v, a, u) != u; -} +#define atomic64_fetch_add_unless atomic64_fetch_add_unless#endif /*
For some reason I remember there being a reason we were doing this in such an odd fashion but I can't remember what it was any more. Assuming this still builds, feel free to add an
Acked-by Palmer Dabbelt <[email protected]>
Thanks!
Mark Rutland
reviewerPATCH v2 12/16 atomics/treewide: make atomic64_fetch_add_unless() optional
In reply to Mark RutlandArchitectures with atomic64_fetch_add_unless provide a preprocessor symbol if they do so, and all other architectures have trivial C implementations of atomic64_add_unless() which are near-identical.
Let's unify the trivial definitions of atomic64_fetch_add_unless() in <linux/atomic.h>, so that we always have both atomic64_fetch_add_unless() and atomic64_add_unless() with less boilerplate code.
This means that atomic64_add_unless() is always implemented in core code, and the instrumented atomics are updated accordingly.
There should be no functional change as a result of this patch.
--- arch/arm64/include/asm/atomic.h | 12 ------------ arch/ia64/include/asm/atomic.h | 15 --------------- arch/mips/include/asm/atomic.h | 24 ------------------------ arch/parisc/include/asm/atomic.h | 24 ------------------------ arch/s390/include/asm/atomic.h | 16 ---------------- arch/sparc/include/asm/atomic_64.h | 15 --------------- arch/x86/include/asm/atomic64_64.h | 19 ------------------- include/asm-generic/atomic-instrumented.h | 6 ------ include/linux/atomic.h | 26 ++++++++++++++++++++++++-- 9 files changed, 24 insertions(+), 133 deletions(-)
arch/arm64/include/asm/atomic.h +0 -12
@@ -40,17 +40,6 @@#include <asm/cmpxchg.h> -#define ___atomic_add_unless (v, a, u, sfx) \ -({ \ - typeof((v)->counter) c, old; \ - \ - c = atomic##sfx##_read(v); \ - while (c != (u) && \ - (old = atomic##sfx##_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c; \ - }) -#define ATOMIC_INIT (i) { (i) }#define atomic_read (v) READ_ONCE((v)->counter)
@@ -200,7 +189,6 @@#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0)#define atomic64_sub_and_test (i, v) (atomic64_sub_return((i), (v)) == 0)#define atomic64_add_negative (i, v) (atomic64_add_return((i), (v)) < 0) -#define atomic64_add_unless (v, a, u) (___atomic_add_unless(v, a, u, 64) != u)#define atomic64_andnot atomic64_andnot#endif
arch/ia64/include/asm/atomic.h +0 -15
@@ -215,21 +215,6 @@ ATOMIC64_FETCH_OP(xor, ^) (cmpxchg(&((v)->counter), old, new))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -static __inline__long atomic64_add_unless (atomic64_t *v,long a,long u) -{ -long c, old; - c =atomic64_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic64_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c != (u); -} -static __inline__long atomic64_dec_if_positive (atomic64_t *v) {long c, old, dec;
arch/mips/include/asm/atomic.h +0 -24
@@ -596,30 +596,6 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) -/** -* atomic64_add_unless - add unless the number is a given value -* @v: pointer of type atomic64_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns true iff @v was not @u. -*/ -static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) -{ -long c, old; - c =atomic64_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic64_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c != (u); -} -#define atomic64_dec_return (v) atomic64_sub_return(1, (v))#define atomic64_inc_return (v) atomic64_add_return(1, (v))
arch/parisc/include/asm/atomic.h +0 -24
@@ -257,30 +257,6 @@ atomic64_read(const atomic64_t *v) ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -/** -* atomic64_add_unless - add unless the number is a given value -* @v: pointer of type atomic64_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v. -*/ -static __inline__int atomic64_add_unless (atomic64_t *v,long a,long u) -{ -long c, old; - c =atomic64_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic64_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c != (u); -} -/* * atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t
arch/s390/include/asm/atomic.h +0 -16
@@ -153,22 +153,6 @@ ATOMIC64_OPS(xor)#undef ATOMIC64_OPS -static inline int atomic64_add_unless (atomic64_t *v,long i,long u) -{ -long c, old; - - c =atomic64_read (v); -for (;;) { -if (unlikely (c == u)) -break ; - old =atomic64_cmpxchg (v, c, c + i); -if (likely (old == c)) -break ; - c = old; - } -return c != u; -} -static inline long atomic64_dec_if_positive (atomic64_t *v) {long c, old, dec;
arch/sparc/include/asm/atomic_64.h +0 -15
@@ -93,21 +93,6 @@ static inline int atomic_xchg(atomic_t *v, int new) ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -static inline long atomic64_add_unless (atomic64_t *v,long a,long u) -{ -long c, old; - c =atomic64_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic64_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c != (u); -} -long atomic64_dec_if_positive (atomic64_t *v);#endif /* !(__ARCH_SPARC64_ATOMIC__) */
arch/x86/include/asm/atomic64_64.h +0 -19
@@ -188,25 +188,6 @@ static inline long arch_atomic64_xchg(atomic64_t *v, long new)return xchg (&v->counter, new); } -/** -* arch_atomic64_add_unless - add unless the number is a given value -* @v: pointer of type atomic64_t -* @a: the amount to add to v... -* @u: ...unless v is equal to u. -* -* Atomically adds @a to @v, so long as it was not @u. -* Returns the old value of @v. -*/ -static inline bool arch_atomic64_add_unless (atomic64_t *v,long a,long u) -{ -s64 c =arch_atomic64_read (v); -do { -if (unlikely (c == u)) -return false; - }while (!arch_atomic64_try_cmpxchg (v, &c, c + a)); -return true; -} -/* * arch_atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t
include/asm-generic/atomic-instrumented.h +0 -6
@@ -100,12 +100,6 @@ static __always_inline int atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 ukasan_check_write (v,sizeof (*v));return arch_atomic64_fetch_add_unless (v, a, u); } -#else -static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) -{ -kasan_check_write (v,sizeof (*v)); -return arch_atomic64_add_unless (v, a, u); -}#endif static __always_inline void atomic_inc(atomic_t *v)
include/linux/atomic.h +24 -2
@@ -1072,6 +1072,30 @@ static inline int atomic_dec_if_positive(atomic_t *v)#endif /* atomic64_try_cmpxchg */ /** +* atomic64_fetch_add_unless - add unless the number is already a given value +* @v: pointer of type atomic64_t +* @a: the amount to add to v... +* @u: ...unless v is equal to u. +* +* Atomically adds @a to @v, so long as @v was not already @u. +* Returns original value of @v +*/ +#ifndef atomic64_fetch_add_unless +static inline long long atomic64_fetch_add_unless (atomic64_t *v,long long a, +long long u) +{ +long long c =atomic64_read (v); + +do { +if (unlikely (c == u)) +break ; + }while (!atomic64_try_cmpxchg (v, &c, c + a)); + +return c; +} +#endif + +/** * atomic64_add_unless - add unless the number is already a given value * @v: pointer of type atomic_t * @a: the amount to add to v...
@@ -1080,12 +1104,10 @@ static inline int atomic_dec_if_positive(atomic_t *v)* Atomically adds @a to @v, so long as @v was not already @u. * Returns non-zero if @v was not @u, and zero otherwise. */ -#ifdef atomic64_fetch_add_unlessstatic inline int atomic64_add_unless (atomic64_t *v,long long a,long long u) {return atomic64_fetch_add_unless (v, a, u) != u; } -#endif /** * atomic64_inc_not_zero - increment unless the number is zero
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 13/16 atomics/treewide: make test ops optional
In reply to Mark RutlandSome of the atomics return the result of a test applied after the atomic operation, and almost all architectures implement these as trivial wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with minor inconsistencies in formatting and documentation, let's make these operations optional, with default fallbacks as above. Implementations must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is, given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 12 --- arch/arc/include/asm/atomic.h | 10 --- arch/arm/include/asm/atomic.h | 9 --- arch/arm64/include/asm/atomic.h | 8 -- arch/h8300/include/asm/atomic.h | 5 -- arch/hexagon/include/asm/atomic.h | 5 -- arch/ia64/include/asm/atomic.h | 23 ------ arch/m68k/include/asm/atomic.h | 4 + arch/mips/include/asm/atomic.h | 84 -------------------- arch/parisc/include/asm/atomic.h | 22 ------ arch/powerpc/include/asm/atomic.h | 30 -------- arch/riscv/include/asm/atomic.h | 46 ----------- arch/s390/include/asm/atomic.h | 8 -- arch/sh/include/asm/atomic.h | 4 - arch/sparc/include/asm/atomic_32.h | 15 ---- arch/sparc/include/asm/atomic_64.h | 20 ----- arch/x86/include/asm/atomic.h | 4 + arch/x86/include/asm/atomic64_32.h | 54 ------------- arch/x86/include/asm/atomic64_64.h | 4 + arch/xtensa/include/asm/atomic.h | 42 ---------- include/asm-generic/atomic-instrumented.h | 24 ++++++ include/asm-generic/atomic.h | 9 --- include/asm-generic/atomic64.h | 4 - include/linux/atomic.h | 124 ++++++++++++++++++++++++++++++ 24 files changed, 160 insertions(+), 410 deletions(-)
arch/alpha/include/asm/atomic.h +0 -12
@@ -297,24 +297,12 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)return old -1 ; } -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0) -#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0) -#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic64_dec_return (v) atomic64_sub_return(1,(v))#define atomic_inc_return (v) atomic_add_return(1,(v))#define atomic64_inc_return (v) atomic64_add_return(1,(v)) -#define atomic_sub_and_test (i,v) (atomic_sub_return((i), (v)) == 0) -#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i), (v)) == 0) - -#define atomic_inc_and_test (v) (atomic_add_return(1, (v)) == 0) -#define atomic64_inc_and_test (v) (atomic64_add_return(1, (v)) == 0) - -#define atomic_dec_and_test (v) (atomic_sub_return(1, (v)) == 0) -#define atomic64_dec_and_test (v) (atomic64_sub_return(1, (v)) == 0) -#define atomic_inc (v) atomic_add(1,(v))#define atomic64_inc (v) atomic64_add(1,(v))
arch/arc/include/asm/atomic.h +0 -10
@@ -311,14 +311,8 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)#define atomic_inc (v) atomic_add(1, v)#define atomic_dec (v) atomic_sub(1, v) -#define atomic_inc_and_test (v) (atomic_add_return(1, v) == 0) -#define atomic_dec_and_test (v) (atomic_sub_return(1, v) == 0)#define atomic_inc_return (v) atomic_add_return(1, (v))#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_sub_and_test (i, v) (atomic_sub_return(i, v) == 0) - -#define atomic_add_negative (i, v) (atomic_add_return(i, v) < 0) -#ifdef CONFIG_GENERIC_ATOMIC64
@@ -566,14 +560,10 @@ static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, }#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)#define atomic64_inc (v) atomic64_add(1LL, (v))#define atomic64_inc_return (v) atomic64_add_return(1LL, (v)) -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) -#define atomic64_sub_and_test (a, v) (atomic64_sub_return((a), (v)) == 0)#define atomic64_dec (v) atomic64_sub(1LL, (v))#define atomic64_dec_return (v) atomic64_sub_return(1LL, (v)) -#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0)#endif /* !CONFIG_GENERIC_ATOMIC64 */
arch/arm/include/asm/atomic.h +0 -9
@@ -248,13 +248,8 @@ ATOMIC_OPS(xor, ^=, eor)#define atomic_inc (v) atomic_add(1, v)#define atomic_dec (v) atomic_sub(1, v) -#define atomic_inc_and_test (v) (atomic_add_return(1, v) == 0) -#define atomic_dec_and_test (v) (atomic_sub_return(1, v) == 0)#define atomic_inc_return_relaxed (v) (atomic_add_return_relaxed(1, v))#define atomic_dec_return_relaxed (v) (atomic_sub_return_relaxed(1, v)) -#define atomic_sub_and_test (i, v) (atomic_sub_return(i, v) == 0) - -#define atomic_add_negative (i,v) (atomic_add_return(i, v) < 0) #ifndef CONFIG_GENERIC_ATOMIC64 typedef struct {
@@ -517,14 +512,10 @@ static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, }#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)#define atomic64_inc (v) atomic64_add(1LL, (v))#define atomic64_inc_return_relaxed (v) atomic64_add_return_relaxed(1LL, (v)) -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) -#define atomic64_sub_and_test (a, v) (atomic64_sub_return((a), (v)) == 0)#define atomic64_dec (v) atomic64_sub(1LL, (v))#define atomic64_dec_return_relaxed (v) atomic64_sub_return_relaxed(1LL, (v)) -#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0)#endif /* !CONFIG_GENERIC_ATOMIC64 */ #endif
arch/arm64/include/asm/atomic.h +0 -8
@@ -110,10 +110,6 @@#define atomic_inc (v) atomic_add(1, (v))#define atomic_dec (v) atomic_sub(1, (v)) -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) -#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0) -#define atomic_sub_and_test (i, v) (atomic_sub_return((i), (v)) == 0) -#define atomic_add_negative (i, v) (atomic_add_return((i), (v)) < 0)#define atomic_andnot atomic_andnot/*
@@ -185,10 +181,6 @@#define atomic64_inc (v) atomic64_add(1, (v))#define atomic64_dec (v) atomic64_sub(1, (v)) -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) -#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0) -#define atomic64_sub_and_test (i, v) (atomic64_sub_return((i), (v)) == 0) -#define atomic64_add_negative (i, v) (atomic64_add_return((i), (v)) < 0)#define atomic64_andnot atomic64_andnot#endif
arch/h8300/include/asm/atomic.h +0 -5
@@ -69,17 +69,12 @@ ATOMIC_OPS(sub, -=)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0) -#define atomic_sub_and_test (i, v) (atomic_sub_return(i, v) == 0) -#define atomic_inc_return (v) atomic_add_return(1, v)#define atomic_dec_return (v) atomic_sub_return(1, v)#define atomic_inc (v) (void)atomic_inc_return(v) -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0)#define atomic_dec (v) (void)atomic_dec_return(v) -#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0)static inline int atomic_cmpxchg (atomic_t *v,int old,int new) {
arch/hexagon/include/asm/atomic.h +0 -5
@@ -201,11 +201,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)#define atomic_inc (v) atomic_add(1, (v))#define atomic_dec (v) atomic_sub(1, (v)) -#define atomic_inc_and_test (v) (atomic_add_return(1, (v)) == 0) -#define atomic_dec_and_test (v) (atomic_sub_return(1, (v)) == 0) -#define atomic_sub_and_test (i, v) (atomic_sub_return(i, (v)) == 0) -#define atomic_add_negative (i, v) (atomic_add_return(i, (v)) < 0) -#define atomic_inc_return (v) (atomic_add_return(1, v))#define atomic_dec_return (v) (atomic_sub_return(1, v))
arch/ia64/include/asm/atomic.h +0 -23
@@ -231,34 +231,11 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v)return dec; } -/* -* Atomically add I to V and return TRUE if the resulting value is -* negative. -*/ -static __inline__int -atomic_add_negative (int i,atomic_t *v) -{ -return atomic_add_return (i, v) <0 ; -} - -static __inline__long -atomic64_add_negative (__s64 i,atomic64_t *v) -{ -return atomic64_add_return (i, v) <0 ; -} -#define atomic_dec_return (v) atomic_sub_return(1, (v))#define atomic_inc_return (v) atomic_add_return(1, (v))#define atomic64_dec_return (v) atomic64_sub_return(1, (v))#define atomic64_inc_return (v) atomic64_add_return(1, (v)) -#define atomic_sub_and_test (i,v) (atomic_sub_return((i), (v)) == 0) -#define atomic_dec_and_test (v) (atomic_sub_return(1, (v)) == 0) -#define atomic_inc_and_test (v) (atomic_add_return(1, (v)) == 0) -#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i), (v)) == 0) -#define atomic64_dec_and_test (v) (atomic64_sub_return(1, (v)) == 0) -#define atomic64_inc_and_test (v) (atomic64_add_return(1, (v)) == 0) -#define atomic_add (i,v) (void)atomic_add_return((i), (v))#define atomic_sub (i,v) (void)atomic_sub_return((i), (v))#define atomic_inc (v) atomic_add(1, (v))
arch/m68k/include/asm/atomic.h +4 -0
@@ -138,6 +138,7 @@ static inline int atomic_dec_and_test(atomic_t *v) __asm__ __volatile__("subql #1,%1; seq %0" :"=d" (c),"+m" (*v));return c !=0 ; } +#define atomic_dec_and_test atomic_dec_and_teststatic inline int atomic_dec_and_test_lt (atomic_t *v) {
@@ -155,6 +156,7 @@ static inline int atomic_inc_and_test(atomic_t *v) __asm__ __volatile__("addql #1,%1; seq %0" :"=d" (c),"+m" (*v));return c !=0 ; } +#define atomic_inc_and_test atomic_inc_and_test#ifdef CONFIG_RMW_INSNS
@@ -201,6 +203,7 @@ static inline int atomic_sub_and_test(int i, atomic_t *v) : ASM_DI (i));return c !=0 ; } +#define atomic_sub_and_test atomic_sub_and_teststatic inline int atomic_add_negative (int i,atomic_t *v) {
@@ -210,5 +213,6 @@ static inline int atomic_add_negative(int i, atomic_t *v) : ASM_DI (i)); return c != 0; } +#define atomic_add_negative atomic_add_negative#endif /* __ARCH_M68K_ATOMIC __ */
arch/mips/include/asm/atomic.h +0 -84
108 lines not on this page · raw message
arch/parisc/include/asm/atomic.h +0 -22
@@ -142,22 +142,6 @@ ATOMIC_OPS(xor, ^=)#define atomic_inc_return (v) (atomic_add_return( 1,(v)))#define atomic_dec_return (v) (atomic_add_return( -1,(v))) -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0) - -/* -* atomic_inc_and_test - increment and test -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1 -* and returns true if the result is zero, or false for all -* other cases. -*/ -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) - -#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0) - -#define atomic_sub_and_test (i,v) (atomic_sub_return((i),(v)) == 0) -#define ATOMIC_INIT (i) { (i) }#ifdef CONFIG_64BIT
@@ -246,12 +230,6 @@ atomic64_read(const atomic64_t *v)#define atomic64_inc_return (v) (atomic64_add_return( 1,(v)))#define atomic64_dec_return (v) (atomic64_add_return( -1,(v))) -#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0) - -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) -#define atomic64_dec_and_test (v) (atomic64_dec_return(v) == 0) -#define atomic64_sub_and_test (i,v) (atomic64_sub_return((i),(v)) == 0) -/* exported interface */ #define atomic64_cmpxchg (v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
arch/powerpc/include/asm/atomic.h +0 -30
@@ -129,8 +129,6 @@ ATOMIC_OPS(xor, xor)#undef ATOMIC_OP_RETURN_RELAXED#undef ATOMIC_OP -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0) -static __inline__void atomic_inc (atomic_t *v) {int t;
@@ -163,16 +161,6 @@ static __inline__ int atomic_inc_return_relaxed(atomic_t *v)return t; } -/* -* atomic_inc_and_test - increment and test -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1 -* and returns true if the result is zero, or false for all -* other cases. -*/ -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) -static __inline__void atomic_dec (atomic_t *v) {int t;
@@ -281,9 +269,6 @@ static __inline__ int atomic_inc_not_zero(atomic_t *v) }#define atomic_inc_not_zero (v) atomic_inc_not_zero((v)) -#define atomic_sub_and_test (a, v) (atomic_sub_return((a), (v)) == 0) -#define atomic_dec_and_test (v) (atomic_dec_return((v)) == 0) -/* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1, even if
@@ -413,8 +398,6 @@ ATOMIC64_OPS(xor, xor)#undef ATOMIC64_OP_RETURN_RELAXED#undef ATOMIC64_OP -#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0) -static __inline__void atomic64_inc (atomic64_t *v) {long t;
@@ -445,16 +428,6 @@ static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v)return t; } -/* -* atomic64_inc_and_test - increment and test -* @v: pointer of type atomic64_t -* -* Atomically increments @v by 1 -* and returns true if the result is zero, or false for all -* other cases. -*/ -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) -static __inline__void atomic64_dec (atomic64_t *v) {long t;
@@ -488,9 +461,6 @@ static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v)#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed -#define atomic64_sub_and_test (a, v) (atomic64_sub_return((a), (v)) == 0) -#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0) -/* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1.
arch/riscv/include/asm/atomic.h +0 -46
58 lines not on this page · raw message
arch/s390/include/asm/atomic.h +0 -8
@@ -55,17 +55,13 @@ static inline void atomic_add(int i, atomic_t *v)__atomic_add (i, &v->counter); } -#define atomic_add_negative (_i, _v) (atomic_add_return(_i, _v) < 0)#define atomic_inc (_v) atomic_add(1, _v)#define atomic_inc_return (_v) atomic_add_return(1, _v) -#define atomic_inc_and_test (_v) (atomic_add_return(1, _v) == 0)#define atomic_sub (_i, _v) atomic_add(-(int)(_i), _v)#define atomic_sub_return (_i, _v) atomic_add_return(-(int)(_i), _v)#define atomic_fetch_sub (_i, _v) atomic_fetch_add(-(int)(_i), _v) -#define atomic_sub_and_test (_i, _v) (atomic_sub_return(_i, _v) == 0)#define atomic_dec (_v) atomic_sub(1, _v)#define atomic_dec_return (_v) atomic_sub_return(1, _v) -#define atomic_dec_and_test (_v) (atomic_sub_return(1, _v) == 0)#define ATOMIC_OPS (op) \ static inline void atomic_##op(int i, atomic_t *v) \
@@ -170,16 +166,12 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) return dec; } -#define atomic64_add_negative (_i, _v) (atomic64_add_return(_i, _v) < 0)#define atomic64_inc (_v) atomic64_add(1, _v)#define atomic64_inc_return (_v) atomic64_add_return(1, _v) -#define atomic64_inc_and_test (_v) (atomic64_add_return(1, _v) == 0)#define atomic64_sub_return (_i, _v) atomic64_add_return(-(long)(_i), _v)#define atomic64_fetch_sub (_i, _v) atomic64_fetch_add(-(long)(_i), _v)#define atomic64_sub (_i, _v) atomic64_add(-(long)(_i), _v) -#define atomic64_sub_and_test (_i, _v) (atomic64_sub_return(_i, _v) == 0)#define atomic64_dec (_v) atomic64_sub(1, _v)#define atomic64_dec_return (_v) atomic64_sub_return(1, _v) -#define atomic64_dec_and_test (_v) (atomic64_sub_return(1, _v) == 0)#endif /* __ARCH_S390_ATOMIC__ */
arch/sh/include/asm/atomic.h +0 -4
@@ -32,12 +32,8 @@#include <asm/atomic-irq.h> #endif -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0)#define atomic_dec_return (v) atomic_sub_return(1, (v))#define atomic_inc_return (v) atomic_add_return(1, (v)) -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) -#define atomic_sub_and_test (i,v) (atomic_sub_return((i), (v)) == 0) -#define atomic_dec_and_test (v) (atomic_sub_return(1, (v)) == 0)#define atomic_inc (v) atomic_add(1, (v))#define atomic_dec (v) atomic_sub(1, (v))
arch/sparc/include/asm/atomic_32.h +0 -15
@@ -51,19 +51,4 @@ void atomic_set(atomic_t *, int);#define atomic_inc_return (v) (atomic_add_return( 1, (v)))#define atomic_dec_return (v) (atomic_add_return( -1, (v))) -#define atomic_add_negative (a, v) (atomic_add_return((a), (v)) < 0) - -/* -* atomic_inc_and_test - increment and test -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1 -* and returns true if the result is zero, or false for all -* other cases. -*/ -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) - -#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0) -#define atomic_sub_and_test (i, v) (atomic_sub_return(i, v) == 0) -#endif /* !(__ARCH_SPARC_ATOMIC__) */
arch/sparc/include/asm/atomic_64.h +0 -20
@@ -56,32 +56,12 @@ ATOMIC_OPS(xor)#define atomic_inc_return (v) atomic_add_return(1, v)#define atomic64_inc_return (v) atomic64_add_return(1, v) -/* -* atomic_inc_and_test - increment and test -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1 -* and returns true if the result is zero, or false for all -* other cases. -*/ -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) - -#define atomic_sub_and_test (i, v) (atomic_sub_return(i, v) == 0) -#define atomic64_sub_and_test (i, v) (atomic64_sub_return(i, v) == 0) - -#define atomic_dec_and_test (v) (atomic_sub_return(1, v) == 0) -#define atomic64_dec_and_test (v) (atomic64_sub_return(1, v) == 0) -#define atomic_inc (v) atomic_add(1, v)#define atomic64_inc (v) atomic64_add(1, v)#define atomic_dec (v) atomic_sub(1, v)#define atomic64_dec (v) atomic64_sub(1, v) -#define atomic_add_negative (i, v) (atomic_add_return(i, v) < 0) -#define atomic64_add_negative (i, v) (atomic64_add_return(i, v) < 0) -#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))static inline int atomic_xchg (atomic_t *v,int new)
arch/x86/include/asm/atomic.h +4 -0
@@ -80,6 +80,7 @@ static __always_inline void arch_atomic_sub(int i, atomic_t *v)* true if the result is zero, or false for all * other cases. */ +#define arch_atomic_sub_and_test arch_atomic_sub_and_test static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v) {GEN_BINARY_RMWcc (LOCK_PREFIX "subl" , v->counter,"er" , i,"%0" , e);
@@ -117,6 +118,7 @@ static __always_inline void arch_atomic_dec(atomic_t *v)* returns true if the result is 0, or false for all other * cases. */ +#define arch_atomic_dec_and_test arch_atomic_dec_and_test static __always_inline bool arch_atomic_dec_and_test(atomic_t *v) {GEN_UNARY_RMWcc (LOCK_PREFIX "decl" , v->counter,"%0" , e);
@@ -130,6 +132,7 @@ static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)* and returns true if the result is zero, or false for all * other cases. */ +#define arch_atomic_inc_and_test arch_atomic_inc_and_test static __always_inline bool arch_atomic_inc_and_test(atomic_t *v) {GEN_UNARY_RMWcc (LOCK_PREFIX "incl" , v->counter,"%0" , e);
@@ -144,6 +147,7 @@ static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)* if the result is negative, or false when * result is greater than or equal to zero. */ +#define arch_atomic_add_negative arch_atomic_add_negative static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v) {GEN_BINARY_RMWcc (LOCK_PREFIX "addl" , v->counter,"er" , i,"%0" , s);
arch/x86/include/asm/atomic64_32.h +0 -54
66 lines not on this page · raw message
arch/x86/include/asm/atomic64_64.h +4 -0
@@ -71,6 +71,7 @@ static inline void arch_atomic64_sub(long i, atomic64_t *v)* true if the result is zero, or false for all * other cases. */ +#define arch_atomic64_sub_and_test arch_atomic64_sub_and_teststatic inline bool arch_atomic64_sub_and_test (long i,atomic64_t *v) {GEN_BINARY_RMWcc (LOCK_PREFIX "subq" , v->counter,"er" , i,"%0" , e);
@@ -110,6 +111,7 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v)* returns true if the result is 0, or false for all other * cases. */ +#define arch_atomic64_dec_and_test arch_atomic64_dec_and_teststatic inline bool arch_atomic64_dec_and_test (atomic64_t *v) {GEN_UNARY_RMWcc (LOCK_PREFIX "decq" , v->counter,"%0" , e);
@@ -123,6 +125,7 @@ static inline bool arch_atomic64_dec_and_test(atomic64_t *v)* and returns true if the result is zero, or false for all * other cases. */ +#define arch_atomic64_inc_and_test arch_atomic64_inc_and_teststatic inline bool arch_atomic64_inc_and_test (atomic64_t *v) {GEN_UNARY_RMWcc (LOCK_PREFIX "incq" , v->counter,"%0" , e);
@@ -137,6 +140,7 @@ static inline bool arch_atomic64_inc_and_test(atomic64_t *v)* if the result is negative, or false when * result is greater than or equal to zero. */ +#define arch_atomic64_add_negative arch_atomic64_add_negativestatic inline bool arch_atomic64_add_negative (long i,atomic64_t *v) {GEN_BINARY_RMWcc (LOCK_PREFIX "addq" , v->counter,"er" , i,"%0" , s);
arch/xtensa/include/asm/atomic.h +0 -42
54 lines not on this page · raw message
include/asm-generic/atomic-instrumented.h +24 -0
@@ -225,29 +225,41 @@ static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v)return arch_atomic64_dec_if_positive (v); } +#ifdef arch_atomic_dec_and_test +#define atomic_dec_and_test atomic_dec_and_test static __always_inline bool atomic_dec_and_test(atomic_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic_dec_and_test (v); } +#endif +#ifdef arch_atomic64_dec_and_test +#define atomic64_dec_and_test atomic64_dec_and_test static __always_inline bool atomic64_dec_and_test(atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_dec_and_test (v); } +#endif +#ifdef arch_atomic_inc_and_test +#define atomic_inc_and_test atomic_inc_and_test static __always_inline bool atomic_inc_and_test(atomic_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic_inc_and_test (v); } +#endif +#ifdef arch_atomic64_inc_and_test +#define atomic64_inc_and_test atomic64_inc_and_test static __always_inline bool atomic64_inc_and_test(atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_inc_and_test (v); } +#endif static __always_inline int atomic_add_return(int i, atomic_t *v) {
@@ -333,29 +345,41 @@ static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v)return arch_atomic64_fetch_xor (i, v); } +#ifdef arch_atomic_sub_and_test +#define atomic_sub_and_test atomic_sub_and_test static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic_sub_and_test (i, v); } +#endif +#ifdef arch_atomic64_sub_and_test +#define atomic64_sub_and_test atomic64_sub_and_test static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_sub_and_test (i, v); } +#endif +#ifdef arch_atomic_add_negative +#define atomic_add_negative atomic_add_negative static __always_inline bool atomic_add_negative(int i, atomic_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic_add_negative (i, v); } +#endif +#ifdef arch_atomic64_add_negative +#define atomic64_add_negative atomic64_add_negative static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_add_negative (i, v); } +#endif static __always_inline unsigned long cmpxchg_size (volatile void *ptr,unsigned long old,unsigned long new,int size)
include/asm-generic/atomic.h +0 -9
@@ -186,11 +186,6 @@ ATOMIC_OP(xor, ^)#include <linux/irqflags.h> -static inline int atomic_add_negative (int i,atomic_t *v) -{ -return atomic_add_return (i, v) <0 ; -} -static inline void atomic_add (int i,atomic_t *v) {atomic_add_return (i, v);
@@ -214,10 +209,6 @@ static inline void atomic_dec(atomic_t *v)#define atomic_dec_return (v) atomic_sub_return(1, (v))#define atomic_inc_return (v) atomic_add_return(1, (v)) -#define atomic_sub_and_test (i, v) (atomic_sub_return((i), (v)) == 0) -#define atomic_dec_and_test (v) (atomic_dec_return(v) == 0) -#define atomic_inc_and_test (v) (atomic_inc_return(v) == 0) -#define atomic_xchg (ptr, v) (xchg(&(ptr)->counter, (v)))#define atomic_cmpxchg (v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
include/asm-generic/atomic64.h +0 -4
@@ -55,13 +55,9 @@ extern long long atomic64_xchg(atomic64_t *v, long long new);extern long long atomic64_fetch_add_unless (atomic64_t *v,long long a,long long u);#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#define atomic64_add_negative (a, v) (atomic64_add_return((a), (v)) < 0)#define atomic64_inc (v) atomic64_add(1LL, (v))#define atomic64_inc_return (v) atomic64_add_return(1LL, (v)) -#define atomic64_inc_and_test (v) (atomic64_inc_return(v) == 0) -#define atomic64_sub_and_test (a, v) (atomic64_sub_return((a), (v)) == 0)#define atomic64_dec (v) atomic64_sub(1LL, (v))#define atomic64_dec_return (v) atomic64_sub_return(1LL, (v)) -#define atomic64_dec_and_test (v) (atomic64_dec_return((v)) == 0)#endif /* _ASM_GENERIC_ATOMIC64_H */
include/linux/atomic.h +124 -0
136 lines not on this page · raw message
Signature · 1 line
2.11.0
Re: [PATCHv2 13/16] atomics/treewide: make test ops optional
Quoted 110 lines · On Tue, 29 May 2018 08:43:43 PDT (-0700), [email protected] wrote:
Some of the atomics return the result of a test applied after the atomic operation, and almost all architectures implement these as trivial wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with minor inconsistencies in formatting and documentation, let's make these operations optional, with default fallbacks as above. Implementations must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is, given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Will Deacon <[email protected]>--- arch/alpha/include/asm/atomic.h | 12 --- arch/arc/include/asm/atomic.h | 10 --- arch/arm/include/asm/atomic.h | 9 --- arch/arm64/include/asm/atomic.h | 8 -- arch/h8300/include/asm/atomic.h | 5 -- arch/hexagon/include/asm/atomic.h | 5 -- arch/ia64/include/asm/atomic.h | 23 ------ arch/m68k/include/asm/atomic.h | 4 + arch/mips/include/asm/atomic.h | 84 -------------------- arch/parisc/include/asm/atomic.h | 22 ------ arch/powerpc/include/asm/atomic.h | 30 -------- arch/riscv/include/asm/atomic.h | 46 ----------- arch/s390/include/asm/atomic.h | 8 -- arch/sh/include/asm/atomic.h | 4 - arch/sparc/include/asm/atomic_32.h | 15 ---- arch/sparc/include/asm/atomic_64.h | 20 ----- arch/x86/include/asm/atomic.h | 4 + arch/x86/include/asm/atomic64_32.h | 54 ------------- arch/x86/include/asm/atomic64_64.h | 4 + arch/xtensa/include/asm/atomic.h | 42 ---------- include/asm-generic/atomic-instrumented.h | 24 ++++++ include/asm-generic/atomic.h | 9 --- include/asm-generic/atomic64.h | 4 - include/linux/atomic.h | 124 ++++++++++++++++++++++++++++++ 24 files changed, 160 insertions(+), 410 deletions(-)[...]
arch/riscv/include/asm/atomic.h +0
-39@@ -209,36 +209,6 @@ ATOMIC_OPS(xor, xor, i)#undef ATOMIC_FETCH_OP#undef ATOMIC_OP_RETURN -/* -* The extra atomic operations that are constructed from one of the core -* AMO-based operations above (aside from sub, which is easier to fit above). -* These are required to perform a full barrier, but they're OK this way -* because atomic_*_return is also required to perform a full barrier. -* -*/ -#define ATOMIC_OP (op, func_op, comp_op,I , c_type, prefix) \ -static __always_inline \ -bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_##func_op##_return(i, v) comp_op I; \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS (op, func_op, comp_op,I ) \ - ATOMIC_OP(op, func_op, comp_op, I, int, ) -#else -#define ATOMIC_OPS (op, func_op, comp_op,I ) \ - ATOMIC_OP(op, func_op, comp_op, I, int, ) \ - ATOMIC_OP(op, func_op, comp_op, I, long, 64) -#endif - -ATOMIC_OPS(add_and_test, add, ==, 0) -ATOMIC_OPS(sub_and_test, sub, ==, 0) -ATOMIC_OPS(add_negative, add, <, 0) - -#undef ATOMIC_OP -#undef ATOMIC_OPS -#define ATOMIC_OP (op, func_op,I , c_type, prefix) \ static __always_inline \ void atomic##prefix##_##op(atomic##prefix##_t *v) \@@ -315,22 +285,6 @@ ATOMIC_OPS(dec, add, +, -1)#undef ATOMIC_FETCH_OP#undef ATOMIC_OP_RETURN -#define ATOMIC_OP (op, func_op, comp_op,I , prefix) \ -static __always_inline \ -bool atomic##prefix##_##op(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_##func_op##_return(v) comp_op I; \ -} - -ATOMIC_OP(inc_and_test, inc, ==, 0, ) -ATOMIC_OP(dec_and_test, dec, ==, 0, )
arch/riscv/include/asm/atomic.h ATOMIC_OPS(dec, add, +, -1) · line 290
Quoted 10 lines
arch/riscv/include/asm/atomic.h +0
-7-#ifndef CONFIG_GENERIC_ATOMIC64 -ATOMIC_OP(inc_and_test, inc, ==, 0, 64) -ATOMIC_OP(dec_and_test, dec, ==, 0, 64) -#endif - -#undef ATOMIC_OP -/* This is required to provide a full barrier on success. */ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) {
Mark Rutland
reviewerPATCH v2 14/16 atomics/treewide: remove atomic_inc_not_zero_hint()
In reply to Mark RutlandWhile documentation suggests atomic_inc_not_zero_hint() will perform better than atomic_inc_not_zero(), this is unlikely to be the case. No architectures implement atomic_inc_not_zero_hint() directly, and thus it either falls back to atomic_inc_not_zero(), or a loop using atomic_cmpxchg().
Whenever the hint does not match the value in memory, the repeated use of atomic_cmpxchg() will be more expensive than the read that atomic_inc_not_zero_hint() attempts to avoid. For architectures with LL/SC atomics, a read cannot be avoided, and it would always be better to use atomic_inc_not_zero() directly. For other architectures, their own atomic_inc_not_zero() is likely to be more optimal than an atomic_cmpxchg() loop regardless.
Generally, atomic_inc_not_zero_hint() is liable to perform worse than atomic_inc_not_zero(). Further, atomic_inc_not_zero_hint() only exists for atomic_t, and not atomic64_t or atomic_long_t, and there is only one user in the kernel tree.
Given all this, let's remove atomic_inc_not_zero_hint(), and migrate the existing user over to atomic_inc_not_zero().
There should be no functional change as a result of this patch.
--- include/linux/atomic.h | 32 -------------------------------- net/atm/pppoatm.c | 2 +- 2 files changed, 1 insertion(+), 33 deletions(-)
include/linux/atomic.h +0 -32
@@ -656,38 +656,6 @@ static inline int atomic_fetch_andnot_release(int i, atomic_t *v) }#endif -/** -* atomic_inc_not_zero_hint - increment if not null -* @v: pointer of type atomic_t -* @hint: probable value of the atomic before the increment -* -* This version of atomic_inc_not_zero() gives a hint of probable -* value of the atomic. This helps processor to not read the memory -* before doing the atomic read/modify/write cycle, lowering -* number of bus transactions on some arches. -* -* Returns: 0 if increment was not done, 1 otherwise. -*/ -#ifndef atomic_inc_not_zero_hint -static inline int atomic_inc_not_zero_hint (atomic_t *v,int hint) -{ -int val, c = hint; - -/* sanity test, should be removed by compiler if hint is a constant */ -if (!hint) -return atomic_inc_not_zero (v); - -do { - val =atomic_cmpxchg (v, c, c +1 ); -if (val == c) -return 1 ; - c = val; - }while (c); - -return 0 ; -} -#endif -#ifndef atomic_inc_unless_negativestatic inline int atomic_inc_unless_negative (atomic_t *p) {
net/atm/pppoatm.c +1 -1
@@ -244,7 +244,7 @@ static int pppoatm_may_send(struct pppoatm_vcc *pvcc, int size)* the packet count limit, so... */ if (atm_may_send (pvcc->atmvcc, size) && -atomic_inc_not_zero_hint (&pvcc->inflight,NONE_INFLIGHT )) +atomic_inc_not_zero (&pvcc->inflight))return 1 ;/*
Signature · 1 line
2.11.0
Mark Rutland
reviewerPATCH v2 15/16 atomics/treewide: make unconditional inc/dec ops optional
In reply to Mark RutlandMany of the inc/dec ops are mandatory, but for most architectures inc/dec are simply trivial wrappers around their corresponding add/sub ops.
Let's make all the inc/dec ops optional, so that we can get rid of these boilerplate wrappers.
The instrumented atomics are updated accordingly.
There should be no functional change as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 12 ----- arch/arc/include/asm/atomic.h | 11 ----- arch/arm/include/asm/atomic.h | 11 ----- arch/arm64/include/asm/atomic.h | 24 ---------- arch/h8300/include/asm/atomic.h | 7 --- arch/hexagon/include/asm/atomic.h | 6 --- arch/ia64/include/asm/atomic.h | 9 ---- arch/m68k/include/asm/atomic.h | 5 +- arch/mips/include/asm/atomic.h | 38 ---------------- arch/parisc/include/asm/atomic.h | 12 ----- arch/powerpc/include/asm/atomic.h | 4 ++ arch/riscv/include/asm/atomic.h | 76 ------------------------------- arch/s390/include/asm/atomic.h | 8 ---- arch/sh/include/asm/atomic.h | 6 --- arch/sparc/include/asm/atomic_32.h | 5 -- arch/sparc/include/asm/atomic_64.h | 12 ----- arch/x86/include/asm/atomic.h | 5 +- arch/x86/include/asm/atomic64_32.h | 4 ++ arch/x86/include/asm/atomic64_64.h | 5 +- arch/xtensa/include/asm/atomic.h | 32 ------------- include/asm-generic/atomic-instrumented.h | 24 ++++++++++ include/asm-generic/atomic.h | 13 ------ include/asm-generic/atomic64.h | 5 -- include/linux/atomic.h | 48 +++++++++++++++++++ 24 files changed, 86 insertions(+), 296 deletions(-)
arch/alpha/include/asm/atomic.h +0 -12
@@ -297,16 +297,4 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) return old - 1; } -#define atomic_dec_return (v) atomic_sub_return(1,(v)) -#define atomic64_dec_return (v) atomic64_sub_return(1,(v)) - -#define atomic_inc_return (v) atomic_add_return(1,(v)) -#define atomic64_inc_return (v) atomic64_add_return(1,(v)) - -#define atomic_inc (v) atomic_add(1,(v)) -#define atomic64_inc (v) atomic64_add(1,(v)) - -#define atomic_dec (v) atomic_sub(1,(v)) -#define atomic64_dec (v) atomic64_sub(1,(v)) -#endif /* _ALPHA_ATOMIC_H */
arch/arc/include/asm/atomic.h +0 -11
@@ -308,12 +308,6 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -#define atomic_inc (v) atomic_add(1, v) -#define atomic_dec (v) atomic_sub(1, v) - -#define atomic_inc_return (v) atomic_add_return(1, (v)) -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h>
@@ -560,11 +554,6 @@ static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, }#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#define atomic64_inc (v) atomic64_add(1LL, (v)) -#define atomic64_inc_return (v) atomic64_add_return(1LL, (v)) -#define atomic64_dec (v) atomic64_sub(1LL, (v)) -#define atomic64_dec_return (v) atomic64_sub_return(1LL, (v)) -#endif /* !CONFIG_GENERIC_ATOMIC64 */ #endif /* !__ASSEMBLY__ */
arch/arm/include/asm/atomic.h +0 -11
@@ -245,12 +245,6 @@ ATOMIC_OPS(xor, ^=, eor)#define atomic_xchg (v, new) (xchg(&((v)->counter), new)) -#define atomic_inc (v) atomic_add(1, v) -#define atomic_dec (v) atomic_sub(1, v) - -#define atomic_inc_return_relaxed (v) (atomic_add_return_relaxed(1, v)) -#define atomic_dec_return_relaxed (v) (atomic_sub_return_relaxed(1, v)) - #ifndef CONFIG_GENERIC_ATOMIC64 typedef struct { long long counter;
@@ -512,11 +506,6 @@ static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, }#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#define atomic64_inc (v) atomic64_add(1LL, (v)) -#define atomic64_inc_return_relaxed (v) atomic64_add_return_relaxed(1LL, (v)) -#define atomic64_dec (v) atomic64_sub(1LL, (v)) -#define atomic64_dec_return_relaxed (v) atomic64_sub_return_relaxed(1LL, (v)) -#endif /* !CONFIG_GENERIC_ATOMIC64 */ #endif #endif
arch/arm64/include/asm/atomic.h +0 -24
@@ -50,21 +50,11 @@#define atomic_add_return_release atomic_add_return_release#define atomic_add_return atomic_add_return -#define atomic_inc_return_relaxed (v) atomic_add_return_relaxed(1, (v)) -#define atomic_inc_return_acquire (v) atomic_add_return_acquire(1, (v)) -#define atomic_inc_return_release (v) atomic_add_return_release(1, (v)) -#define atomic_inc_return (v) atomic_add_return(1, (v)) -#define atomic_sub_return_relaxed atomic_sub_return_relaxed#define atomic_sub_return_acquire atomic_sub_return_acquire#define atomic_sub_return_release atomic_sub_return_release#define atomic_sub_return atomic_sub_return -#define atomic_dec_return_relaxed (v) atomic_sub_return_relaxed(1, (v)) -#define atomic_dec_return_acquire (v) atomic_sub_return_acquire(1, (v)) -#define atomic_dec_return_release (v) atomic_sub_return_release(1, (v)) -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed#define atomic_fetch_add_acquire atomic_fetch_add_acquire#define atomic_fetch_add_release atomic_fetch_add_release
@@ -108,8 +98,6 @@ cmpxchg_release(&((v)->counter), (old), (new))#define atomic_cmpxchg (v, old, new) cmpxchg(&((v)->counter), (old), (new)) -#define atomic_inc (v) atomic_add(1, (v)) -#define atomic_dec (v) atomic_sub(1, (v))#define atomic_andnot atomic_andnot/*
@@ -124,21 +112,11 @@#define atomic64_add_return_release atomic64_add_return_release#define atomic64_add_return atomic64_add_return -#define atomic64_inc_return_relaxed (v) atomic64_add_return_relaxed(1, (v)) -#define atomic64_inc_return_acquire (v) atomic64_add_return_acquire(1, (v)) -#define atomic64_inc_return_release (v) atomic64_add_return_release(1, (v)) -#define atomic64_inc_return (v) atomic64_add_return(1, (v)) -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed#define atomic64_sub_return_acquire atomic64_sub_return_acquire#define atomic64_sub_return_release atomic64_sub_return_release#define atomic64_sub_return atomic64_sub_return -#define atomic64_dec_return_relaxed (v) atomic64_sub_return_relaxed(1, (v)) -#define atomic64_dec_return_acquire (v) atomic64_sub_return_acquire(1, (v)) -#define atomic64_dec_return_release (v) atomic64_sub_return_release(1, (v)) -#define atomic64_dec_return (v) atomic64_sub_return(1, (v)) -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire#define atomic64_fetch_add_release atomic64_fetch_add_release
@@ -179,8 +157,6 @@#define atomic64_cmpxchg_release atomic_cmpxchg_release#define atomic64_cmpxchg atomic_cmpxchg -#define atomic64_inc (v) atomic64_add(1, (v)) -#define atomic64_dec (v) atomic64_sub(1, (v))#define atomic64_andnot atomic64_andnot#endif
arch/h8300/include/asm/atomic.h +0 -7
@@ -69,13 +69,6 @@ ATOMIC_OPS(sub, -=)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -#define atomic_inc_return (v) atomic_add_return(1, v) -#define atomic_dec_return (v) atomic_sub_return(1, v) - -#define atomic_inc (v) (void)atomic_inc_return(v) - -#define atomic_dec (v) (void)atomic_dec_return(v) -static inline int atomic_cmpxchg (atomic_t *v,int old,int new) {int ret;
arch/hexagon/include/asm/atomic.h +0 -6
@@ -198,10 +198,4 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) }#define atomic_fetch_add_unless atomic_fetch_add_unless -#define atomic_inc (v) atomic_add(1, (v)) -#define atomic_dec (v) atomic_sub(1, (v)) - -#define atomic_inc_return (v) (atomic_add_return(1, v)) -#define atomic_dec_return (v) (atomic_sub_return(1, v)) -#endif
arch/ia64/include/asm/atomic.h +0 -9
@@ -231,19 +231,10 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) return dec; } -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_inc_return (v) atomic_add_return(1, (v)) -#define atomic64_dec_return (v) atomic64_sub_return(1, (v)) -#define atomic64_inc_return (v) atomic64_add_return(1, (v)) -#define atomic_add (i,v) (void)atomic_add_return((i), (v))#define atomic_sub (i,v) (void)atomic_sub_return((i), (v)) -#define atomic_inc (v) atomic_add(1, (v)) -#define atomic_dec (v) atomic_sub(1, (v))#define atomic64_add (i,v) (void)atomic64_add_return((i), (v))#define atomic64_sub (i,v) (void)atomic64_sub_return((i), (v)) -#define atomic64_inc (v) atomic64_add(1, (v)) -#define atomic64_dec (v) atomic64_sub(1, (v))#endif /* _ASM_IA64_ATOMIC_H */
arch/m68k/include/asm/atomic.h +2 -3
@@ -126,11 +126,13 @@ static inline void atomic_inc(atomic_t *v) { __asm__ __volatile__("addql #1,%0" :"+m" (*v)); } +#define atomic_inc atomic_incstatic inline void atomic_dec (atomic_t *v) { __asm__ __volatile__("subql #1,%0" :"+m" (*v)); } +#define atomic_dec atomic_decstatic inline int atomic_dec_and_test (atomic_t *v) {
@@ -192,9 +194,6 @@ static inline int atomic_xchg(atomic_t *v, int new)#endif /* !CONFIG_RMW_INSNS */ -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_inc_return (v) atomic_add_return(1, (v)) -static inline int atomic_sub_and_test (int i,atomic_t *v) {char c;
arch/mips/include/asm/atomic.h +0 -38
@@ -274,31 +274,12 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), (new))) -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_inc_return (v) atomic_add_return(1, (v)) -/* * atomic_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t */ #define atomic_dec_if_positive (v) atomic_sub_if_positive(1, v) -/* -* atomic_inc - increment atomic variable -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1. -*/ -#define atomic_inc (v) atomic_add(1, (v)) - -/* -* atomic_dec - decrement and test -* @v: pointer of type atomic_t -* -* Atomically decrements @v by 1. -*/ -#define atomic_dec (v) atomic_sub(1, (v)) -#ifdef CONFIG_64BIT #define ATOMIC64_INIT (i) { (i) }
@@ -554,31 +535,12 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic64_xchg (v, new) (xchg(&((v)->counter), (new))) -#define atomic64_dec_return (v) atomic64_sub_return(1, (v)) -#define atomic64_inc_return (v) atomic64_add_return(1, (v)) -/* * atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic64_t */ #define atomic64_dec_if_positive (v) atomic64_sub_if_positive(1, v) -/* -* atomic64_inc - increment atomic variable -* @v: pointer of type atomic64_t -* -* Atomically increments @v by 1. -*/ -#define atomic64_inc (v) atomic64_add(1, (v)) - -/* -* atomic64_dec - decrement and test -* @v: pointer of type atomic64_t -* -* Atomically decrements @v by 1. -*/ -#define atomic64_dec (v) atomic64_sub(1, (v)) -#endif /* CONFIG_64BIT */ #endif /* _ASM_ATOMIC_H */
arch/parisc/include/asm/atomic.h +0 -12
@@ -136,12 +136,6 @@ ATOMIC_OPS(xor, ^=)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -#define atomic_inc (v) (atomic_add( 1,(v))) -#define atomic_dec (v) (atomic_add( -1,(v))) - -#define atomic_inc_return (v) (atomic_add_return( 1,(v))) -#define atomic_dec_return (v) (atomic_add_return( -1,(v))) -#define ATOMIC_INIT (i) { (i) }#ifdef CONFIG_64BIT
@@ -224,12 +218,6 @@ atomic64_read(const atomic64_t *v)return READ_ONCE ((v)->counter); } -#define atomic64_inc (v) (atomic64_add( 1,(v))) -#define atomic64_dec (v) (atomic64_add( -1,(v))) - -#define atomic64_inc_return (v) (atomic64_add_return( 1,(v))) -#define atomic64_dec_return (v) (atomic64_add_return( -1,(v))) -/* exported interface */ #define atomic64_cmpxchg (v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
arch/powerpc/include/asm/atomic.h +4 -0
@@ -143,6 +143,7 @@ static __inline__ void atomic_inc(atomic_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic_inc atomic_incstatic __inline__int atomic_inc_return_relaxed (atomic_t *v) {
@@ -175,6 +176,7 @@ static __inline__ void atomic_dec(atomic_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic_dec atomic_decstatic __inline__int atomic_dec_return_relaxed (atomic_t *v) {
@@ -411,6 +413,7 @@ static __inline__ void atomic64_inc(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic64_inc atomic64_incstatic __inline__long atomic64_inc_return_relaxed (atomic64_t *v) {
@@ -441,6 +444,7 @@ static __inline__ void atomic64_dec(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic64_dec atomic64_decstatic __inline__long atomic64_dec_return_relaxed (atomic64_t *v) {
arch/riscv/include/asm/atomic.h +0 -76
82 lines not on this page · raw message
arch/s390/include/asm/atomic.h +0 -8
@@ -55,13 +55,9 @@ static inline void atomic_add(int i, atomic_t *v)__atomic_add (i, &v->counter); } -#define atomic_inc (_v) atomic_add(1, _v) -#define atomic_inc_return (_v) atomic_add_return(1, _v)#define atomic_sub (_i, _v) atomic_add(-(int)(_i), _v)#define atomic_sub_return (_i, _v) atomic_add_return(-(int)(_i), _v)#define atomic_fetch_sub (_i, _v) atomic_fetch_add(-(int)(_i), _v) -#define atomic_dec (_v) atomic_sub(1, _v) -#define atomic_dec_return (_v) atomic_sub_return(1, _v)#define ATOMIC_OPS (op) \ static inline void atomic_##op(int i, atomic_t *v) \
@@ -166,12 +162,8 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) return dec; } -#define atomic64_inc (_v) atomic64_add(1, _v) -#define atomic64_inc_return (_v) atomic64_add_return(1, _v)#define atomic64_sub_return (_i, _v) atomic64_add_return(-(long)(_i), _v)#define atomic64_fetch_sub (_i, _v) atomic64_fetch_add(-(long)(_i), _v)#define atomic64_sub (_i, _v) atomic64_add(-(long)(_i), _v) -#define atomic64_dec (_v) atomic64_sub(1, _v) -#define atomic64_dec_return (_v) atomic64_sub_return(1, _v)#endif /* __ARCH_S390_ATOMIC__ */
arch/sh/include/asm/atomic.h +0 -6
@@ -32,12 +32,6 @@#include <asm/atomic-irq.h> #endif -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_inc_return (v) atomic_add_return(1, (v)) - -#define atomic_inc (v) atomic_add(1, (v)) -#define atomic_dec (v) atomic_sub(1, (v)) -#define atomic_xchg (v, new) (xchg(&((v)->counter), new))#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
arch/sparc/include/asm/atomic_32.h +0 -5
@@ -38,8 +38,6 @@ void atomic_set(atomic_t *, int);#define atomic_add (i, v) ((void)atomic_add_return( (int)(i), (v)))#define atomic_sub (i, v) ((void)atomic_add_return(-(int)(i), (v))) -#define atomic_inc (v) ((void)atomic_add_return( 1, (v))) -#define atomic_dec (v) ((void)atomic_add_return( -1, (v)))#define atomic_and (i, v) ((void)atomic_fetch_and((i), (v)))#define atomic_or (i, v) ((void)atomic_fetch_or((i), (v)))
@@ -48,7 +46,4 @@ void atomic_set(atomic_t *, int);#define atomic_sub_return (i, v) (atomic_add_return(-(int)(i), (v)))#define atomic_fetch_sub (i, v) (atomic_fetch_add (-(int)(i), (v))) -#define atomic_inc_return (v) (atomic_add_return( 1, (v))) -#define atomic_dec_return (v) (atomic_add_return( -1, (v))) -#endif /* !(__ARCH_SPARC_ATOMIC__) */
arch/sparc/include/asm/atomic_64.h +0 -12
@@ -50,18 +50,6 @@ ATOMIC_OPS(xor)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -#define atomic_dec_return (v) atomic_sub_return(1, v) -#define atomic64_dec_return (v) atomic64_sub_return(1, v) - -#define atomic_inc_return (v) atomic_add_return(1, v) -#define atomic64_inc_return (v) atomic64_add_return(1, v) - -#define atomic_inc (v) atomic_add(1, v) -#define atomic64_inc (v) atomic64_add(1, v) - -#define atomic_dec (v) atomic_sub(1, v) -#define atomic64_dec (v) atomic64_sub(1, v) -#define atomic_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))static inline int atomic_xchg (atomic_t *v,int new)
arch/x86/include/asm/atomic.h +2 -3
@@ -92,6 +92,7 @@ static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)* * Atomically increments @v by 1. */ +#define arch_atomic_inc arch_atomic_inc static __always_inline void arch_atomic_inc(atomic_t *v) { asm volatile(LOCK_PREFIX "incl %0"
@@ -104,6 +105,7 @@ static __always_inline void arch_atomic_inc(atomic_t *v)* * Atomically decrements @v by 1. */ +#define arch_atomic_dec arch_atomic_dec static __always_inline void arch_atomic_dec(atomic_t *v) { asm volatile(LOCK_PREFIX "decl %0"
@@ -177,9 +179,6 @@ static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)return arch_atomic_add_return (-i, v); } -#define arch_atomic_inc_return (v) (arch_atomic_add_return(1, v)) -#define arch_atomic_dec_return (v) (arch_atomic_sub_return(1, v)) - static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v) {return xadd (&v->counter, i);
arch/x86/include/asm/atomic64_32.h +4 -0
@@ -158,6 +158,7 @@ static inline long long arch_atomic64_inc_return(atomic64_t *v) "S" (v) : "memory", "ecx"); return a; } +#define arch_atomic64_inc_return arch_atomic64_inc_returnstatic inline long long arch_atomic64_dec_return (atomic64_t *v) {
@@ -166,6 +167,7 @@ static inline long long arch_atomic64_dec_return(atomic64_t *v) "S" (v) : "memory", "ecx"); return a; } +#define arch_atomic64_dec_return arch_atomic64_dec_return/** * arch_atomic64_add - add integer to atomic64 variable
@@ -203,6 +205,7 @@ static inline long long arch_atomic64_sub(long long i, atomic64_t *v)* * Atomically increments @v by 1. */ +#define arch_atomic64_inc arch_atomic64_incstatic inline void arch_atomic64_inc (atomic64_t *v) { __alternative_atomic64(inc, inc_return, /* no output */,
@@ -215,6 +218,7 @@ static inline void arch_atomic64_inc(atomic64_t *v)* * Atomically decrements @v by 1. */ +#define arch_atomic64_dec arch_atomic64_decstatic inline void arch_atomic64_dec (atomic64_t *v) { __alternative_atomic64(dec, dec_return, /* no output */,
arch/x86/include/asm/atomic64_64.h +2 -3
@@ -83,6 +83,7 @@ static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)* * Atomically increments @v by 1. */ +#define arch_atomic64_inc arch_atomic64_inc static __always_inline void arch_atomic64_inc(atomic64_t *v) { asm volatile(LOCK_PREFIX "incq %0"
@@ -96,6 +97,7 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v)* * Atomically decrements @v by 1. */ +#define arch_atomic64_dec arch_atomic64_dec static __always_inline void arch_atomic64_dec(atomic64_t *v) { asm volatile(LOCK_PREFIX "decq %0"
@@ -173,9 +175,6 @@ static inline long arch_atomic64_fetch_sub(long i, atomic64_t *v)return xadd (&v->counter, -i); } -#define arch_atomic64_inc_return (v) (arch_atomic64_add_return(1, (v))) -#define arch_atomic64_dec_return (v) (arch_atomic64_sub_return(1, (v))) -static inline long arch_atomic64_cmpxchg (atomic64_t *v,long old,long new) {return arch_cmpxchg (&v->counter, old, new);
arch/xtensa/include/asm/atomic.h +0 -32
@@ -197,38 +197,6 @@ ATOMIC_OPS(xor)#undef ATOMIC_OP_RETURN#undef ATOMIC_OP -/** -* atomic_inc - increment atomic variable -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1. -*/ -#define atomic_inc (v) atomic_add(1,(v)) - -/** -* atomic_inc - increment atomic variable -* @v: pointer of type atomic_t -* -* Atomically increments @v by 1. -*/ -#define atomic_inc_return (v) atomic_add_return(1,(v)) - -/** -* atomic_dec - decrement atomic variable -* @v: pointer of type atomic_t -* -* Atomically decrements @v by 1. -*/ -#define atomic_dec (v) atomic_sub(1,(v)) - -/** -* atomic_dec_return - decrement atomic variable -* @v: pointer of type atomic_t -* -* Atomically decrements @v by 1. -*/ -#define atomic_dec_return (v) atomic_sub_return(1,(v)) -#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))#define atomic_xchg (v, new) (xchg(&((v)->counter), new))
include/asm-generic/atomic-instrumented.h +24 -0
@@ -102,29 +102,41 @@ static __always_inline int atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u }#endif +#ifdef arch_atomic_inc +#define atomic_inc atomic_inc static __always_inline void atomic_inc(atomic_t *v) {kasan_check_write (v,sizeof (*v));arch_atomic_inc (v); } +#endif +#ifdef arch_atomic64_inc +#define atomic64_inc atomic64_inc static __always_inline void atomic64_inc(atomic64_t *v) {kasan_check_write (v,sizeof (*v));arch_atomic64_inc (v); } +#endif +#ifdef arch_atomic_dec +#define atomic_dec atomic_dec static __always_inline void atomic_dec(atomic_t *v) {kasan_check_write (v,sizeof (*v));arch_atomic_dec (v); } +#endif +#ifdef atch_atomic64_dec +#define atomic64_dec static __always_inline void atomic64_dec(atomic64_t *v) {kasan_check_write (v,sizeof (*v));arch_atomic64_dec (v); } +#endif static __always_inline void atomic_add(int i, atomic_t *v) {
@@ -186,29 +198,41 @@ static __always_inline void atomic64_xor(s64 i, atomic64_t *v)arch_atomic64_xor (i, v); } +#ifdef arch_atomic_inc_return +#define atomic_inc_return atomic_inc_return static __always_inline int atomic_inc_return(atomic_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic_inc_return (v); } +#endif +#ifdef arch_atomic64_in_return +#define atomic64_inc_return atomic64_inc_return static __always_inline s64 atomic64_inc_return(atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_inc_return (v); } +#endif +#ifdef arch_atomic_dec_return +#define atomic_dec_return atomic_dec_return static __always_inline int atomic_dec_return(atomic_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic_dec_return (v); } +#endif +#ifdef arch_atomic64_dec_return +#define atomic64_dec_return atomic64_dec_return static __always_inline s64 atomic64_dec_return(atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_dec_return (v); } +#endif #ifdef arch_atomic64_inc_not_zero#define atomic64_inc_not_zero atomic64_inc_not_zero
include/asm-generic/atomic.h +0 -13
@@ -196,19 +196,6 @@ static inline void atomic_sub(int i, atomic_t *v)atomic_sub_return (i, v); } -static inline void atomic_inc (atomic_t *v) -{ -atomic_add_return (1 , v); -} - -static inline void atomic_dec (atomic_t *v) -{ -atomic_sub_return (1 , v); -} - -#define atomic_dec_return (v) atomic_sub_return(1, (v)) -#define atomic_inc_return (v) atomic_add_return(1, (v)) -#define atomic_xchg (ptr, v) (xchg(&(ptr)->counter, (v)))#define atomic_cmpxchg (v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
include/asm-generic/atomic64.h +0 -5
@@ -55,9 +55,4 @@ extern long long atomic64_xchg(atomic64_t *v, long long new);extern long long atomic64_fetch_add_unless (atomic64_t *v,long long a,long long u);#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#define atomic64_inc (v) atomic64_add(1LL, (v)) -#define atomic64_inc_return (v) atomic64_add_return(1LL, (v)) -#define atomic64_dec (v) atomic64_sub(1LL, (v)) -#define atomic64_dec_return (v) atomic64_sub_return(1LL, (v)) -#endif /* _ASM_GENERIC_ATOMIC64_H */
include/linux/atomic.h +48 -0
92 lines not on this page · raw message
Signature · 1 line
2.11.0
Re: [PATCHv2 15/16] atomics/treewide: make unconditional inc/dec ops optional
Quoted 118 lines · On Tue, 29 May 2018 08:43:45 PDT (-0700), [email protected] wrote:
Many of the inc/dec ops are mandatory, but for most architectures inc/dec are simply trivial wrappers around their corresponding add/sub ops.
Let's make all the inc/dec ops optional, so that we can get rid of these boilerplate wrappers.
The instrumented atomics are updated accordingly.
There should be no functional change as a result of this patch.
Signed-off-by Mark Rutland <[email protected]>Cc Boqun Feng <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Will Deacon <[email protected]>--- arch/alpha/include/asm/atomic.h | 12 ----- arch/arc/include/asm/atomic.h | 11 ----- arch/arm/include/asm/atomic.h | 11 ----- arch/arm64/include/asm/atomic.h | 24 ---------- arch/h8300/include/asm/atomic.h | 7 --- arch/hexagon/include/asm/atomic.h | 6 --- arch/ia64/include/asm/atomic.h | 9 ---- arch/m68k/include/asm/atomic.h | 5 +- arch/mips/include/asm/atomic.h | 38 ---------------- arch/parisc/include/asm/atomic.h | 12 ----- arch/powerpc/include/asm/atomic.h | 4 ++ arch/riscv/include/asm/atomic.h | 76 ------------------------------- arch/s390/include/asm/atomic.h | 8 ---- arch/sh/include/asm/atomic.h | 6 --- arch/sparc/include/asm/atomic_32.h | 5 -- arch/sparc/include/asm/atomic_64.h | 12 ----- arch/x86/include/asm/atomic.h | 5 +- arch/x86/include/asm/atomic64_32.h | 4 ++ arch/x86/include/asm/atomic64_64.h | 5 +- arch/xtensa/include/asm/atomic.h | 32 ------------- include/asm-generic/atomic-instrumented.h | 24 ++++++++++ include/asm-generic/atomic.h | 13 ------ include/asm-generic/atomic64.h | 5 -- include/linux/atomic.h | 48 +++++++++++++++++++ 24 files changed, 86 insertions(+), 296 deletions(-)[...]
arch/riscv/include/asm/atomic.h +0
-69@@ -209,82 +209,6 @@ ATOMIC_OPS(xor, xor, i)#undef ATOMIC_FETCH_OP#undef ATOMIC_OP_RETURN -#define ATOMIC_OP (op, func_op,I , c_type, prefix) \ -static __always_inline \ -void atomic##prefix##_##op(atomic##prefix##_t *v) \ -{ \ - atomic##prefix##_##func_op(I, v); \ -} - -#define ATOMIC_FETCH_OP (op, func_op,I , c_type, prefix) \ -static __always_inline \ -c_type atomic##prefix##_fetch_##op##_relaxed(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##func_op##_relaxed(I, v); \ -} \ -static __always_inline \ -c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##func_op(I, v); \ -} - -#define ATOMIC_OP_RETURN (op, asm_op, c_op,I , c_type, prefix) \ -static __always_inline \ -c_type atomic##prefix##_##op##_return_relaxed(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##op##_relaxed(v) c_op I; \ -} \ -static __always_inline \ -c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##op(v) c_op I; \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS (op, asm_op, c_op,I ) \ - ATOMIC_OP( op, asm_op, I, int, ) \ - ATOMIC_FETCH_OP( op, asm_op, I, int, ) \ - ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) -#else -#define ATOMIC_OPS (op, asm_op, c_op,I ) \ - ATOMIC_OP( op, asm_op, I, int, ) \ - ATOMIC_FETCH_OP( op, asm_op, I, int, ) \ - ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \ - ATOMIC_OP( op, asm_op, I, long, 64) \ - ATOMIC_FETCH_OP( op, asm_op, I, long, 64) \ - ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64) -#endif - -ATOMIC_OPS(inc, add, +, 1) -ATOMIC_OPS(dec, add, +, -1) - -#define atomic_inc_return_relaxed atomic_inc_return_relaxed -#define atomic_dec_return_relaxed atomic_dec_return_relaxed -#define atomic_inc_return atomic_inc_return -#define atomic_dec_return atomic_dec_return - -#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed -#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed -#define atomic_fetch_inc atomic_fetch_inc -#define atomic_fetch_dec atomic_fetch_dec - -#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed -#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed -#define atomic64_inc_return atomic64_inc_return -#define atomic64_dec_return atomic64_dec_return - -#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed -#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed -#define atomic64_fetch_inc atomic64_fetch_inc -#define atomic64_fetch_dec atomic64_fetch_dec
arch/riscv/include/asm/atomic.h ATOMIC_OPS(xor, xor, i) · line 214
Quoted 10 lines
arch/riscv/include/asm/atomic.h +0
-7-#endif - -#undef ATOMIC_OPS -#undef ATOMIC_OP -#undef ATOMIC_FETCH_OP -#undef ATOMIC_OP_RETURN -/* This is required to provide a full barrier on success. */ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) {
As far as the RISC-V stuff goes: this works for me, as I always like to have less code in the arch port.
Thanks!
Mark Rutland
reviewerPATCH v2 16/16 atomics/treewide: make conditional inc/dec ops optional
In reply to Mark RutlandThe conditional inc/dec ops differ for atomic_t and atomic64_t:
* atomic_inc_unless_positive() is optional for atomic_t, and doesn't exist for atomic64_t.
* atomic_dec_unless_negative() is optional for atomic_t, and doesn't exist for atomic64_t.
* atomic_dec_if_positive is optional for atomic_t, and is mandatory for atomic64_t.
Let's make these consistently optional for both. At the same time, let's clean up the exisiting fallbacks to use atomic_try_cmpxchg().
The instrumented atomics are updated accordingly.
There should be no functional change as a result of this patch.
--- arch/alpha/include/asm/atomic.h | 1 + arch/arc/include/asm/atomic.h | 1 + arch/arm/include/asm/atomic.h | 1 + arch/arm64/include/asm/atomic.h | 2 + arch/ia64/include/asm/atomic.h | 16 ----- arch/parisc/include/asm/atomic.h | 23 -------- arch/powerpc/include/asm/atomic.h | 1 + arch/s390/include/asm/atomic.h | 17 ------ arch/sparc/include/asm/atomic_64.h | 1 + arch/x86/include/asm/atomic64_32.h | 1 + arch/x86/include/asm/atomic64_64.h | 18 ------ include/asm-generic/atomic-instrumented.h | 3 + include/asm-generic/atomic64.h | 1 + include/linux/atomic.h | 97 +++++++++++++++++++++++-------- 14 files changed, 85 insertions(+), 98 deletions(-)
arch/alpha/include/asm/atomic.h +1 -0
@@ -296,5 +296,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)smp_mb (); return old - 1; } +#define atomic64_dec_if_positive atomic64_dec_if_positive#endif /* _ALPHA_ATOMIC_H */
arch/arc/include/asm/atomic.h +1 -0
@@ -517,6 +517,7 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v)return val; } +#define atomic64_dec_if_positive atomic64_dec_if_positive/** * atomic64_fetch_add_unless - add unless the number is a given value
arch/arm/include/asm/atomic.h +1 -0
@@ -474,6 +474,7 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v)return result; } +#define atomic64_dec_if_positive atomic64_dec_if_positivestatic inline long long atomic64_fetch_add_unless (atomic64_t *v,long long a,long long u)
arch/arm64/include/asm/atomic.h +2 -0
@@ -159,5 +159,7 @@#define atomic64_andnot atomic64_andnot +#define atomic64_dec_if_positive atomic64_dec_if_positive +#endif #endif
arch/ia64/include/asm/atomic.h +0 -16
@@ -215,22 +215,6 @@ ATOMIC64_FETCH_OP(xor, ^) (cmpxchg(&((v)->counter), old, new))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -static __inline__long atomic64_dec_if_positive (atomic64_t *v) -{ -long c, old, dec; - c =atomic64_read (v); -for (;;) { - dec = c -1 ; -if (unlikely (dec <0 )) -break ; - old =atomic64_cmpxchg ((v), c, dec); -if (likely (old == c)) -break ; - c = old; - } -return dec; -} -#define atomic_add (i,v) (void)atomic_add_return((i), (v))#define atomic_sub (i,v) (void)atomic_sub_return((i), (v))
arch/parisc/include/asm/atomic.h +0 -23
@@ -223,29 +223,6 @@ atomic64_read(const atomic64_t *v) ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))#define atomic64_xchg (v, new) (xchg(&((v)->counter), new)) -/* -* atomic64_dec_if_positive - decrement by 1 if old value positive -* @v: pointer of type atomic_t -* -* The function returns the old value of *v minus 1, even if -* the atomic variable, v, was not decremented. -*/ -static inline long atomic64_dec_if_positive (atomic64_t *v) -{ -long c, old, dec; - c =atomic64_read (v); -for (;;) { - dec = c -1 ; -if (unlikely (dec <0 )) -break ; - old =atomic64_cmpxchg ((v), c, dec); -if (likely (old == c)) -break ; - c = old; - } -return dec; -} -#endif /* !CONFIG_64BIT */
arch/powerpc/include/asm/atomic.h +1 -0
@@ -488,6 +488,7 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v)return t; } +#define atomic64_dec_if_positive atomic64_dec_if_positive#define atomic64_cmpxchg (v, o, n) (cmpxchg(&((v)->counter), (o), (n)))#define atomic64_cmpxchg_relaxed (v, o, n) \
arch/s390/include/asm/atomic.h +0 -17
@@ -145,23 +145,6 @@ ATOMIC64_OPS(xor)#undef ATOMIC64_OPS -static inline long atomic64_dec_if_positive (atomic64_t *v) -{ -long c, old, dec; - - c =atomic64_read (v); -for (;;) { - dec = c -1 ; -if (unlikely (dec <0 )) -break ; - old =atomic64_cmpxchg ((v), c, dec); -if (likely (old == c)) -break ; - c = old; - } -return dec; -} -#define atomic64_sub_return (_i, _v) atomic64_add_return(-(long)(_i), _v)#define atomic64_fetch_sub (_i, _v) atomic64_fetch_add(-(long)(_i), _v)#define atomic64_sub (_i, _v) atomic64_add(-(long)(_i), _v)
arch/sparc/include/asm/atomic_64.h +1 -0
@@ -62,5 +62,6 @@ static inline int atomic_xchg(atomic_t *v, int new)#define atomic64_xchg (v, new) (xchg(&((v)->counter), new))long atomic64_dec_if_positive (atomic64_t *v); +#define atomic64_dec_if_positive atomic64_dec_if_positive#endif /* !(__ARCH_SPARC64_ATOMIC__) */
arch/x86/include/asm/atomic64_32.h +1 -0
@@ -254,6 +254,7 @@ static inline int arch_atomic64_inc_not_zero(atomic64_t *v)return r; } +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positivestatic inline long long arch_atomic64_dec_if_positive (atomic64_t *v) {long long r;
arch/x86/include/asm/atomic64_64.h +0 -18
@@ -191,24 +191,6 @@ static inline long arch_atomic64_xchg(atomic64_t *v, long new)return xchg (&v->counter, new); } -/* -* arch_atomic64_dec_if_positive - decrement by 1 if old value positive -* @v: pointer of type atomic_t -* -* The function returns the old value of *v minus 1, even if -* the atomic variable, v, was not decremented. -*/ -static inline long arch_atomic64_dec_if_positive (atomic64_t *v) -{ -s64 dec, c =arch_atomic64_read (v); -do { - dec = c -1 ; -if (unlikely (dec <0 )) -break ; - }while (!arch_atomic64_try_cmpxchg (v, &c, dec)); -return dec; -} -static inline void arch_atomic64_and (long i,atomic64_t *v) { asm volatile(LOCK_PREFIX "andq %1,%0"
include/asm-generic/atomic-instrumented.h +3 -0
@@ -243,11 +243,14 @@ static __always_inline s64 atomic64_inc_not_zero(atomic64_t *v) }#endif +#ifdef arch_atomic64_dec_if_positive +#define atomic64_dec_if_positive atomic64_dec_if_positive static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) {kasan_check_write (v,sizeof (*v));return arch_atomic64_dec_if_positive (v); } +#endif #ifdef arch_atomic_dec_and_test#define atomic_dec_and_test atomic_dec_and_test
include/asm-generic/atomic64.h +1 -0
@@ -50,6 +50,7 @@ ATOMIC64_OPS(xor)#undef ATOMIC64_OPextern long long atomic64_dec_if_positive (atomic64_t *v); +#define atomic64_dec_if_positive atomic64_dec_if_positiveextern long long atomic64_cmpxchg (atomic64_t *v,long long o,long long n);extern long long atomic64_xchg (atomic64_t *v,long long new);extern long long atomic64_fetch_add_unless (atomic64_t *v,long long a,long long u);
include/linux/atomic.h +73 -24
124 lines not on this page · raw message
Signature · 1 line
2.11.0
Mark Rutland
reviewerQuoted 14 lines · Mark Rutland
This series contains a few cleanups of the atomic API, fixing inconsistencies between atomic_* and atomic64_*, and minimizing repetition in arch code. This is nicer for arch code, and the improved regularity will help when generating the atomic headers in future.
Mark Rutland (16):
atomics/treewide: s/__atomic_add_unless/atomic_fetch_add_unless/ atomics/treewide: remove redundantatomic_inc_not_zero()definitions atomics/treewide: makeatomic64_inc_not_zero()optional atomics/treewide: makeatomic_fetch_add_unless()optional
atomics: prepare foratomic64_fetch_add_unless()atomics/generic: defineatomic64_fetch_add_unless()atomics/alpha: defineatomic64_fetch_add_unless()atomics/arc: defineatomic64_fetch_add_unless()
Quoted 8 lines
atomics/arm: define
atomic64_fetch_add_unless()atomics/powerpc: defineatomic64_fetch_add_unless()atomics/riscv: defineatomic64_fetch_add_unless()atomics/treewide: makeatomic64_fetch_add_unless()optional atomics/treewide: make test ops optional atomics/treewide: removeatomic_inc_not_zero_hint()atomics/treewide: make unconditional inc/dec ops optional atomics/treewide: make conditional inc/dec ops optional
While testing the subsqeuent generated header patches, I discovered that the andnot ifdeffery was inconsistent to that for all other atomics, so I'm going to append the below patch to this series, if it needs a v3.
Otherwise, that's pushed out to my atomics/api-cleanup branch.
Thanks,
Mark,
---->8----
Quoted 1 line
From
da1cef7ec7c1ad143a06723164263ef308bfb569Mon Sep 17 00:00:00 2001
From: Mark Rutland <[email protected]>
Date: Wed, 30 May 2018 11:55:28 +0100
Subject: [PATCH] atomics/treewide: clean up andnot ifdeffery
The ifdeffery for atomic*_{fetch_,}andnot() is unlike that for all the other atomics. If atomic*_andnot() is not defined, the corresponding atomic*_fetch_andnot() is assumed to not be defined.
Additionally, the fallbacks for the various orering cases are written much later in atomic.h as static inlines.
This isn't problematic today, but gets in the way of scripting the generation of atomics. To prepare for scripting, this patch:
* Switches to separate ifdefs for atomic*_andnot() and atomic*_fetch_andnot(), updating implementations as appropriate.
* Moves the fallbacks into the standards ifdefs, as macro expansions rather than static inlines.
* Removes trivial andnot implementations from architectures, where these are superceded by core code.
There should be no functional change as a result of this patch.
--- arch/arc/include/asm/atomic.h | 8 ++-- arch/arm/include/asm/atomic.h | 2 + include/linux/atomic.h | 96 ++++++++++++++----------------------------- 3 files changed, 36 insertions(+), 70 deletions(-)
arch/arc/include/asm/atomic.h +4 -4
@@ -187,7 +187,8 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define atomic_andnot atomic_andnot +#define atomic_andnot atomic_andnot +#define atomic_fetch_andnot atomic_fetch_andnot#undef ATOMIC_OPS#define ATOMIC_OPS (op, c_op, asm_op) \
@@ -296,8 +297,6 @@ ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) -#define atomic_andnot (mask, v) atomic_and(~(mask), (v)) -#define atomic_fetch_andnot (mask, v) atomic_fetch_and(~(mask), (v)) ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
@@ -430,7 +429,8 @@ static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \ ATOMIC64_OP_RETURN(op, op1, op2) \ ATOMIC64_FETCH_OP(op, op1, op2) -#define atomic64_andnot atomic64_andnot +#define atomic64_andnot atomic64_andnot +#define atomic64_fetch_andnot atomic64_fetch_andnot ATOMIC64_OPS(add, add.f, adc)ATOMIC64_OPS (sub, sub.f, sbc)
arch/arm/include/asm/atomic.h +2 -0
@@ -216,6 +216,8 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } +#define atomic_fetch_andnot atomic_fetch_andnot +#endif /* __LINUX_ARM_ARCH__ */ #define ATOMIC_OPS (op, c_op, asm_op) \
include/linux/atomic.h +30 -66
134 lines not on this page · raw message
Signature · 1 line
2.11.0
Mark Rutland
reviewerinclude/linux/atomic.h static inline int atomic_dec_if_positive(atomic_t *v) · line 1013
Quoted 5 lines · Mark Rutland
include/linux/atomic.h +3
-2-#ifdef atomic64_andnot -/* atomic64_fetch_andnot_relaxed */ +#ifndef atomic_andnot +#define atomic_andnot (i, v) atomic_and(~(int)(i), (v)) +#endif
Ugh; s/atomic/atomic64 here.
I've pushed out a corrected branch.
Mark.
Looking good, thanks!