atomic: introduce atomic_cmpxchg
Subsystems (10)
PATCH 1/5 atomic: introduce atomic_cmpxchg
This patch still needs work on arm (v6) and m32r. I would just be shooting in the dark if I attempted either myself.
Signature · 1 line
SUSE Labs, Novell Inc.
Introduce an atomic_cmpxchg operation. Implement this for i386 and ppc64.
include/asm-i386/atomic.h +2 -0
@@ -215,6 +215,8 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } +#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_inc_return (v) (atomic_add_return(1,v))#define atomic_dec_return (v) (atomic_sub_return(1,v))
include/asm-ppc64/atomic.h +2 -0
@@ -162,6 +162,8 @@ static __inline__ int atomic_dec_return(return t; } +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_sub_and_test (a, v) (atomic_sub_return((a), (v)) == 0)#define atomic_dec_and_test (v) (atomic_dec_return((v)) == 0)
include/asm-ia64/atomic.h +2 -0
@@ -88,6 +88,8 @@ ia64_atomic64_sub (__s64 i, atomic64_t *return new; } +#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \
include/asm-x86_64/atomic.h +2 -0
@@ -360,6 +360,8 @@ static __inline__ int atomic_sub_return(return atomic_add_return (-i,v); } +#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_inc_return (v) (atomic_add_return(1,v))#define atomic_dec_return (v) (atomic_sub_return(1,v))
Documentation/atomic_ops.txt +15 -0
@@ -115,6 +115,21 @@ boolean is return which indicates whethe is negative. It requires explicit memory barrier semantics around the operation. +Finally: + + int atomic_cmpxchg(atomic_t *v, int old, int new); + +This performs an atomic compare exchange operation on the atomic value v, +with the given old and new values. Like all atomic_xxx operations, +atomic_cmpxchg will only satisfy its atomicity semantics as long as all +other accesses of *v are performed through atomic_xxx operations. + +atomic_cmpxchg requires explicit memory barriers around the operation. + +The semantics for atomic_cmpxchg are the same as those defined for 'cas' +below. + + If a caller requires memory barrier semantics around an atomic_t operation which does not return a value, a set of interfaces are defined which accomplish this:
arch/sparc/lib/atomic32.c +14 -0
@@ -38,6 +38,20 @@ int __atomic_add_return(int i, atomic_treturn ret; } +int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; +spin_lock_irqsave (ATOMIC_HASH (v), flags); + + ret = v->counter; +if (likely (ret == old)) + v->counter = new; + +spin_unlock_irqrestore (ATOMIC_HASH (v), flags); +return ret; +} +void atomic_set (atomic_t *v,int i) {unsigned long flags;
include/asm-sparc/atomic.h +1 -0
@@ -19,6 +19,7 @@ typedef struct { volatile int counter; }#define ATOMIC_INIT (i) { (i) }extern int __atomic_add_return (int ,atomic_t *); +extern int atomic_cmpxchg (atomic_t *,int ,int );extern void atomic_set (atomic_t *,int );#define atomic_read (v) ((v)->counter)
include/asm-alpha/atomic.h +2 -0
@@ -171,6 +171,8 @@ static __inline__ long atomic64_sub_retureturn result; } +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic64_dec_return (v) atomic64_sub_return(1,(v))
include/asm-m68k/atomic.h +2 -0
@@ -139,6 +139,8 @@ static inline void atomic_set_mask(unsig __asm__ __volatile__("orl %1,%0" :"+m" (*v) :"id" (mask)); } +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/* Atomic operations are already serializing */ #define smp_mb__before_atomic_dec () barrier()#define smp_mb__after_atomic_dec () barrier()
include/asm-m68knommu/atomic.h +2 -0
@@ -128,6 +128,8 @@ extern __inline__ int atomic_sub_return(return temp; } +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic_inc_return (v) atomic_add_return(1,(v))
include/asm-mips/atomic.h +2 -0
@@ -267,6 +267,8 @@ static __inline__ int atomic_sub_if_posireturn result; } +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic_inc_return (v) atomic_add_return(1,(v))
include/asm-parisc/atomic.h +1 -0
@@ -164,6 +164,7 @@ static __inline__ int atomic_read(const }/* exported interface */ +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))#define atomic_add (i,v) ((void)(__atomic_add_return( ((int)i),(v))))#define atomic_sub (i,v) ((void)(__atomic_add_return(-((int)i),(v))))
include/asm-ppc/atomic.h +2 -0
@@ -177,6 +177,8 @@ static __inline__ int atomic_dec_return(return t; } +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_sub_and_test (a, v) (atomic_sub_return((a), (v)) == 0)#define atomic_dec_and_test (v) (atomic_dec_return((v)) == 0)
include/asm-s390/atomic.h +2 -0
@@ -198,6 +198,8 @@ atomic_compare_and_swap(int expected_oldreturn retval; } +#define atomic_cmpxchg (v, o, n) (atomic_compare_and_swap((o), (n), &((v)->counter))) +#define smp_mb__before_atomic_dec () smp_mb()#define smp_mb__after_atomic_dec () smp_mb()#define smp_mb__before_atomic_inc () smp_mb()
include/asm-sparc64/atomic.h +2 -0
@@ -70,6 +70,8 @@ extern int atomic64_sub_ret(int, atomic6#define atomic_add_negative (i, v) (atomic_add_ret(i, v) < 0)#define atomic64_add_negative (i, v) (atomic64_add_ret(i, v) < 0) +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();
include/asm-arm26/atomic.h +14 -0
@@ -62,6 +62,20 @@ static inline int atomic_sub_return(intreturn val; } +static inline int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +local_irq_restore (flags); + +return ret; +} +static inline void atomic_clear_mask (unsigned long mask,unsigned long *addr) {unsigned long flags;
include/asm-frv/atomic.h +2 -0
@@ -414,4 +414,6 @@ extern uint32_t __cmpxchg_32(uint32_t *v#endif +#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#endif /* _ASM_ATOMIC_H */
include/asm-h8300/atomic.h +12 -0
@@ -82,6 +82,18 @@ static __inline__ int atomic_dec_and_tesreturn ret ==0 ; } +static __inline__int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; +local_irq_save (flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +local_irq_restore (flags); +return ret; +} +static __inline__void atomic_clear_mask (unsigned long mask,unsigned long *v) { __asm__ __volatile__("stc ccr,r1l\n\t"
include/asm-sh64/atomic.h +14 -0
@@ -99,6 +99,20 @@ static __inline__ int atomic_sub_return(#define atomic_inc (v) atomic_add(1,(v))#define atomic_dec (v) atomic_sub(1,(v)) +static __inline__int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +local_irq_restore (flags); + +return ret; +} +static __inline__void atomic_clear_mask (unsigned int mask,atomic_t *v) {unsigned long flags;
include/asm-v850/atomic.h +14 -0
@@ -90,6 +90,20 @@ static __inline__ void atomic_clear_mask#define atomic_dec_and_test (v) (atomic_sub_return (1, (v)) == 0)#define atomic_add_negative (i,v) (atomic_add_return ((i), (v)) < 0) +static __inline__int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +local_irq_restore (flags); + +return ret; +} +/* Atomic operations are already serializing on ARM */ #define smp_mb__before_atomic_dec () barrier()#define smp_mb__after_atomic_dec () barrier()
include/asm-xtensa/atomic.h +1 -0
@@ -223,6 +223,7 @@ static inline int atomic_sub_return(int*/ #define atomic_add_negative (i,v) (atomic_add_return((i),(v)) < 0) +#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))static inline void atomic_clear_mask (unsigned int mask,atomic_t *v) {
include/asm-sh/atomic.h +14 -0
@@ -87,6 +87,20 @@ static __inline__ int atomic_sub_return(#define atomic_inc (v) atomic_add(1,(v))#define atomic_dec (v) atomic_sub(1,(v)) +static __inline__int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +local_irq_restore (flags); + +return ret; +} +static __inline__void atomic_clear_mask (unsigned int mask,atomic_t *v) {unsigned long flags;
include/asm-arm/atomic.h +15 -0
@@ -131,6 +131,21 @@ static inline int atomic_sub_return(intreturn val; } +static inline int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +local_irq_restore (flags); + +return ret; +} + +static inline void atomic_clear_mask (unsigned long mask,unsigned long *addr) static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) {unsigned long flags;
include/asm-cris/atomic.h +13 -0
@@ -123,6 +123,19 @@ extern __inline__ int atomic_inc_and_tesreturn retval; } +static __inline__int atomic_cmpxchg (atomic_t *v,int old,int new) +{ +int ret; +unsigned long flags; +cris_atomic_save (v, flags); + ret = v->counter; +if (likely (ret == old)) + v->counter = new; +cris_atomic_restore (v, flags); +return ret; +} + +static inline void atomic_clear_mask (unsigned long mask,unsigned long *addr) /* Atomic operations are already serializing */#define smp_mb__before_atomic_dec () barrier()#define smp_mb__after_atomic_dec () barrier()
PATCH 3/5 rcu file: use atomic primitives
In reply to Nick PigginThis removes rcuref.h in favour of using the new atomic primitives. Lightly tested on i386.
Signature · 1 line
SUSE Labs, Novell Inc.
include/linux/rcuref.h deleted +0 -220
@@ -1,220 +0,0 @@ -/* -* rcuref.h -* -* Reference counting for elements of lists/arrays protected by -* RCU. -* -* This program is free software; you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation; either version 2 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* -* Copyright (C) IBM Corporation, 2005 -* -* Author: Dipankar Sarma <[email protected]> -* Ravikiran Thirumalai <[email protected]> -* -* See Documentation/RCU/rcuref.txt for detailed user guide. -* -*/ - -#ifndef _RCUREF_H_ -#define _RCUREF_H_ - -#ifdef __KERNEL__ - -#include <linux/types.h> -#include <linux/interrupt.h> -#include <linux/spinlock.h> -#include <asm/atomic.h> - -/* -* These APIs work on traditional atomic_t counters used in the -* kernel for reference counting. Under special circumstances -* where a lock-free get() operation races with a put() operation -* these APIs can be used. See Documentation/RCU/rcuref.txt. -*/ - -#ifdef __HAVE_ARCH_CMPXCHG - -/** -* rcuref_inc - increment refcount for object. -* @rcuref: reference counter in the object in question. -* -* This should be used only for objects where we use RCU and -* use the rcuref_inc_lf() api to acquire a reference -* in a lock-free reader-side critical section. -*/ -static inline void rcuref_inc (atomic_t *rcuref) -{ -atomic_inc (rcuref); -} - -/** -* rcuref_dec - decrement refcount for object. -* @rcuref: reference counter in the object in question. -* -* This should be used only for objects where we use RCU and -* use the rcuref_inc_lf() api to acquire a reference -* in a lock-free reader-side critical section. -*/ -static inline void rcuref_dec (atomic_t *rcuref) -{ -atomic_dec (rcuref); -} - -/** -* rcuref_dec_and_test - decrement refcount for object and test -* @rcuref: reference counter in the object. -* @release: pointer to the function that will clean up the object -* when the last reference to the object is released. -* This pointer is required. -* -* Decrement the refcount, and if 0, return 1. Else return 0. -* -* This should be used only for objects where we use RCU and -* use the rcuref_inc_lf() api to acquire a reference -* in a lock-free reader-side critical section. -*/ -static inline int rcuref_dec_and_test (atomic_t *rcuref) -{ -return atomic_dec_and_test (rcuref); -} - -/* -* cmpxchg is needed on UP too, if deletions to the list/array can happen -* in interrupt context. -*/ - -/** -* rcuref_inc_lf - Take reference to an object in a read-side -* critical section protected by RCU. -* @rcuref: reference counter in the object in question. -* -* Try and increment the refcount by 1. The increment might fail if -* the reference counter has been through a 1 to 0 transition and -* is no longer part of the lock-free list. -* Returns non-zero on successful increment and zero otherwise. -*/ -static inline int rcuref_inc_lf (atomic_t *rcuref) -{ -int c, old; - c =atomic_read (rcuref); -while (c && (old =cmpxchg (&rcuref->counter, c, c +1 )) != c) - c = old; -return c; -} - -#else /* !__HAVE_ARCH_CMPXCHG */ - -extern spinlock_t __rcuref_hash[]; - -/* -* Use a hash table of locks to protect the reference count -* since cmpxchg is not available in this arch. -*/ -#ifdef CONFIG_SMP -#define RCUREF_HASH_SIZE 4 -#define RCUREF_HASH (k) \ - (&__rcuref_hash[(((unsigned long)k)>>8) & (RCUREF_HASH_SIZE-1)]) -#else -#define RCUREF_HASH_SIZE 1 -#define RCUREF_HASH (k) &__rcuref_hash[0] -#endif /* CONFIG_SMP */ - -/** -* rcuref_inc - increment refcount for object. -* @rcuref: reference counter in the object in question. -* -* This should be used only for objects where we use RCU and -* use the rcuref_inc_lf() api to acquire a reference in a lock-free -* reader-side critical section. -*/ -static inline void rcuref_inc (atomic_t *rcuref) -{ -unsigned long flags; -spin_lock_irqsave (RCUREF_HASH (rcuref), flags); - rcuref->counter +=1 ; -spin_unlock_irqrestore (RCUREF_HASH (rcuref), flags); -} - -/** -* rcuref_dec - decrement refcount for object. -* @rcuref: reference counter in the object in question. -* -* This should be used only for objects where we use RCU and -* use the rcuref_inc_lf() api to acquire a reference in a lock-free -* reader-side critical section. -*/ -static inline void rcuref_dec (atomic_t *rcuref) -{ -unsigned long flags; -spin_lock_irqsave (RCUREF_HASH (rcuref), flags); - rcuref->counter -=1 ; -spin_unlock_irqrestore (RCUREF_HASH (rcuref), flags); -} - -/** -* rcuref_dec_and_test - decrement refcount for object and test -* @rcuref: reference counter in the object. -* @release: pointer to the function that will clean up the object -* when the last reference to the object is released. -* This pointer is required. -* -* Decrement the refcount, and if 0, return 1. Else return 0. -* -* This should be used only for objects where we use RCU and -* use the rcuref_inc_lf() api to acquire a reference in a lock-free -* reader-side critical section. -*/ -static inline int rcuref_dec_and_test (atomic_t *rcuref) -{ -unsigned long flags; -spin_lock_irqsave (RCUREF_HASH (rcuref), flags); - rcuref->counter--; -if (!rcuref->counter) { -spin_unlock_irqrestore (RCUREF_HASH (rcuref), flags); -return 1 ; - }else { -spin_unlock_irqrestore (RCUREF_HASH (rcuref), flags); -return 0 ; - } -} - -/** -* rcuref_inc_lf - Take reference to an object of a lock-free collection -* by traversing a lock-free list/array. -* @rcuref: reference counter in the object in question. -* -* Try and increment the refcount by 1. The increment might fail if -* the reference counter has been through a 1 to 0 transition and -* object is no longer part of the lock-free list. -* Returns non-zero on successful increment and zero otherwise. -*/ -static inline int rcuref_inc_lf (atomic_t *rcuref) -{ -int ret; -unsigned long flags; -spin_lock_irqsave (RCUREF_HASH (rcuref), flags); -if (rcuref->counter) - ret = rcuref->counter++; -else - ret =0 ; -spin_unlock_irqrestore (RCUREF_HASH (rcuref), flags); -return ret; -} - - -#endif /* !__HAVE_ARCH_CMPXCHG */ - -#endif /* __KERNEL__ */ -#endif /* _RCUREF_H_ */
fs/aio.c +1 -2
@@ -29,7 +29,6 @@#include <linux/highmem.h> #include <linux/workqueue.h> #include <linux/security.h> -#include <linux/rcuref.h> #include <asm/kmap_types.h> #include <asm/uaccess.h>
@@ -500,7 +499,7 @@ static int __aio_put_req(struct kioctx */* Must be done under the lock to serialise against cancellation. * Call this aio_fput as it duplicates fput via the fput_work. */ -if (unlikely (rcuref_dec_and_test (&req->ki_filp->f_count))) { +if (unlikely (atomic_dec_and_test (&req->ki_filp->f_count))) {get_ioctx (ctx);spin_lock (&fput_lock);list_add (&req->ki_list, &fput_head);
fs/file_table.c +4 -4
@@ -117,7 +117,7 @@ EXPORT_SYMBOL(get_empty_filp); void fastcall fput(struct file *file) { -if (rcuref_dec_and_test (&file->f_count)) +if (atomic_dec_and_test (&file->f_count))__fput (file); }
@@ -166,7 +166,7 @@ struct file fastcall *fget(unsigned intrcu_read_lock (); file =fcheck_files (files, fd);if (file) { -if (!rcuref_inc_lf (&file->f_count)) { +if (!atomic_inc_not_zero (&file->f_count)) {/* File object ref couldn't be taken */ rcu_read_unlock ();return NULL ;
@@ -198,7 +198,7 @@ struct file fastcall *fget_light(unsignercu_read_lock (); file =fcheck_files (files, fd);if (file) { -if (rcuref_inc_lf (&file->f_count)) +if (atomic_inc_not_zero (&file->f_count)) *fput_needed =1 ;else /* Didn't get the reference, someone's freed */
@@ -213,7 +213,7 @@ struct file fastcall *fget_light(unsignevoid put_filp (struct file *file) { -if (rcuref_dec_and_test (&file->f_count)) { +if (atomic_dec_and_test (&file->f_count)) {security_file_free (file);file_kill (file);file_free (file);
include/linux/fs.h +1 -2
@@ -9,7 +9,6 @@#include <linux/config.h> #include <linux/limits.h> #include <linux/ioctl.h> -#include <linux/rcuref.h> /* * It's silly to have NR_OPEN bigger than NR_FILE, but you can change
@@ -604,7 +603,7 @@ extern spinlock_t files_lock;#define file_list_lock () spin_lock(&files_lock);#define file_list_unlock () spin_unlock(&files_lock); -#define get_file (x) rcuref_inc(&(x)->f_count) +#define get_file (x) atomic_inc(&(x)->f_count)#define file_count (x) atomic_read(&(x)->f_count)#define MAX_NON_LFS ((1UL<<31) - 1)
Documentation/RCU/rcuref.txt +12 -12
@@ -31,9 +31,9 @@ release_referenced() delete() If this list/array is made lock free using rcu as in changing the write_lock in add() and delete() to spin_lock and changing read_lock -in search_and_reference to rcu_read_lock(), the rcuref_get in +in search_and_reference to rcu_read_lock(), the atomic_get in search_and_reference could potentially hold reference to an element which -has already been deleted from the list/array. rcuref_lf_get_rcu takes +has already been deleted from the list/array. atomic_inc_not_zero takes care of this scenario. search_and_reference should look as; 1. 2.
@@ -41,7 +41,7 @@ add() search_and_reference() { { alloc_object rcu_read_lock(); ... search_for_element - atomic_set(&el->rc, 1); if (rcuref_inc_lf(&el->rc)) { + atomic_set(&el->rc, 1); if (atomic_inc_not_zero(&el->rc)) { write_lock(&list_lock); rcu_read_unlock(); return FAIL; add_element }
@@ -52,23 +52,23 @@ add() search_and_reference() release_referenced() delete() { { ... write_lock(&list_lock); - rcuref_dec(&el->rc, relfunc) ... + atomic_dec(&el->rc, relfunc) ... ... delete_element } write_unlock(&list_lock); ... - if (rcuref_dec_and_test(&el->rc)) + if (atomic_dec_and_test(&el->rc)) call_rcu(&el->head, el_free); ... } Sometimes, reference to the element need to be obtained in the -update (write) stream. In such cases, rcuref_inc_lf might be an overkill -since the spinlock serialising list updates are held. rcuref_inc +update (write) stream. In such cases, atomic_inc_not_zero might be an overkill +since the spinlock serialising list updates are held. atomic_inc is to be used in such cases. -For arches which do not have cmpxchg rcuref_inc_lf +For arches which do not have cmpxchg atomic_inc_not_zero api uses a hashed spinlock implementation and the same hashed spinlock -is acquired in all rcuref_xxx primitives to preserve atomicity. -Note: Use rcuref_inc api only if you need to use rcuref_inc_lf on the -refcounter atleast at one place. Mixing rcuref_inc and atomic_xxx api -might lead to races. rcuref_inc_lf() must be used in lockfree +is acquired in all atomic_xxx primitives to preserve atomicity. +Note: Use atomic_inc api only if you need to use atomic_inc_not_zero on the +refcounter atleast at one place. Mixing atomic_inc and atomic_xxx api +might lead to races. atomic_inc_not_zero() must be used in lockfree RCU critical sections only.
PATCH 2/5 atomic: introduce atomic_inc_not_zero
In reply to Nick PigginAlso needs work on those same architectures. Other architectures might want to look at providing a more optimal implementation.
Signature · 1 line
SUSE Labs, Novell Inc.
include/asm-ppc64/atomic.h +17 -0
@@ -164,6 +164,23 @@ static __inline__ int atomic_dec_return(#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) + +#define atomic_sub_and_test (a, v) (atomic_sub_return((a), (v)) == 0)#define atomic_dec_and_test (v) (atomic_dec_return((v)) == 0)
include/asm-alpha/atomic.h +9 -0
@@ -173,6 +173,15 @@ static __inline__ long atomic64_sub_retu#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic64_dec_return (v) atomic64_sub_return(1,(v))
include/asm-arm26/atomic.h +14 -0
@@ -76,6 +76,20 @@ static inline int atomic_cmpxchg(atomic_return ret; } +static inline int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +local_irq_restore (flags); + +return ret; +} +static inline void atomic_clear_mask (unsigned long mask,unsigned long *addr) {unsigned long flags;
include/asm-frv/atomic.h +9 -0
@@ -416,4 +416,13 @@ extern uint32_t __cmpxchg_32(uint32_t *v#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#endif /* _ASM_ATOMIC_H */
include/asm-h8300/atomic.h +12 -0
@@ -94,6 +94,18 @@ static __inline__ int atomic_cmpxchg(atoreturn ret; } +static __inline__int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; +local_irq_save (flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +local_irq_restore (flags); +return ret; +} +static __inline__void atomic_clear_mask (unsigned long mask,unsigned long *v) { __asm__ __volatile__("stc ccr,r1l\n\t"
include/asm-i386/atomic.h +16 -0
@@ -217,6 +217,22 @@ static __inline__ int atomic_sub_return(#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_inc_return (v) (atomic_add_return(1,v))#define atomic_dec_return (v) (atomic_sub_return(1,v))
include/asm-ia64/atomic.h +9 -0
@@ -90,6 +90,15 @@ ia64_atomic64_sub (__s64 i, atomic64_t *#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_add_return (i,v) \ ({ \ int __ia64_aar_i = (i); \
include/asm-m68k/atomic.h +9 -0
@@ -141,6 +141,15 @@ static inline void atomic_set_mask(unsig#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +/* Atomic operations are already serializing */ #define smp_mb__before_atomic_dec () barrier()#define smp_mb__after_atomic_dec () barrier()
include/asm-m68knommu/atomic.h +9 -0
@@ -130,6 +130,15 @@ extern __inline__ int atomic_sub_return(#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic_inc_return (v) atomic_add_return(1,(v))
include/asm-mips/atomic.h +16 -0
@@ -269,6 +269,22 @@ static __inline__ int atomic_sub_if_posi#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_dec_return (v) atomic_sub_return(1,(v))#define atomic_inc_return (v) atomic_add_return(1,(v))
include/asm-parisc/atomic.h +16 -0
@@ -166,6 +166,22 @@ static __inline__ int atomic_read(const/* exported interface */ #define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_add (i,v) ((void)(__atomic_add_return( ((int)i),(v))))#define atomic_sub (i,v) ((void)(__atomic_add_return(-((int)i),(v))))#define atomic_inc (v) ((void)(__atomic_add_return( 1,(v))))
include/asm-ppc/atomic.h +16 -0
@@ -179,6 +179,22 @@ static __inline__ int atomic_dec_return(#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_sub_and_test (a, v) (atomic_sub_return((a), (v)) == 0)#define atomic_dec_and_test (v) (atomic_dec_return((v)) == 0)
include/asm-s390/atomic.h +9 -0
@@ -200,6 +200,15 @@ atomic_compare_and_swap(int expected_old#define atomic_cmpxchg (v, o, n) (atomic_compare_and_swap((o), (n), &((v)->counter))) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define smp_mb__before_atomic_dec () smp_mb()#define smp_mb__after_atomic_dec () smp_mb()#define smp_mb__before_atomic_inc () smp_mb()
include/asm-sh/atomic.h +14 -0
@@ -101,6 +101,20 @@ static __inline__ int atomic_cmpxchg(atoreturn ret; } +static __inline__int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +local_irq_restore (flags); + +return ret; +} +static __inline__void atomic_clear_mask (unsigned int mask,atomic_t *v) {unsigned long flags;
include/asm-sh64/atomic.h +14 -0
@@ -113,6 +113,20 @@ static __inline__ int atomic_cmpxchg(atoreturn ret; } +static __inline__int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +local_irq_restore (flags); + +return ret; +} +static __inline__void atomic_clear_mask (unsigned int mask,atomic_t *v) {unsigned long flags;
include/asm-sparc/atomic.h +1 -0
@@ -20,6 +20,7 @@ typedef struct { volatile int counter; }extern int __atomic_add_return (int ,atomic_t *);extern int atomic_cmpxchg (atomic_t *,int ,int ); +extern int atomic_inc_not_zero (atomic_t *);extern void atomic_set (atomic_t *,int );#define atomic_read (v) ((v)->counter)
include/asm-sparc64/atomic.h +9 -0
@@ -72,6 +72,15 @@ extern int atomic64_sub_ret(int, atomic6#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +/* Atomic operations are already serializing */ #ifdef CONFIG_SMP #define smp_mb__before_atomic_dec () membar_storeload_loadload();
include/asm-v850/atomic.h +14 -0
@@ -104,6 +104,20 @@ static __inline__ int atomic_cmpxchg(atoreturn ret; } +static __inline__int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; + +local_irq_save (flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +local_irq_restore (flags); + +return ret; +} +/* Atomic operations are already serializing on ARM */ #define smp_mb__before_atomic_dec () barrier()#define smp_mb__after_atomic_dec () barrier()
include/asm-x86_64/atomic.h +16 -0
@@ -362,6 +362,22 @@ static __inline__ int atomic_sub_return(#define atomic_cmpxchg (v, old, new) ((int)cmpxchg(&((v)->counter), old, new)) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +#define atomic_inc_return (v) (atomic_add_return(1,v))#define atomic_dec_return (v) (atomic_sub_return(1,v))
include/asm-xtensa/atomic.h +16 -0
@@ -225,6 +225,22 @@ static inline int atomic_sub_return(int#define atomic_cmpxchg (v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +/** +* atomic_inc_not_zero - increment if not zero +* @v: pointer of type atomic_t +* +* Atomically increments @v by 1, so long as it was not 0. +* Returns non-zero on successful increment and zero otherwise. +*/ +#define atomic_inc_not_zero (v) \ +({ \ + int c, old; \ + c = atomic_read(v); \ + while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \ + c = old; \ + c; \ +}) +static inline void atomic_clear_mask (unsigned int mask,atomic_t *v) {unsigned int all_f =-1 ;
include/asm-cris/atomic.h +12 -0
@@ -135,6 +135,18 @@ static __inline__ int atomic_cmpxchg(atoreturn ret; } +static __inline__int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; +cris_atomic_save (v, flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +cris_atomic_restore (v, flags); +return ret; +} +static inline void atomic_clear_mask (unsigned long mask,unsigned long *addr) /* Atomic operations are already serializing */#define smp_mb__before_atomic_dec () barrier()
arch/sparc/lib/atomic32.c +14 -0
@@ -52,6 +52,20 @@ int atomic_cmpxchg(atomic_t *v, int old,return ret; } +int atomic_inc_not_zero (atomic_t *v) +{ +int ret; +unsigned long flags; +spin_lock_irqsave (ATOMIC_HASH (v), flags); + ret = v->counter; +if (ret >0 ) + v->counter++; +spin_unlock_irqrestore (ATOMIC_HASH (v), flags); +return ret; +} + +static inline void atomic_clear_mask (unsigned long mask,unsigned long *addr) +/* Atomic operations are already serializing */ void atomic_set(atomic_t *v, int i) {unsigned long flags;
Documentation/atomic_ops.txt +11 -1
@@ -115,7 +115,7 @@ boolean is return which indicates whethe is negative. It requires explicit memory barrier semantics around the operation. -Finally: +Then: int atomic_cmpxchg(atomic_t *v, int old, int new);
@@ -129,6 +129,16 @@ atomic_cmpxchg requires explicit memory The semantics for atomic_cmpxchg are the same as those defined for 'cas' below. +Finally: + + int atomic_inc_not_zero(atomic_t *v); + +If the atomic value v is not zero, this function increments v and returns +non zero. If v is zero then it returns zero. This is done as an atomic +operation. + +atomic_inc_not_zero requires explicit memory barriers around the operation. + If a caller requires memory barrier semantics around an atomic_t operation which does not return a value, a set of interfaces are
PATCH 4/5 atomic: dec_and_lock use cmpxchg
In reply to Nick PigginI noticed David posted a patch to do something similar the other day. With a generic atomic_cmpxchg available, such a patch is no longer objectionable to those architectures that don't #define __HAVE_ARCH_CMPXCHG
Do we want this David? Any other architecture have a super optimised atomic_dec_and_lock that surpasses this implementation?
Signature · 1 line
SUSE Labs, Novell Inc.
lib/dec_and_lock.c +22 -9
@@ -3,10 +3,21 @@#include <asm/atomic.h> /* -* This is an architecture-neutral, but slow, -* implementation of the notion of "decrement -* a reference count, and return locked if it -* decremented to zero". +* Decrement v if it is not equal to one, and return non zero. +* Otherwise return zero. +*/ +static int atomic_dec_not_one (atomic_t *v) +{ +int c, old; + c =atomic_read (v); +while (c >1 && (old =atomic_cmpxchg ((v), c, c -1 )) != c) + c = old; +return c >1 ; +} + +/* +* Implement an atomic "decrement a reference count, and +* return locked if it decremented to zero" operation. * * NOTE NOTE NOTE! This is _not_ equivalent to *
@@ -19,14 +30,16 @@* because the spin-lock and the decrement must be * "atomic". * -* This slow version gets the spinlock unconditionally, -* and releases it if it isn't needed. Architectures -* are encouraged to come up with better approaches, -* this is trivially done efficiently using a load-locked -* store-conditional approach, for example. +* This version shortcuts the unconditional spin lock on +* CONFIG_SMP builds to avoid the expensive lock operation. */ int _atomic_dec_and_lock (atomic_t *atomic,spinlock_t *lock) { +#ifdef CONFIG_SMP +if (atomic_dec_not_one (atomic)) +return 0 ; +#endif +spin_lock (lock);if (atomic_dec_and_test (atomic))return 1 ;
arch/alpha/lib/dec_and_lock.c deleted +0 -42
42 lines not on this page · raw message
arch/i386/lib/dec_and_lock.c deleted +0 -42
42 lines not on this page · raw message
arch/ia64/lib/dec_and_lock.c deleted +0 -42
42 lines not on this page · raw message
arch/mips/lib/dec_and_lock.c deleted +0 -47
47 lines not on this page · raw message
arch/ppc/lib/dec_and_lock.c deleted +0 -38
@@ -1,38 +0,0 @@ -#include <linux/module.h> -#include <linux/spinlock.h> -#include <asm/atomic.h> -#include <asm/system.h> - -/* -* This is an implementation of the notion of "decrement a -* reference count, and return locked if it decremented to zero". -* -* This implementation can be used on any architecture that -* has a cmpxchg, and where atomic->value is an int holding -* the value of the atomic (i.e. the high bits aren't used -* for a lock or anything like that). -*/ -int _atomic_dec_and_lock (atomic_t *atomic,spinlock_t *lock) -{ -int counter; -int newcount; - -for (;;) { - counter =atomic_read (atomic); - newcount = counter -1 ; -if (!newcount) -break ;/* do it the slow way */ - - newcount =cmpxchg (&atomic->counter, counter, newcount); -if (newcount == counter) -return 0 ; - } - -spin_lock (lock); -if (atomic_dec_and_test (atomic)) -return 1 ; -spin_unlock (lock); -return 0 ; -} - -EXPORT_SYMBOL (_atomic_dec_and_lock);
arch/ppc64/lib/dec_and_lock.c deleted +0 -47
47 lines not on this page · raw message
arch/x86_64/lib/dec_and_lock.c deleted +0 -40
@@ -1,40 +0,0 @@ -/* -* x86 version of "atomic_dec_and_lock()" using -* the atomic "cmpxchg" instruction. -* -* (For CPU's lacking cmpxchg, we use the slow -* generic version, and this one never even gets -* compiled). -*/ - -#include <linux/spinlock.h> -#include <asm/atomic.h> - -int _atomic_dec_and_lock (atomic_t *atomic,spinlock_t *lock) -{ -int counter; -int newcount; - -repeat: - counter =atomic_read (atomic); - newcount = counter-1 ; - -if (!newcount) - goto slow_path; - - asmvolatile ("lock; cmpxchgl %1,%2" - :"=a" (newcount) - :"r" (newcount),"m" (atomic->counter),"0" (counter)); - -/* If the above failed, "eax" will have changed */ -if (newcount != counter) - goto repeat; -return 0 ; - -slow_path: -spin_lock (lock); -if (atomic_dec_and_test (atomic)) -return 1 ; -spin_unlock (lock); -return 0 ; -}
lib/Makefile +1 -5
@@ -5,7 +5,7 @@ lib-y := errno.o ctype.o string.o vsprintf.o cmdline.o \ bust_spinlocks.o rbtree.o radix-tree.o dump_stack.o \ idr.o div64.o int_sqrt.o bitmap.o extable.o prio_tree.o \ - sha1.o + sha1.o dec_and_lock.o lib-y += kobject.o kref.o kobject_uevent.o klist.o
@@ -24,10 +24,6 @@ lib-$(CONFIG_GENERIC_FIND_NEXT_BIT) += f obj-$(CONFIG_LOCK_KERNEL) += kernel_lock.o obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o -ifneq ($(CONFIG_HAVE_DEC_LOCK),y) - lib-y += dec_and_lock.o -endif - obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o obj-$(CONFIG_CRC16) += crc16.o obj-$(CONFIG_CRC32) += crc32.o
arch/alpha/Kconfig +0 -5
@@ -501,11 +501,6 @@ config SMP If you don't know what to do here, say N. -config HAVE_DEC_LOCK - bool - depends on SMP - default y - config NR_CPUS int "Maximum number of CPUs (2-64)" range 2 64
arch/i386/Kconfig +0 -5
@@ -908,11 +908,6 @@ config IRQBALANCE The default yes will allow the kernel to do irq load balancing. Saying no will keep the kernel from doing irq load balancing. -config HAVE_DEC_LOCK - bool - depends on (SMP || PREEMPT) && X86_CMPXCHG - default y - # turning this on wastes a bunch of space. # Summit needs it only when NUMA is on config BOOT_IOREMAP
arch/i386/lib/Makefile +0 -1
@@ -7,4 +7,3 @@ lib-y = checksum.o delay.o usercopy.o ge bitops.o lib-$(CONFIG_X86_USE_3DNOW) += mmx.o -lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
arch/ia64/Kconfig +0 -5
@@ -298,11 +298,6 @@ config PREEMPT source "mm/Kconfig" -config HAVE_DEC_LOCK - bool - depends on (SMP || PREEMPT) - default y - config IA32_SUPPORT bool "Support for Linux/x86 binaries" help
arch/ia64/lib/Makefile +0 -1
@@ -15,7 +15,6 @@ lib-$(CONFIG_ITANIUM) += copy_page.o cop lib-$(CONFIG_MCKINLEY) += copy_page_mck.o memcpy_mck.o lib-$(CONFIG_PERFMON) += carta_random.o lib-$(CONFIG_MD_RAID5) += xor.o -lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o AFLAGS___divdi3.o = AFLAGS___udivdi3.o = -DUNSIGNED
arch/m32r/Kconfig +0 -5
@@ -220,11 +220,6 @@ config PREEMPT Say Y here if you are building a kernel for a desktop, embedded or real-time system. Say N if you are unsure. -config HAVE_DEC_LOCK - bool - depends on (SMP || PREEMPT) - default n - config SMP bool "Symmetric multi-processing support" ---help---
arch/mips/Kconfig +0 -4
@@ -1009,10 +1009,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config HAVE_DEC_LOCK - bool - default y - # # Select some configuration options automatically based on user selections. #
arch/ppc/Kconfig +0 -4
@@ -26,10 +26,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config HAVE_DEC_LOCK - bool - default y - config PPC bool default y
arch/ppc64/Kconfig +0 -4
@@ -28,10 +28,6 @@ config GENERIC_ISA_DMA bool default y -config HAVE_DEC_LOCK - bool - default y - config EARLY_PRINTK bool default y
arch/sparc64/lib/Makefile +0 -2
@@ -14,6 +14,4 @@ lib-y := PeeCeeI.o copy_page.o clear_pag copy_in_user.o user_fixup.o memmove.o \ mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o -lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o - obj-y += iomap.o
arch/x86_64/Kconfig +0 -5
@@ -277,11 +277,6 @@ source "mm/Kconfig" config HAVE_ARCH_EARLY_PFN_TO_NID def_bool y -config HAVE_DEC_LOCK - bool - depends on SMP - default y - config NR_CPUS int "Maximum number of CPUs (2-256)" range 2 256
arch/x86_64/lib/Makefile +0 -2
@@ -10,5 +10,3 @@ lib-y := csum-partial.o csum-copy.o csum usercopy.o getuser.o putuser.o \ thunk.o clear_page.o copy_page.o bitstr.o bitops.o lib-y += memcpy.o memmove.o memset.o copy_user.o - -lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
arch/xtensa/Kconfig +0 -4
@@ -26,10 +26,6 @@ config RWSEM_XCHGADD_ALGORITHM bool default y -config HAVE_DEC_LOCK - bool - default y - config GENERIC_HARDIRQS bool default y
arch/sparc64/lib/dec_and_lock.S deleted +0 -80
80 lines not on this page · raw message
arch/alpha/kernel/alpha_ksyms.c +0 -1
@@ -184,7 +184,6 @@ EXPORT_SYMBOL(cpu_data);EXPORT_SYMBOL (smp_num_cpus);EXPORT_SYMBOL (smp_call_function);EXPORT_SYMBOL (smp_call_function_on_cpu); -EXPORT_SYMBOL (_atomic_dec_and_lock);EXPORT_SYMBOL (cpu_present_mask);#endif /* CONFIG_SMP */
arch/sparc64/kernel/sparc64_ksyms.c +0 -3
@@ -163,9 +163,6 @@ EXPORT_SYMBOL(atomic64_add);EXPORT_SYMBOL (atomic64_add_ret);EXPORT_SYMBOL (atomic64_sub);EXPORT_SYMBOL (atomic64_sub_ret); -#ifdef CONFIG_SMP -EXPORT_SYMBOL (_atomic_dec_and_lock); -#endif /* Atomic bit operations. */ EXPORT_SYMBOL (test_and_set_bit);
arch/x86_64/kernel/x8664_ksyms.c +0 -4
@@ -178,10 +178,6 @@ EXPORT_SYMBOL(rwsem_down_write_failed_thEXPORT_SYMBOL (empty_zero_page); -#ifdef CONFIG_HAVE_DEC_LOCK -EXPORT_SYMBOL (_atomic_dec_and_lock); -#endif -EXPORT_SYMBOL (die_chain);EXPORT_SYMBOL (register_die_notifier);
arch/alpha/lib/Makefile +0 -2
@@ -40,8 +40,6 @@ lib-y = __divqu.o __remqu.o __divlu.o __ fpreg.o \ callback_srm.o srm_puts.o srm_printk.o -lib-$(CONFIG_SMP) += dec_and_lock.o - # The division routines are built from single source, with different defines. AFLAGS___divqu.o = -DDIV AFLAGS___remqu.o = -DREM
arch/mips/lib/Makefile +1 -2
@@ -2,8 +2,7 @@ # Makefile for MIPS-specific library files.. # -lib-y += csum_partial_copy.o dec_and_lock.o memcpy.o promlib.o \ - strlen_user.o strncpy_user.o strnlen_user.o +lib-y += csum_partial_copy.o memcpy.o promlib.o strlen_user.o strncpy_user.o strnlen_user.o obj-y += iomap.o
arch/ppc/lib/Makefile +1 -1
@@ -2,7 +2,7 @@ # Makefile for ppc-specific library files.. # -obj-y := checksum.o string.o strcase.o dec_and_lock.o div64.o +obj-y := checksum.o string.o strcase.o div64.o obj-$(CONFIG_8xx) += rheap.o obj-$(CONFIG_CPM2) += rheap.o
arch/ppc64/lib/Makefile +1 -1
@@ -2,7 +2,7 @@ # Makefile for ppc64-specific library files.. # -lib-y := checksum.o dec_and_lock.o string.o strcase.o +lib-y := checksum.o string.o strcase.o lib-y += copypage.o memcpy.o copyuser.o usercopy.o # Lock primitives are defined as no-ops in include/linux/spinlock.h
arch/sparc64/Kconfig.debug +0 -8
@@ -33,14 +33,6 @@ config DEBUG_BOOTMEM depends on DEBUG_KERNEL bool "Debug BOOTMEM initialization" -# We have a custom atomic_dec_and_lock() implementation but it's not -# compatible with spinlock debugging so we need to fall back on -# the generic version in that case. -config HAVE_DEC_LOCK - bool - depends on SMP && !DEBUG_SPINLOCK - default y - config MCOUNT bool depends on STACK_DEBUG
kernel/rcupdate.c +0 -14
@@ -45,7 +45,6 @@#include <linux/percpu.h> #include <linux/notifier.h> #include <linux/rcupdate.h> -#include <linux/rcuref.h> #include <linux/cpu.h> /* Definition for rcupdate control block. */
@@ -73,19 +72,6 @@ DEFINE_PER_CPU(struct rcu_data, rcu_bh_d static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL};static int maxbatch =10 ; -#ifndef __HAVE_ARCH_CMPXCHG -/* -* We use an array of spinlocks for the rcurefs -- similar to ones in sparc -* 32 bit atomic_t implementations, and a hash function similar to that -* for our refcounting needs. -* Can't help multiprocessors which donot have cmpxchg :( -*/ - -spinlock_t __rcuref_hash[RCUREF_HASH_SIZE ] = { - [0 ... (RCUREF_HASH_SIZE -1 )] =SPIN_LOCK_UNLOCKED -}; -#endif -/** * call_rcu - Queue an RCU callback for invocation after a grace period. * @head: structure to be used for queueing the RCU updates.
PATCH 5/5 remove HAVE_ARCH_CMPXCHG
Is there any point in keeping this around?
Signature · 1 line
SUSE Labs, Novell Inc.
include/asm-alpha/system.h +0 -2
@@ -477,8 +477,6 @@ extern void __xchg_called_with_bad_point* we don't need any memory barrier as far I can tell. */ -#define __HAVE_ARCH_CMPXCHG 1 -static inline unsigned long __cmpxchg_u8 (volatile char *m,long old,long new) {
include/asm-i386/system.h +0 -4
@@ -257,10 +257,6 @@ static inline unsigned long __xchg(unsig* indicated by comparing RETURN with OLD. */ -#ifdef CONFIG_X86_CMPXCHG -#define __HAVE_ARCH_CMPXCHG 1 -#endif -static inline unsigned long __cmpxchg (volatile void *ptr,unsigned long old,unsigned long new,int size) {
include/asm-m68k/system.h +0 -1
@@ -164,7 +164,6 @@ static inline unsigned long __xchg(unsig* indicated by comparing RETURN with OLD. */ #ifdef CONFIG_RMW_INSNS -#define __HAVE_ARCH_CMPXCHG 1static inline unsigned long __cmpxchg (volatile void *p,unsigned long old,unsigned long new,int size)
include/asm-m68knommu/system.h +0 -1
@@ -195,7 +195,6 @@ static inline unsigned long __xchg(unsig* store NEW in MEM. Return the initial value in MEM. Success is * indicated by comparing RETURN with OLD. */ -#define __HAVE_ARCH_CMPXCHG 1static __inline__unsigned long cmpxchg (volatile int *p,int old,int new)
include/asm-mips/system.h +0 -2
@@ -277,8 +277,6 @@ static inline unsigned long __xchg(unsig#define xchg (ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))#define tas (ptr) (xchg((ptr),1)) -#define __HAVE_ARCH_CMPXCHG 1 -static inline unsigned long __cmpxchg_u32 (volatile int * m,unsigned long old,unsigned long new) {
include/asm-ppc/system.h +0 -2
@@ -155,8 +155,6 @@ extern inline void * xchg_ptr(void * m, } -#define __HAVE_ARCH_CMPXCHG 1 -static __inline__unsigned long __cmpxchg_u32 (volatile unsigned int *p,unsigned int old,unsigned int new) {
include/asm-ppc64/system.h +0 -2
@@ -223,8 +223,6 @@ __xchg(volatile void *ptr, unsigned long#define tas (ptr) (xchg((ptr),1)) -#define __HAVE_ARCH_CMPXCHG 1 -static __inline__unsigned long __cmpxchg_u32 (volatile unsigned int *p,unsigned long old,unsigned long new) {
include/asm-s390/system.h +0 -2
@@ -189,8 +189,6 @@ static inline unsigned long __xchg(unsig* indicated by comparing RETURN with OLD. */ -#define __HAVE_ARCH_CMPXCHG 1 -#define cmpxchg (ptr,o,n)\ ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ (unsigned long)(n),sizeof(*(ptr))))
include/asm-sparc64/system.h +0 -2
@@ -321,8 +321,6 @@ extern void die_if_kernel(char *str, str* indicated by comparing RETURN with OLD. */ -#define __HAVE_ARCH_CMPXCHG 1 -static __inline__unsigned long __cmpxchg_u32 (volatile int *m,int old,int new) {
include/asm-x86_64/system.h +0 -2
@@ -237,8 +237,6 @@ static inline unsigned long __xchg(unsig* indicated by comparing RETURN with OLD. */ -#define __HAVE_ARCH_CMPXCHG 1 -static inline unsigned long __cmpxchg (volatile void *ptr,unsigned long old,unsigned long new,int size) {
arch/i386/kernel/acpi/boot.c +1 -1
@@ -83,7 +83,7 @@ int acpi_skip_timer_override __initdata;static u64 acpi_lapic_addr__initdata =APIC_DEFAULT_PHYS_BASE ;#endif -#ifndef __HAVE_ARCH_CMPXCHG +#ifndef CONFIG_X86_CMPXCHG #warning ACPI uses CMPXCHG, i486 and later hardware#endif
include/asm-i386/mc146818rtc.h +1 -1
@@ -13,7 +13,7 @@#define RTC_ALWAYS_BCD 1/* RTC operates in binary mode */ #endif -#ifdef __HAVE_ARCH_CMPXCHG +#ifdef CONFIG_X86_CMPXCHG /* * This lock provides nmi access to the CMOS/RTC registers. It has some * special properties. It is owned by a CPU and stores the index register
include/asm-ia64/intrinsics.h +0 -2
@@ -111,8 +111,6 @@ extern void ia64_xchg_called_with_bad_po* indicated by comparing RETURN with OLD. */ -#define __HAVE_ARCH_CMPXCHG 1 -/* * This function doesn't exist, so you'll get a linker error * if something tries to do an invalid cmpxchg().
include/asm-parisc/atomic.h +0 -2
@@ -98,8 +98,6 @@ static __inline__ unsigned long __xchg(u ((__typeof__ (*(ptr )))__xchg ((unsigned long )(x),(ptr),sizeof (*(ptr)))) -#define __HAVE_ARCH_CMPXCHG 1 - /* bug catcher for when unsupported size is used - won't link */extern void __cmpxchg_called_with_bad_pointer (void );
include/asm-um/system-i386.h +0 -2
@@ -3,6 +3,4 @@#include "asm/system-generic.h" -#define __HAVE_ARCH_CMPXCHG 1 -#endif
Quoted 2 lines · Nick Piggin
This patch still needs work on arm (v6) and m32r. I would just be shooting in the dark if I attempted either myself.
ARMv6, something like:
int atomic_cmpxchg(atomic_t *ptr, int old, int new)
{
u32 oldval, res;
do {
asm(
"ldrex %1, [%2]\n\t"
"teq %1, %3\n\t"
"strexeq %0, %4, [%2]\n\t"
: "=&r" (res), "=&r" (oldval)
: "r" (&ptr->counter), "r" (old), "r" (new)
: "cc");
} while (res);
return oldval;
}should do what you require.
Signature · 3 lines
Russell King Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/ maintainer of: 2.6 Serial core
Re: [PATCH 4/5] atomic: dec_and_lock use cmpxchg
In reply to Nick Pigginkernel/rcupdate.c
Quoted 4 lines · Nick Piggin
Index:
linux-2.6/kernel/rcupdate.c
===================================================================kernel/rcupdate.c +0
-0
Dang, this leaked in from 3/5, and 1/5 also has some funny whitespace I introduced.
They'll all still apply just fine, and the end result will look the same, so they're fine for testing and comments. I'll submit cleaned up and signed off versions if/when we're ready (speaking of which, David I should get you to sign this as well if you agree with it)
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Nick PigginHi,
Quoted 2 lines · Nick Piggin
Also needs work on those same architectures. Other architectures might want to look at providing a more optimal implementation.
IMO a rather pointless primitive, unless there is a cpu architecture which has a inc_not_zero instruction, otherwise it will always be the same as using cmpxchg.
bye, Roman
Quoted 8 lines · Russell King
Quoted 2 lines · Nick Piggin
This patch still needs work on arm (v6) and m32r. I would just be shooting in the dark if I attempted either myself.
ARMv6, something like:
Thanks very much, I've updated the patchset.
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
David S. Miller
maintainerRe: [PATCH 4/5] atomic: dec_and_lock use cmpxchg
In reply to Nick PigginFrom: Nick Piggin <[email protected]>
Date: Thu, 15 Sep 2005 00:55:45 +1000
Quoted 8 lines
I noticed David posted a patch to do something similar the other day. With a generic
atomic_cmpxchgavailable, such a patch is no longer objectionable to those architectures that don't #define__HAVE_ARCH_CMPXCHGDo we want this David? Any other architecture have a super optimised
atomic_dec_and_lockthat surpasses this implementation?
I talked to Linus about my patch and he made the good point that the Alpha optimization is very valid (it avoids loading the GP and other stuff by doing that fancy assembler entry point). So we decided that we'd make the cmpxchg() generic version, but make moving over to that gradual instead of mandatorily forcing everyone to use the new thing.
So, for example, we'd convert sparc64 (because I want to) and the architectures using the exact same ppc64 C code, but leave the others alone. It'd be up to the remaining platform's maintainers to move over if they wanted to.
I'll submit that updated patch to him later today.
Re: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to Nick PigginHi,
Quoted 1 line · Nick Piggin
Is there any point in keeping this around?
Yes, for drivers which want to use it to synchronize with userspace. Alternatively it could be changed into a Kconfig definition.
bye, Roman
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Roman ZippelQuoted 11 lines · Roman Zippel
Hi,
Quoted 2 lines · Nick Piggin
Also needs work on those same architectures. Other architectures might want to look at providing a more optimal implementation.
IMO a rather pointless primitive, unless there is a cpu architecture which has a
inc_not_zeroinstruction, otherwise it will always be the same as using cmpxchg.
It will always be *implemented* with cmpxchg you mean, which is a bit different. But even then, no, you definitely don't need an inc_not_zero instruction to make this primitive faster than the cmpxchg version. Just look at all the !SMP architectures that just turn off interrupts while doing the op. Look at the architectures that use hashed spinlocks.
Or here is possible pseudo code for an architecture with ll/sc instructions:
do {
tmp = load_locked(v);
if (!tmp)
break;
tmp++;
} while (!store_cond(v, tmp));return tmp;
As opposed to using the cmpxchg version, which would have more loads and conditional branches, AFAIKS.
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
Re: [PATCH 4/5] atomic: dec_and_lock use cmpxchg
In reply to David S. MillerQuoted 6 lines · David S. Miller
I talked to Linus about my patch and he made the good point that the Alpha optimization is very valid (it avoids loading the GP and other stuff by doing that fancy assembler entry point). So we decided that we'd make the
cmpxchg()generic version, but make moving over to that gradual instead of mandatorily forcing everyone to use the new thing.
Quoted 7 lines
So, for example, we'd convert sparc64 (because I want to) and the architectures using the exact same ppc64 C code, but leave the others alone. It'd be up to the remaining platform's maintainers to move over if they wanted to.
I'll submit that updated patch to him later today.
OK that's fair enough. I'll submit whatever cleanups are possible on top of your patch after the atomic_cmpxchg patch goes in (if it goes in). Thanks.
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Nick PigginHi,
Quoted 10 lines · Nick Piggin
Or here is possible pseudo code for an architecture with ll/sc instructions:
do { tmp = load_locked(v); if (!tmp) break; tmp++; } while (!store_cond(v, tmp));
Quoted 4 lines
return tmp;
As opposed to using the cmpxchg version, which would have more loads and conditional branches, AFAIKS.
I'd prefer to generalize this construct, than polluting atomic.h with all kinds of esoteric atomic operations.
So you would get:
do {
old = atomic_load_locked(v);
if (!old)
break;
new = old + 1;
} while (!atomic_store_lock(v, old, new));bye, Roman
Re: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to Roman ZippelQuoted 11 lines · Roman Zippel
Hi,
Quoted 1 line · Nick Piggin
Is there any point in keeping this around?
Yes, for drivers which want to use it to synchronize with userspace. Alternatively it could be changed into a
Kconfigdefinition.
I think it already is. At least, I did grep for it and didn't see anything.
I think userspace synchronization may be quite a valid use of atomic cmpxchg, but Kconfig is a far better place to do it than testing HAVE_ARCH_CMPXCHG.
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Nick PigginQuoted 7 lines · Nick Piggin
Quoted 5 lines · Roman Zippel
Hi,
On Thu, 15 Sep 2005, Nick Piggin wrote:
Quoted 8 lines
Quoted 7 lines
Quoted 2 lines
Also needs work on those same architectures. Other architectures might want to look at providing a more optimal implementation.
IMO a rather pointless primitive, unless there is a cpu architecture which has a
inc_not_zeroinstruction, otherwise it will always be the same as using cmpxchg.
[snip]
But even supposing the cmpxchg variant was the highest performing implementation available on any architecture, I would still consider exporting the inc_not_zero instruction.
The reason is that cmpxchg is not nearly so readable as inc_not_zero when used inline in the code.
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
David S. Miller
maintainerRe: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to Nick PigginFrom: Nick Piggin <[email protected]>
Date: Thu, 15 Sep 2005 02:49:58 +1000
Quoted 2 lines
I think it already is. At least, I did grep for it and didn't see anything.
Things in this class, such as DRM, just happen to only get enabled on platforms that have a cmpxchg() available, they don't actually check.
That being said, I agree that Kconfig is the place to do this and then DRM's Kconfig can get the correct dependency.
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Roman ZippelQuoted 20 lines · Roman Zippel
Quoted 10 lines · Nick Piggin
Or here is possible pseudo code for an architecture with ll/sc instructions:
do { tmp = load_locked(v); if (!tmp) break; tmp++; } while (!store_cond(v, tmp));Quoted 4 lines
return tmp;
As opposed to using the cmpxchg version, which would have more loads and conditional branches, AFAIKS.
I'd prefer to generalize this construct, than polluting
atomic.hwith all kinds of esoteric atomic operations.
So you would get:
Quoted 6 lines
do {
old = atomic_load_locked(v);
if (!old)
break;
new = old + 1;
} while (!atomic_store_lock(v, old, new));How do you propose architectures which don't have locked loads implement this, where the only atomic instruction is an unconditional atomic swap between memory and CPU register?
Signature · 3 lines
Russell King Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/ maintainer of: 2.6 Serial core
Re: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to Nick PigginQuoted 13 lines · Nick Piggin
Quoted 11 lines · Roman Zippel
Hi,
Quoted 1 line · Nick Piggin
Is there any point in keeping this around?
Yes, for drivers which want to use it to synchronize with userspace. Alternatively it could be changed into a
Kconfigdefinition.
Quoted 6 lines
I think it already is. At least, I did grep for it and didn't see anything.
I think userspace synchronization may be quite a valid use of atomic cmpxchg, but
Kconfigis a far better place to do it than testingHAVE_ARCH_CMPXCHG.
What business has userspace got of telling whether cmpxchg works on an architecture by looking at kernel headers?
Even if an architecture provides an implementation of it, it might rely on turning IRQs off, which may not be possible in userspace, leading to the userspace version actually being non-atomic.
*Forget* kernel includes telling userspace what architecture features are available. It's extremely buggy by design.
If you want to remove HAVE_ARCH_CMPXCHG that's fine. If userspace complains, you've found a bug for them. 8)
Signature · 3 lines
Russell King Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/ maintainer of: 2.6 Serial core
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Russell KingHi,
Quoted 10 lines · Russell King
Quoted 6 lines
do { old = atomic_load_locked(v); if (!old) break; new = old + 1; } while (!atomic_store_lock(v, old, new));How do you propose architectures which don't have locked loads implement this, where the only atomic instruction is an unconditional atomic swap between memory and CPU register?
#define atomic_store_lock atomic_cmpxchg
bye, Roman
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
Quoted 11 lines · Roman Zippel
Hi,
Quoted 7 lines · Russell King
Quoted 6 lines
do { old = atomic_load_locked(v); if (!old) break; new = old + 1; } while (!atomic_store_lock(v, old, new));
Quoted 5 lines
Quoted 3 lines
How do you propose architectures which don't have locked loads implement this, where the only atomic instruction is an unconditional atomic swap between memory and CPU register?
#define
atomic_store_lockatomic_cmpxchg
No. "unconditional atomic swap" does not mean cmpxchg - it means that atomic_cmpxchg itself would have to be open coded, which is inefficient.
What you're asking architectures to do is:
retry:
load
operation
save interrupts
load
compare
store if equal
restore interrupts
goto retry if not equal
whereas they could have done the far simpler version of:
save interrupts
load
operation
store
restore interrupts
which they do today.
The whole point about architecture specific includes is not to provide a frenzied feeding ground for folk who like to "clean code up" but to allow architectures to do things in the most efficient way for them without polluting the kernel too much.
It seems that aspect is being lost sight of here.
Signature · 3 lines
Russell King Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/ maintainer of: 2.6 Serial core
David S. Miller
maintainerRe: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to Russell KingFrom: Russell King <[email protected]>
Date: Wed, 14 Sep 2005 23:03:53 +0100
Quoted 2 lines
What business has userspace got of telling whether cmpxchg works on an architecture by looking at kernel headers?
Russell, please don't fly off the handle like this.
Nick is talking about something slightly different.
Things, for example, like DRM assume there is a cmpxchg() they can use in the kernel and userland for interlocking.
DRM doesn't actually _check_ anything to see if this is the case, it just so happens to only get enabled on platforms where cmpxchg() is available in this fashion.
What Nick is suggesting is to actually move HAVE_ARCH_CMPXCHG or something like it into the Kconfig so that things like DRM can actually do the correct dependency check. If you want something like "HAVE_ARCH_CMPXCHG_WHICH_CAN_INTERFACE_WITH_USERSPACE" that DRM can check too, all the better.
It's not about whether userspace can include some kernel header and get cmpxchg(), it's whether there is some way that a cmpxchg() shared semaphore between userspace and kernel is possible, which things like DRM depend upon having available.
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Russell KingQuoted 7 lines · Russell King
The whole point about architecture specific includes is not to provide a frenzied feeding ground for folk who like to "clean code up" but to allow architectures to do things in the most efficient way for them without polluting the kernel too much.
It seems that aspect is being lost sight of here.
Yep. We've got atomic_add, atomic_sub, atomic_inc, atomic_dec, atomic_inc_return, atomic_add_return, atomic_dec_return, atomic_sub_return to start with.
Not only that, but we can probably emulate all the atomic_ operations with atomic_cmpxchg, not just atomic_inc_not_zero.
Roman: any ideas about what you would prefer? You'll notice atomic_inc_not_zero replaces rcuref_inc_lf, which is used several times in the VFS.
Thanks,
Nick
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com
Re: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to David S. MillerQuoted 7 lines · David S. Miller
From: Russell King <[email protected]>
Date: Wed, 14 Sep 2005 23:03:53 +0100Quoted 2 lines
What business has userspace got of telling whether cmpxchg works on an architecture by looking at kernel headers?
Russell, please don't fly off the handle like this.
Sigh, I wasn't. I was making a valid point. Maybe if you read the bit I quoted you'd have realised that, which was:
Quoted 1 line
I think userspace synchronization may be quite a valid use of
^^^^^^^^^
Quoted 2 lines
atomic cmpxchg, but
Kconfigis a far better place to do it than testingHAVE_ARCH_CMPXCHG.
But maybe I'm not capable of interpreting the above english? To me at least it's definitely talking about user space, not kernel space.
Signature · 3 lines
Russell King Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/ maintainer of: 2.6 Serial core
Re: [PATCH 5/5] remove HAVE_ARCH_CMPXCHG
In reply to Nick PigginQuoted 2 lines · Nick Piggin
Is there any point in keeping this around?
Not all platforms have CMPXCHG and some code wants to know about that - eg DRI
David S. Miller
maintainerRe: [PATCH 4/5] atomic: dec_and_lock use cmpxchg
In reply to Nick PigginFrom: Nick Piggin <[email protected]>
Date: Thu, 15 Sep 2005 02:52:08 +1000
Quoted 3 lines
OK that's fair enough. I'll submit whatever cleanups are possible on top of your patch after the
atomic_cmpxchgpatch goes in (if it goes in). Thanks.
The updated patch is in Linus's current tree, FYI. The GIT commit is: 4db2ce0199f04b6e99999f22e28ef9a0ae5f0d2f
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Russell KingHi,
Quoted 23 lines · Russell King
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
do { old = atomic_load_locked(v); if (!old) break; new = old + 1; } while (!atomic_store_lock(v, old, new));What you're asking architectures to do is:
retry:
load
operation
save interrupts
load
compare
store if equal
restore interrupts
goto retry if not equalwhereas they could have done the far simpler version of:
Quoted 7 lines
save interrupts
load
operation
store
restore interruptswhich they do today.
So modify it this way:
atomic_lock(flags);
do {
old = atomic_load_locked(v);
if (!old)
break;
new = old + 1;
} while (!atomic_store_locked(v, old, new));
atomic_unlock(flags);Quoted 4 lines
The whole point about architecture specific includes is not to provide a frenzied feeding ground for folk who like to "clean code up" but to allow architectures to do things in the most efficient way for them without polluting the kernel too much.
In this case it's massively repeated code which only differs slighty. My biggest problem here is the lack of gcc support to get the condition code out of an asm. Especially architectures with locked load/store instruction could then have just a single implementation in asm-generic.
A long time ago I was playing with something like this:
#define atomic_exchange(ptr, old, new) ({ \
old = *(ptr); \
asm volatile ("1:" \
: "=&a" (old), "=m" (*(ptr)) \
: "0" (old), "m" (*(ptr)) \
: "memory"); \
asm volatile (__LOCK "cmpxchg %2,%0; jne 1b" \
: "=m" (*(ptr)), "=&a" (old) \
: "r" (new), "m" (*(ptr)), "1" (old) \
: "memory"); \
})so e.g. an atomic_add_return() would be atomic_exchange(ptr, old, old + val), but I guess gcc people would kill me for that. :-)
bye, Roman
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Nick PigginHi,
Quoted 3 lines · Nick Piggin
Roman: any ideas about what you would prefer? You'll notice
atomic_inc_not_zeroreplacesrcuref_inc_lf, which is used several times in the VFS.
In the larger picture I'm not completely happy with these scalibilty patches, as they add extra overhead at the lower end. On a UP system in general nothing beats:
spin_lock(); if (*ptr) ptr += 1; spin_unlock();
The main problem is here that the atomic functions are used in two basic situation:
1) interrupt synchronization
2) multiprocessor synchronization
The atomic functions have to assume both, but on UP systems it often is a lot cheaper if they don't have to synchronize with interrupts. So replacing a spinlock with a few atomic operations can hurt UP performance.
bye, Roman
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
Quoted 21 lines · Roman Zippel
Hi,
Quoted 3 lines · Nick Piggin
Roman: any ideas about what you would prefer? You'll notice
atomic_inc_not_zeroreplacesrcuref_inc_lf, which is used several times in the VFS.In the larger picture I'm not completely happy with these scalibilty patches, as they add extra overhead at the lower end. On a UP system in general nothing beats:
spin_lock(); if (*ptr) ptr += 1; spin_unlock();The main problem is here that the atomic functions are used in two basic situation:
Quoted 7 lines
1) interrupt synchronization
2) multiprocessor synchronizationThe atomic functions have to assume both, but on UP systems it often is a lot cheaper if they don't have to synchronize with interrupts. So replacing a spinlock with a few atomic operations can hurt UP performance.
Nope. On uniprocessor systems, atomic_foo() doesn't actually do the buslocked atomic thing.
#ifdef CONFIG_SMP
#define LOCK "lock ; "
#else
#define LOCK ""
#endif
On x86, at least. Other architectures can do the same thing if they have an atomic-wrt-IRQs add and sub.
David S. Miller
maintainerRe: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Roman ZippelFrom: Roman Zippel <[email protected]>
Date: Sat, 17 Sep 2005 02:59:50 +0200 (CEST)
Quoted 2 lines
My biggest problem here is the lack of gcc support to get the condition code out of an asm.
I agree, this is the biggest deficiency in gcc inline assembly and I run into it all the time.
Things like __builtin_trap() would never even be needed if we could communicate condition code state into and out of inline asm statements, for example.
Sparc32 and sparc64 both have "trap on condition code" instructions. __builtin_trap() will do the right thing, _BUT_ I want to emit nice bug table entries like ppc64 does in asm-ppc64/bug.h, but I can't do that without generating really crappy code. And the whole limitation comes from the fact that I can't tell ask gcc for "the condition codes that result from test X" as the input for an asm.
So what would be great is something akin to:
__asm__ __volatile__("t%cc 0x5"
: "=C" (test));or something like that. The "%cc" thing would be expand to the appropriate two/three letter condition test type code, for example "ne", "eq", "ge", "ltu" and the like. GCC could choose the best comparison + %cc code, as it already does for conditional branch generation.
It could even help on platforms like ppc64 where the trap condition instruction does a test of a register against an immediate value. Something like "BUG_ON(a != 2)" is probably emitting something like:
sub %a, 2, %tmp tdnei %tmp, 0
whereas this could be:
tdnei %tmp, 2
if GCC has some inline asm condition code input facility like the above.
David S. Miller
maintainerRe: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Roman ZippelFrom: Roman Zippel <[email protected]>
Date: Sat, 17 Sep 2005 03:15:29 +0200 (CEST)
Quoted 5 lines
The main problem is here that the atomic functions are used in two basic situation:
1) interrupt synchronization
2) multiprocessor synchronization
3) preempt synchronization
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to David S. MillerQuoted 5 lines · David S. Miller
Quoted 2 lines
My biggest problem here is the lack of gcc support to get the condition code out of an asm.
I agree, this is the biggest deficiency in gcc inline assembly and I run into it all the time.
gcc did have some support to pass condition codes into assembly. On ARM, you used to be able to do things like:
if (foo)
asm("blah%? whatever");and gcc would replace %? with whatever condition was appropriate for the current block of code. You can still write it as the above.
However, this optimisation was disabled on ARM apparantly because it was very hard to for people to get correct - if you forgot the %?, you need to add a "cc" clobber, and if you forget that as well you might get unconditional behaviour.
Signature · 3 lines
Russell King Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/ maintainer of: 2.6 Serial core
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Roman ZippelQuoted 12 lines · Roman Zippel
Hi,
Quoted 3 lines · Nick Piggin
Roman: any ideas about what you would prefer? You'll notice
atomic_inc_not_zeroreplacesrcuref_inc_lf, which is used several times in the VFS.In the larger picture I'm not completely happy with these scalibilty patches, as they add extra overhead at the lower end. On a UP system in general nothing beats:
Quoted 7 lines
spin_lock(); if (*ptr) ptr += 1; spin_unlock();The main problem is here that the atomic functions are used in two basic situation:
Are you talking about the lock-free fdtable patches ? They don't replace non-atomic locked critical sections by atomic operations. Reference counting is already there to extend the life of objects beyond locked critical setions.
Thanks
Dipankar
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Andrew MortonHi,
Quoted 11 lines · Andrew Morton
Nope. On uniprocessor systems,
atomic_foo()doesn't actually do the buslocked atomic thing.#ifdef
CONFIG_SMP
#define LOCK "lock ; "
#else
#define LOCK ""
#endifOn x86, at least. Other architectures can do the same thing if they have an atomic-wrt-IRQs add and sub.
That's true on x86, but if these functions have to be emulated using spinlocks, they always have to disable interrupts, whether the caller needs it or not. Also x86 has the lock attribute, which helps a lot, but on other archs a cmpxchg instruction can be quite expensive, so it could be cheaper by just using {disable,enable}_preempt.
bye, Roman
David S. Miller
maintainerRe: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Russell KingFrom: Russell King <[email protected]>
Date: Sat, 17 Sep 2005 08:27:36 +0100
Quoted 6 lines
gcc did have some support to pass condition codes into assembly. On ARM, you used to be able to do things like:
if (foo) asm("blah%? whatever");
Quoted 8 lines
and gcc would replace %? with whatever condition was appropriate for the current block of code. You can still write it as the above.
However, this optimisation was disabled on ARM apparantly because it was very hard to for people to get correct - if you forgot the %?, you need to add a "cc" clobber, and if you forget that as well you might get unconditional behaviour.
Yes, that is an error prone syntax to use, that's for sure. That is, incidentally, why I said the condition test should be an explicit input arg to the asm.
Re: [PATCH 2/5] atomic: introduce atomic_inc_not_zero
In reply to Roman ZippelQuoted 17 lines · Roman Zippel
Hi,
Quoted 3 lines · Nick Piggin
Roman: any ideas about what you would prefer? You'll notice
atomic_inc_not_zeroreplacesrcuref_inc_lf, which is used several times in the VFS.In the larger picture I'm not completely happy with these scalibilty patches, as they add extra overhead at the lower end. On a UP system in general nothing beats:
spin_lock(); if (*ptr) ptr += 1; spin_unlock();
Quoted 7 lines
The main problem is here that the atomic functions are used in two basic situation:
1) interrupt synchronization
2) multiprocessor synchronizationThe atomic functions have to assume both, but on UP systems it often is a lot cheaper if they don't have to synchronize with interrupts. So replacing a spinlock with a few atomic operations can hurt UP performance.
Maybe so, but what I'm doing is introducing a slightly better implementation of what is currently in tree, and attempting to follow current standards as far as possible. I don't think you could say that is a bad thing.
Now I don't think anyone would be flat out opposed to 1 - reworking the atomic.h code to allow some genericity (is that a word?); 2 - reworking atomic.h code to allow combining of atomic ops, or allowing interrupt unsafe ops...
Of course, neither is going to be merged unless done tastefully, and I imagine both would be difficult to get right, with probably a low cost/benefit ratio.
Thanks,
Nick
Signature · 3 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com