Skip to contentSkip to conversation

riscv: Coding convention for xchg

PATCH In review [email protected] 4 participants 16 messages Nov 24, 2020 – Nov 26, 2020

Subsystems (4)
lore
Share & tools
lore

[email protected]

author maintainer
#0 lore raw

PATCH 1/5 riscv: Coding convention for xchg

From: Guo Ren <[email protected]>

This is prepare for QUEUED_SPINLOCKS which need xchg support short type value.
- Remove unused codes (xchg32, xchg64, cmpxchg32 ...)
- Combine xchg_relaxed, xchg_acquire, xchg_release into one asm
- Make atomic.aq/rl with seperated fence acquire & release

Cc Peter Zijlstra <[email protected]>
Cc Michael Clark <[email protected]>
---
 arch/riscv/include/asm/atomic.h  |  31 ++++-----
 arch/riscv/include/asm/cmpxchg.h | 135 +++------------------------------------
 arch/riscv/include/asm/fence.h   |   6 ++
 3 files changed, 30 insertions(+), 142 deletions(-)
arch/riscv/include/asm/atomic.h +14 -17
index 400a8c8..632fbe4 100644
@@ -19,11 +19,8 @@
 #include <asm/cmpxchg.h>
 #include <asm/barrier.h>
 
-#define __atomic_acquire_fence()					\
-	__asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
-
-#define __atomic_release_fence()					\
-	__asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
+#define __atomic_acquire_fence() __acquire_fence()
+#define __atomic_release_fence() __release_fence()
 
 static __always_inline int atomic_read(const atomic_t *v)
 {
@@ -242,58 +239,58 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u
  * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
  * {cmp,}xchg and the operations that return, so they need a full barrier.
  */
-#define ATOMIC_OP(c_t, prefix, size)					\
+#define ATOMIC_OP(c_t, prefix)						\
 static __always_inline							\
 c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n)		\
 {									\
-	return __xchg_relaxed(&(v->counter), n, size);			\
+	return xchg_relaxed(&(v->counter), n);				\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n)		\
 {									\
-	return __xchg_acquire(&(v->counter), n, size);			\
+	return xchg_acquire(&(v->counter), n);				\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n)		\
 {									\
-	return __xchg_release(&(v->counter), n, size);			\
+	return xchg_release(&(v->counter), n);				\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n)			\
 {									\
-	return __xchg(&(v->counter), n, size);				\
+	return xchg(&(v->counter), n);					\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v,		\
 				     c_t o, c_t n)			\
 {									\
-	return __cmpxchg_relaxed(&(v->counter), o, n, size);		\
+	return cmpxchg_relaxed(&(v->counter), o, n);			\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v,		\
 				     c_t o, c_t n)			\
 {									\
-	return __cmpxchg_acquire(&(v->counter), o, n, size);		\
+	return cmpxchg_acquire(&(v->counter), o, n);			\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v,		\
 				     c_t o, c_t n)			\
 {									\
-	return __cmpxchg_release(&(v->counter), o, n, size);		\
+	return cmpxchg_release(&(v->counter), o, n);			\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n)	\
 {									\
-	return __cmpxchg(&(v->counter), o, n, size);			\
+	return cmpxchg(&(v->counter), o, n);				\
 }
 
 #ifdef CONFIG_GENERIC_ATOMIC64
 #define ATOMIC_OPS()							\
-	ATOMIC_OP(int,   , 4)
+	ATOMIC_OP(int,   )
 #else
 #define ATOMIC_OPS()							\
-	ATOMIC_OP(int,   , 4)						\
-	ATOMIC_OP(s64, 64, 8)
+	ATOMIC_OP(int,   )						\
+	ATOMIC_OP(s64, 64)
 #endif
 
 ATOMIC_OPS()
arch/riscv/include/asm/cmpxchg.h +10 -125

160 lines not on this page · raw message

arch/riscv/include/asm/fence.h +6 -0
index 2b443a3..3832601 100644
@@ -9,4 +9,10 @@
 #define RISCV_RELEASE_BARRIER
 #endif
 
+#define __acquire_fence() \
+	__asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
+
+#define __release_fence() \
+	__asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory")
+
 #endif	/* _ASM_RISCV_FENCE_H */
Signature · 1 line
2.7.4
author maintainer
#1 lore raw

PATCH 2/5 riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

From: Guo Ren <[email protected]>

riscv only support lr.wd/s(c).w(d) with word(double word) size & align access. There are not lr.h/sc.h instructions. But qspinlock.c need xchg with short type variable:

xchg_tail -> xchg_releaxed(&lock->tail, ...

typedef struct qspinlock {
        union {
                atomic_t val;

                /*
                 * By using the whole 2nd least significant byte for the
                 * pending bit, we can allow better optimization of the lock
                 * acquisition for the pending bit holder.
                 */
                struct {
                        u8      locked;
                        u8      pending;
                };
                struct {
                        u16     locked_pending;
                        u16     tail; /* half word*/
                };
        };
} arch_spinlock_t;

So we add short emulation in xchg with word length and it only solve qspinlock's requirement.

Michael have sent qspinlock before, ref to Link below.

---
 arch/riscv/Kconfig                      |   2 +
 arch/riscv/include/asm/Kbuild           |   3 +
 arch/riscv/include/asm/cmpxchg.h        |  36 +++++++++
 arch/riscv/include/asm/spinlock.h       | 126 +-------------------------------
 arch/riscv/include/asm/spinlock_types.h |  15 +---
 5 files changed, 46 insertions(+), 136 deletions(-)
arch/riscv/Kconfig +2 -0
index 2d61c4c..d757ba4 100644
@@ -36,6 +36,8 @@ config RISCV
 	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU
 	select ARCH_WANT_FRAME_POINTERS
 	select ARCH_WANT_HUGE_PMD_SHARE if 64BIT
+	select ARCH_USE_QUEUED_RWLOCKS
+	select ARCH_USE_QUEUED_SPINLOCKS
 	select CLONE_BACKWARDS
 	select CLINT_TIMER if !MMU
 	select COMMON_CLK
arch/riscv/include/asm/Kbuild +3 -0
index 59dd7be..6f5f438 100644
@@ -6,3 +6,6 @@ generic-y += kvm_para.h
 generic-y += local64.h
 generic-y += user.h
 generic-y += vmlinux.lds.h
+generic-y += mcs_spinlock.h
+generic-y += qrwlock.h
+generic-y += qspinlock.h
arch/riscv/include/asm/cmpxchg.h +36 -0
index 5609185..e178700 100644
@@ -16,7 +16,43 @@
 	__typeof__(ptr) __ptr = (ptr);					\
 	__typeof__(new) __new = (new);					\
 	__typeof__(*(ptr)) __ret;					\
+	register unsigned long __rc, tmp, align, addr;			\
 	switch (size) {							\
+	case 2:								\
+		align = ((unsigned long) __ptr & 0x3);			\
+		addr = ((unsigned long) __ptr & ~0x3);			\
+		if (align) {						\
+		__asm__ __volatile__ (					\
+			"0:	lr.w %0, 0(%z4)\n"			\
+			"	move %1, %0\n"				\
+			"	slli %1, %1, 16\n"			\
+			"	srli %1, %1, 16\n"			\
+			"	move %2, %z3\n"				\
+			"	slli %2, %2, 16\n"			\
+			"	or   %1, %2, %1\n"			\
+			"	sc.w %2, %1, 0(%z4)\n"			\
+			"	bnez %2, 0b\n"				\
+			"	srli %0, %0, 16\n"			\
+			: "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)	\
+			: "rJ" (__new), "rJ"(addr)			\
+			: "memory");					\
+		} else {						\
+		__asm__ __volatile__ (					\
+			"0:	lr.w %0, (%z4)\n"			\
+			"	move %1, %0\n"				\
+			"	srli %1, %1, 16\n"			\
+			"	slli %1, %1, 16\n"			\
+			"	move %2, %z3\n"				\
+			"	or   %1, %2, %1\n"			\
+			"	sc.w %2, %1, 0(%z4)\n"			\
+			"	bnez %2, 0b\n"				\
+			"	slli %0, %0, 16\n"			\
+			"	srli %0, %0, 16\n"			\
+			: "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)	\
+			: "rJ" (__new), "rJ"(addr)			\
+			: "memory");					\
+		}							\
+		break;							\
 	case 4:								\
 		__asm__ __volatile__ (					\
 			"	amoswap.w %0, %2, %1\n"			\
arch/riscv/include/asm/spinlock.h +2 -124

131 lines not on this page · raw message

arch/riscv/include/asm/spinlock_types.h +3 -12
index f398e76..d033a97 100644
@@ -6,20 +6,11 @@
 #ifndef _ASM_RISCV_SPINLOCK_TYPES_H
 #define _ASM_RISCV_SPINLOCK_TYPES_H
 
-#ifndef __LINUX_SPINLOCK_TYPES_H
+#if !defined(__LINUX_SPINLOCK_TYPES_H) && !defined(_ASM_RISCV_SPINLOCK_H)
 # error "please don't include this file directly"
 #endif
 
-typedef struct {
-	volatile unsigned int lock;
-} arch_spinlock_t;
-
-#define __ARCH_SPIN_LOCK_UNLOCKED	{ 0 }
-
-typedef struct {
-	volatile unsigned int lock;
-} arch_rwlock_t;
-
-#define __ARCH_RW_LOCK_UNLOCKED		{ 0 }
+#include <asm-generic/qspinlock_types.h>
+#include <asm-generic/qrwlock_types.h>
 
 #endif /* _ASM_RISCV_SPINLOCK_TYPES_H */
Signature · 1 line
2.7.4
maintainer
#2 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Quoted 49 lines · On Tue, Nov 24, 2020 at 01:43:54PM +0000, [email protected] wrote:
arch/riscv/include/asm/Kbuild +3 -0
index 59dd7be..6f5f438 100644
@@ -6,3 +6,6 @@ generic-y += kvm_para.h
 generic-y += local64.h
 generic-y += user.h
 generic-y += vmlinux.lds.h
+generic-y += mcs_spinlock.h
+generic-y += qrwlock.h
+generic-y += qspinlock.h
arch/riscv/include/asm/cmpxchg.h +29 -0
index 5609185..e178700 100644
@@ -16,7 +16,43 @@
 	__typeof__(ptr) __ptr = (ptr);					\
 	__typeof__(new) __new = (new);					\
 	__typeof__(*(ptr)) __ret;					\
+	register unsigned long __rc, tmp, align, addr;			\
 	switch (size) {							\
+	case 2:								\
+		align = ((unsigned long) __ptr & 0x3);			\
+		addr = ((unsigned long) __ptr & ~0x3);			\
+		if (align) {						\
+		__asm__ __volatile__ (					\
+			"0:	lr.w %0, 0(%z4)\n"			\
+			"	move %1, %0\n"				\
+			"	slli %1, %1, 16\n"			\
+			"	srli %1, %1, 16\n"			\
+			"	move %2, %z3\n"				\
+			"	slli %2, %2, 16\n"			\
+			"	or   %1, %2, %1\n"			\
+			"	sc.w %2, %1, 0(%z4)\n"			\
+			"	bnez %2, 0b\n"				\
+			"	srli %0, %0, 16\n"			\
+			: "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)	\
+			: "rJ" (__new), "rJ"(addr)			\
+			: "memory");					\
+		} else {						\
+		__asm__ __volatile__ (					\
+			"0:	lr.w %0, (%z4)\n"			\
+			"	move %1, %0\n"				\
+			"	srli %1, %1, 16\n"			\
+			"	slli %1, %1, 16\n"			\
+			"	move %2, %z3\n"				\
+			"	or   %1, %2, %1\n"			\
+			"	sc.w %2, %1, 0(%z4)\n"			\
+			"	bnez %2, 0b\n"				\

arch/riscv/include/asm/cmpxchg.h line 58

Quoted 10 lines
arch/riscv/include/asm/cmpxchg.h +7 -0
+			"	slli %0, %0, 16\n"			\
+			"	srli %0, %0, 16\n"			\
+			: "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)	\
+			: "rJ" (__new), "rJ"(addr)			\
+			: "memory");					\
+		}							\
+		break;							\
 	case 4:								\
 		__asm__ __volatile__ (					\
 			"	amoswap.w %0, %2, %1\n"			\

I'm pretty sure there's a handfull of implementations like this out there... if only we could share.

Anyway, this too should be an independent patch.

_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv

#3 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Quoted 19 lines · Peter Zijlstra
Quoted 7 lines · On Tue, Nov 24, 2020 at 01:43:54PM +0000, [email protected] wrote:
arch/riscv/include/asm/Kbuild +5 -0
 
+             if (align) {                                            \
+             __asm__ __volatile__ (                                  \
+                     "0:     lr.w %0, 0(%z4)\n"                      \
+                     "       move %1, %0\n"                          \
+                     "       slli %1, %1, 16\n"                      \

arch/riscv/include/asm/Kbuild

Quoted 10 lines
arch/riscv/include/asm/Kbuild +10 -0
+                     "       srli %1, %1, 16\n"                      \
+                     "       move %2, %z3\n"                         \
+                     "       slli %2, %2, 16\n"                      \
+                     "       or   %1, %2, %1\n"                      \
+                     "       sc.w %2, %1, 0(%z4)\n"                  \
+                     "       bnez %2, 0b\n"                          \
+                     "       srli %0, %0, 16\n"                      \
+                     : "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)      \
+                     : "rJ" (__new), "rJ"(addr)                      \
+                     : "memory");                                    \
Quoted 2 lines

I'm pretty sure there's a handfull of implementations like this out there... if only we could share.

Isn't this effectively the same as the "_Q_PENDING_BITS != 8" version of xchg_tail()?

If nothing else needs xchg() on a 16-bit value, maybe changing the #ifdef in the qspinlock code is enough.

Only around half the architectures actually implement 8-bit and 16-bit cmpxchg() and xchg(), it might even be worth trying to eventually change the interface to not do it at all, but instead have explicit cmpxchg8() and cmpxchg16() helpers for the few files that do use them.

Arnd

_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv

author maintainer
#4 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Quoted 21 lines · Arnd Bergmann
Quoted 19 lines · Peter Zijlstra
Quoted 17 lines · On Tue, Nov 24, 2020 at 01:43:54PM +0000, [email protected] wrote:
arch/riscv/include/asm/Kbuild +15 -0
 
+             if (align) {                                            \
+             __asm__ __volatile__ (                                  \
+                     "0:     lr.w %0, 0(%z4)\n"                      \
+                     "       move %1, %0\n"                          \
+                     "       slli %1, %1, 16\n"                      \
+                     "       srli %1, %1, 16\n"                      \
+                     "       move %2, %z3\n"                         \
+                     "       slli %2, %2, 16\n"                      \
+                     "       or   %1, %2, %1\n"                      \
+                     "       sc.w %2, %1, 0(%z4)\n"                  \
+                     "       bnez %2, 0b\n"                          \
+                     "       srli %0, %0, 16\n"                      \
+                     : "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)      \
+                     : "rJ" (__new), "rJ"(addr)                      \
+                     : "memory");                                    \
Quoted 5 lines
Quoted 2 lines

I'm pretty sure there's a handfull of implementations like this out there... if only we could share.

Isn't this effectively the same as the "_Q_PENDING_BITS != 8" version of xchg_tail()?

This can be concluded as the different effectiveness between cmpxchg and xchg. For the arch which only has lr/sc instructions, the cmpxchg & xchg are similar.

#if _Q_PENDING_BITS == 8

static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)

{
        /*
         * We can use relaxed semantics since the caller ensures that the
         * MCS node is properly initialized before updating the tail.
         */
        return (u32)xchg_relaxed(&lock->tail,
                                 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
}

#else /* _Q_PENDING_BITS == 8 */

static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)

{
        u32 old, new, val = atomic_read(&lock->val);

        for (;;) {
                new = (val & _Q_LOCKED_PENDING_MASK) | tail;
                /*
                 * We can use relaxed semantics since the caller ensures that
                 * the MCS node is properly initialized before updating the
                 * tail.
                 */
                old = atomic_cmpxchg_relaxed(&lock->val, val, new);
                if (old == val)
                        break;

                val = old;
        }
        return old;
}

#endif /* _Q_PENDING_BITS == 8 */

Quoted 11 lines

If nothing else needs xchg() on a 16-bit value, maybe changing the #ifdef in the qspinlock code is enough.

Only around half the architectures actually implement 8-bit and 16-bit cmpxchg() and xchg(), it might even be worth trying to eventually change the interface to not do it at all, but instead have explicit cmpxchg8() and cmpxchg16() helpers for the few files that do use them.

Arnd

Signature · 4 lines
Best Regards
 Guo Ren

ML: https://lore.kernel.org/linux-csky/
maintainer
#5 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

In reply to Arnd Bergmann
Quoted 20 lines · Arnd Bergmann
Quoted 19 lines · Peter Zijlstra
Quoted 17 lines · On Tue, Nov 24, 2020 at 01:43:54PM +0000, [email protected] wrote:
arch/riscv/include/asm/Kbuild +15 -0
 
+             if (align) {                                            \
+             __asm__ __volatile__ (                                  \
+                     "0:     lr.w %0, 0(%z4)\n"                      \
+                     "       move %1, %0\n"                          \
+                     "       slli %1, %1, 16\n"                      \
+                     "       srli %1, %1, 16\n"                      \
+                     "       move %2, %z3\n"                         \
+                     "       slli %2, %2, 16\n"                      \
+                     "       or   %1, %2, %1\n"                      \
+                     "       sc.w %2, %1, 0(%z4)\n"                  \
+                     "       bnez %2, 0b\n"                          \
+                     "       srli %0, %0, 16\n"                      \
+                     : "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)      \
+                     : "rJ" (__new), "rJ"(addr)                      \
+                     : "memory");                                    \
Quoted 5 lines
Quoted 2 lines

I'm pretty sure there's a handfull of implementations like this out there... if only we could share.

Isn't this effectively the same as the "_Q_PENDING_BITS != 8" version of xchg_tail()?

Pretty much I suppose.

Quoted 2 lines

If nothing else needs xchg() on a 16-bit value, maybe changing the #ifdef in the qspinlock code is enough.

Like the below? I think the native LL/SC variant is slightly better than the cmpxchg loop. The problem is that cmpxchg() is ll/sc and then we loop that, instead of only having the ll/sc loop.

Quoted 5 lines

Only around half the architectures actually implement 8-bit and 16-bit cmpxchg() and xchg(), it might even be worth trying to eventually change the interface to not do it at all, but instead have explicit cmpxchg8() and cmpxchg16() helpers for the few files that do use them.

Yeah, many RISCs don't have sub-word atomics. Not sure how many other users there are. qspinlock certainly is the most popular I suppose.

---

kernel/locking/qspinlock.c +28 -21
index cbff6ba53d56..7049fb2af0b2 100644
@@ -163,26 +163,6 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
 	WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
 }
 
-/*
- * xchg_tail - Put in the new queue tail code word & retrieve previous one
- * @lock : Pointer to queued spinlock structure
- * @tail : The new queue tail code word
- * Return: The previous queue tail code word
- *
- * xchg(lock, tail), which heads an address dependency
- *
- * p,*,* -> n,*,* ; prev = xchg(lock, node)
- */
-static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
-{
-	/*
-	 * We can use relaxed semantics since the caller ensures that the
-	 * MCS node is properly initialized before updating the tail.
-	 */
-	return (u32)xchg_relaxed(&lock->tail,
-				 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
-}
-
 #else /* _Q_PENDING_BITS == 8 */
 
 /**
@@ -207,6 +187,32 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
 	atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
 }
 
+#endif /* _Q_PENDING_BITS == 8 */
+
+#if _Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queued spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail), which heads an address dependency
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+	/*
+	 * We can use relaxed semantics since the caller ensures that the
+	 * MCS node is properly initialized before updating the tail.
+	 */
+	return (u32)xchg_relaxed(&lock->tail,
+				 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* !(_Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16) */
+
 /**
  * xchg_tail - Put in the new queue tail code word & retrieve previous one
  * @lock : Pointer to queued spinlock structure
@@ -236,7 +242,8 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
 	}
 	return old;
 }
-#endif /* _Q_PENDING_BITS == 8 */
+
+#endif /* _Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16 */
 
 /**
  * queued_fetch_set_pending_acquire - fetch the whole lock value and set pending

_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv

maintainer
#6 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Quoted 19 lines · Peter Zijlstra
kernel/locking/qspinlock.c +15 -0
@@ -207,6 +187,32 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
 	atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
 }
 
+#endif /* _Q_PENDING_BITS == 8 */
+
+#if _Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queued spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail), which heads an address dependency
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)

kernel/locking/qspinlock.c static __always_inline void clear_pending_set_locked(struct qspinlock *lock) · line 214

Quoted 10 lines
kernel/locking/qspinlock.c +10 -0
+{
+	/*
+	 * We can use relaxed semantics since the caller ensures that the
+	 * MCS node is properly initialized before updating the tail.
+	 */
+	return (u32)xchg_relaxed(&lock->tail,
+				 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* !(_Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16) */

Why can't architectures just implement this with a 32-bit xchg instruction if they don't have one that operates on 16 bits? Sure, they'll store more data, but it's atomic so you shouldn't be able to tell... (ignoring parisc crazy).

Also, I'm surprised qspinlock benefits riscv. On arm64, there's nothing in it over tickets for <= 16 CPUs.

Will

_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv

author maintainer
#7 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Hi Will,

Quoted 37 lines · Will Deacon
Quoted 19 lines · Peter Zijlstra
kernel/locking/qspinlock.c +15 -0
@@ -207,6 +187,32 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
      atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
 }
 
+#endif /* _Q_PENDING_BITS == 8 */
+
+#if _Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queued spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail), which heads an address dependency
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)

kernel/locking/qspinlock.c static __always_inline void clear_pending_set_locked(struct qspinlock *lock) · line 214

Quoted 10 lines
kernel/locking/qspinlock.c +10 -0
+{
+     /*
+      * We can use relaxed semantics since the caller ensures that the
+      * MCS node is properly initialized before updating the tail.
+      */
+     return (u32)xchg_relaxed(&lock->tail,
+                              tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* !(_Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16) */

Why can't architectures just implement this with a 32-bit xchg instruction if they don't have one that operates on 16 bits? Sure, they'll store more data, but it's atomic so you shouldn't be able to tell... (ignoring parisc crazy).

Quoted 2 lines

Also, I'm surprised qspinlock benefits riscv. On arm64, there's nothing in it over tickets for <= 16 CPUs.

NUMA is on the way:
https://lore.kernel.org/linux-riscv/[email protected]/

With your advice, I think we could using tickets lock when <= 16 CPUs and using qspinlock when > 16 CPUs.
Is that right?

The next patchset plan is:
- Using tickets & qspinlock together in riscv. Abandon 16bits
xchg/cmpxchg implementation.
- Abanden qspinlock in csky, because it only could 4 CPUs' SMP.

Quoted 2 lines

Will

Signature · 4 lines
Best Regards
 Guo Ren

ML: https://lore.kernel.org/linux-csky/
maintainer
#8 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Quoted 37 lines · Guo Ren
Quoted 36 lines · Will Deacon
Quoted 19 lines · Peter Zijlstra
kernel/locking/qspinlock.c +15 -0
@@ -207,6 +187,32 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
      atomic_add(-_Q_PENDING_VAL + _Q_LOCKED_VAL, &lock->val);
 }
 
+#endif /* _Q_PENDING_BITS == 8 */
+
+#if _Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queued spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail), which heads an address dependency
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)

kernel/locking/qspinlock.c static __always_inline void clear_pending_set_locked(struct qspinlock *lock) · line 214

Quoted 10 lines
kernel/locking/qspinlock.c +10 -0
+{
+     /*
+      * We can use relaxed semantics since the caller ensures that the
+      * MCS node is properly initialized before updating the tail.
+      */
+     return (u32)xchg_relaxed(&lock->tail,
+                              tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* !(_Q_PENDING_BITS == 8 && ARCH_HAS_XCHG16) */

Why can't architectures just implement this with a 32-bit xchg instruction if they don't have one that operates on 16 bits? Sure, they'll store more data, but it's atomic so you shouldn't be able to tell... (ignoring parisc crazy).

Quoted 4 lines
Quoted 2 lines

Also, I'm surprised qspinlock benefits riscv. On arm64, there's nothing in it over tickets for <= 16 CPUs.

NUMA is on the way:
https://lore.kernel.org/linux-riscv/[email protected]/

Well, they're patches not hardware (and I only see mention of QEMU over there for the RISCV platform) ;)

Quoted 3 lines

With your advice, I think we could using tickets lock when <= 16 CPUs and using qspinlock when > 16 CPUs.
Is that right?

No, when I say "there's nothing in it", it means they're interchangeable. It's just that qspinlock introduces a lot of complexity and I'm not keen massively keen on changing the core code (which is used by many other architectures) just because you don't have a 16-bit xchg() implementation.

So if you need qspinlock on riscv (not sure you do), then go ahead and make that your one true lock implementation, but implement 16-bit xchg() at the same time. Bonus points if you implement that in terms of 32-bit xchg() in generic code (might need to watch out for endianness when shifting the new value after aligning the base pointer).

Will

author maintainer
#9 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

In reply to Peter Zijlstra

Thx Peter,

Quoted 56 lines · Peter Zijlstra
Quoted 54 lines · On Tue, Nov 24, 2020 at 01:43:54PM +0000, [email protected] wrote:
arch/riscv/include/asm/Kbuild +3 -0
index 59dd7be..6f5f438 100644
@@ -6,3 +6,6 @@ generic-y += kvm_para.h
 generic-y += local64.h
 generic-y += user.h
 generic-y += vmlinux.lds.h
+generic-y += mcs_spinlock.h
+generic-y += qrwlock.h
+generic-y += qspinlock.h
arch/riscv/include/asm/cmpxchg.h +34 -0
index 5609185..e178700 100644
@@ -16,7 +16,43 @@
      __typeof__(ptr) __ptr = (ptr);                                  \
      __typeof__(new) __new = (new);                                  \
      __typeof__(*(ptr)) __ret;                                       \
+     register unsigned long __rc, tmp, align, addr;                  \
      switch (size) {                                                 \
+     case 2:                                                         \
+             align = ((unsigned long) __ptr & 0x3);                  \
+             addr = ((unsigned long) __ptr & ~0x3);                  \
+             if (align) {                                            \
+             __asm__ __volatile__ (                                  \
+                     "0:     lr.w %0, 0(%z4)\n"                      \
+                     "       move %1, %0\n"                          \
+                     "       slli %1, %1, 16\n"                      \
+                     "       srli %1, %1, 16\n"                      \
+                     "       move %2, %z3\n"                         \
+                     "       slli %2, %2, 16\n"                      \
+                     "       or   %1, %2, %1\n"                      \
+                     "       sc.w %2, %1, 0(%z4)\n"                  \
+                     "       bnez %2, 0b\n"                          \
+                     "       srli %0, %0, 16\n"                      \
+                     : "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)      \
+                     : "rJ" (__new), "rJ"(addr)                      \
+                     : "memory");                                    \
+             } else {                                                \
+             __asm__ __volatile__ (                                  \
+                     "0:     lr.w %0, (%z4)\n"                       \
+                     "       move %1, %0\n"                          \
+                     "       srli %1, %1, 16\n"                      \
+                     "       slli %1, %1, 16\n"                      \
+                     "       move %2, %z3\n"                         \
+                     "       or   %1, %2, %1\n"                      \
+                     "       sc.w %2, %1, 0(%z4)\n"                  \
+                     "       bnez %2, 0b\n"                          \
+                     "       slli %0, %0, 16\n"                      \
+                     "       srli %0, %0, 16\n"                      \
+                     : "=&r" (__ret), "=&r" (tmp), "=&r" (__rc)      \
+                     : "rJ" (__new), "rJ"(addr)                      \
+                     : "memory");                                    \
Quoted 8 lines

arch/riscv/include/asm/cmpxchg.h line 58

Quoted 5 lines
arch/riscv/include/asm/cmpxchg.h +2 -0
+             }                                                       \
+             break;                                                  \
      case 4:                                                         \
              __asm__ __volatile__ (                                  \
                      "       amoswap.w %0, %2, %1\n"                 \

I'm pretty sure there's a handfull of implementations like this out there... if only we could share.

Michael has sent qspinlock before, ref to Link below. He reused mips' code.

Which short xchg implementation do you prefer (Mine or his)?

Quoted 2 lines

Anyway, this too should be an independent patch.

Ok, I'll separate it into two patches,
1. implement short xchg
2. qspinlock enabled based on Michael's patch

Signature · 4 lines
Best Regards
 Guo Ren

ML: https://lore.kernel.org/linux-csky/
maintainer
#10 lore raw

Re: [PATCH 2/5] riscv: Add QUEUED_SPINLOCKS & QUEUED_RWLOCKS supported

Quoted 7 lines · Guo Ren
Quoted 2 lines

I'm pretty sure there's a handfull of implementations like this out there... if only we could share.

Michael has sent qspinlock before, ref to Link below. He reused mips' code.

Which short xchg implementation do you prefer (Mine or his)?

Well, it would be very nice to have mips/riscv/csky all use the same header to implement these I suppose.

But then we're back to a cmpxchg-loop, in which case Arnd's suggestion isn't worse.

_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv

author maintainer
#11 lore raw

PATCH 3/5 csky: Remove simple spinlock implementation

In reply to [email protected]

From: Guo Ren <[email protected]>

There are two implementation of spinlock in csky:
- simple one (NR_CPU = 1,2)
- tick's one (NR_CPU = 3,4,5...)

We needn't the baby codes of simple one now.

---
 arch/csky/Kconfig                      |   2 +-
 arch/csky/include/asm/spinlock.h       | 164 ---------------------------------
 arch/csky/include/asm/spinlock_types.h |  10 --
 3 files changed, 1 insertion(+), 175 deletions(-)
arch/csky/Kconfig +1 -1
index 268fad5..14ee229 100644
@@ -7,7 +7,7 @@ config CSKY
 	select ARCH_HAS_SYNC_DMA_FOR_CPU
 	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select ARCH_USE_BUILTIN_BSWAP
-	select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2
+	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_WANT_FRAME_POINTERS if !CPU_CK610
 	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
 	select COMMON_CLK
arch/csky/include/asm/spinlock.h +0 -164

174 lines not on this page · raw message

arch/csky/include/asm/spinlock_types.h +0 -10
index 88b8243..8ff0f6f 100644
@@ -22,16 +22,6 @@ typedef struct {
 
 #define __ARCH_SPIN_LOCK_UNLOCKED	{ { 0 } }
 
-#ifdef CONFIG_QUEUED_RWLOCKS
 #include <asm-generic/qrwlock_types.h>
 
-#else /* CONFIG_NR_CPUS > 2 */
-
-typedef struct {
-	u32 lock;
-} arch_rwlock_t;
-
-#define __ARCH_RW_LOCK_UNLOCKED		{ 0 }
-
-#endif /* CONFIG_QUEUED_RWLOCKS */
 #endif /* __ASM_CSKY_SPINLOCK_TYPES_H */
Signature · 1 line
2.7.4
author maintainer
#12 lore raw

PATCH 4/5 csky: Add QUEUED_SPINLOCKS supported

In reply to [email protected]

From: Guo Ren <[email protected]>

Abiv2 only support ldex.w/stex.w with word(double word) size & align access. There are not short type instructions. But qspinlock.c need xchg with short type variable:

xchg_tail -> xchg_releaxed(&lock->tail, ...

typedef struct qspinlock {
        union {
                atomic_t val;

                /*
                 * By using the whole 2nd least significant byte for the
                 * pending bit, we can allow better optimization of the lock
                 * acquisition for the pending bit holder.
                 */
                struct {
                        u8      locked;
                        u8      pending;
                };
                struct {
                        u16     locked_pending;
                        u16     tail; /* half word*/
                };
        };
} arch_spinlock_t;

So we add short emulation in xchg with word length and it only solve qspinlock's requirement.

Cc Peter Zijlstra <[email protected]>
Cc Paul E. McKenney <[email protected]>
Cc Arnd Bergmann <[email protected]>
---
 arch/csky/Kconfig                      |  1 +
 arch/csky/include/asm/Kbuild           |  2 +
 arch/csky/include/asm/cmpxchg.h        | 43 ++++++++++++++++--
 arch/csky/include/asm/spinlock.h       | 82 +---------------------------------
 arch/csky/include/asm/spinlock_types.h | 18 +-------
 5 files changed, 46 insertions(+), 100 deletions(-)
arch/csky/Kconfig +1 -0
index 14ee229..ac02b17 100644
@@ -8,6 +8,7 @@ config CSKY
 	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_QUEUED_RWLOCKS
+	select ARCH_USE_QUEUED_SPINLOCKS
 	select ARCH_WANT_FRAME_POINTERS if !CPU_CK610
 	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
 	select COMMON_CLK
arch/csky/include/asm/Kbuild +2 -0
index 64876e59..f814d46 100644
@@ -3,7 +3,9 @@ generic-y += asm-offsets.h
 generic-y += gpio.h
 generic-y += kvm_para.h
 generic-y += local64.h
+generic-y += mcs_spinlock.h
 generic-y += qrwlock.h
+generic-y += qspinlock.h
 generic-y += seccomp.h
 generic-y += user.h
 generic-y += vmlinux.lds.h
arch/csky/include/asm/cmpxchg.h +40 -3

56 lines not on this page · raw message

arch/csky/include/asm/spinlock.h +1 -81

89 lines not on this page · raw message

arch/csky/include/asm/spinlock_types.h +2 -16
index 8ff0f6f..82f5fd5 100644
@@ -3,25 +3,11 @@
 #ifndef __ASM_CSKY_SPINLOCK_TYPES_H
 #define __ASM_CSKY_SPINLOCK_TYPES_H
 
-#ifndef __LINUX_SPINLOCK_TYPES_H
+#if !defined(__LINUX_SPINLOCK_TYPES_H) && !defined(__ASM_CSKY_SPINLOCK_H)
 # error "please don't include this file directly"
 #endif
 
-#define TICKET_NEXT	16
-
-typedef struct {
-	union {
-		u32 lock;
-		struct __raw_tickets {
-			/* little endian */
-			u16 owner;
-			u16 next;
-		} tickets;
-	};
-} arch_spinlock_t;
-
-#define __ARCH_SPIN_LOCK_UNLOCKED	{ { 0 } }
-
+#include <asm-generic/qspinlock_types.h>
 #include <asm-generic/qrwlock_types.h>
 
 #endif /* __ASM_CSKY_SPINLOCK_TYPES_H */
Signature · 1 line
2.7.4
author maintainer
#13 lore raw

PATCH 5/5 csky: Optimize atomic operations with correct barrier usage

In reply to [email protected]

From: Guo Ren <[email protected]>

The implementation of csky atomic operations in the past was very rough. Frankly speaking, the implementation is wrong and limits hardware performance. Optimize the performance of atomic, spinlock, cmpxchg more fine-grained by increasing acquire/release memory barriers. Here are the details of the modification:

- Add acquire/release barrier for cmpxchg.h.
- Remove custom atomic.h implementations.

Cc Peter Zijlstra <[email protected]>
Cc Paul E. McKenney <[email protected]>
Cc Arnd Bergmann <[email protected]>
---
 arch/csky/include/asm/atomic.h  | 203 +---------------------------------------
 arch/csky/include/asm/barrier.h |  64 ++++++++++---
 arch/csky/include/asm/cmpxchg.h |  85 ++++++++++++++---
 3 files changed, 128 insertions(+), 224 deletions(-)
arch/csky/include/asm/atomic.h +2 -201
index e369d73..c699c41 100644
@@ -3,209 +3,10 @@
 #ifndef __ASM_CSKY_ATOMIC_H
 #define __ASM_CSKY_ATOMIC_H
 
-#include <linux/version.h>
-#include <asm/cmpxchg.h>
 #include <asm/barrier.h>
 
-#ifdef CONFIG_CPU_HAS_LDSTEX
-
-#define __atomic_add_unless __atomic_add_unless
-static inline int __atomic_add_unless(atomic_t *v, int a, int u)
-{
-	unsigned long tmp, ret;
-
-	smp_mb();
-
-	asm volatile (
-	"1:	ldex.w		%0, (%3) \n"
-	"	mov		%1, %0   \n"
-	"	cmpne		%0, %4   \n"
-	"	bf		2f	 \n"
-	"	add		%0, %2   \n"
-	"	stex.w		%0, (%3) \n"
-	"	bez		%0, 1b   \n"
-	"2:				 \n"
-		: "=&r" (tmp), "=&r" (ret)
-		: "r" (a), "r"(&v->counter), "r"(u)
-		: "memory");
-
-	if (ret != u)
-		smp_mb();
-
-	return ret;
-}
-
-#define ATOMIC_OP(op, c_op)						\
-static inline void atomic_##op(int i, atomic_t *v)			\
-{									\
-	unsigned long tmp;						\
-									\
-	asm volatile (							\
-	"1:	ldex.w		%0, (%2) \n"				\
-	"	" #op "		%0, %1   \n"				\
-	"	stex.w		%0, (%2) \n"				\
-	"	bez		%0, 1b   \n"				\
-		: "=&r" (tmp)						\
-		: "r" (i), "r"(&v->counter)				\
-		: "memory");						\
-}
-
-#define ATOMIC_OP_RETURN(op, c_op)					\
-static inline int atomic_##op##_return(int i, atomic_t *v)		\
-{									\
-	unsigned long tmp, ret;						\
-									\
-	smp_mb();							\
-	asm volatile (							\
-	"1:	ldex.w		%0, (%3) \n"				\
-	"	" #op "		%0, %2   \n"				\
-	"	mov		%1, %0   \n"				\
-	"	stex.w		%0, (%3) \n"				\
-	"	bez		%0, 1b   \n"				\
-		: "=&r" (tmp), "=&r" (ret)				\
-		: "r" (i), "r"(&v->counter)				\
-		: "memory");						\
-	smp_mb();							\
-									\
-	return ret;							\
-}
-
-#define ATOMIC_FETCH_OP(op, c_op)					\
-static inline int atomic_fetch_##op(int i, atomic_t *v)			\
-{									\
-	unsigned long tmp, ret;						\
-									\
-	smp_mb();							\
-	asm volatile (							\
-	"1:	ldex.w		%0, (%3) \n"				\
-	"	mov		%1, %0   \n"				\
-	"	" #op "		%0, %2   \n"				\
-	"	stex.w		%0, (%3) \n"				\
-	"	bez		%0, 1b   \n"				\
-		: "=&r" (tmp), "=&r" (ret)				\
-		: "r" (i), "r"(&v->counter)				\
-		: "memory");						\
-	smp_mb();							\
-									\
-	return ret;							\
-}
-
-#else /* CONFIG_CPU_HAS_LDSTEX */
-
-#include <linux/irqflags.h>
-
-#define __atomic_add_unless __atomic_add_unless
-static inline int __atomic_add_unless(atomic_t *v, int a, int u)
-{
-	unsigned long tmp, ret, flags;
-
-	raw_local_irq_save(flags);
-
-	asm volatile (
-	"	ldw		%0, (%3) \n"
-	"	mov		%1, %0   \n"
-	"	cmpne		%0, %4   \n"
-	"	bf		2f	 \n"
-	"	add		%0, %2   \n"
-	"	stw		%0, (%3) \n"
-	"2:				 \n"
-		: "=&r" (tmp), "=&r" (ret)
-		: "r" (a), "r"(&v->counter), "r"(u)
-		: "memory");
-
-	raw_local_irq_restore(flags);
-
-	return ret;
-}
-
-#define ATOMIC_OP(op, c_op)						\
-static inline void atomic_##op(int i, atomic_t *v)			\
-{									\
-	unsigned long tmp, flags;					\
-									\
-	raw_local_irq_save(flags);					\
-									\
-	asm volatile (							\
-	"	ldw		%0, (%2) \n"				\
-	"	" #op "		%0, %1   \n"				\
-	"	stw		%0, (%2) \n"				\
-		: "=&r" (tmp)						\
-		: "r" (i), "r"(&v->counter)				\
-		: "memory");						\
-									\
-	raw_local_irq_restore(flags);					\
-}
-
-#define ATOMIC_OP_RETURN(op, c_op)					\
-static inline int atomic_##op##_return(int i, atomic_t *v)		\
-{									\
-	unsigned long tmp, ret, flags;					\
-									\
-	raw_local_irq_save(flags);					\
-									\
-	asm volatile (							\
-	"	ldw		%0, (%3) \n"				\
-	"	" #op "		%0, %2   \n"				\
-	"	stw		%0, (%3) \n"				\
-	"	mov		%1, %0   \n"				\
-		: "=&r" (tmp), "=&r" (ret)				\
-		: "r" (i), "r"(&v->counter)				\
-		: "memory");						\
-									\
-	raw_local_irq_restore(flags);					\
-									\
-	return ret;							\
-}
-
-#define ATOMIC_FETCH_OP(op, c_op)					\
-static inline int atomic_fetch_##op(int i, atomic_t *v)			\
-{									\
-	unsigned long tmp, ret, flags;					\
-									\
-	raw_local_irq_save(flags);					\
-									\
-	asm volatile (							\
-	"	ldw		%0, (%3) \n"				\
-	"	mov		%1, %0   \n"				\
-	"	" #op "		%0, %2   \n"				\
-	"	stw		%0, (%3) \n"				\
-		: "=&r" (tmp), "=&r" (ret)				\
-		: "r" (i), "r"(&v->counter)				\
-		: "memory");						\
-									\
-	raw_local_irq_restore(flags);					\
-									\
-	return ret;							\
-}
-
-#endif /* CONFIG_CPU_HAS_LDSTEX */
-
-#define atomic_add_return atomic_add_return
-ATOMIC_OP_RETURN(add, +)
-#define atomic_sub_return atomic_sub_return
-ATOMIC_OP_RETURN(sub, -)
-
-#define atomic_fetch_add atomic_fetch_add
-ATOMIC_FETCH_OP(add, +)
-#define atomic_fetch_sub atomic_fetch_sub
-ATOMIC_FETCH_OP(sub, -)
-#define atomic_fetch_and atomic_fetch_and
-ATOMIC_FETCH_OP(and, &)
-#define atomic_fetch_or atomic_fetch_or
-ATOMIC_FETCH_OP(or, |)
-#define atomic_fetch_xor atomic_fetch_xor
-ATOMIC_FETCH_OP(xor, ^)
-
-#define atomic_and atomic_and
-ATOMIC_OP(and, &)
-#define atomic_or atomic_or
-ATOMIC_OP(or, |)
-#define atomic_xor atomic_xor
-ATOMIC_OP(xor, ^)
-
-#undef ATOMIC_FETCH_OP
-#undef ATOMIC_OP_RETURN
-#undef ATOMIC_OP
+#define __atomic_acquire_fence() __smp_acquire_fence()
+#define __atomic_release_fence() __smp_release_fence()
 
 #include <asm-generic/atomic.h>
 
arch/csky/include/asm/barrier.h +52 -12

76 lines not on this page · raw message

arch/csky/include/asm/cmpxchg.h +74 -11

136 lines not on this page · raw message

Signature · 1 line
2.7.4
maintainer
#14 lore raw
In reply to [email protected]
Quoted 7 lines · On Tue, Nov 24, 2020 at 01:43:53PM +0000, [email protected] wrote:

From: Guo Ren <[email protected]>

This is prepare for QUEUED_SPINLOCKS which need xchg support short type value.
- Remove unused codes (xchg32, xchg64, cmpxchg32 ...)
- Combine xchg_relaxed, xchg_acquire, xchg_release into one asm
- Make atomic.aq/rl with seperated fence acquire & release

Every time you find yourself doing multiple things, make it multiple patches.

Quoted 74 lines
arch/riscv/include/asm/atomic.h +9 -0
@@ -242,58 +239,58 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u
  * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
  * {cmp,}xchg and the operations that return, so they need a full barrier.
  */
+#define ATOMIC_OP(c_t, prefix)						\
 static __always_inline							\
 c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n)		\
 {									\
+	return xchg_relaxed(&(v->counter), n);				\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n)		\
 {									\
+	return xchg_acquire(&(v->counter), n);				\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n)		\
 {									\
+	return xchg_release(&(v->counter), n);				\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n)			\
 {									\
+	return xchg(&(v->counter), n);					\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v,		\
 				     c_t o, c_t n)			\
 {									\
+	return cmpxchg_relaxed(&(v->counter), o, n);			\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v,		\
 				     c_t o, c_t n)			\
 {									\
+	return cmpxchg_acquire(&(v->counter), o, n);			\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v,		\
 				     c_t o, c_t n)			\
 {									\
+	return cmpxchg_release(&(v->counter), o, n);			\
 }									\
 static __always_inline							\
 c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n)	\
 {									\
+	return cmpxchg(&(v->counter), o, n);				\
 }
 
arch/riscv/include/asm/cmpxchg.h +5 -0
index 262e5bb..5609185 100644
@@ -44,118 +44,31 @@
 					    _x_, sizeof(*(ptr)));	\
 })
 
 #define xchg_acquire(ptr, x)						\
 ({									\
+	__typeof__(*(ptr)) _x_ = (x);					\
+	__ret = __xchg_relaxed((ptr), _x_, sizeof(*(ptr)));		\
+	__acquire_fence();						\
 	__ret;								\
 })
 
 #define xchg_release(ptr, x)						\
 ({									\
 	__typeof__(*(ptr)) _x_ = (x);					\
+	__release_fence();						\
+	(__typeof__(*(ptr))) __xchg_relaxed((ptr),			\
 					    _x_, sizeof(*(ptr)));	\
 })
 
 #define xchg(ptr, x)							\

arch/riscv/include/asm/cmpxchg.h line 73

Quoted 10 lines
arch/riscv/include/asm/cmpxchg.h +5 -0
 ({									\
+	__typeof__(*(ptr)) __ret;					\
 	__typeof__(*(ptr)) _x_ = (x);					\
+	__smp_mb();							\
+	__ret = __xchg_relaxed((ptr), _x_, sizeof(*(ptr)));		\
+	__smp_mb();							\
+	__ret;								\
 })
 
 /*

Why are you defining *{,_acquire,_release}() at all, doesn't atomic-fallback.h DTRT for you?

_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv

author maintainer
#15 lore raw
Quoted 4 lines · Peter Zijlstra
Quoted 2 lines · On Tue, Nov 24, 2020 at 01:43:53PM +0000, [email protected] wrote:

From: Guo Ren <[email protected]>

Quoted 8 lines
Quoted 5 lines

This is prepare for QUEUED_SPINLOCKS which need xchg support short type value.
- Remove unused codes (xchg32, xchg64, cmpxchg32 ...)
- Combine xchg_relaxed, xchg_acquire, xchg_release into one asm
- Make atomic.aq/rl with seperated fence acquire & release

Every time you find yourself doing multiple things, make it multiple patches.

Ok.

Quoted 84 lines
Quoted 84 lines
arch/riscv/include/asm/atomic.h +9 -0
@@ -242,58 +239,58 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u
  * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
  * {cmp,}xchg and the operations that return, so they need a full barrier.
  */
+#define ATOMIC_OP(c_t, prefix)                                               \
 static __always_inline                                                       \
 c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n)              \
 {                                                                    \
+     return xchg_relaxed(&(v->counter), n);                          \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n)              \
 {                                                                    \
+     return xchg_acquire(&(v->counter), n);                          \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n)              \
 {                                                                    \
+     return xchg_release(&(v->counter), n);                          \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n)                      \
 {                                                                    \
+     return xchg(&(v->counter), n);                                  \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v,          \
                                   c_t o, c_t n)                      \
 {                                                                    \
+     return cmpxchg_relaxed(&(v->counter), o, n);                    \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v,          \
                                   c_t o, c_t n)                      \
 {                                                                    \
+     return cmpxchg_acquire(&(v->counter), o, n);                    \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v,          \
                                   c_t o, c_t n)                      \
 {                                                                    \
+     return cmpxchg_release(&(v->counter), o, n);                    \
 }                                                                    \
 static __always_inline                                                       \
 c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n)    \
 {                                                                    \
+     return cmpxchg(&(v->counter), o, n);                            \
 }
 
arch/riscv/include/asm/cmpxchg.h +10 -0
index 262e5bb..5609185 100644
@@ -44,118 +44,31 @@
                                          _x_, sizeof(*(ptr)));       \
 })
 
 #define xchg_acquire(ptr, x)                                         \
 ({                                                                   \
+     __typeof__(*(ptr)) _x_ = (x);                                   \
+     __ret = __xchg_relaxed((ptr), _x_, sizeof(*(ptr)));             \
+     __acquire_fence();                                              \
      __ret;                                                          \
 })
 
 #define xchg_release(ptr, x)                                         \
 ({                                                                   \
      __typeof__(*(ptr)) _x_ = (x);                                   \
+     __release_fence();                                              \
+     (__typeof__(*(ptr))) __xchg_relaxed((ptr),                      \
                                          _x_, sizeof(*(ptr)));       \
 })
 
 #define xchg(ptr, x)                                                 \
 ({                                                                   \
+     __typeof__(*(ptr)) __ret;                                       \
      __typeof__(*(ptr)) _x_ = (x);                                   \
+     __smp_mb();                                                     \
+     __ret = __xchg_relaxed((ptr), _x_, sizeof(*(ptr)));             \
+     __smp_mb();                                                     \
+     __ret;                                                          \
 })
 
Quoted 4 lines

arch/riscv/include/asm/cmpxchg.h line 73

Quoted 1 line
arch/riscv/include/asm/cmpxchg.h +0 -0
 /*

Why are you defining *{,_acquire,_release}() at all, doesn't atomic-fallback.h DTRT for you?

Yes, you are right. I could reuse that. Thx

#define cmpxchg_acquire(...) \
        __atomic_op_acquire(cmpxchg, __VA_ARGS__)

#endif

#define __atomic_op_acquire(op, args...)                                \
({                                                                      \
        typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
        __atomic_acquire_fence();                                       \
        __ret;                                                          \

})

Signature · 4 lines
Best Regards
 Guo Ren

ML: https://lore.kernel.org/linux-csky/