Skip to contentSkip to conversation

LoongArch: Add 128-bit atomic cmpxchg support (v2)

PATCH Superseded George Guo 3 participants 8 messages Nov 24, 2025 – Nov 25, 2025 1 message not in the archive LoongArch

v2 of 10 Superseded Latest is v10 →

lore
Share & tools
lore

George Guo

author
#0 lore raw

v2 cover LoongArch: Add 128-bit atomic cmpxchg support (v2)

This patch series adds 128-bit atomic compare-and-exchange support for LoongArch architecture, which fixes BPF scheduler test failures caused by missing 128-bit atomics support.

The series consists of two patches:

1. "LoongArch: Add 128-bit atomic cmpxchg support"
- Implements 128-bit atomic compare-and-exchange using LoongArch's LL.D/SC.Q instructions
- Fixes BPF scheduler test failures (scx_central scx_qmap) where kmalloc_nolock_noprof returns NULL due to missing 128-bit atomics, leading to -ENOMEM errors during scheduler initialization

2. "LoongArch: Enable 128-bit atomics cmpxchg support"
- Adds select HAVE_CMPXCHG_DOUBLE and select HAVE_ALIGNED_STRUCT_PAGE in Kconfig to enable 128-bit atomic cmpxchg support

The issue was identified through BPF scheduler test failures where scx_central and scx_qmap schedulers would fail to initialize. Testing was performed using the scx_qmap scheduler from tools/sched_ext/, confirming that the patches resolve the initialization failures.

---
Changes in v2:
- Use a normal ld.d for the high word instead of ll.d to avoid race condition
- Insert a dbar between ll.d and ld.d to prevent reordering
- Simply __cmpxchg128_asm("ll.d", "sc.q", ptr, o, n) to __cmpxchg128_asm(ptr, o, n)
- Fix address operand constraints after testing different approaches:
* ld.d with "m"
* ll.d with "ZC",
* sc.q with "ZB"(alternative constraints caused issues:
- "r" caused system hang
- "ZC" caused compiler error:
{standard input}: Assembler messages:
{standard input}:10037: Fatal error: Immediate overflow.
format: u0:0 )
- Link to v1: https://lore.kernel.org/r/[email protected]

---
George Guo (2):
LoongArch: Add 128-bit atomic cmpxchg support
LoongArch: Enable 128-bit atomics cmpxchg support

 arch/loongarch/Kconfig               |  2 ++
 arch/loongarch/include/asm/cmpxchg.h | 47 ++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)

---
base-commit: d5ae5ac32615e4af729f0610fdc11ff4f4798aef
change-id: 20251120-2-d03862b2cf6d

Best regards,

Signature · 1 line
George Guo <[email protected]>
author
#1 lore raw

PATCH v2 1/2 LoongArch: Add 128-bit atomic cmpxchg support

From: George Guo <[email protected]>

Implement 128-bit atomic compare-and-exchange using LoongArch's LL.D/SC.Q instructions.

At the same time, fix BPF scheduler test failures (scx_central scx_qmap) caused by kmalloc_nolock_noprof returning NULL due to missing 128-bit atomics. The NULL returns led to -ENOMEM errors during scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler (located in tools/sched_ext/). Building with make and running ./tools/sched_ext/build/bin/scx_qmap.

---
 arch/loongarch/include/asm/cmpxchg.h | 47 ++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
arch/loongarch/include/asm/cmpxchg.h +47 -0
index 979fde61bba8a42cb4f019f13ded2a3119d4aaf4..757f6e82b9880d04f4883dc9a802312111aa4588 100644
@@ -111,6 +111,44 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size)
 	__ret;								\
 })
 
+union __u128_halves {
+	u128 full;
+	struct {
+		u64 low;
+		u64 high;
+	};
+};
+
+#define __cmpxchg128_asm(ptr, old, new)					\
+({									\
+	union __u128_halves __old, __new, __ret;			\
+	volatile u64 *__ptr = (volatile u64 *)(ptr);			\
+									\
+	__old.full = (old);                                             \
+	__new.full = (new);						\
+									\
+	__asm__ __volatile__(						\
+	"1:   ll.d    %0, %3		# 128-bit cmpxchg low	\n"	\
+	"     dbar    0			# memory barrier	\n"	\
+	"     ld.d    %1, %4		# 128-bit cmpxchg high	\n"	\
+	"     bne     %0, %z5, 2f				\n"	\
+	"     bne     %1, %z6, 2f				\n"	\
+	"     move    $t0, %z7					\n"	\
+	"     move    $t1, %z8					\n"	\
+	"     sc.q    $t0, $t1, %2				\n"	\
+	"     beqz    $t0, 1b					\n"	\
+	"2:							\n"	\
+	__WEAK_LLSC_MB							\
+	: "=&r" (__ret.low), "=&r" (__ret.high),			\
+	  "=ZB" (__ptr[0])						\
+	: "ZC" (__ptr[0]), "m" (__ptr[1]),				\
+	  "Jr" (__old.low), "Jr" (__old.high),				\
+	  "Jr" (__new.low), "Jr" (__new.high)				\
+	: "t0", "t1", "memory");					\
+									\
+	__ret.full;							\
+})
+
 static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old,
 					   unsigned int new, unsigned int size)
 {
@@ -198,6 +236,15 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int
 	__res;								\
 })
 
+/* cmpxchg128 */
+#define system_has_cmpxchg128()		1
+
+#define arch_cmpxchg128(ptr, o, n)					\
+({									\
+	BUILD_BUG_ON(sizeof(*(ptr)) != 16);				\
+	__cmpxchg128_asm(ptr, o, n);			\
+})
+
 #ifdef CONFIG_64BIT
 #define arch_cmpxchg64_local(ptr, o, n)					\
   ({									\
Signature · 1 line
2.48.1
#2 lore raw

Re: [PATCH v2 1/2] LoongArch: Add 128-bit atomic cmpxchg support

Quoted 48 lines · George Guo

From: George Guo <[email protected]>

Implement 128-bit atomic compare-and-exchange using LoongArch's LL.D/SC.Q instructions.

At the same time, fix BPF scheduler test failures (scx_central scx_qmap) caused by kmalloc_nolock_noprof returning NULL due to missing 128-bit atomics. The NULL returns led to -ENOMEM errors during scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler (located in tools/sched_ext/). Building with make and running ./tools/sched_ext/build/bin/scx_qmap.

---
 arch/loongarch/include/asm/cmpxchg.h | 47 ++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
arch/loongarch/include/asm/cmpxchg.h +20 -0
index 979fde61bba8a42cb4f019f13ded2a3119d4aaf4..757f6e82b9880d04f4883dc9a802312111aa4588 100644
@@ -111,6 +111,44 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size)
        __ret;                                                          \
 })
 
+union __u128_halves {
+       u128 full;
+       struct {
+               u64 low;
+               u64 high;
+       };
+};
+
+#define __cmpxchg128_asm(ptr, old, new)                                        \
+({                                                                     \
+       union __u128_halves __old, __new, __ret;                        \
+       volatile u64 *__ptr = (volatile u64 *)(ptr);                    \
+                                                                       \
+       __old.full = (old);                                             \
+       __new.full = (new);                                             \
+                                                                       \
+       __asm__ __volatile__(                                           \
+       "1:   ll.d    %0, %3            # 128-bit cmpxchg low   \n"     \
+       "     dbar    0                 # memory barrier        \n"     \
+       "     ld.d    %1, %4            # 128-bit cmpxchg high  \n"     \

arch/loongarch/include/asm/cmpxchg.h __arch_xchg(volatile void *ptr, unsigned long x, int size) · line 143

Quoted 10 lines
arch/loongarch/include/asm/cmpxchg.h +10 -0
+       "     bne     %0, %z5, 2f                               \n"     \
+       "     bne     %1, %z6, 2f                               \n"     \
+       "     move    $t0, %z7                                  \n"     \
+       "     move    $t1, %z8                                  \n"     \
+       "     sc.q    $t0, $t1, %2                              \n"     \
+       "     beqz    $t0, 1b                                   \n"     \
+       "2:                                                     \n"     \
+       __WEAK_LLSC_MB                                                  \
+       : "=&r" (__ret.low), "=&r" (__ret.high),                        \
+         "=ZB" (__ptr[0])                                              \

"ZB" isn't a legal constraint for the address operand in sc.q. When assembled, it turns into something like sc.q $r,$r,$r,0, which clearly doesn't match the instruction format, yet gas happily accepts it wheil clang rightfully rejects it. Classic GNU-as leniency biting again. :)

Quoted 32 lines
arch/loongarch/include/asm/cmpxchg.h +17 -1
+       : "ZC" (__ptr[0]), "m" (__ptr[1]),                              \
+         "Jr" (__old.low), "Jr" (__old.high),                          \
+         "Jr" (__new.low), "Jr" (__new.high)                           \
+       : "t0", "t1", "memory");                                        \
+                                                                       \
+       __ret.full;                                                     \
+})
+
 static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old,
                                           unsigned int new, unsigned int size)
 {
@@ -198,6 +236,15 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int
        __res;                                                          \
 })
 
+/* cmpxchg128 */
+#define system_has_cmpxchg128()                1
+
+#define arch_cmpxchg128(ptr, o, n)                                     \
+({                                                                     \
+       BUILD_BUG_ON(sizeof(*(ptr)) != 16);                             \
+       __cmpxchg128_asm(ptr, o, n);                    \
+})
+
 #ifdef CONFIG_64BIT
 #define arch_cmpxchg64_local(ptr, o, n)                                        \
   ({                                                                   \
 
--

2.48.1

author
#3 lore raw

Re: [PATCH v2 1/2] LoongArch: Add 128-bit atomic cmpxchg support

Quoted 79 lines · On Mon, 24 Nov 2025 19:37:40 +0800 hev <[email protected]> wrote:
Quoted 77 lines · George Guo

From: George Guo <[email protected]>

Implement 128-bit atomic compare-and-exchange using LoongArch's LL.D/SC.Q instructions.

At the same time, fix BPF scheduler test failures (scx_central scx_qmap) caused by kmalloc_nolock_noprof returning NULL due to missing 128-bit atomics. The NULL returns led to -ENOMEM errors during scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler (located in tools/sched_ext/). Building with make and running ./tools/sched_ext/build/bin/scx_qmap.

---
 arch/loongarch/include/asm/cmpxchg.h | 47

++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+)

(quoted diff) +0 -0

b/arch/loongarch/include/asm/cmpxchg.h index 979fde61bba8a42cb4f019f13ded2a3119d4aaf4..757f6e82b9880d04f4883dc9a802312111aa4588 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -111,6 +111,44 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size) __ret;

                                                       \ })
(quoted diff) +28 -0
+union __u128_halves {
+       u128 full;
+       struct {
+               u64 low;
+               u64 high;
+       };
+};
+
+#define __cmpxchg128_asm(ptr, old, new)
            \ +({
                  \
+       union __u128_halves __old, __new, __ret;
    \
+       volatile u64 *__ptr = (volatile u64 *)(ptr);
    \
+
    \
+       __old.full = (old);
    \
+       __new.full = (new);
    \
+
    \
+       __asm__ __volatile__(
    \
+       "1:   ll.d    %0, %3            # 128-bit cmpxchg low   \n"
    \
+       "     dbar    0                 # memory barrier        \n"
    \
+       "     ld.d    %1, %4            # 128-bit cmpxchg high  \n"
    \
+       "     bne     %0, %z5, 2f                               \n"
    \
+       "     bne     %1, %z6, 2f                               \n"
    \
+       "     move    $t0, %z7                                  \n"
    \
+       "     move    $t1, %z8                                  \n"
    \
+       "     sc.q    $t0, $t1, %2                              \n"
    \
+       "     beqz    $t0, 1b                                   \n"
    \
+       "2:                                                     \n"
    \
+       __WEAK_LLSC_MB
    \
+       : "=&r" (__ret.low), "=&r" (__ret.high),
Quoted 9 lines
Quoted 3 lines
    \
+         "=ZB" (__ptr[0])
    \

"ZB" isn't a legal constraint for the address operand in sc.q. When assembled, it turns into something like sc.q $r,$r,$r,0, which clearly doesn't match the instruction format, yet gas happily accepts it wheil clang rightfully rejects it. Classic GNU-as leniency biting again. :)

Hi Hev,

Thanks for your advice, I tried sc.q with r or ZC. the result as
below: (with gcc 14.2.1 in fedora-42)
- sc.q with "r" caused system hang
- sc.q with "ZC" caused compiler error:
{standard input}: Assembler messages:
{standard input}:10037: Fatal error: Immediate overflow.
format: u0:0 )

Quoted 41 lines
Quoted 40 lines
(quoted diff) +7 -0
+       : "ZC" (__ptr[0]), "m" (__ptr[1]),
    \
+         "Jr" (__old.low), "Jr" (__old.high),
    \
+         "Jr" (__new.low), "Jr" (__new.high)
    \
+       : "t0", "t1", "memory");
    \
+
    \
+       __ret.full;
    \ +})
+
 static inline unsigned int __cmpxchg_small(volatile void *ptr,

unsigned int old, unsigned int new, unsigned int size)
{

(quoted diff) +8 -1
@@ -198,6 +236,15 @@ __cmpxchg(volatile void *ptr, unsigned long
 old, unsigned long new, unsigned int __res;
                                 \ })
 
+/* cmpxchg128 */
+#define system_has_cmpxchg128()                1
+
+#define arch_cmpxchg128(ptr, o, n)
    \ +({
          \
+       BUILD_BUG_ON(sizeof(*(ptr)) != 16);
    \
+       __cmpxchg128_asm(ptr, o, n);                    \
+})
+
 #ifdef CONFIG_64BIT
 #define arch_cmpxchg64_local(ptr, o, n)
            \ ({
               \
 
--

2.48.1

#4 lore raw

Re: [PATCH v2 1/2] LoongArch: Add 128-bit atomic cmpxchg support

Quoted 7 lines · George Guo
Quoted 7 lines
Quoted 2 lines

+         "=ZB" (__ptr[0])
     \

"ZB" isn't a legal constraint for the address operand in sc.q. When assembled, it turns into something like sc.q $r,$r,$r,0, which clearly doesn't match the instruction format, yet gas happily accepts it wheil clang rightfully rejects it. Classic GNU-as leniency biting again. :)

I clearly remember when Jiajie submitted the sc.q support to GAS Qinggang was really insistent on supporting the additional ",0" here. But I don't really understand why we must support it...

Quoted 4 lines

Thanks for your advice, I tried sc.q with r or ZC. the result as
below: (with gcc 14.2.1 in fedora-42)
   - sc.q with "r"  caused system hang

It won't work because it'll pass the value (not address) of __ptr[0].

Quoted 3 lines

   - sc.q with "ZC" caused compiler error:
     {standard input}: Assembler messages:
     {standard input}:10037: Fatal error: Immediate overflow.

It won't work because the only accepted immediate of sc.q is 0, but ZC would allow any factor of 4 in [-32768, 32768). I.e. ZC is for {ldptr,stptr,ll,sc}.{w,d}.

As ZB is only used for sc.q (yet) in GCC backend maybe we can change ZB to print simply $rX instead of $rX,0 and make LLVM do the same. Would someone submit a GCC patch for that? Or is there already such a constraint but I don't know?

BTW for the barrier between ll.d and ld.d, "dbar 0x700" is enough to order two loads on the same address, and a Loongson hardware engineer just confirmed me privately that "same address" can be read as "in the same cacheline" here. Thus it's enough in our case and it has a lower overhead than "dbar 0".

Signature · 1 line
Xi Ruoyao <[email protected]>

A message that is not in the archive (sent off-list or not yet received): [email protected]

#6 lore raw

Re: [PATCH v2 1/2] LoongArch: Add 128-bit atomic cmpxchg support

Quoted 5 lines · mengqinggang
Quoted 3 lines

I clearly remember when Jiajie submitted the sc.q support to GAS Qinggang was really insistent on supporting the additional ",0" here. But I don't really understand why we must support it...

Because gcc output sc.q $r,$r,$r,0.

At that time GCC had not started to use sc.q yet. Some unfortunate communication error, I guess.

Signature · 1 line
Xi Ruoyao <[email protected]>
#7 lore raw

Re: [PATCH v2 1/2] LoongArch: Add 128-bit atomic cmpxchg support

In reply to George Guo
Quoted 92 lines · George Guo
Quoted 79 lines · On Mon, 24 Nov 2025 19:37:40 +0800 hev <[email protected]> wrote:
Quoted 77 lines · George Guo

From: George Guo <[email protected]>

Implement 128-bit atomic compare-and-exchange using LoongArch's LL.D/SC.Q instructions.

At the same time, fix BPF scheduler test failures (scx_central scx_qmap) caused by kmalloc_nolock_noprof returning NULL due to missing 128-bit atomics. The NULL returns led to -ENOMEM errors during scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler (located in tools/sched_ext/). Building with make and running ./tools/sched_ext/build/bin/scx_qmap.

---
 arch/loongarch/include/asm/cmpxchg.h | 47

++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+)

(quoted diff) +0 -0

b/arch/loongarch/include/asm/cmpxchg.h index 979fde61bba8a42cb4f019f13ded2a3119d4aaf4..757f6e82b9880d04f4883dc9a802312111aa4588 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -111,6 +111,44 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size) __ret;

                                                       \ })
(quoted diff) +28 -0
+union __u128_halves {
+       u128 full;
+       struct {
+               u64 low;
+               u64 high;
+       };
+};
+
+#define __cmpxchg128_asm(ptr, old, new)
            \ +({
                  \
+       union __u128_halves __old, __new, __ret;
    \
+       volatile u64 *__ptr = (volatile u64 *)(ptr);
    \
+
    \
+       __old.full = (old);
    \
+       __new.full = (new);
    \
+
    \
+       __asm__ __volatile__(
    \
+       "1:   ll.d    %0, %3            # 128-bit cmpxchg low   \n"
    \
+       "     dbar    0                 # memory barrier        \n"
    \
+       "     ld.d    %1, %4            # 128-bit cmpxchg high  \n"
    \
+       "     bne     %0, %z5, 2f                               \n"
    \
+       "     bne     %1, %z6, 2f                               \n"
    \
+       "     move    $t0, %z7                                  \n"
    \
+       "     move    $t1, %z8                                  \n"
    \
+       "     sc.q    $t0, $t1, %2                              \n"
    \
+       "     beqz    $t0, 1b                                   \n"
    \
+       "2:                                                     \n"
    \
+       __WEAK_LLSC_MB
    \
+       : "=&r" (__ret.low), "=&r" (__ret.high),
Quoted 9 lines
Quoted 3 lines
    \
+         "=ZB" (__ptr[0])
    \

"ZB" isn't a legal constraint for the address operand in sc.q. When assembled, it turns into something like sc.q $r,$r,$r,0, which clearly doesn't match the instruction format, yet gas happily accepts it wheil clang rightfully rejects it. Classic GNU-as leniency biting again. :)

Quoted 5 lines

Hi Hev,

Thanks for your advice, I tried sc.q with r or ZC. the result as
below: (with gcc 14.2.1 in fedora-42)
- sc.q with "r" caused system hang

input operands:
: "r" (__ptr), ...

Quoted 46 lines

- sc.q with "ZC" caused compiler error:
{standard input}: Assembler messages:
{standard input}:10037: Fatal error: Immediate overflow.
format: u0:0 )

Quoted 41 lines
Quoted 40 lines
(quoted diff) +7 -0
+       : "ZC" (__ptr[0]), "m" (__ptr[1]),
    \
+         "Jr" (__old.low), "Jr" (__old.high),
    \
+         "Jr" (__new.low), "Jr" (__new.high)
    \
+       : "t0", "t1", "memory");
    \
+
    \
+       __ret.full;
    \ +})
+
 static inline unsigned int __cmpxchg_small(volatile void *ptr,

unsigned int old, unsigned int new, unsigned int size)
{

(quoted diff) +8 -1
@@ -198,6 +236,15 @@ __cmpxchg(volatile void *ptr, unsigned long
 old, unsigned long new, unsigned int __res;
                                 \ })
 
+/* cmpxchg128 */
+#define system_has_cmpxchg128()                1
+
+#define arch_cmpxchg128(ptr, o, n)
    \ +({
          \
+       BUILD_BUG_ON(sizeof(*(ptr)) != 16);
    \
+       __cmpxchg128_asm(ptr, o, n);                    \
+})
+
 #ifdef CONFIG_64BIT
 #define arch_cmpxchg64_local(ptr, o, n)
            \ ({
               \
 
--

2.48.1

author
#8 lore raw

PATCH v2 2/2 LoongArch: Enable 128-bit atomics cmpxchg support

In reply to George Guo

From: George Guo <[email protected]>

Add select HAVE_CMPXCHG_DOUBLE and select HAVE_ALIGNED_STRUCT_PAGE in Kconfig to enable 128-bit atomic cmpxchg support on LoongArch.

---
 arch/loongarch/Kconfig | 2 ++
 1 file changed, 2 insertions(+)
arch/loongarch/Kconfig +2 -0
index 5b1116733d881bc2b1b43fb93f20367add4dbc54..6fb2c253969f9ddece5478920423d7326c3ec046 100644
@@ -114,6 +114,7 @@ config LOONGARCH
 	select GENERIC_TIME_VSYSCALL
 	select GPIOLIB
 	select HAS_IOPORT
+	select HAVE_ALIGNED_STRUCT_PAGE
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_JUMP_LABEL_RELATIVE
@@ -140,6 +141,7 @@ config LOONGARCH
 	select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_EBPF_JIT
+	select HAVE_CMPXCHG_DOUBLE
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN
 	select HAVE_EXIT_THREAD
 	select HAVE_GENERIC_TIF_BITS
Signature · 1 line
2.48.1