libarena: Use compiler load-acquire/store-release in bpf_atomic.h
Subsystems (3)
Kumar Kartikeya Dwivedi
maintainerPATCH 1/1 libarena: Use compiler load-acquire/store-release in bpf_atomic.h
Teach libarena's BPF atomic primitives to use compiler builtins for load-acquire and store-release when Clang advertises __BPF_FEATURE_LOAD_ACQ_STORE_REL. Older compilers continue to use the existing barrier-based fallback.
Notably, as BPF programs begin running on arm64, it is better to use the more appropriate variants since we can no longer rely on x86 TSO ordering.
Commit 880442305a39 ("bpf: Introduce load-acquire and store-release instructions") introduced support, hence kernels from 6.15 onwards are needed when compiling with compilers supporting these instructions. We have relatively recent kernel version requirements in libarena anyway, and have not cut first release, hence declare such a dependency.
--- .../selftests/bpf/libarena/include/bpf_atomic.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +16 -0
@@ -86,6 +86,21 @@ extern bool CONFIG_X86_64 __kconfig __weak;/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ #define smp_acquire__after_ctrl_dep () ({ smp_rmb(); }) +#if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) +#define __BPF_HAVE_LOAD_ACQUIRE_STORE_RELEASE 1 +#endif + +#ifdef __BPF_HAVE_LOAD_ACQUIRE_STORE_RELEASE +/* +* Clang advertises this feature when it can lower acquire/release atomic +* builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the +* barrier-based fallback below. The generated instructions require kernel +* verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to +* keep using the fallback when targeting older kernels. +*/ +#define smp_load_acquire (p) __atomic_load_n((p), __ATOMIC_ACQUIRE) +#define smp_store_release (p, val) __atomic_store_n((p), (val), __ATOMIC_RELEASE) +#else #define smp_load_acquire (p) \ ({ \ __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \
@@ -102,6 +117,7 @@ extern bool CONFIG_X86_64 __kconfig __weak;barrier (); \WRITE_ONCE (*(p), val); \ }) +#endif #define smp_cond_load_relaxed_label (p, cond_expr, label) \ ({ \
--
2.53.0
Quoted 31 lines · Kumar Kartikeya Dwivedi
Teach libarena's BPF atomic primitives to use compiler builtins for load-acquire and store-release when Clang advertises
__BPF_FEATURE_LOAD_ACQ_STORE_REL. Older compilers continue to use the existing barrier-based fallback.Notably, as BPF programs begin running on arm64, it is better to use the more appropriate variants since we can no longer rely on x86 TSO ordering.
Commit
880442305a39("bpf: Introduce load-acquire and store-release instructions") introduced support, hence kernels from 6.15 onwards are needed when compiling with compilers supporting these instructions. We have relatively recent kernel version requirements in libarena anyway, and have not cut first release, hence declare such a dependency.Signed-off-by Kumar Kartikeya Dwivedi <[email protected]>--- .../selftests/bpf/libarena/include/bpf_atomic.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +4
-0@@ -86,6 +86,21 @@ extern bool CONFIG_X86_64 __kconfig __weak;/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ #define smp_acquire__after_ctrl_dep () ({ smp_rmb(); }) +#if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) +#define __BPF_HAVE_LOAD_ACQUIRE_STORE_RELEASE 1 +#endif +
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h extern bool CONFIG_X86_64 __kconfig __weak; · line 102
Quoted 10 lines
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +10
-0+#ifdef __BPF_HAVE_LOAD_ACQUIRE_STORE_RELEASE +/* +* Clang advertises this feature when it can lower acquire/release atomic +* builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the +* barrier-based fallback below. The generated instructions require kernel +* verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to +* keep using the fallback when targeting older kernels. +*/ +#define smp_load_acquire (p) __atomic_load_n((p), __ATOMIC_ACQUIRE) +#define smp_store_release (p, val) __atomic_store_n((p), (val), __ATOMIC_RELEASE)
__atomic_load_n(&some_struct, …) is a compile error. we should do the more generic version:
#define smp_load_acquire(p) \
({ \
__unqual_scalar_typeof(*p) __val; \
__atomic_load(p, &__val, __ATOMIC_ACQUIRE); \
__val; \
})#define smp_store_release(p, v) \
({ \
__unqual_scalar_typeof(*p) __val = (v); \
__atomic_store(p, &__val, __ATOMIC_RELEASE); \
})Thanks,
Puranjay
[email protected]
AI reviewCan the __atomic_load_n / __atomic_store_n form here handle a pointer to a struct?
Quoted 12 lines
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +4
-0@@ -86,6 +86,21 @@ extern bool CONFIG_X86_64 __kconfig __weak;/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ #define smp_acquire__after_ctrl_dep () ({ smp_rmb(); }) +#if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) +#define __BPF_HAVE_LOAD_ACQUIRE_STORE_RELEASE 1 +#endif +
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h extern bool CONFIG_X86_64 __kconfig __weak; · line 102
Quoted 10 lines
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +10
-0+#ifdef __BPF_HAVE_LOAD_ACQUIRE_STORE_RELEASE +/* +* Clang advertises this feature when it can lower acquire/release atomic +* builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the +* barrier-based fallback below. The generated instructions require kernel +* verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to +* keep using the fallback when targeting older kernels. +*/ +#define smp_load_acquire (p) __atomic_load_n((p), __ATOMIC_ACQUIRE) +#define smp_store_release (p, val) __atomic_store_n((p), (val), __ATOMIC_RELEASE)
Can the __atomic_load_n / __atomic_store_n form here handle a pointer to a struct?
__atomic_load_n(&some_struct, ...) is a compile error when p points to an aggregate type, since these builtins only accept scalar-typed pointers.
Would the more generic builtin form work for both scalar and struct pointers, for example:
#define smp_load_acquire(p) ({ \
__unqual_scalar_typeof(*p) __val; \
__atomic_load(p, &__val, __ATOMIC_ACQUIRE); \
__val; \
})
#define smp_store_release(p, v) ({ \
__unqual_scalar_typeof(*p) __val = (v); \
__atomic_store(p, &__val, __ATOMIC_RELEASE); \
})tools/testing/selftests/bpf/libarena/include/bpf_atomic.h extern bool CONFIG_X86_64 __kconfig __weak; · line 120
Quoted 9 lines
tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +2
-0+#else #define smp_load_acquire (p) \ ({ \ __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \@@ -102,6 +117,7 @@ extern bool CONFIG_X86_64 __kconfig __weak;barrier (); \WRITE_ONCE (*(p), val); \ }) +#endif
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/29860654839