Introduce cmpxchg128() -- aka. the demise of cmpxchg_double()
Subsystems (15)
Peter Zijlstra
maintainerPATCH 12/12 s390/cpum_sf: Convert to cmpxchg128()
In reply to Peter ZijlstraNow that there is a cross arch u128 and cmpxchg128(), use those instead of the custom CDSG helper.
--- arch/s390/include/asm/cpu_mf.h | 2 +- arch/s390/kernel/perf_cpum_sf.c | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-)
arch/s390/include/asm/cpu_mf.h +1 -1
@@ -140,7 +140,7 @@ union hws_trailer_header { unsigned int dsdes:16; /* 48-63: size of diagnostic SDE */unsigned long long overflow;/* 64 - Overflow Count */ }; -__uint128_t val; +u128 val; };struct hws_trailer_entry {
arch/s390/kernel/perf_cpum_sf.c +3 -13
@@ -1271,16 +1271,6 @@ static void hw_collect_samples(struct pe } } -static inline __uint128_t __cdsg (__uint128_t *ptr,__uint128_t old,__uint128_t new) -{ - asmvolatile ( -" cdsg %[old],%[new],%[ptr]\n" - : [old]"+d" (old), [ptr]"+QS" (*ptr) - : [new]"d" (new) - :"memory" ,"cc" ); -return old; -} -/* hw_perf_event_update() - Process sampling buffer * @event: The perf event * @flush_all: Flag to also flush partially filled sample-data-blocks
@@ -1352,7 +1342,7 @@ static void hw_perf_event_update(struct new.f =0 ; new.a =1 ; new.overflow =0 ; - prev.val =__cdsg (&te->header.val, old.val, new.val); + prev.val =cmpxchg128 (&te->header.val, old.val, new.val); }while (prev.val != old.val);/* Advance to next sample-data-block */
@@ -1562,7 +1552,7 @@ static bool aux_set_alert(struct aux_buf } new.a =1 ; new.overflow =0 ; - prev.val =__cdsg (&te->header.val, old.val, new.val); + prev.val =cmpxchg128 (&te->header.val, old.val, new.val); }while (prev.val != old.val);return true; }
@@ -1636,7 +1626,7 @@ static bool aux_reset_buffer(struct aux_ new.a =1 ; else new.a = 0; - prev.val =__cdsg (&te->header.val, old.val, new.val); + prev.val =cmpxchg128 (&te->header.val, old.val, new.val); }while (prev.val != old.val); *overflow += orig_overflow; }
Peter Zijlstra
maintainercover Introduce cmpxchg128() -- aka. the demise of cmpxchg_double()
Hi!
After much breaking of things, find here the improved version.
Since v3:
- unbreak everything that does *NOT* have cmpxchg128()
Notably this_cpu_cmpxchg_double() is used unconditionally by SLUB which means that this_cpu_try_cmpxchg128() needs to be unconditionally available on all 64bit architectures.
- fixed up x86/x86_64 cmpxchg{8,16}b emulation for this_cpu_cmpxchg{64,128}()
- introduce {raw,this}_cpu_try_cmpxchg*()
- add fallback for !__SIZEOF_INT128__ 64bit architectures
Sadly there are supported 64bit architecture/compiler combinations that do not have __SIZEOF_INT128__, specifically it was found that HPPA64 only added this with GCC-11.
this is yuck, and ideally we'd simply raise compiler requirements, but this 'works'.
My plan is to re-add this to tip/locking/core and thus -next later this week.
Also available at:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git locking/core
--- Documentation/core-api/this_cpu_ops.rst | 2 - arch/arm64/include/asm/atomic_ll_sc.h | 56 +++--- arch/arm64/include/asm/atomic_lse.h | 39 ++--- arch/arm64/include/asm/cmpxchg.h | 48 ++---- arch/arm64/include/asm/percpu.h | 30 ++-- arch/s390/include/asm/cmpxchg.h | 32 +--- arch/s390/include/asm/cpu_mf.h | 2 +- arch/s390/include/asm/percpu.h | 34 ++-- arch/s390/kernel/perf_cpum_sf.c | 16 +- arch/x86/include/asm/cmpxchg.h | 25 --- arch/x86/include/asm/cmpxchg_32.h | 2 +- arch/x86/include/asm/cmpxchg_64.h | 63 ++++++- arch/x86/include/asm/percpu.h | 102 ++++++----- arch/x86/lib/Makefile | 3 +- arch/x86/lib/cmpxchg16b_emu.S | 43 +++-- arch/x86/lib/cmpxchg8b_emu.S | 67 ++++++-- drivers/iommu/amd/amd_iommu_types.h | 9 +- drivers/iommu/amd/iommu.c | 10 +- drivers/iommu/intel/irq_remapping.c | 8 +- include/asm-generic/percpu.h | 257 ++++++++++++++++++++++------ include/crypto/b128ops.h | 14 +- include/linux/atomic/atomic-arch-fallback.h | 95 +++++++++- include/linux/atomic/atomic-instrumented.h | 93 ++++++++-- include/linux/dmar.h | 125 +++++++------- include/linux/percpu-defs.h | 45 ++--- include/linux/slub_def.h | 12 +- include/linux/types.h | 12 ++ include/uapi/linux/types.h | 4 + lib/crypto/curve25519-hacl64.c | 2 - lib/crypto/poly1305-donna64.c | 2 - mm/slab.h | 53 +++++- mm/slub.c | 139 +++++++++------ scripts/atomic/gen-atomic-fallback.sh | 4 +- scripts/atomic/gen-atomic-instrumented.sh | 19 +- 34 files changed, 952 insertions(+), 515 deletions(-)
Peter Zijlstra
maintainerPATCH 7/12 percpu: #ifndef __SIZEOF_INT128__
Some 64bit architectures do not advertise __SIZEOF_INT128__ on all supported compiler versions. Notably the HPPA64 only started doing with GCC-11.
Since the per-cpu ops are universally availably, and this_cpu_{,try_}cmpxchg128() is expected to be available on all 64bit architectures a wee bodge is in order.
Sadly, while C reverts to memcpy() for assignment of POD types, it does not revert to memcmp() for for equality. Therefore frob that manually.
--- include/asm-generic/percpu.h | 56 +++++++++++++++++++++++++++++++++++++++++++ include/linux/types.h | 7 +++++ 2 files changed, 63 insertions(+)
include/asm-generic/percpu.h +56 -0
@@ -313,6 +313,35 @@ do { \#define raw_cpu_xchg_8 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)#endif +#ifndef __SIZEOF_INT128__ +#define raw_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (!__builtin_memcmp(&__val, &__old, sizeof(pcp))) { \ + *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ + __ret; \ +}) + +#define raw_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg_memcpy(pcp, &__old, nval); \ + __old; \ +}) + +#define raw_cpu_cmpxchg128 (pcp, oval, nval) \ + raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) +#define raw_cpu_try_cmpxchg128 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) +#endif +#ifndef raw_cpu_try_cmpxchg_1#ifdef raw_cpu_cmpxchg_1#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \
@@ -503,6 +532,33 @@ do { \#define this_cpu_xchg_8 (pcp, nval) this_cpu_generic_xchg(pcp, nval)#endif +#ifndef __SIZEOF_INT128__ +#define this_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + bool __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) + +#define this_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) + +#define this_cpu_cmpxchg128 (pcp, oval, nval) \ + this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) +#define this_cpu_try_cmpxchg128 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) +#endif +#ifndef this_cpu_try_cmpxchg_1#ifdef this_cpu_cmpxchg_1#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \
include/linux/types.h +7 -0
@@ -13,6 +13,13 @@#ifdef __SIZEOF_INT128__typedef __s128 s128 ;typedef __u128 u128 ; +#else +#ifdef CONFIG_64BIT +/* hack for this_cpu_cmpxchg128 */ +typedef struct { +u64 a, b; +}u128 __attribute__((aligned (16 ))); +#endif #endif typedef u32 __kernel_dev_t ;
Peter Zijlstra
maintainerPATCH 1/12 cyrpto/b128ops: Remove struct u128
In reply to Peter ZijlstraPer git-grep u128_xor() and its related struct u128 are unused except to implement {be,le}128_xor(). Remove them to free up the namespace.
--- include/crypto/b128ops.h | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-)
include/crypto/b128ops.h +3 -11
@@ -50,10 +50,6 @@#include <linux/types.h> typedef struct { -u64 a, b; -}u128 ; - -typedef struct {__be64 a, b; }be128 ;
@@ -61,20 +57,16 @@ typedef struct { __le64 b, a; } le128; -static inline void u128_xor (u128 *r,const u128 *p,const u128 *q) +static inline void be128_xor (be128 *r,const be128 *p,const be128 *q) { r->a = p->a ^ q->a; r->b = p->b ^ q->b; } -static inline void be128_xor (be128 *r,const be128 *p,const be128 *q) -{ -u128_xor ((u128 *)r, (u128 *)p, (u128 *)q); -} -static inline void le128_xor (le128 *r,const le128 *p,const le128 *q) { -u128_xor ((u128 *)r, (u128 *)p, (u128 *)q); + r->a = p->a ^ q->a; + r->b = p->b ^ q->b; }#endif /* _CRYPTO_B128OPS_H */
Peter Zijlstra
maintainerPATCH 5/12 percpu: Add {raw,this}_cpu_try_cmpxchg()
In reply to Peter ZijlstraAdd the try_cmpxchg() form to the per-cpu ops.
--- include/asm-generic/percpu.h | 113 +++++++++++++++++++++++++++++++++++++++++-- include/linux/percpu-defs.h | 19 +++++++ 2 files changed, 128 insertions(+), 4 deletions(-)
include/asm-generic/percpu.h +109 -4
@@ -89,16 +89,37 @@ do { \ __ret; \ }) -#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ +#define __cpu_fallback_try_cmpxchg (pcp, ovalp, nval, _cmpxchg) \ +({ \ + typeof(pcp) __val, __old = *(ovalp); \ + __val = _cmpxchg(pcp, __old, nval); \ + if (__val != __old) \ + *(ovalp) = __val; \ + __val == __old; \ +}) + +#define raw_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ ({ \ typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __ret; \ - __ret = *__p; \ - if (__ret == (oval)) \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (__val == __old) { \ *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ __ret; \ }) +#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \ + __old; \ +}) +#define raw_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \
@@ -170,6 +191,16 @@ do { \ __ret; \ }) +#define this_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ +({ \ + bool __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) +#define this_cpu_generic_cmpxchg (pcp, oval, nval) \ ({ \ typeof(pcp) __ret; \
@@ -282,6 +313,43 @@ do { \#define raw_cpu_xchg_8 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)#endif +#ifndef raw_cpu_try_cmpxchg_1 +#ifdef raw_cpu_cmpxchg_1 +#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1) +#else +#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_2 +#ifdef raw_cpu_cmpxchg_2 +#define raw_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2) +#else +#define raw_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_4 +#ifdef raw_cpu_cmpxchg_4 +#define raw_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4) +#else +#define raw_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_8 +#ifdef raw_cpu_cmpxchg_8 +#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8) +#else +#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_cmpxchg_1#define raw_cpu_cmpxchg_1 (pcp, oval, nval) \ raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -407,6 +475,43 @@ do { \#define this_cpu_xchg_8 (pcp, nval) this_cpu_generic_xchg(pcp, nval)#endif +#ifndef this_cpu_try_cmpxchg_1 +#ifdef this_cpu_cmpxchg_1 +#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1) +#else +#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_2 +#ifdef this_cpu_cmpxchg_2 +#define this_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2) +#else +#define this_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_4 +#ifdef this_cpu_cmpxchg_4 +#define this_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4) +#else +#define this_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_8 +#ifdef this_cpu_cmpxchg_8 +#define this_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8) +#else +#define this_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_cmpxchg_1#define this_cpu_cmpxchg_1 (pcp, oval, nval) \ this_cpu_generic_cmpxchg(pcp, oval, nval)
include/linux/percpu-defs.h +19 -0
@@ -343,6 +343,21 @@ static __always_inline void __this_cpu_p pscr2_ret__; \ }) +#define __pcpu_size_call_return2bool (stem, variable, ...) \ +({ \ + bool pscr2_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ + case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ + case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ + case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pscr2_ret__; \ +}) +/* * Special handling for cmpxchg_double. cmpxchg_double is passed two * percpu variables. The first has to be aligned to a double word
@@ -426,6 +441,8 @@ do { \#define raw_cpu_xchg (pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)#define raw_cpu_cmpxchg (pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) +#define raw_cpu_try_cmpxchg (pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)#define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
@@ -513,6 +530,8 @@ do { \#define this_cpu_xchg (pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)#define this_cpu_cmpxchg (pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) +#define this_cpu_try_cmpxchg (pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)#define this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
Peter Zijlstra
maintainerPATCH 6/12 percpu: Wire up cmpxchg128
In reply to Peter ZijlstraIn order to replace cmpxchg_double() with the newly minted cmpxchg128() family of functions, wire it up in this_cpu_cmpxchg().
--- arch/arm64/include/asm/percpu.h | 20 ++++++++++ arch/s390/include/asm/percpu.h | 16 ++++++++ arch/x86/include/asm/percpu.h | 74 ++++++++++++++++++++++++++++++++++++---- arch/x86/lib/Makefile | 3 + arch/x86/lib/cmpxchg16b_emu.S | 43 +++++++++++++---------- arch/x86/lib/cmpxchg8b_emu.S | 67 ++++++++++++++++++++++++++++-------- include/asm-generic/percpu.h | 56 ++++++++++++++++++++++++++++++ 7 files changed, 240 insertions(+), 39 deletions(-)
arch/arm64/include/asm/percpu.h +20 -0
@@ -140,6 +140,10 @@ PERCPU_RET_OP(add, add, ldadd)* re-enabling preemption for preemptible kernels, but doing that in a way * which builds inside a module would mean messing directly with the preempt * count. If you do this, peterz and tglx will hunt you down. +* +* Not to mention it'll break the actual preemption model for missing a +* preemption point when TIF_NEED_RESCHED gets set while preemption is +* disabled. */ #define this_cpu_cmpxchg_double_8 (ptr1, ptr2, o1, o2, n1, n2) \ ({ \
@@ -240,6 +244,22 @@ PERCPU_RET_OP(add, add, ldadd)#define this_cpu_cmpxchg_8 (pcp, o, n) \ _pcp_protect_return(cmpxchg_relaxed, pcp, o, n) +#define this_cpu_cmpxchg64 (pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) + +#define this_cpu_cmpxchg128 (pcp, o, n) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + u128 old__, new__, ret__; \ + pcp_op_T__ *ptr__; \ + old__ = o; \ + new__ = n; \ + preempt_disable_notrace(); \ + ptr__ = raw_cpu_ptr(&(pcp)); \ + ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \ + preempt_enable_notrace(); \ + ret__; \ +}) +#ifdef __KVM_NVHE_HYPERVISOR__extern unsigned long __hyp_per_cpu_offset (unsigned int cpu);#define __per_cpu_offset
arch/s390/include/asm/percpu.h +16 -0
@@ -148,6 +148,22 @@#define this_cpu_cmpxchg_4 (pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)#define this_cpu_cmpxchg_8 (pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) +#define this_cpu_cmpxchg64 (pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) + +#define this_cpu_cmpxchg128 (pcp, oval, nval) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + u128 old__, new__, ret__; \ + pcp_op_T__ *ptr__; \ + old__ = oval; \ + new__ = nval; \ + preempt_disable_notrace(); \ + ptr__ = raw_cpu_ptr(&(pcp)); \ + ret__ = cmpxchg128((void *)ptr__, old__, new__); \ + preempt_enable_notrace(); \ + ret__; \ +}) +#define arch_this_cpu_xchg (pcp, nval) \ ({ \ typeof(pcp) *ptr__; \
arch/x86/include/asm/percpu.h +68 -6
86 lines not on this page · raw message
arch/x86/lib/Makefile +2 -1
@@ -61,8 +61,9 @@ ifeq ($(CONFIG_X86_32),y) lib-y += strstr_32.o lib-y += string_32.o lib-y += memmove_32.o + lib-y += cmpxchg8b_emu.o ifneq ($(CONFIG_X86_CMPXCHG64),y) - lib-y += cmpxchg8b_emu.o atomic64_386_32.o + lib-y += atomic64_386_32.o endif else obj-y += iomap_copy_64.o
arch/x86/lib/cmpxchg16b_emu.S +25 -18
72 lines not on this page · raw message
arch/x86/lib/cmpxchg8b_emu.S +53 -14
95 lines not on this page · raw message
include/asm-generic/percpu.h +56 -0
80 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 4/12 instrumentation: Wire up cmpxchg128()
In reply to Peter ZijlstraWire up the cmpxchg128 family in the atomic wrapper scripts.
These provide the generic cmpxchg128 family of functions from the arch_ prefixed version, adding explicit instrumentation where needed.
--- include/linux/atomic/atomic-arch-fallback.h | 95 +++++++++++++++++++++++++++- include/linux/atomic/atomic-instrumented.h | 86 +++++++++++++++++++++++++ scripts/atomic/gen-atomic-fallback.sh | 4 - scripts/atomic/gen-atomic-instrumented.sh | 4 - 4 files changed, 183 insertions(+), 6 deletions(-)
include/linux/atomic/atomic-arch-fallback.h +94 -1
@@ -77,6 +77,29 @@#endif /* arch_cmpxchg64_relaxed */ +#ifndef arch_cmpxchg128_relaxed +#define arch_cmpxchg128_acquire arch_cmpxchg128 +#define arch_cmpxchg128_release arch_cmpxchg128 +#define arch_cmpxchg128_relaxed arch_cmpxchg128 +#else /* arch_cmpxchg128_relaxed */ + +#ifndef arch_cmpxchg128_acquire +#define arch_cmpxchg128_acquire (...) \ + __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg128_release +#define arch_cmpxchg128_release (...) \ + __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg128 +#define arch_cmpxchg128 (...) \ + __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) +#endif + +#endif /* arch_cmpxchg128_relaxed */ +#ifndef arch_try_cmpxchg_relaxed#ifdef arch_try_cmpxchg#define arch_try_cmpxchg_acquire arch_try_cmpxchg
@@ -217,6 +240,76 @@#endif /* arch_try_cmpxchg64_relaxed */ +#ifndef arch_try_cmpxchg128_relaxed +#ifdef arch_try_cmpxchg128 +#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128 +#define arch_try_cmpxchg128_release arch_try_cmpxchg128 +#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128 +#endif /* arch_try_cmpxchg128 */ + +#ifndef arch_try_cmpxchg128 +#define arch_try_cmpxchg128 (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128 */ + +#ifndef arch_try_cmpxchg128_acquire +#define arch_try_cmpxchg128_acquire (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_acquire */ + +#ifndef arch_try_cmpxchg128_release +#define arch_try_cmpxchg128_release (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_release */ + +#ifndef arch_try_cmpxchg128_relaxed +#define arch_try_cmpxchg128_relaxed (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_relaxed */ + +#else /* arch_try_cmpxchg128_relaxed */ + +#ifndef arch_try_cmpxchg128_acquire +#define arch_try_cmpxchg128_acquire (...) \ + __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg128_release +#define arch_try_cmpxchg128_release (...) \ + __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg128 +#define arch_try_cmpxchg128 (...) \ + __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#endif /* arch_try_cmpxchg128_relaxed */ +#ifndef arch_try_cmpxchg_local#define arch_try_cmpxchg_local (_ptr, _oldp, _new) \ ({ \
@@ -2668,4 +2761,4 @@ arch_atomic64_dec_if_positive(atomic64_t#endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// ad2e2b4d168dbc60a73922616047a9bfa446af36 +// 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53
include/linux/atomic/atomic-instrumented.h +85 -1
113 lines not on this page · raw message
scripts/atomic/gen-atomic-fallback.sh +2 -2
@@ -217,11 +217,11 @@ cat << EOF EOF -for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do +for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64" "arch_cmpxchg128"; do gen_xchg_fallbacks "${xchg}" done -for cmpxchg in "cmpxchg" "cmpxchg64"; do +for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do gen_try_cmpxchg_fallbacks "${cmpxchg}" done
scripts/atomic/gen-atomic-instrumented.sh +2 -2
@@ -166,14 +166,14 @@ grep '^[a-z]' "$1" | while read name met done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do +for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_xchg "${xchg}" "${order}" "" printf "\n" done done -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do +for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do gen_xchg "${xchg}" "" "" printf "\n" done
Peter Zijlstra
maintainerPATCH 10/12 slub: Replace cmpxchg_double()
In reply to Peter Zijlstra--- include/linux/slub_def.h | 12 +++- mm/slab.h | 53 +++++++++++++++-- mm/slub.c | 139 +++++++++++++++++++++++++++-------------------- 3 files changed, 137 insertions(+), 67 deletions(-)
include/linux/slub_def.h +9 -3
@@ -39,7 +39,8 @@ enum stat_item {CPU_PARTIAL_FREE ,/* Refill cpu partial on free */ CPU_PARTIAL_NODE ,/* Refill cpu partial from node partial */ CPU_PARTIAL_DRAIN ,/* Drain cpu partial to node partial */ - NR_SLUB_STAT_ITEMS }; + NR_SLUB_STAT_ITEMS +};#ifndef CONFIG_SLUB_TINY /*
@@ -47,8 +48,13 @@ enum stat_item {* with this_cpu_cmpxchg_double() alignment requirements. */ struct kmem_cache_cpu { -void **freelist;/* Pointer to next available object */ -unsigned long tid;/* Globally unique transaction id */ +union { +struct { +void **freelist;/* Pointer to next available object */ +unsigned long tid;/* Globally unique transaction id */ + }; +freelist_aba_t freelist_tid; + };struct slab *slab;/* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL struct slab *partial;/* Partially allocated frozen slabs */
mm/slab.h +46 -7
74 lines not on this page · raw message
mm/slub.c +82 -57
238 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 11/12 arch: Remove cmpxchg_double
In reply to Peter ZijlstraNo moar users, remove the monster.
--- Documentation/core-api/this_cpu_ops.rst | 2 - arch/arm64/include/asm/atomic_ll_sc.h | 33 ---------------- arch/arm64/include/asm/atomic_lse.h | 36 ------------------ arch/arm64/include/asm/cmpxchg.h | 46 ----------------------- arch/arm64/include/asm/percpu.h | 10 ----- arch/s390/include/asm/cmpxchg.h | 34 ----------------- arch/s390/include/asm/percpu.h | 18 --------- arch/x86/include/asm/cmpxchg.h | 25 ------------ arch/x86/include/asm/cmpxchg_32.h | 1 arch/x86/include/asm/cmpxchg_64.h | 1 arch/x86/include/asm/percpu.h | 42 --------------------- include/asm-generic/percpu.h | 58 ----------------------------- include/linux/atomic/atomic-instrumented.h | 17 -------- include/linux/percpu-defs.h | 38 ------------------- scripts/atomic/gen-atomic-instrumented.sh | 15 ++----- 15 files changed, 5 insertions(+), 371 deletions(-)
Documentation/core-api/this_cpu_ops.rst +0 -2
@@ -53,7 +53,6 @@ are defined. These operations can be use this_cpu_add_return(pcp, val) this_cpu_xchg(pcp, nval) this_cpu_cmpxchg(pcp, oval, nval) - this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) this_cpu_sub(pcp, val) this_cpu_inc(pcp) this_cpu_dec(pcp)
@@ -242,7 +241,6 @@ modifies the variable, then RMW actions __this_cpu_add_return(pcp, val) __this_cpu_xchg(pcp, nval) __this_cpu_cmpxchg(pcp, oval, nval) - __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) __this_cpu_sub(pcp, val) __this_cpu_inc(pcp) __this_cpu_dec(pcp)
arch/arm64/include/asm/atomic_ll_sc.h +0 -33
@@ -294,39 +294,6 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish,#undef __CMPXCHG_CASE -#define __CMPXCHG_DBL (name, mb, rel, cl) \ -static __always_inline long \ -__ll_sc__cmpxchg_double##name(unsigned long old1, \ - unsigned long old2, \ - unsigned long new1, \ - unsigned long new2, \ - volatile void *ptr) \ -{ \ - unsigned long tmp, ret; \ - \ - asm volatile("// __cmpxchg_double" #name "\n" \ - " prfm pstl1strm, %2\n" \ - "1: ldxp %0, %1, %2\n" \ - " eor %0, %0, %3\n" \ - " eor %1, %1, %4\n" \ - " orr %1, %0, %1\n" \ - " cbnz %1, 2f\n" \ - " st" #rel "xp %w0, %5, %6, %2\n" \ - " cbnz %w0, 1b\n" \ - " " #mb "\n" \ - "2:" \ - : "=&r" (tmp), "=&r" (ret), "+Q" (*(__uint128_t *)ptr) \ - : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \ - : cl); \ - \ - return ret; \ -} - -__CMPXCHG_DBL( , , , ) -__CMPXCHG_DBL(_mb, dmb ish, l, "memory") - -#undef __CMPXCHG_DBL -union __u128_halves {u128 full;struct {
arch/arm64/include/asm/atomic_lse.h +0 -36
@@ -281,42 +281,6 @@ __CMPXCHG_CASE(x, , mb_, 64, al, "memo#undef __CMPXCHG_CASE -#define __CMPXCHG_DBL(name, mb, cl...) \ -static __always_inline long \ -__lse__cmpxchg_double##name(unsigned long old1, \ - unsigned long old2, \ - unsigned long new1, \ - unsigned long new2, \ - volatile void *ptr) \ -{ \ - unsigned long oldval1 = old1; \ - unsigned long oldval2 = old2; \ - register unsigned long x0 asm ("x0") = old1; \ - register unsigned long x1 asm ("x1") = old2; \ - register unsigned long x2 asm ("x2") = new1; \ - register unsigned long x3 asm ("x3") = new2; \ - register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ - \ - asm volatile( \ - __LSE_PREAMBLE \ - " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ - " eor %[old1], %[old1], %[oldval1]\n" \ - " eor %[old2], %[old2], %[oldval2]\n" \ - " orr %[old1], %[old1], %[old2]" \ - : [old1] "+&r" (x0), [old2] "+&r" (x1), \ - [v] "+Q" (*(__uint128_t *)ptr) \ - : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ - [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ - : cl); \ - \ - return x0; \ -} - -__CMPXCHG_DBL( , ) -__CMPXCHG_DBL(_mb, al, "memory") - -#undef __CMPXCHG_DBL - #define __CMPXCHG128(name, mb, cl...) \ static __always_inline u128 \ __lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
arch/arm64/include/asm/cmpxchg.h +0 -46
58 lines not on this page · raw message
arch/arm64/include/asm/percpu.h +0 -10
@@ -145,16 +145,6 @@ PERCPU_RET_OP(add, add, ldadd)* preemption point when TIF_NEED_RESCHED gets set while preemption is * disabled. */ -#define this_cpu_cmpxchg_double_8 (ptr1, ptr2, o1, o2, n1, n2) \ -({ \ - int __ret; \ - preempt_disable_notrace(); \ - __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \ - raw_cpu_ptr(&(ptr2)), \ - o1, o2, n1, n2); \ - preempt_enable_notrace(); \ - __ret; \ -})#define _pcp_protect (op, pcp, ...) \ ({ \
arch/s390/include/asm/cmpxchg.h +0 -34
@@ -190,40 +190,6 @@ static __always_inline unsigned long __c#define arch_cmpxchg_local arch_cmpxchg#define arch_cmpxchg64_local arch_cmpxchg -#define system_has_cmpxchg_double () 1 - -static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2, -unsigned long o1,unsigned long o2, -unsigned long n1,unsigned long n2) -{ -union register_pair old = { .even = o1, .odd = o2, }; -union register_pair new = { .even = n1, .odd = n2, }; -int cc; - - asmvolatile ( -" cdsg %[old],%[new],%[ptr]\n" -" ipm %[cc]\n" -" srl %[cc],28\n" - : [cc]"=&d" (cc), [old]"+&d" (old.pair) - : [new]"d" (new.pair), - [ptr]"QS" (*(unsigned long *)p1),"Q" (*(unsigned long *)p2) - :"memory" ,"cc" ); -return !cc; -} - -#define arch_cmpxchg_double (p1, p2, o1, o2, n1, n2) \ -({ \ - typeof(p1) __p1 = (p1); \ - typeof(p2) __p2 = (p2); \ - \ - BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ - BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ - VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ - __cmpxchg_double((unsigned long)__p1, (unsigned long)__p2, \ - (unsigned long)(o1), (unsigned long)(o2), \ - (unsigned long)(n1), (unsigned long)(n2)); \ -}) -#define system_has_cmpxchg128 () 1 static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
arch/s390/include/asm/percpu.h +0 -18
@@ -180,24 +180,6 @@#define this_cpu_xchg_4 (pcp, nval) arch_this_cpu_xchg(pcp, nval)#define this_cpu_xchg_8 (pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define arch_this_cpu_cmpxchg_double (pcp1, pcp2, o1, o2, n1, n2) \ -({ \ - typeof(pcp1) *p1__; \ - typeof(pcp2) *p2__; \ - int ret__; \ - \ - preempt_disable_notrace(); \ - p1__ = raw_cpu_ptr(&(pcp1)); \ - p2__ = raw_cpu_ptr(&(pcp2)); \ - ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \ - (unsigned long)(o1), (unsigned long)(o2), \ - (unsigned long)(n1), (unsigned long)(n2)); \ - preempt_enable_notrace(); \ - ret__; \ -}) - -#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double -#include <asm-generic/percpu.h> #endif /* __ARCH_S390_PERCPU__ */
arch/x86/include/asm/cmpxchg.h +0 -25
@@ -239,29 +239,4 @@ extern void __add_wrong_size(void)#define __xadd (ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)#define xadd (ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX) -#define __cmpxchg_double (pfx, p1, p2, o1, o2, n1, n2) \ -({ \ - bool __ret; \ - __typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \ - __typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \ - BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ - BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ - VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \ - VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \ - asm volatile(pfx "cmpxchg%c5b %1" \ - CC_SET(e) \ - : CC_OUT(e) (__ret), \ - "+m" (*(p1)), "+m" (*(p2)), \ - "+a" (__old1), "+d" (__old2) \ - : "i" (2 * sizeof(long)), \ - "b" (__new1), "c" (__new2)); \ - __ret; \ -}) - -#define arch_cmpxchg_double (p1, p2, o1, o2, n1, n2) \ - __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2) - -#define arch_cmpxchg_double_local (p1, p2, o1, o2, n1, n2) \ - __cmpxchg_double(, p1, p2, o1, o2, n1, n2) -#endif /* ASM_X86_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h +0 -1
@@ -103,7 +103,6 @@ static inline bool __try_cmpxchg64(volat#endif -#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8)#define system_has_cmpxchg64 () boot_cpu_has(X86_FEATURE_CX8)#endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h +0 -1
@@ -81,7 +81,6 @@ static __always_inline bool arch_try_cmp return __arch_try_cmpxchg128(ptr, oldp, new,); } -#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX16)#define system_has_cmpxchg128 () boot_cpu_has(X86_FEATURE_CX16)#endif /* _ASM_X86_CMPXCHG_64_H */
arch/x86/include/asm/percpu.h +0 -42
54 lines not on this page · raw message
include/asm-generic/percpu.h +0 -58
80 lines not on this page · raw message
include/linux/atomic/atomic-instrumented.h +1 -16
@@ -2234,21 +2234,6 @@ atomic_long_dec_if_positive(atomic_long_arch_try_cmpxchg128_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#define cmpxchg_double (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kcsan_mb(); \ - instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ - arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ -}) - - -#define cmpxchg_double_local (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ - arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ -})#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 82d1be694fab30414527d0877c29fa75ed5a0b74 +// 3611991b015450e119bcd7417a9431af7f3ba13c
include/linux/percpu-defs.h +0 -38
@@ -358,33 +358,6 @@ static __always_inline void __this_cpu_p pscr2_ret__; \ }) -/* -* Special handling for cmpxchg_double. cmpxchg_double is passed two -* percpu variables. The first has to be aligned to a double word -* boundary and the second has to follow directly thereafter. -* We enforce this on all architectures even if they don't support -* a double cmpxchg instruction, since it's a cheap requirement, and it -* avoids breaking the requirement for architectures with the instruction. -*/ -#define __pcpu_double_call_return_bool (stem, pcp1, pcp2, ...) \ -({ \ - bool pdcrb_ret__; \ - __verify_pcpu_ptr(&(pcp1)); \ - BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ - VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \ - VM_BUG_ON((unsigned long)(&(pcp2)) != \ - (unsigned long)(&(pcp1)) + sizeof(pcp1)); \ - switch(sizeof(pcp1)) { \ - case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ - case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ - case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ - case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ - default: \ - __bad_size_call_parameter(); break; \ - } \ - pdcrb_ret__; \ -}) -#define __pcpu_size_call (stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \
@@ -443,9 +416,6 @@ do { \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)#define raw_cpu_try_cmpxchg (pcp, ovalp, nval) \ __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval) -#define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) -#define raw_cpu_sub (pcp, val) raw_cpu_add(pcp, -(val))#define raw_cpu_inc (pcp) raw_cpu_add(pcp, 1)#define raw_cpu_dec (pcp) raw_cpu_sub(pcp, 1)
@@ -505,11 +475,6 @@ do { \raw_cpu_cmpxchg (pcp, oval, nval); \ }) -#define __this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ -({ __this_cpu_preempt_check("cmpxchg_double"); \ - raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \ -}) -#define __this_cpu_sub (pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))#define __this_cpu_inc (pcp) __this_cpu_add(pcp, 1)#define __this_cpu_dec (pcp) __this_cpu_sub(pcp, 1)
@@ -532,9 +497,6 @@ do { \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)#define this_cpu_try_cmpxchg (pcp, ovalp, nval) \ __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval) -#define this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) -#define this_cpu_sub (pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))#define this_cpu_inc (pcp) this_cpu_add(pcp, 1)#define this_cpu_dec (pcp) this_cpu_sub(pcp, 1)
scripts/atomic/gen-atomic-instrumented.sh +4 -11
@@ -84,7 +84,6 @@ gen_xchg() { local xchg="$1"; shift local order="$1"; shift - local mult="$1"; shift kcsan_barrier="" if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,8 +103,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF
@@ -168,22 +167,16 @@ done for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do for order in "" "_acquire" "_release" "_relaxed"; do - gen_xchg "${xchg}" "${order}" "" + gen_xchg "${xchg}" "${order}" printf "\n" done done for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do - gen_xchg "${xchg}" "" "" + gen_xchg "${xchg}" "" printf "\n" done -gen_xchg "cmpxchg_double" "" "2 * " - -printf "\n\n" - -gen_xchg "cmpxchg_double_local" "" "2 * " - cat <<EOF #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
Peter Zijlstra
maintainerPATCH 2/12 types: Introduce [us]128
In reply to Peter ZijlstraIntroduce [us]128 (when available). Unlike [us]64, ensure they are always naturally aligned.
This also enables 128bit wide atomics (which require natural alignment) such as cmpxchg128().
--- include/linux/types.h | 5 +++++ include/uapi/linux/types.h | 4 ++++ lib/crypto/curve25519-hacl64.c | 2 -- lib/crypto/poly1305-donna64.c | 2 -- 4 files changed, 9 insertions(+), 4 deletions(-)
include/linux/types.h +5 -0
@@ -10,6 +10,11 @@#define DECLARE_BITMAP (name,bits) \ unsigned long name[BITS_TO_LONGS(bits)] +#ifdef __SIZEOF_INT128__ +typedef __s128 s128 ; +typedef __u128 u128 ; +#endif +typedef u32 __kernel_dev_t ;typedef __kernel_fd_set fd_set ;
include/uapi/linux/types.h +4 -0
@@ -13,6 +13,10 @@#include <linux/posix_types.h> +#ifdef __SIZEOF_INT128__ +typedef __signed__ __int128 __s128 __attribute__((aligned(16))); +typedef unsigned __int128 __u128 __attribute__((aligned (16 ))); +#endif /* * Below are truly Linux-specific types that should never collide with
lib/crypto/curve25519-hacl64.c +0 -2
@@ -14,8 +14,6 @@#include <crypto/curve25519.h> #include <linux/string.h> -typedef __uint128_t u128 ; - static __always_inline u64 u64_eq_mask(u64 a, u64 b) {u64 x = a ^ b;
lib/crypto/poly1305-donna64.c +0 -2
@@ -10,8 +10,6 @@#include <asm/unaligned.h> #include <crypto/internal/poly1305.h> -typedef __uint128_t u128 ; -void poly1305_core_setkey (struct poly1305_core_key *key,const u8 raw_key[POLY1305_BLOCK_SIZE ]) {
Peter Zijlstra
maintainerPATCH 9/12 x86,intel_iommu: Replace cmpxchg_double()
In reply to Peter Zijlstra--- drivers/iommu/intel/irq_remapping.c | 8 -- include/linux/dmar.h | 125 +++++++++++++++++++----------------- 2 files changed, 68 insertions(+), 65 deletions(-)
drivers/iommu/intel/irq_remapping.c +2 -6
@@ -175,18 +175,14 @@ static int modify_irte(struct irq_2_iomm irte = &iommu->ir_table->base[index];if ((irte->pst ==1 ) || (irte_modified->pst ==1 )) { -bool ret; - - ret =cmpxchg_double (&irte->low, &irte->high, - irte->low, irte->high, - irte_modified->low, irte_modified->high);/* * We use cmpxchg16 to atomically update the 128-bit IRTE, * and it cannot be updated by the hardware or other processors * behind us, so the return value of cmpxchg16 should be the * same as the old value. */ -WARN_ON (!ret); +u128 old = irte->irte; +WARN_ON (!try_cmpxchg128 (&irte->irte, &old, irte_modified->irte)); }else {WRITE_ONCE (irte->low, irte_modified->low);WRITE_ONCE (irte->high, irte_modified->high);
include/linux/dmar.h +66 -59
133 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 3/12 arch: Introduce arch_{,try_}_cmpxchg128{,_local}()
In reply to Peter ZijlstraFor all architectures that currently support cmpxchg_double() implement the cmpxchg128() family of functions that is basically the same but with a saner interface.
--- arch/arm64/include/asm/atomic_ll_sc.h | 41 +++++++++++++++++++++ arch/arm64/include/asm/atomic_lse.h | 31 ++++++++++++++++ arch/arm64/include/asm/cmpxchg.h | 26 +++++++++++++ arch/s390/include/asm/cmpxchg.h | 14 +++++++ arch/x86/include/asm/cmpxchg_32.h | 3 + arch/x86/include/asm/cmpxchg_64.h | 64 +++++++++++++++++++++++++++++++++- 6 files changed, 177 insertions(+), 2 deletions(-)
arch/arm64/include/asm/atomic_ll_sc.h +41 -0
@@ -326,6 +326,47 @@ __CMPXCHG_DBL( , , , ) __CMPXCHG_DBL(_mb, dmb ish, l, "memory")#undef __CMPXCHG_DBL + +union __u128_halves { +u128 full; +struct { +u64 low, high; + }; +}; + +#define __CMPXCHG128(name, mb, rel, cl...) \ +static __always_inline u128 \ +__ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ +{ \ + union __u128_halves r, o = { .full = (old) }, \ + n = { .full = (new) }; \ + unsigned int tmp; \ + \ + asm volatile("// __cmpxchg128" #name "\n" \ + " prfm pstl1strm, %[v]\n" \ + "1: ldxp %[rl], %[rh], %[v]\n" \ + " cmp %[rl], %[ol]\n" \ + " ccmp %[rh], %[oh], 0, eq\n" \ + " b.ne 2f\n" \ + " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \ + " cbnz %w[tmp], 1b\n" \ + " " #mb "\n" \ + "2:" \ + : [v] "+Q" (*(u128 *)ptr), \ + [rl] "=&r" (r.low), [rh] "=&r" (r.high), \ + [tmp] "=&r" (tmp) \ + : [ol] "r" (o.low), [oh] "r" (o.high), \ + [nl] "r" (n.low), [nh] "r" (n.high) \ + : "cc", ##cl); \ + \ + return r.full; \ +} + +__CMPXCHG128( , , ) +__CMPXCHG128(_mb, dmb ish, l, "memory") + +#undef __CMPXCHG128 +#undef K#endif /* __ASM_ATOMIC_LL_SC_H */
arch/arm64/include/asm/atomic_lse.h +31 -0
@@ -317,4 +317,35 @@ __CMPXCHG_DBL(_mb, al, "memory")#undef __CMPXCHG_DBL +#define __CMPXCHG128(name, mb, cl...) \ +static __always_inline u128 \ +__lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ +{ \ + union __u128_halves r, o = { .full = (old) }, \ + n = { .full = (new) }; \ + register unsigned long x0 asm ("x0") = o.low; \ + register unsigned long x1 asm ("x1") = o.high; \ + register unsigned long x2 asm ("x2") = n.low; \ + register unsigned long x3 asm ("x3") = n.high; \ + register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ + \ + asm volatile( \ + __LSE_PREAMBLE \ + " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ + : [old1] "+&r" (x0), [old2] "+&r" (x1), \ + [v] "+Q" (*(u128 *)ptr) \ + : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ + [oldval1] "r" (o.low), [oldval2] "r" (o.high) \ + : cl); \ + \ + r.low = x0; r.high = x1; \ + \ + return r.full; \ +} + +__CMPXCHG128( , ) +__CMPXCHG128(_mb, al, "memory") + +#undef __CMPXCHG128 +#endif /* __ASM_ATOMIC_LSE_H */
arch/arm64/include/asm/cmpxchg.h +26 -0
@@ -146,6 +146,19 @@ __CMPXCHG_DBL(_mb)#undef __CMPXCHG_DBL +#define __CMPXCHG128 (name) \ +static inline u128 __cmpxchg128##name(volatile u128 *ptr, \ + u128 old, u128 new) \ +{ \ + return __lse_ll_sc_body(_cmpxchg128##name, \ + ptr, old, new); \ +} + +__CMPXCHG128( ) +__CMPXCHG128(_mb) + +#undef __CMPXCHG128 +#define __CMPXCHG_GEN (sfx) \ static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \ unsigned long old, \
@@ -228,6 +241,19 @@ __CMPXCHG_GEN(_mb) __ret; \ }) +/* cmpxchg128 */ +#define system_has_cmpxchg128 () 1 + +#define arch_cmpxchg128 (ptr, o, n) \ +({ \ + __cmpxchg128_mb((ptr), (o), (n)); \ +}) + +#define arch_cmpxchg128_local (ptr, o, n) \ +({ \ + __cmpxchg128((ptr), (o), (n)); \ +}) +#define __CMPWAIT_CASE (w, sfx, sz) \ static inline void __cmpwait_case_##sz(volatile void *ptr, \ unsigned long val) \
arch/s390/include/asm/cmpxchg.h +14 -0
@@ -224,4 +224,18 @@ static __always_inline int __cmpxchg_dou (unsigned long)(n1), (unsigned long)(n2)); \ }) +#define system_has_cmpxchg128 () 1 + +static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new) +{ + asmvolatile ( +" cdsg %[old],%[new],%[ptr]\n" + : [old]"+d" (old), [ptr]"+QS" (*ptr) + : [new]"d" (new) + :"memory" ,"cc" ); +return old; +} + +#define arch_cmpxchg128 arch_cmpxchg128 +#endif /* __ASM_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h +2 -1
@@ -103,6 +103,7 @@ static inline bool __try_cmpxchg64(volat#endif -#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8) +#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8) +#define system_has_cmpxchg64 () boot_cpu_has(X86_FEATURE_CX8)#endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h +63 -1
69 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 8/12 x86,amd_iommu: Replace cmpxchg_double()
In reply to Peter Zijlstra--- drivers/iommu/amd/amd_iommu_types.h | 9 +++++++-- drivers/iommu/amd/iommu.c | 10 ++++------ 2 files changed, 11 insertions(+), 8 deletions(-)
drivers/iommu/amd/amd_iommu_types.h +7 -2
@@ -986,8 +986,13 @@ union irte_ga_hi { };struct irte_ga { -union irte_ga_lo lo; -union irte_ga_hi hi; +union { +struct { +union irte_ga_lo lo; +union irte_ga_hi hi; + }; +u128 irte; + }; };struct irq_2_irte {
drivers/iommu/amd/iommu.c +4 -6
@@ -3003,10 +3003,10 @@ static int alloc_irq_index(struct amd_iostatic int modify_irte_ga (struct amd_iommu *iommu,u16 devid,int index,struct irte_ga *irte,struct amd_ir_data *data) { -bool ret;struct irq_remap_table *table; -unsigned long flags;struct irte_ga *entry; +unsigned long flags; +u128 old; table =get_irq_table (iommu, devid);if (!table)
@@ -3017,16 +3017,14 @@ static int modify_irte_ga(struct amd_iom entry = (struct irte_ga *)table->table; entry = &entry[index]; - ret =cmpxchg_double (&entry->lo.val, &entry->hi.val, - entry->lo.val, entry->hi.val, - irte->lo.val, irte->hi.val);/* * We use cmpxchg16 to atomically update the 128-bit IRTE, * and it cannot be updated by the hardware or other processors * behind us, so the return value of cmpxchg16 should be the * same as the old value. */ -WARN_ON (!ret); + old = entry->irte; +WARN_ON (!try_cmpxchg128 (&entry->irte, &old, irte->irte));if (data) data->ref = entry;
Arnd Bergmann
maintainerRe: [PATCH 07/12] percpu: #ifndef __SIZEOF_INT128__
In reply to Peter ZijlstraQuoted 3 lines · Peter Zijlstra
Some 64bit architectures do not advertise
__SIZEOF_INT128__on all supported compiler versions. Notably the HPPA64 only started doing with GCC-11.
I checked the other compilers to be sure that anything else we support (gcc-5.1 and up) across all 64-bit architectures does support int128.
It would be nice to have the hack more localized to parisc and guarded with a CONFIG_GCC_VERSION check so we can kill it off in the future, once we drop either gcc-10 or parisc support.
Quoted 12 lines
include/asm-generic/percpu.h +12
-0+#ifndef __SIZEOF_INT128__ +#define raw_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (!__builtin_memcmp(&__val, &__old, sizeof(pcp))) { \ + *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \
include/asm-generic/percpu.h do { \ · line 337
Quoted 10 lines
include/asm-generic/percpu.h +10
-0+ } \ + __ret; \ +}) + +#define raw_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg_memcpy(pcp, &__old, nval); \ + __old; \ +})
Instead of having this in include/asm-generic under !__SIZEOF_INT128__, could you just move this into the parisc files with a compiler version check?
Arnd
Quoted 16 lines · Peter Zijlstra
Hi!
After much breaking of things, find here the improved version.
Since v3:
- unbreak everything that does *NOT* have
cmpxchg128()Notably
this_cpu_cmpxchg_double()is used unconditionally by SLUB which means thatthis_cpu_try_cmpxchg128()needs to be unconditionally available on all 64bit architectures.- fixed up x86/x86_64 cmpxchg{8,16}b emulation for this_cpu_cmpxchg{64,128}()
- introduce {raw,this}_cpu_try_cmpxchg*()
Quoted 8 lines
- add fallback for !__SIZEOF_INT128__ 64bit architectures
Sadly there are supported 64bit architecture/compiler combinations that do not have
__SIZEOF_INT128__, specifically it was found that HPPA64 only added this with GCC-11.this is yuck, and ideally we'd simply raise compiler requirements, but this 'works'.
The patches look good to me, and I used my local cross-build script to build test this with the kernel.org GCC 10.3.0 cross toolchain for all of the following arch/triplet/config combinations:
alpha alpha-linux defconfig arc arc-linux defconfig arm arm-linux-gnueabi multi_v4t_defconfig arm arm-linux-gnueabi multi_v5_defconfig arm arm-linux-gnueabi multi_v7_defconfig arm arm-linux-gnueabi omap1_defconfig arm64 aarch64-linux defconfig csky csky-linux defconfig i386 i386-linux defconfig ia64 ia64-linux defconfig m68k m68k-linux defconfig microblaze microblaze-linux defconfig mips mips-linux 32r1_defconfig mips mips-linux 32r2_defconfig mips mips-linux 32r6_defconfig mips mips64-linux 64r1_defconfig mips mips64-linux 64r2_defconfig mips mips64-linux 64r6_defconfig nios2 nios2-linux defconfig openrisc or1k-linux defconfig parisc hppa-linux generic-32bit_defconfig parisc hppa64-linux generic-64bit_defconfig powerpc powerpc-linux ppc40x_defconfig powerpc powerpc64-linux ppc64_defconfig powerpc powerpc64-linux ppc64e_defconfig riscv riscv32-linux rv32_defconfig riscv riscv64-linux defconfig s390 s390-linux defconfig sh sh4-linux defconfig sparc sparc-linux sparc32_defconfig sparc sparc64-linux sparc64_defconfig x86_64 x86_64-linux defconfig xtensa xtensa-linux defconfig
... and everything seemed happy.
I've also boot-tested arm64 defconfig.
So FWIW, for the series:
Quoted 1 line
My plan is to re-add this to tip/locking/core and thus -next later this week.
I'll need to rebase my kerneldoc series atop this, so getting this into a stable branch soon would be great!
Thanks,
Mark.
Quoted 42 lines
Also available at:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.gitlocking/core--- Documentation/core-api/this_cpu_ops.rst | 2 - arch/arm64/include/asm/atomic_ll_sc.h | 56 +++--- arch/arm64/include/asm/atomic_lse.h | 39 ++--- arch/arm64/include/asm/cmpxchg.h | 48 ++---- arch/arm64/include/asm/percpu.h | 30 ++-- arch/s390/include/asm/cmpxchg.h | 32 +--- arch/s390/include/asm/cpu_mf.h | 2 +- arch/s390/include/asm/percpu.h | 34 ++-- arch/s390/kernel/perf_cpum_sf.c | 16 +- arch/x86/include/asm/cmpxchg.h | 25 --- arch/x86/include/asm/cmpxchg_32.h | 2 +- arch/x86/include/asm/cmpxchg_64.h | 63 ++++++- arch/x86/include/asm/percpu.h | 102 ++++++----- arch/x86/lib/Makefile | 3 +- arch/x86/lib/cmpxchg16b_emu.S | 43 +++-- arch/x86/lib/cmpxchg8b_emu.S | 67 ++++++-- drivers/iommu/amd/amd_iommu_types.h | 9 +- drivers/iommu/amd/iommu.c | 10 +- drivers/iommu/intel/irq_remapping.c | 8 +- include/asm-generic/percpu.h | 257 ++++++++++++++++++++++------ include/crypto/b128ops.h | 14 +- include/linux/atomic/atomic-arch-fallback.h | 95 +++++++++- include/linux/atomic/atomic-instrumented.h | 93 ++++++++-- include/linux/dmar.h | 125 +++++++------- include/linux/percpu-defs.h | 45 ++--- include/linux/slub_def.h | 12 +- include/linux/types.h | 12 ++ include/uapi/linux/types.h | 4 + lib/crypto/curve25519-hacl64.c | 2 - lib/crypto/poly1305-donna64.c | 2 - mm/slab.h | 53 +++++- mm/slub.c | 139 +++++++++------ scripts/atomic/gen-atomic-fallback.sh | 4 +- scripts/atomic/gen-atomic-instrumented.sh | 19 +- 34 files changed, 952 insertions(+), 515 deletions(-)
Peter Zijlstra
maintainerRe: [PATCH 07/12] percpu: #ifndef __SIZEOF_INT128__
In reply to Arnd BergmannQuoted 8 lines · Arnd Bergmann
Quoted 3 lines · Peter Zijlstra
Some 64bit architectures do not advertise
__SIZEOF_INT128__on all supported compiler versions. Notably the HPPA64 only started doing with GCC-11.I checked the other compilers to be sure that anything else we support (gcc-5.1 and up) across all 64-bit architectures does support int128.
Oh excellent -- I didn't have sufficient old cross compilers to verify, hence me not doing what you suggest below.
If HPPA64 really is the only one so affected, then yes, I can move this hack into arch/parisc.
Peter Zijlstra
maintainerPATCH v2 7/12 parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Arnd BergmannQuoted 4 lines · Arnd Bergmann
It would be nice to have the hack more localized to parisc and guarded with a
CONFIG_GCC_VERSIONcheck so we can kill it off in the future, once we drop either gcc-10 or parisc support.
I vote for dropping parisc -- it's the only 64bit arch that doesn't have sane atomics.
Anyway, the below seems to work -- build tested with GCC-10.1
---
Subject: parisc/percpu: Work around the lack of __SIZEOF_INT128__
From: Peter Zijlstra <[email protected]>
Date: Tue May 30 22:27:40 CEST 2023
HPPA64 is unique in not providing __SIZEOF_INT128__ across all supported compilers, specifically it only started doing this with GCC-11.
Since the per-cpu ops are universally availably, and this_cpu_{,try_}cmpxchg128() is expected to be available on all 64bit architectures a wee bodge is in order.
Sadly, while C reverts to memcpy() for assignment of POD types, it does not revert to memcmp() for for equality. Therefore frob that manually.
--- arch/parisc/include/asm/percpu.h | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+)
arch/parisc/include/asm/percpu.h new file +77 -0
@@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PARISC_PERCPU_H +#define _ASM_PARISC_PERCPU_H + +#include <linux/types.h> + +#if defined(CONFIG_64BIT ) &&CONFIG_GCC_VERSION <1100000 + +/* +* GCC prior to 11 does not provide __SIZEOF_INT128__ on HPPA64 +* as such we need to provide an alternative implementation of +* {raw,this}_cpu_{,try_}cmpxchg128(). +* +* This obviously doesn't function as u128 should, but for the purpose +* of per-cpu cmpxchg128 it might just do. +*/ +typedef struct { +u64 a, b; +}u128 __attribute__((aligned (16 ))); + +#define raw_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (!__builtin_memcmp(&__val, &__old, sizeof(pcp))) { \ + *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ + __ret; \ +}) + +#define raw_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg_memcpy(pcp, &__old, nval); \ + __old; \ +}) + +#define raw_cpu_cmpxchg128 (pcp, oval, nval) \ + raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) +#define raw_cpu_try_cmpxchg128 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) + +#define this_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + bool __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) + +#define this_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) + +#define this_cpu_cmpxchg128 (pcp, oval, nval) \ + this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) +#define this_cpu_try_cmpxchg128 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) + +#endif /* !__SIZEOF_INT128__ */ + +#include <asm-generic/percpu.h> + +#endif /* _ASM_PARISC_PERCPU_H */
Helge Deller
maintainerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 9 lines · Peter Zijlstra
Quoted 4 lines · Arnd Bergmann
It would be nice to have the hack more localized to parisc and guarded with a
CONFIG_GCC_VERSIONcheck so we can kill it off in the future, once we drop either gcc-10 or parisc support.I vote for dropping parisc -- it's the only 64bit arch that doesn't have sane atomics.
Of course I'm against dropping parisc.
Quoted 1 line
Anyway, the below seems to work -- build tested with GCC-10.1
I don't think we need to care about gcc-10 on parisc.
Debian and Gentoo are the only supported distributions, while Debian requires gcc-12 to build > 6.x kernels, and I assume Gentoo uses at least gcc-12 as well.
So raising the gcc limit for parisc only (at least temporarily for now) should be fine and your workaround below wouldn't be necessary, right?
Helge
Quoted 101 lines
---
Subject: parisc/percpu: Work around the lack of__SIZEOF_INT128__
From: Peter Zijlstra <[email protected]>
Date: Tue May 30 22:27:40 CEST 2023HPPA64 is unique in not providing
__SIZEOF_INT128__across all supported compilers, specifically it only started doing this with GCC-11.Since the per-cpu ops are universally availably, and this_cpu_{,try_}cmpxchg128() is expected to be available on all 64bit architectures a wee bodge is in order.
Sadly, while C reverts to
memcpy()for assignment of POD types, it does not revert tomemcmp()for for equality. Therefore frob that manually.Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/parisc/include/asm/percpu.h | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+)arch/parisc/include/asm/percpu.h new file +77
-0@@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PARISC_PERCPU_H +#define _ASM_PARISC_PERCPU_H + +#include <linux/types.h> + +#if defined(CONFIG_64BIT ) &&CONFIG_GCC_VERSION <1100000 + +/* +* GCC prior to 11 does not provide __SIZEOF_INT128__ on HPPA64 +* as such we need to provide an alternative implementation of +* {raw,this}_cpu_{,try_}cmpxchg128(). +* +* This obviously doesn't function as u128 should, but for the purpose +* of per-cpu cmpxchg128 it might just do. +*/ +typedef struct { +u64 a, b; +}u128 __attribute__((aligned (16 ))); + +#define raw_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (!__builtin_memcmp(&__val, &__old, sizeof(pcp))) { \ + *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ + __ret; \ +}) + +#define raw_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg_memcpy(pcp, &__old, nval); \ + __old; \ +}) + +#define raw_cpu_cmpxchg128 (pcp, oval, nval) \ + raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) +#define raw_cpu_try_cmpxchg128 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) + +#define this_cpu_generic_try_cmpxchg_memcmp (pcp, ovalp, nval) \ +({ \ + bool __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) + +#define this_cpu_generic_cmpxchg_memcmp (pcp, oval, nval) \ +({ \ + typeof(pcp) __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) + +#define this_cpu_cmpxchg128 (pcp, oval, nval) \ + this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) +#define this_cpu_try_cmpxchg128 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) + +#endif /* !__SIZEOF_INT128__ */ + +#include <asm-generic/percpu.h> + +#endif /* _ASM_PARISC_PERCPU_H */
Peter Zijlstra
maintainerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 8 lines · Helge Deller
Quoted 7 lines · Peter Zijlstra
Quoted 4 lines · Arnd Bergmann
It would be nice to have the hack more localized to parisc and guarded with a
CONFIG_GCC_VERSIONcheck so we can kill it off in the future, once we drop either gcc-10 or parisc support.
Quoted 4 lines
Quoted 2 lines
I vote for dropping parisc -- it's the only 64bit arch that doesn't have sane atomics.
Of course I'm against dropping parisc.
:-)
Quoted 9 lines
Quoted 1 line
Anyway, the below seems to work -- build tested with GCC-10.1
I don't think we need to care about gcc-10 on parisc.
Debian and Gentoo are the only supported distributions, while Debian requires gcc-12 to build > 6.x kernels, and I assume Gentoo uses at least gcc-12 as well.So raising the gcc limit for parisc only (at least temporarily for now) should be fine and your workaround below wouldn't be necessary, right?
Correct, if you're willing to set minimum GCC version to 11 for parisc all is well and this patch can go play in the bit bucket.
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Helge DellerQuoted 8 lines · Helge Deller
I don't think we need to care about gcc-10 on parisc.
Debian and Gentoo are the only supported distributions, while Debian requires gcc-12 to build > 6.x kernels, and I assume Gentoo uses at least gcc-12 as well.So raising the gcc limit for parisc only (at least temporarily for now) should be fine and your workaround below wouldn't be necessary, right?
This absolutely sounds like the right option. Let's simplify the problem space by just saying that parisc needs the newer compiler.
Right now we have that "minimum gcc version" in a somewhat annoying
place: it's in the ./scripts/min-tool-version.sh file as a shell script.
I wonder if we could move the gcc minimum version check into the Kconfig file instead, and make it easier to let architectures override the minimum version.
I don't quite know how to do that sanely, though. I don't think we have a sane way to error out at Kconfig time (except by forcing some syntax error inside an 'if' statement or something horrendously hacky like that).
Added Masahiro to the (already overlong) participants list.
Linus
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 14 lines · Linus Torvalds
Quoted 8 lines · Helge Deller
I don't think we need to care about gcc-10 on parisc.
Debian and Gentoo are the only supported distributions, while Debian requires gcc-12 to build > 6.x kernels, and I assume Gentoo uses at least gcc-12 as well.So raising the gcc limit for parisc only (at least temporarily for now) should be fine and your workaround below wouldn't be necessary, right?
This absolutely sounds like the right option. Let's simplify the problem space by just saying that parisc needs the newer compiler.
Quoted 7 lines
Right now we have that "minimum gcc version" in a somewhat annoying
place: it's in the./scripts/min-tool-version.shfile as a shell script.I wonder if we could move the gcc minimum version check into the
Kconfigfile instead, and make it easier to let architectures override the minimum version.
Currently, it is invoked in the Kconfig time,
but not directly in Kconfig files.
scripts/Kconfig.include
-> scripts/cc-version.sh
-> scripts/min-tool-version.sh
It would be ugly if we wrote the equivalent code
directly in Kconfig files.
Quoted 5 lines
I don't quite know how to do that sanely, though. I don't think we have a sane way to error out at
Kconfigtime (except by forcing some syntax error inside an 'if' statement or something horrendously hacky like that).
The parse stage can fail by $(error-if ) macro, but the evaluation stage never fails. I think it is a design.
I think checking the compiler version during the parse stage makes sense given the current situation. The compiler version is fixed when Kconfig starts. If the compiler is found to be too old, there is no meaning to proceed.
You suggested to choose a compiler in the Kconfig time:
https://lore.kernel.org/lkml/CAHk-=whdrvCkSWh=BRrwZwNo3=yLBXXM88NGx8VEpP1VTgmkyQ@mail.gmail.com/
When we achieve that, moving the min version to Kconfig files will be the right thing to do. Then, everything will be evaluated dynamically.
Quoted 4 lines
Added Masahiro to the (already overlong) participants list.
Linus
Signature · 2 lines
Best Regards Masahiro Yamada
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Peter ZijlstraQuoted 23 lines · Peter Zijlstra
Quoted 8 lines · Helge Deller
Quoted 7 lines · Peter Zijlstra
Quoted 4 lines · Arnd Bergmann
It would be nice to have the hack more localized to parisc and guarded with a
CONFIG_GCC_VERSIONcheck so we can kill it off in the future, once we drop either gcc-10 or parisc support.Quoted 4 lines
Quoted 2 lines
I vote for dropping parisc -- it's the only 64bit arch that doesn't have sane atomics.
Of course I'm against dropping parisc.
:-)
Quoted 7 lines
Quoted 1 line
Anyway, the below seems to work -- build tested with GCC-10.1
I don't think we need to care about gcc-10 on parisc.
Debian and Gentoo are the only supported distributions, while Debian requires gcc-12 to build > 6.x kernels, and I assume Gentoo uses at least gcc-12 as well.
Quoted 5 lines
Quoted 2 lines
So raising the gcc limit for parisc only (at least temporarily for now) should be fine and your workaround below wouldn't be necessary, right?
Correct, if you're willing to set minimum GCC version to 11 for parisc all is well and this patch can go play in the bit bucket.
It's fine for us in Gentoo, thanks!
Peter Zijlstra
maintainerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Linus TorvaldsQuoted 3 lines · Linus Torvalds
Right now we have that "minimum gcc version" in a somewhat annoying
place: it's in the./scripts/min-tool-version.shfile as a shell script.
Something like so then?
---
Subject: parisc: Raise minimal GCC version
From: Peter Zijlstra <[email protected]>
Date: Fri Jun 2 16:33:54 CEST 2023
With 64bit builds depending on __SIZEOF_INT128__ raise the parisc minimum compiler version to gcc-11.0.0.
All other 64bit architectures provide this from GCC-5.1.0 (and probably before), except hppa64 which only started advertising this with GCC-11.
--- scripts/min-tool-version.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
scripts/min-tool-version.sh +5 -1
@@ -17,7 +17,11 @@ binutils) echo 2.25.0 ;; gcc) - echo 5.1.0 + if [ "$SRCARCH" = parisc ]; then + echo 11.0.0 + else + echo 5.1.0 + fi ;; llvm) if [ "$SRCARCH" = s390 ]; then
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 30 lines · Peter Zijlstra
Quoted 3 lines · Linus Torvalds
Right now we have that "minimum gcc version" in a somewhat annoying
place: it's in the./scripts/min-tool-version.shfile as a shell script.Something like so then?
---
Subject: parisc: Raise minimal GCC version
From: Peter Zijlstra <[email protected]>
Date: Fri Jun 2 16:33:54 CEST 2023With 64bit builds depending on
__SIZEOF_INT128__raise the parisc minimum compiler version to gcc-11.0.0.All other 64bit architectures provide this from GCC-5.1.0 (and probably before), except hppa64 which only started advertising this with GCC-11.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- scripts/min-tool-version.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)scripts/min-tool-version.sh +0
-0@@ -17,7 +17,11 @@ binutils) echo 2.25.0 ;;
scripts/min-tool-version.sh binutils) · line 27
Quoted 10 lines
scripts/min-tool-version.sh +5
-1gcc) - echo 5.1.0 + if [ "$SRCARCH" = parisc ]; then + echo 11.0.0 + else + echo 5.1.0 + fi ;; llvm) if [ "$SRCARCH" = s390 ]; then
I gave this a spin and it looks good to me:
[mark@lakrids:~/src/linux]% usekorg 10.3.0 make ARCH=arm64 CROSS_COMPILE=aarch64-linux- defconfig
HOSTCC scripts/basic/fixdep HOSTCC scripts/kconfig/conf.o HOSTCC scripts/kconfig/confdata.o HOSTCC scripts/kconfig/expr.o LEX scripts/kconfig/lexer.lex.c YACC scripts/kconfig/parser.tab.[ch] HOSTCC scripts/kconfig/lexer.lex.o HOSTCC scripts/kconfig/menu.o HOSTCC scripts/kconfig/parser.tab.o HOSTCC scripts/kconfig/preprocess.o HOSTCC scripts/kconfig/symbol.o HOSTCC scripts/kconfig/util.o HOSTLD scripts/kconfig/conf *** Default configuration is based on 'defconfig' # # configuration written to .config # [mark@lakrids:~/src/linux]% usekorg 10.3.0 make ARCH=parisc CROSS_COMPILE=hppa64-linux- generic-64bit_defconfig *** *** C compiler is too old. *** Your GCC version: 10.3.0 *** Minimum GCC version: 11.0.0 *** scripts/Kconfig.include:44: Sorry, this C compiler is not supported. make[1]: *** [scripts/kconfig/Makefile:94: generic-64bit_defconfig] Error 1 make: *** [Makefile:692: generic-64bit_defconfig] Error 2 [mark@lakrids:~/src/linux]% usekorg 11.3.0 make ARCH=parisc CROSS_COMPILE=hppa64-linux- generic-64bit_defconfig # # configuration written to .config #
FWIW:
Mark.
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Peter ZijlstraQuoted 2 lines · Peter Zijlstra
Something like so then?
Ack. I think it would be much cleaner if we would have it as part of the Kconfig file and architectures could just override some GCC_MIN_VERSION value, but that's not the universe we currently have, so your patch looks like the best thing to do.
Linus
H. Peter Anvin
reviewerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Peter ZijlstraQuoted 30 lines · Peter Zijlstra
Quoted 3 lines · Linus Torvalds
Right now we have that "minimum gcc version" in a somewhat annoying
place: it's in the./scripts/min-tool-version.shfile as a shell script.Something like so then?
---
Subject: parisc: Raise minimal GCC version
From: Peter Zijlstra <[email protected]>
Date: Fri Jun 2 16:33:54 CEST 2023With 64bit builds depending on
__SIZEOF_INT128__raise the parisc minimum compiler version to gcc-11.0.0.All other 64bit architectures provide this from GCC-5.1.0 (and probably before), except hppa64 which only started advertising this with GCC-11.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
scripts/min-tool-version.sh| 6 +++++-1 file changed, 5 insertions(+), 1 deletion(-)scripts/min-tool-version.sh +0
-0@@ -17,7 +17,11 @@ binutils) echo 2.25.0 ;;
scripts/min-tool-version.sh binutils) · line 27
Quoted 10 lines
scripts/min-tool-version.sh +5
-1gcc) - echo 5.1.0 + if [ "$SRCARCH" = parisc ]; then + echo 11.0.0 + else + echo 5.1.0 + fi ;; llvm) if [ "$SRCARCH" = s390 ]; then
Dumb question: is this only about the cpp macro or is it about __int128 existing at all?
Peter Zijlstra
maintainerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 1 line · H. Peter Anvin
Dumb question: is this only about the cpp macro or is it about
__int128existing at all?
It's mostly about __int128 being there, __SIZEOF_INT128__ is just the way we go about detecting if it's there or not.
H. Peter Anvin
reviewerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Ok. So the patch description needs to be fixed. Otherwise the solution would be far simpler :)
Peter Zijlstra
maintainerRe: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 1 line · H. Peter Anvin
Ok. So the patch description needs to be fixed. Otherwise the solution would be far simpler :)
"With 64bit builds depending on __SIZEOF_INT128__ to detect the presence of __int128 raise the parisc minimum compiler version to gcc-11.0.0."
better?
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
Quoted 6 lines · Peter Zijlstra
"With 64bit builds depending on
__SIZEOF_INT128__to detect the presence of__int128raise the parisc minimum compiler version to gcc-11.0.0."better?
I'd just say "64-bit targets need the __int128 type, which for pa-risc means raising the minimum gcc version to 11".
The __SIZEOF_INT128__ part isn't the important part. That's just the symptom.
Linus
Re: [PATCH v2 07/12] parisc/percpu: Work around the lack of __SIZEOF_INT128__
In reply to Peter ZijlstraQuoted 13 lines · Peter Zijlstra
Quoted 3 lines · Linus Torvalds
Right now we have that "minimum gcc version" in a somewhat annoying
place: it's in the./scripts/min-tool-version.shfile as a shell script.Something like so then?
---
Subject: parisc: Raise minimal GCC version
From: Peter Zijlstra <[email protected]>
Date: Fri Jun 2 16:33:54 CEST 2023
Quoted 8 lines
With 64bit builds depending on
__SIZEOF_INT128__raise the parisc minimum compiler version to gcc-11.0.0.All other 64bit architectures provide this from GCC-5.1.0 (and probably before), except hppa64 which only started advertising this with GCC-11.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
The patch raises the compiler for 32- and 64-bit parisc builds, but that's OK.
So:
Thank you!
Helge
Quoted 19 lines
--- scripts/min-tool-version.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)scripts/min-tool-version.sh +5
-1@@ -17,7 +17,11 @@ binutils) echo 2.25.0 ;; gcc) - echo 5.1.0 + if [ "$SRCARCH" = parisc ]; then + echo 11.0.0 + else + echo 5.1.0 + fi ;; llvm) if [ "$SRCARCH" = s390 ]; then
[tip: locking/core] s390/cpum_sf: Convert to cmpxchg128()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 497cc42bf53b55185ab3d39c634fbf09eb6681ae Gitweb: https://git.kernel.org/tip/497cc42bf53b55185ab3d39c634fbf09eb6681ae Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:45 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:40 +02:00
s390/cpum_sf: Convert to cmpxchg128()
Now that there is a cross arch u128 and cmpxchg128(), use those instead of the custom CDSG helper.
--- arch/s390/include/asm/cpu_mf.h | 2 +- arch/s390/kernel/perf_cpum_sf.c | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-)
arch/s390/include/asm/cpu_mf.h +1 -1
@@ -140,7 +140,7 @@ union hws_trailer_header { unsigned int dsdes:16; /* 48-63: size of diagnostic SDE */unsigned long long overflow;/* 64 - Overflow Count */ }; -__uint128_t val; +u128 val; };struct hws_trailer_entry {
arch/s390/kernel/perf_cpum_sf.c +3 -13
@@ -1271,16 +1271,6 @@ static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, } } -static inline __uint128_t __cdsg (__uint128_t *ptr,__uint128_t old,__uint128_t new) -{ - asmvolatile ( -" cdsg %[old],%[new],%[ptr]\n" - : [old]"+d" (old), [ptr]"+QS" (*ptr) - : [new]"d" (new) - :"memory" ,"cc" ); -return old; -} -/* hw_perf_event_update() - Process sampling buffer * @event: The perf event * @flush_all: Flag to also flush partially filled sample-data-blocks
@@ -1352,7 +1342,7 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) new.f =0 ; new.a =1 ; new.overflow =0 ; - prev.val =__cdsg (&te->header.val, old.val, new.val); + prev.val =cmpxchg128 (&te->header.val, old.val, new.val); }while (prev.val != old.val);/* Advance to next sample-data-block */
@@ -1562,7 +1552,7 @@ static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index, } new.a =1 ; new.overflow =0 ; - prev.val =__cdsg (&te->header.val, old.val, new.val); + prev.val =cmpxchg128 (&te->header.val, old.val, new.val); }while (prev.val != old.val);return true; }
@@ -1636,7 +1626,7 @@ static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, new.a =1 ; else new.a = 0; - prev.val =__cdsg (&te->header.val, old.val, new.val); + prev.val =cmpxchg128 (&te->header.val, old.val, new.val); }while (prev.val != old.val); *overflow += orig_overflow; }
[tip: locking/core] x86,amd_iommu: Replace cmpxchg_double()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 0a0a6800b02489c1288f963ad290b6a4876a2cc8 Gitweb: https://git.kernel.org/tip/0a0a6800b02489c1288f963ad290b6a4876a2cc8 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:41 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:38 +02:00
x86,amd_iommu: Replace cmpxchg_double()
--- drivers/iommu/amd/amd_iommu_types.h | 9 +++++++-- drivers/iommu/amd/iommu.c | 10 ++++------ 2 files changed, 11 insertions(+), 8 deletions(-)
drivers/iommu/amd/amd_iommu_types.h +7 -2
@@ -986,8 +986,13 @@ union irte_ga_hi { };struct irte_ga { -union irte_ga_lo lo; -union irte_ga_hi hi; +union { +struct { +union irte_ga_lo lo; +union irte_ga_hi hi; + }; +u128 irte; + }; };struct irq_2_irte {
drivers/iommu/amd/iommu.c +4 -6
@@ -3003,10 +3003,10 @@ out:static int modify_irte_ga (struct amd_iommu *iommu,u16 devid,int index,struct irte_ga *irte,struct amd_ir_data *data) { -bool ret;struct irq_remap_table *table; -unsigned long flags;struct irte_ga *entry; +unsigned long flags; +u128 old; table =get_irq_table (iommu, devid);if (!table)
@@ -3017,16 +3017,14 @@ static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, entry = (struct irte_ga *)table->table; entry = &entry[index]; - ret =cmpxchg_double (&entry->lo.val, &entry->hi.val, - entry->lo.val, entry->hi.val, - irte->lo.val, irte->hi.val);/* * We use cmpxchg16 to atomically update the 128-bit IRTE, * and it cannot be updated by the hardware or other processors * behind us, so the return value of cmpxchg16 should be the * same as the old value. */ -WARN_ON (!ret); + old = entry->irte; +WARN_ON (!try_cmpxchg128 (&entry->irte, &old, irte->irte));if (data) data->ref = entry;
[tip: locking/core] x86,intel_iommu: Replace cmpxchg_double()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: b1fe7f2cda2a003afe316ce8dfe8d3645694a67e Gitweb: https://git.kernel.org/tip/b1fe7f2cda2a003afe316ce8dfe8d3645694a67e Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:42 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:38 +02:00
x86,intel_iommu: Replace cmpxchg_double()
--- drivers/iommu/intel/irq_remapping.c | 8 +-- include/linux/dmar.h | 125 ++++++++++++++------------- 2 files changed, 68 insertions(+), 65 deletions(-)
drivers/iommu/intel/irq_remapping.c +2 -6
@@ -175,18 +175,14 @@ static int modify_irte(struct irq_2_iommu *irq_iommu, irte = &iommu->ir_table->base[index];if ((irte->pst ==1 ) || (irte_modified->pst ==1 )) { -bool ret; - - ret =cmpxchg_double (&irte->low, &irte->high, - irte->low, irte->high, - irte_modified->low, irte_modified->high);/* * We use cmpxchg16 to atomically update the 128-bit IRTE, * and it cannot be updated by the hardware or other processors * behind us, so the return value of cmpxchg16 should be the * same as the old value. */ -WARN_ON (!ret); +u128 old = irte->irte; +WARN_ON (!try_cmpxchg128 (&irte->irte, &old, irte_modified->irte)); }else {WRITE_ONCE (irte->low, irte_modified->low);WRITE_ONCE (irte->high, irte_modified->high);
include/linux/dmar.h +66 -59
133 lines not on this page · raw message
[tip: locking/core] slub: Replace cmpxchg_double()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 6801be4f2653e5fdddca73b527cf0728284ba8a3 Gitweb: https://git.kernel.org/tip/6801be4f2653e5fdddca73b527cf0728284ba8a3 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:43 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:39 +02:00
slub: Replace cmpxchg_double()
--- include/linux/slub_def.h | 12 ++- mm/slab.h | 53 ++++++++++++-- mm/slub.c | 139 ++++++++++++++++++++++---------------- 3 files changed, 137 insertions(+), 67 deletions(-)
include/linux/slub_def.h +9 -3
@@ -39,7 +39,8 @@ enum stat_item {CPU_PARTIAL_FREE ,/* Refill cpu partial on free */ CPU_PARTIAL_NODE ,/* Refill cpu partial from node partial */ CPU_PARTIAL_DRAIN ,/* Drain cpu partial to node partial */ - NR_SLUB_STAT_ITEMS }; + NR_SLUB_STAT_ITEMS +};#ifndef CONFIG_SLUB_TINY /*
@@ -47,8 +48,13 @@ enum stat_item {* with this_cpu_cmpxchg_double() alignment requirements. */ struct kmem_cache_cpu { -void **freelist;/* Pointer to next available object */ -unsigned long tid;/* Globally unique transaction id */ +union { +struct { +void **freelist;/* Pointer to next available object */ +unsigned long tid;/* Globally unique transaction id */ + }; +freelist_aba_t freelist_tid; + };struct slab *slab;/* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL struct slab *partial;/* Partially allocated frozen slabs */
mm/slab.h +46 -7
74 lines not on this page · raw message
mm/slub.c +82 -57
238 lines not on this page · raw message
[tip: locking/core] arch: Remove cmpxchg_double
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: febe950dbfb464799beb0339cc6fb10699f4a5da Gitweb: https://git.kernel.org/tip/febe950dbfb464799beb0339cc6fb10699f4a5da Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:44 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:39 +02:00
arch: Remove cmpxchg_double
No moar users, remove the monster.
--- Documentation/core-api/this_cpu_ops.rst | 2 +- arch/arm64/include/asm/atomic_ll_sc.h | 33 +------------ arch/arm64/include/asm/atomic_lse.h | 36 +------------- arch/arm64/include/asm/cmpxchg.h | 46 +----------------- arch/arm64/include/asm/percpu.h | 10 +---- arch/s390/include/asm/cmpxchg.h | 34 +------------ arch/s390/include/asm/percpu.h | 18 +------- arch/x86/include/asm/cmpxchg.h | 25 +--------- arch/x86/include/asm/cmpxchg_32.h | 1 +- arch/x86/include/asm/cmpxchg_64.h | 1 +- arch/x86/include/asm/percpu.h | 42 +--------------- include/asm-generic/percpu.h | 58 +--------------------- include/linux/atomic/atomic-instrumented.h | 17 +------ include/linux/percpu-defs.h | 38 +-------------- scripts/atomic/gen-atomic-instrumented.sh | 15 +---- 15 files changed, 5 insertions(+), 371 deletions(-)
Documentation/core-api/this_cpu_ops.rst +0 -2
@@ -53,7 +53,6 @@ preemption and interrupts:: this_cpu_add_return(pcp, val) this_cpu_xchg(pcp, nval) this_cpu_cmpxchg(pcp, oval, nval) - this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) this_cpu_sub(pcp, val) this_cpu_inc(pcp) this_cpu_dec(pcp)
@@ -242,7 +241,6 @@ safe:: __this_cpu_add_return(pcp, val) __this_cpu_xchg(pcp, nval) __this_cpu_cmpxchg(pcp, oval, nval) - __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) __this_cpu_sub(pcp, val) __this_cpu_inc(pcp) __this_cpu_dec(pcp)
arch/arm64/include/asm/atomic_ll_sc.h +0 -33
@@ -294,39 +294,6 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L)#undef __CMPXCHG_CASE -#define __CMPXCHG_DBL (name, mb, rel, cl) \ -static __always_inline long \ -__ll_sc__cmpxchg_double##name(unsigned long old1, \ - unsigned long old2, \ - unsigned long new1, \ - unsigned long new2, \ - volatile void *ptr) \ -{ \ - unsigned long tmp, ret; \ - \ - asm volatile("// __cmpxchg_double" #name "\n" \ - " prfm pstl1strm, %2\n" \ - "1: ldxp %0, %1, %2\n" \ - " eor %0, %0, %3\n" \ - " eor %1, %1, %4\n" \ - " orr %1, %0, %1\n" \ - " cbnz %1, 2f\n" \ - " st" #rel "xp %w0, %5, %6, %2\n" \ - " cbnz %w0, 1b\n" \ - " " #mb "\n" \ - "2:" \ - : "=&r" (tmp), "=&r" (ret), "+Q" (*(__uint128_t *)ptr) \ - : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \ - : cl); \ - \ - return ret; \ -} - -__CMPXCHG_DBL( , , , ) -__CMPXCHG_DBL(_mb, dmb ish, l, "memory") - -#undef __CMPXCHG_DBL -union __u128_halves {u128 full;struct {
arch/arm64/include/asm/atomic_lse.h +0 -36
@@ -281,42 +281,6 @@ __CMPXCHG_CASE(x, , mb_, 64, al, "memory")#undef __CMPXCHG_CASE -#define __CMPXCHG_DBL(name, mb, cl...) \ -static __always_inline long \ -__lse__cmpxchg_double##name(unsigned long old1, \ - unsigned long old2, \ - unsigned long new1, \ - unsigned long new2, \ - volatile void *ptr) \ -{ \ - unsigned long oldval1 = old1; \ - unsigned long oldval2 = old2; \ - register unsigned long x0 asm ("x0") = old1; \ - register unsigned long x1 asm ("x1") = old2; \ - register unsigned long x2 asm ("x2") = new1; \ - register unsigned long x3 asm ("x3") = new2; \ - register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ - \ - asm volatile( \ - __LSE_PREAMBLE \ - " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ - " eor %[old1], %[old1], %[oldval1]\n" \ - " eor %[old2], %[old2], %[oldval2]\n" \ - " orr %[old1], %[old1], %[old2]" \ - : [old1] "+&r" (x0), [old2] "+&r" (x1), \ - [v] "+Q" (*(__uint128_t *)ptr) \ - : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ - [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ - : cl); \ - \ - return x0; \ -} - -__CMPXCHG_DBL( , ) -__CMPXCHG_DBL(_mb, al, "memory") - -#undef __CMPXCHG_DBL - #define __CMPXCHG128(name, mb, cl...) \ static __always_inline u128 \ __lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
arch/arm64/include/asm/cmpxchg.h +0 -46
58 lines not on this page · raw message
arch/arm64/include/asm/percpu.h +0 -10
@@ -145,16 +145,6 @@ PERCPU_RET_OP(add, add, ldadd)* preemption point when TIF_NEED_RESCHED gets set while preemption is * disabled. */ -#define this_cpu_cmpxchg_double_8 (ptr1, ptr2, o1, o2, n1, n2) \ -({ \ - int __ret; \ - preempt_disable_notrace(); \ - __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \ - raw_cpu_ptr(&(ptr2)), \ - o1, o2, n1, n2); \ - preempt_enable_notrace(); \ - __ret; \ -})#define _pcp_protect (op, pcp, ...) \ ({ \
arch/s390/include/asm/cmpxchg.h +0 -34
@@ -190,40 +190,6 @@ static __always_inline unsigned long __cmpxchg(unsigned long address,#define arch_cmpxchg_local arch_cmpxchg#define arch_cmpxchg64_local arch_cmpxchg -#define system_has_cmpxchg_double () 1 - -static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2, -unsigned long o1,unsigned long o2, -unsigned long n1,unsigned long n2) -{ -union register_pair old = { .even = o1, .odd = o2, }; -union register_pair new = { .even = n1, .odd = n2, }; -int cc; - - asmvolatile ( -" cdsg %[old],%[new],%[ptr]\n" -" ipm %[cc]\n" -" srl %[cc],28\n" - : [cc]"=&d" (cc), [old]"+&d" (old.pair) - : [new]"d" (new.pair), - [ptr]"QS" (*(unsigned long *)p1),"Q" (*(unsigned long *)p2) - :"memory" ,"cc" ); -return !cc; -} - -#define arch_cmpxchg_double (p1, p2, o1, o2, n1, n2) \ -({ \ - typeof(p1) __p1 = (p1); \ - typeof(p2) __p2 = (p2); \ - \ - BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ - BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ - VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ - __cmpxchg_double((unsigned long)__p1, (unsigned long)__p2, \ - (unsigned long)(o1), (unsigned long)(o2), \ - (unsigned long)(n1), (unsigned long)(n2)); \ -}) -#define system_has_cmpxchg128 () 1 static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
arch/s390/include/asm/percpu.h +0 -18
@@ -180,24 +180,6 @@#define this_cpu_xchg_4 (pcp, nval) arch_this_cpu_xchg(pcp, nval)#define this_cpu_xchg_8 (pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define arch_this_cpu_cmpxchg_double (pcp1, pcp2, o1, o2, n1, n2) \ -({ \ - typeof(pcp1) *p1__; \ - typeof(pcp2) *p2__; \ - int ret__; \ - \ - preempt_disable_notrace(); \ - p1__ = raw_cpu_ptr(&(pcp1)); \ - p2__ = raw_cpu_ptr(&(pcp2)); \ - ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \ - (unsigned long)(o1), (unsigned long)(o2), \ - (unsigned long)(n1), (unsigned long)(n2)); \ - preempt_enable_notrace(); \ - ret__; \ -}) - -#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double -#include <asm-generic/percpu.h> #endif /* __ARCH_S390_PERCPU__ */
arch/x86/include/asm/cmpxchg.h +0 -25
@@ -239,29 +239,4 @@ extern void __add_wrong_size(void)#define __xadd (ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)#define xadd (ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX) -#define __cmpxchg_double (pfx, p1, p2, o1, o2, n1, n2) \ -({ \ - bool __ret; \ - __typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \ - __typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \ - BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ - BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ - VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \ - VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \ - asm volatile(pfx "cmpxchg%c5b %1" \ - CC_SET(e) \ - : CC_OUT(e) (__ret), \ - "+m" (*(p1)), "+m" (*(p2)), \ - "+a" (__old1), "+d" (__old2) \ - : "i" (2 * sizeof(long)), \ - "b" (__new1), "c" (__new2)); \ - __ret; \ -}) - -#define arch_cmpxchg_double (p1, p2, o1, o2, n1, n2) \ - __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2) - -#define arch_cmpxchg_double_local (p1, p2, o1, o2, n1, n2) \ - __cmpxchg_double(, p1, p2, o1, o2, n1, n2) -#endif /* ASM_X86_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h +0 -1
@@ -103,7 +103,6 @@ static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)#endif -#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8)#define system_has_cmpxchg64 () boot_cpu_has(X86_FEATURE_CX8)#endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h +0 -1
@@ -81,7 +81,6 @@ static __always_inline bool arch_try_cmpxchg128_local(volatile u128 *ptr, u128 * return __arch_try_cmpxchg128(ptr, oldp, new,); } -#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX16)#define system_has_cmpxchg128 () boot_cpu_has(X86_FEATURE_CX16)#endif /* _ASM_X86_CMPXCHG_64_H */
arch/x86/include/asm/percpu.h +0 -42
54 lines not on this page · raw message
include/asm-generic/percpu.h +0 -58
80 lines not on this page · raw message
include/linux/atomic/atomic-instrumented.h +1 -16
@@ -2234,21 +2234,6 @@ atomic_long_dec_if_positive(atomic_long_t *v)arch_try_cmpxchg128_local (__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#define cmpxchg_double (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kcsan_mb(); \ - instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ - arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ -}) - - -#define cmpxchg_double_local (ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ - arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ -})#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 82d1be694fab30414527d0877c29fa75ed5a0b74 +// 3611991b015450e119bcd7417a9431af7f3ba13c
include/linux/percpu-defs.h +0 -38
@@ -358,33 +358,6 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } pscr2_ret__; \ }) -/* -* Special handling for cmpxchg_double. cmpxchg_double is passed two -* percpu variables. The first has to be aligned to a double word -* boundary and the second has to follow directly thereafter. -* We enforce this on all architectures even if they don't support -* a double cmpxchg instruction, since it's a cheap requirement, and it -* avoids breaking the requirement for architectures with the instruction. -*/ -#define __pcpu_double_call_return_bool (stem, pcp1, pcp2, ...) \ -({ \ - bool pdcrb_ret__; \ - __verify_pcpu_ptr(&(pcp1)); \ - BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ - VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \ - VM_BUG_ON((unsigned long)(&(pcp2)) != \ - (unsigned long)(&(pcp1)) + sizeof(pcp1)); \ - switch(sizeof(pcp1)) { \ - case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ - case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ - case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ - case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ - default: \ - __bad_size_call_parameter(); break; \ - } \ - pdcrb_ret__; \ -}) -#define __pcpu_size_call (stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \
@@ -443,9 +416,6 @@ do { \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)#define raw_cpu_try_cmpxchg (pcp, ovalp, nval) \ __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval) -#define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) -#define raw_cpu_sub (pcp, val) raw_cpu_add(pcp, -(val))#define raw_cpu_inc (pcp) raw_cpu_add(pcp, 1)#define raw_cpu_dec (pcp) raw_cpu_sub(pcp, 1)
@@ -505,11 +475,6 @@ do { \raw_cpu_cmpxchg (pcp, oval, nval); \ }) -#define __this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ -({ __this_cpu_preempt_check("cmpxchg_double"); \ - raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \ -}) -#define __this_cpu_sub (pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))#define __this_cpu_inc (pcp) __this_cpu_add(pcp, 1)#define __this_cpu_dec (pcp) __this_cpu_sub(pcp, 1)
@@ -532,9 +497,6 @@ do { \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)#define this_cpu_try_cmpxchg (pcp, ovalp, nval) \ __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval) -#define this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) -#define this_cpu_sub (pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))#define this_cpu_inc (pcp) this_cpu_add(pcp, 1)#define this_cpu_dec (pcp) this_cpu_sub(pcp, 1)
scripts/atomic/gen-atomic-instrumented.sh +4 -11
@@ -84,7 +84,6 @@ gen_xchg() { local xchg="$1"; shift local order="$1"; shift - local mult="$1"; shift kcsan_barrier="" if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,8 +103,8 @@ cat <<EOF EOF [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" cat <<EOF - instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ - instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\ arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ }) EOF
@@ -168,22 +167,16 @@ done for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do for order in "" "_acquire" "_release" "_relaxed"; do - gen_xchg "${xchg}" "${order}" "" + gen_xchg "${xchg}" "${order}" printf "\n" done done for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do - gen_xchg "${xchg}" "" "" + gen_xchg "${xchg}" "" printf "\n" done -gen_xchg "cmpxchg_double" "" "2 * " - -printf "\n\n" - -gen_xchg "cmpxchg_double_local" "" "2 * " - cat <<EOF #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
[tip: locking/core] types: Introduce [us]128
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 224d80c584d3016cb8d83d1c33914fdd3508aa8c Gitweb: https://git.kernel.org/tip/224d80c584d3016cb8d83d1c33914fdd3508aa8c Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:35 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:35 +02:00
types: Introduce [us]128
Introduce [us]128 (when available). Unlike [us]64, ensure they are always naturally aligned.
This also enables 128bit wide atomics (which require natural alignment) such as cmpxchg128().
--- include/linux/types.h | 5 +++++ include/uapi/linux/types.h | 4 ++++ lib/crypto/curve25519-hacl64.c | 2 -- lib/crypto/poly1305-donna64.c | 2 -- 4 files changed, 9 insertions(+), 4 deletions(-)
include/linux/types.h +5 -0
@@ -10,6 +10,11 @@#define DECLARE_BITMAP (name,bits) \ unsigned long name[BITS_TO_LONGS(bits)] +#ifdef __SIZEOF_INT128__ +typedef __s128 s128 ; +typedef __u128 u128 ; +#endif +typedef u32 __kernel_dev_t ;typedef __kernel_fd_set fd_set ;
include/uapi/linux/types.h +4 -0
@@ -13,6 +13,10 @@#include <linux/posix_types.h> +#ifdef __SIZEOF_INT128__ +typedef __signed__ __int128 __s128 __attribute__((aligned(16))); +typedef unsigned __int128 __u128 __attribute__((aligned (16 ))); +#endif /* * Below are truly Linux-specific types that should never collide with
lib/crypto/curve25519-hacl64.c +0 -2
@@ -14,8 +14,6 @@#include <crypto/curve25519.h> #include <linux/string.h> -typedef __uint128_t u128 ; - static __always_inline u64 u64_eq_mask(u64 a, u64 b) {u64 x = a ^ b;
lib/crypto/poly1305-donna64.c +0 -2
@@ -10,8 +10,6 @@#include <asm/unaligned.h> #include <crypto/internal/poly1305.h> -typedef __uint128_t u128 ; -void poly1305_core_setkey (struct poly1305_core_key *key,const u8 raw_key[POLY1305_BLOCK_SIZE ]) {
[tip: locking/core] instrumentation: Wire up cmpxchg128()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 8c8b096a23d12fedf3c0f50524f30113ef97aa8c Gitweb: https://git.kernel.org/tip/8c8b096a23d12fedf3c0f50524f30113ef97aa8c Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:37 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:36 +02:00
instrumentation: Wire up cmpxchg128()
Wire up the cmpxchg128 family in the atomic wrapper scripts.
These provide the generic cmpxchg128 family of functions from the arch_ prefixed version, adding explicit instrumentation where needed.
--- include/linux/atomic/atomic-arch-fallback.h | 95 +++++++++++++++++++- include/linux/atomic/atomic-instrumented.h | 86 +++++++++++++++++- scripts/atomic/gen-atomic-fallback.sh | 4 +- scripts/atomic/gen-atomic-instrumented.sh | 4 +- 4 files changed, 183 insertions(+), 6 deletions(-)
include/linux/atomic/atomic-arch-fallback.h +94 -1
@@ -77,6 +77,29 @@#endif /* arch_cmpxchg64_relaxed */ +#ifndef arch_cmpxchg128_relaxed +#define arch_cmpxchg128_acquire arch_cmpxchg128 +#define arch_cmpxchg128_release arch_cmpxchg128 +#define arch_cmpxchg128_relaxed arch_cmpxchg128 +#else /* arch_cmpxchg128_relaxed */ + +#ifndef arch_cmpxchg128_acquire +#define arch_cmpxchg128_acquire (...) \ + __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg128_release +#define arch_cmpxchg128_release (...) \ + __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg128 +#define arch_cmpxchg128 (...) \ + __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) +#endif + +#endif /* arch_cmpxchg128_relaxed */ +#ifndef arch_try_cmpxchg_relaxed#ifdef arch_try_cmpxchg#define arch_try_cmpxchg_acquire arch_try_cmpxchg
@@ -217,6 +240,76 @@#endif /* arch_try_cmpxchg64_relaxed */ +#ifndef arch_try_cmpxchg128_relaxed +#ifdef arch_try_cmpxchg128 +#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128 +#define arch_try_cmpxchg128_release arch_try_cmpxchg128 +#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128 +#endif /* arch_try_cmpxchg128 */ + +#ifndef arch_try_cmpxchg128 +#define arch_try_cmpxchg128 (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128 */ + +#ifndef arch_try_cmpxchg128_acquire +#define arch_try_cmpxchg128_acquire (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_acquire */ + +#ifndef arch_try_cmpxchg128_release +#define arch_try_cmpxchg128_release (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_release */ + +#ifndef arch_try_cmpxchg128_relaxed +#define arch_try_cmpxchg128_relaxed (_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_relaxed */ + +#else /* arch_try_cmpxchg128_relaxed */ + +#ifndef arch_try_cmpxchg128_acquire +#define arch_try_cmpxchg128_acquire (...) \ + __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg128_release +#define arch_try_cmpxchg128_release (...) \ + __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg128 +#define arch_try_cmpxchg128 (...) \ + __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#endif /* arch_try_cmpxchg128_relaxed */ +#ifndef arch_try_cmpxchg_local#define arch_try_cmpxchg_local (_ptr, _oldp, _new) \ ({ \
@@ -2668,4 +2761,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)#endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// ad2e2b4d168dbc60a73922616047a9bfa446af36 +// 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53
include/linux/atomic/atomic-instrumented.h +85 -1
113 lines not on this page · raw message
scripts/atomic/gen-atomic-fallback.sh +2 -2
@@ -217,11 +217,11 @@ cat << EOF EOF -for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do +for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64" "arch_cmpxchg128"; do gen_xchg_fallbacks "${xchg}" done -for cmpxchg in "cmpxchg" "cmpxchg64"; do +for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do gen_try_cmpxchg_fallbacks "${cmpxchg}" done
scripts/atomic/gen-atomic-instrumented.sh +2 -2
@@ -166,14 +166,14 @@ grep '^[a-z]' "$1" | while read name meta args; do done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do +for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_xchg "${xchg}" "${order}" "" printf "\n" done done -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do +for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do gen_xchg "${xchg}" "" "" printf "\n" done
[tip: locking/core] percpu: Add {raw,this}_cpu_try_cmpxchg()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: c5c0ba953b8c969c5d51bf1c57f239866a97c47c Gitweb: https://git.kernel.org/tip/c5c0ba953b8c969c5d51bf1c57f239866a97c47c Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:38 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:36 +02:00
percpu: Add {raw,this}_cpu_try_cmpxchg()
Add the try_cmpxchg() form to the per-cpu ops.
--- include/asm-generic/percpu.h | 113 ++++++++++++++++++++++++++++++++-- include/linux/percpu-defs.h | 19 ++++++- 2 files changed, 128 insertions(+), 4 deletions(-)
include/asm-generic/percpu.h +109 -4
@@ -89,16 +89,37 @@ do { \ __ret; \ }) -#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ +#define __cpu_fallback_try_cmpxchg (pcp, ovalp, nval, _cmpxchg) \ +({ \ + typeof(pcp) __val, __old = *(ovalp); \ + __val = _cmpxchg(pcp, __old, nval); \ + if (__val != __old) \ + *(ovalp) = __val; \ + __val == __old; \ +}) + +#define raw_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ ({ \ typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __ret; \ - __ret = *__p; \ - if (__ret == (oval)) \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (__val == __old) { \ *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ __ret; \ }) +#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \ + __old; \ +}) +#define raw_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \
@@ -170,6 +191,16 @@ do { \ __ret; \ }) +#define this_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ +({ \ + bool __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) +#define this_cpu_generic_cmpxchg (pcp, oval, nval) \ ({ \ typeof(pcp) __ret; \
@@ -282,6 +313,43 @@ do { \#define raw_cpu_xchg_8 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)#endif +#ifndef raw_cpu_try_cmpxchg_1 +#ifdef raw_cpu_cmpxchg_1 +#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1) +#else +#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_2 +#ifdef raw_cpu_cmpxchg_2 +#define raw_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2) +#else +#define raw_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_4 +#ifdef raw_cpu_cmpxchg_4 +#define raw_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4) +#else +#define raw_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_8 +#ifdef raw_cpu_cmpxchg_8 +#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8) +#else +#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_cmpxchg_1#define raw_cpu_cmpxchg_1 (pcp, oval, nval) \ raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -407,6 +475,43 @@ do { \#define this_cpu_xchg_8 (pcp, nval) this_cpu_generic_xchg(pcp, nval)#endif +#ifndef this_cpu_try_cmpxchg_1 +#ifdef this_cpu_cmpxchg_1 +#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1) +#else +#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_2 +#ifdef this_cpu_cmpxchg_2 +#define this_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2) +#else +#define this_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_4 +#ifdef this_cpu_cmpxchg_4 +#define this_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4) +#else +#define this_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_8 +#ifdef this_cpu_cmpxchg_8 +#define this_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8) +#else +#define this_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_cmpxchg_1#define this_cpu_cmpxchg_1 (pcp, oval, nval) \ this_cpu_generic_cmpxchg(pcp, oval, nval)
include/linux/percpu-defs.h +19 -0
@@ -343,6 +343,21 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } pscr2_ret__; \ }) +#define __pcpu_size_call_return2bool (stem, variable, ...) \ +({ \ + bool pscr2_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ + case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ + case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ + case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pscr2_ret__; \ +}) +/* * Special handling for cmpxchg_double. cmpxchg_double is passed two * percpu variables. The first has to be aligned to a double word
@@ -426,6 +441,8 @@ do { \#define raw_cpu_xchg (pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)#define raw_cpu_cmpxchg (pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) +#define raw_cpu_try_cmpxchg (pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)#define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
@@ -513,6 +530,8 @@ do { \#define this_cpu_xchg (pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)#define this_cpu_cmpxchg (pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) +#define this_cpu_try_cmpxchg (pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)#define this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
[tip: locking/core] percpu: Wire up cmpxchg128
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 6d12c8d308e68b9b0fa98ca2df4f83db4b4c965d Gitweb: https://git.kernel.org/tip/6d12c8d308e68b9b0fa98ca2df4f83db4b4c965d Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:39 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:37 +02:00
percpu: Wire up cmpxchg128
In order to replace cmpxchg_double() with the newly minted cmpxchg128() family of functions, wire it up in this_cpu_cmpxchg().
--- arch/arm64/include/asm/percpu.h | 20 +++++++++- arch/s390/include/asm/percpu.h | 16 +++++++- arch/x86/include/asm/percpu.h | 74 +++++++++++++++++++++++++++++--- arch/x86/lib/Makefile | 3 +- arch/x86/lib/cmpxchg16b_emu.S | 43 +++++++++++-------- arch/x86/lib/cmpxchg8b_emu.S | 67 ++++++++++++++++++++++------- include/asm-generic/percpu.h | 56 ++++++++++++++++++++++++- 7 files changed, 240 insertions(+), 39 deletions(-)
arch/arm64/include/asm/percpu.h +20 -0
@@ -140,6 +140,10 @@ PERCPU_RET_OP(add, add, ldadd)* re-enabling preemption for preemptible kernels, but doing that in a way * which builds inside a module would mean messing directly with the preempt * count. If you do this, peterz and tglx will hunt you down. +* +* Not to mention it'll break the actual preemption model for missing a +* preemption point when TIF_NEED_RESCHED gets set while preemption is +* disabled. */ #define this_cpu_cmpxchg_double_8 (ptr1, ptr2, o1, o2, n1, n2) \ ({ \
@@ -240,6 +244,22 @@ PERCPU_RET_OP(add, add, ldadd)#define this_cpu_cmpxchg_8 (pcp, o, n) \ _pcp_protect_return(cmpxchg_relaxed, pcp, o, n) +#define this_cpu_cmpxchg64 (pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) + +#define this_cpu_cmpxchg128 (pcp, o, n) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + u128 old__, new__, ret__; \ + pcp_op_T__ *ptr__; \ + old__ = o; \ + new__ = n; \ + preempt_disable_notrace(); \ + ptr__ = raw_cpu_ptr(&(pcp)); \ + ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \ + preempt_enable_notrace(); \ + ret__; \ +}) +#ifdef __KVM_NVHE_HYPERVISOR__extern unsigned long __hyp_per_cpu_offset (unsigned int cpu);#define __per_cpu_offset
arch/s390/include/asm/percpu.h +16 -0
@@ -148,6 +148,22 @@#define this_cpu_cmpxchg_4 (pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)#define this_cpu_cmpxchg_8 (pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) +#define this_cpu_cmpxchg64 (pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) + +#define this_cpu_cmpxchg128 (pcp, oval, nval) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + u128 old__, new__, ret__; \ + pcp_op_T__ *ptr__; \ + old__ = oval; \ + new__ = nval; \ + preempt_disable_notrace(); \ + ptr__ = raw_cpu_ptr(&(pcp)); \ + ret__ = cmpxchg128((void *)ptr__, old__, new__); \ + preempt_enable_notrace(); \ + ret__; \ +}) +#define arch_this_cpu_xchg (pcp, nval) \ ({ \ typeof(pcp) *ptr__; \
arch/x86/include/asm/percpu.h +68 -6
86 lines not on this page · raw message
arch/x86/lib/Makefile +2 -1
@@ -61,8 +61,9 @@ ifeq ($(CONFIG_X86_32),y) lib-y += strstr_32.o lib-y += string_32.o lib-y += memmove_32.o + lib-y += cmpxchg8b_emu.o ifneq ($(CONFIG_X86_CMPXCHG64),y) - lib-y += cmpxchg8b_emu.o atomic64_386_32.o + lib-y += atomic64_386_32.o endif else obj-y += iomap_copy_64.o
arch/x86/lib/cmpxchg16b_emu.S +25 -18
72 lines not on this page · raw message
arch/x86/lib/cmpxchg8b_emu.S +53 -14
95 lines not on this page · raw message
include/asm-generic/percpu.h +56 -0
80 lines not on this page · raw message
[tip: locking/core] cyrpto/b128ops: Remove struct u128
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: f413e724818c6482146218b3bcaf3d75b1317fc4 Gitweb: https://git.kernel.org/tip/f413e724818c6482146218b3bcaf3d75b1317fc4 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:34 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:35 +02:00
cyrpto/b128ops: Remove struct u128
Per git-grep u128_xor() and its related struct u128 are unused except to implement {be,le}128_xor(). Remove them to free up the namespace.
--- include/crypto/b128ops.h | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-)
include/crypto/b128ops.h +3 -11
@@ -50,10 +50,6 @@#include <linux/types.h> typedef struct { -u64 a, b; -}u128 ; - -typedef struct {__be64 a, b; }be128 ;
@@ -61,20 +57,16 @@ typedef struct { __le64 b, a; } le128; -static inline void u128_xor (u128 *r,const u128 *p,const u128 *q) +static inline void be128_xor (be128 *r,const be128 *p,const be128 *q) { r->a = p->a ^ q->a; r->b = p->b ^ q->b; } -static inline void be128_xor (be128 *r,const be128 *p,const be128 *q) -{ -u128_xor ((u128 *)r, (u128 *)p, (u128 *)q); -} -static inline void le128_xor (le128 *r,const le128 *p,const le128 *q) { -u128_xor ((u128 *)r, (u128 *)p, (u128 *)q); + r->a = p->a ^ q->a; + r->b = p->b ^ q->b; }#endif /* _CRYPTO_B128OPS_H */
[tip: locking/core] arch: Introduce arch_{,try_}_cmpxchg128{,_local}()
In reply to Peter ZijlstraThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: b23e139d0b66c0216e7e9361a5021290395f504c Gitweb: https://git.kernel.org/tip/b23e139d0b66c0216e7e9361a5021290395f504c Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 31 May 2023 15:08:36 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 05 Jun 2023 09:36:35 +02:00
arch: Introduce arch_{,try_}_cmpxchg128{,_local}()
For all architectures that currently support cmpxchg_double() implement the cmpxchg128() family of functions that is basically the same but with a saner interface.
--- arch/arm64/include/asm/atomic_ll_sc.h | 41 +++++++++++++++++- arch/arm64/include/asm/atomic_lse.h | 31 +++++++++++++- arch/arm64/include/asm/cmpxchg.h | 26 +++++++++++- arch/s390/include/asm/cmpxchg.h | 14 ++++++- arch/x86/include/asm/cmpxchg_32.h | 3 +- arch/x86/include/asm/cmpxchg_64.h | 64 +++++++++++++++++++++++++- 6 files changed, 177 insertions(+), 2 deletions(-)
arch/arm64/include/asm/atomic_ll_sc.h +41 -0
@@ -326,6 +326,47 @@ __CMPXCHG_DBL( , , , ) __CMPXCHG_DBL(_mb, dmb ish, l, "memory")#undef __CMPXCHG_DBL + +union __u128_halves { +u128 full; +struct { +u64 low, high; + }; +}; + +#define __CMPXCHG128(name, mb, rel, cl...) \ +static __always_inline u128 \ +__ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ +{ \ + union __u128_halves r, o = { .full = (old) }, \ + n = { .full = (new) }; \ + unsigned int tmp; \ + \ + asm volatile("// __cmpxchg128" #name "\n" \ + " prfm pstl1strm, %[v]\n" \ + "1: ldxp %[rl], %[rh], %[v]\n" \ + " cmp %[rl], %[ol]\n" \ + " ccmp %[rh], %[oh], 0, eq\n" \ + " b.ne 2f\n" \ + " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \ + " cbnz %w[tmp], 1b\n" \ + " " #mb "\n" \ + "2:" \ + : [v] "+Q" (*(u128 *)ptr), \ + [rl] "=&r" (r.low), [rh] "=&r" (r.high), \ + [tmp] "=&r" (tmp) \ + : [ol] "r" (o.low), [oh] "r" (o.high), \ + [nl] "r" (n.low), [nh] "r" (n.high) \ + : "cc", ##cl); \ + \ + return r.full; \ +} + +__CMPXCHG128( , , ) +__CMPXCHG128(_mb, dmb ish, l, "memory") + +#undef __CMPXCHG128 +#undef K#endif /* __ASM_ATOMIC_LL_SC_H */
arch/arm64/include/asm/atomic_lse.h +31 -0
@@ -317,4 +317,35 @@ __CMPXCHG_DBL(_mb, al, "memory")#undef __CMPXCHG_DBL +#define __CMPXCHG128(name, mb, cl...) \ +static __always_inline u128 \ +__lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ +{ \ + union __u128_halves r, o = { .full = (old) }, \ + n = { .full = (new) }; \ + register unsigned long x0 asm ("x0") = o.low; \ + register unsigned long x1 asm ("x1") = o.high; \ + register unsigned long x2 asm ("x2") = n.low; \ + register unsigned long x3 asm ("x3") = n.high; \ + register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ + \ + asm volatile( \ + __LSE_PREAMBLE \ + " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ + : [old1] "+&r" (x0), [old2] "+&r" (x1), \ + [v] "+Q" (*(u128 *)ptr) \ + : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ + [oldval1] "r" (o.low), [oldval2] "r" (o.high) \ + : cl); \ + \ + r.low = x0; r.high = x1; \ + \ + return r.full; \ +} + +__CMPXCHG128( , ) +__CMPXCHG128(_mb, al, "memory") + +#undef __CMPXCHG128 +#endif /* __ASM_ATOMIC_LSE_H */
arch/arm64/include/asm/cmpxchg.h +26 -0
@@ -146,6 +146,19 @@ __CMPXCHG_DBL(_mb)#undef __CMPXCHG_DBL +#define __CMPXCHG128 (name) \ +static inline u128 __cmpxchg128##name(volatile u128 *ptr, \ + u128 old, u128 new) \ +{ \ + return __lse_ll_sc_body(_cmpxchg128##name, \ + ptr, old, new); \ +} + +__CMPXCHG128( ) +__CMPXCHG128(_mb) + +#undef __CMPXCHG128 +#define __CMPXCHG_GEN (sfx) \ static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \ unsigned long old, \
@@ -228,6 +241,19 @@ __CMPXCHG_GEN(_mb) __ret; \ }) +/* cmpxchg128 */ +#define system_has_cmpxchg128 () 1 + +#define arch_cmpxchg128 (ptr, o, n) \ +({ \ + __cmpxchg128_mb((ptr), (o), (n)); \ +}) + +#define arch_cmpxchg128_local (ptr, o, n) \ +({ \ + __cmpxchg128((ptr), (o), (n)); \ +}) +#define __CMPWAIT_CASE (w, sfx, sz) \ static inline void __cmpwait_case_##sz(volatile void *ptr, \ unsigned long val) \
arch/s390/include/asm/cmpxchg.h +14 -0
@@ -224,4 +224,18 @@ static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2, (unsigned long)(n1), (unsigned long)(n2)); \ }) +#define system_has_cmpxchg128 () 1 + +static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new) +{ + asmvolatile ( +" cdsg %[old],%[new],%[ptr]\n" + : [old]"+d" (old), [ptr]"+QS" (*ptr) + : [new]"d" (new) + :"memory" ,"cc" ); +return old; +} + +#define arch_cmpxchg128 arch_cmpxchg128 +#endif /* __ASM_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h +2 -1
@@ -103,6 +103,7 @@ static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)#endif -#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8) +#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8) +#define system_has_cmpxchg64 () boot_cpu_has(X86_FEATURE_CX8)#endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h +63 -1
69 lines not on this page · raw message
Re: [PATCH v2 5/12] percpu: Add {raw,this}_cpu_try_cmpxchg()
In reply to Peter ZijlstraQuoted 4 lines · Peter Zijlstra
Add the
try_cmpxchg()form to the per-cpu ops.Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
+CC Nathan, llvm list
Hi all, this patch seems to break booting on Qualcomm ARM64 platforms when compiled with clang (GCC works fine) for some reason..:
next-20230605 - works
next-20230606 - doesn't
grev -m 1 dc4e51fd9846 on next-20230606 - works again b4 shazam <this_msgid> -P 1-4 - still works b4 shazam <this_msgid> -P 5 - breaks
Confirmed on at least Qualcomm QCM2290, SM8250.
Checking the serial console, it hits a BUG_ON:
[ 0.000000] ------------[ cut here ]------------ [ 0.000000] kernel BUG at mm/vmalloc.c:1638! [ 0.000000] Internal error: Oops - BUG: 00000000f2000800 [#1] SMP [ 0.000000] Modules linked in: [ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted [snip] [ 0.000000] Hardware name: Qualcomm Technologies, Inc. Robotics RB1 (DT) [ 0.000000] pstate: 000000c5 (nzcv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.000000] pc : alloc_vmap_area+0xafc/0xb08 [ 0.000000] lr : alloc_vmap_area+0x9e4/0xb08 [ 0.000000] sp : ffffa50137f53c20 [ 0.000000] x29: ffffa50137f53c60 x28: ffffa50137f30c18 x27: 0000000000000000 [ 0.000000] x26: 0000000000007fff x25: ffff800080000000 x24: 000000000000cfff [ 0.000000] x23: ffffffffffff8000 x22: ffffa50137fef970 x21: fffffbfff0000000 [ 0.000000] x20: ffff022982003208 x19: ffff0229820031f8 x18: ffffa50137f64f70 [ 0.000000] x17: ffffa50137fef980 x16: ffffa501375e6d08 x15: 0000000000000001 [ 0.000000] x14: ffffa5013831e1a0 x13: ffffa50137f30c18 x12: 0000000000402dc2 [ 0.000000] x11: 0000000000000000 x10: ffff022982003018 x9 : ffffa5013831e188 [ 0.000000] x8 : ffffcb55ff003228 x7 : 0000000000000000 x6 : 0000000000000048 [ 0.000000] x5 : 0000000000000000 x4 : ffffa50137f53bd0 x3 : ffffa50136490000 [ 0.000000] x2 : 0000000000000001 x1 : ffffa5013831e190 x0 : ffff022982003208 [ 0.000000] Call trace: [ 0.000000] alloc_vmap_area+0xafc/0xb08 [ 0.000000] __get_vm_area_node+0x108/0x1e8 [ 0.000000] __vmalloc_node_range+0x1fc/0x728 [ 0.000000] __vmalloc_node+0x5c/0x70 [ 0.000000] init_IRQ+0x90/0x11c [ 0.000000] start_kernel+0x1ac/0x3bc [ 0.000000] __primary_switched+0xc4/0xcc [ 0.000000] Code: f000e300 91062000 943bd9ba 17ffff8f (d4210000) [ 0.000000] ---[ end trace 0000000000000000 ]--- [ 0.000000] Kernel panic - not syncing: Attempted to kill the idle task!
Compiled with clang 15.0.7 from Arch repos, with
make ARCH=arm64 LLVM=1
Konrad
Quoted 195 lines
include/asm-generic/percpu.h | 113 +++++++++++++++++++++++++++++++++++++++++-- include/linux/percpu-defs.h | 19 +++++++ 2 files changed, 128 insertions(+), 4 deletions(-)include/asm-generic/percpu.h +109
-4@@ -89,16 +89,37 @@ do { \ __ret; \ }) -#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ +#define __cpu_fallback_try_cmpxchg (pcp, ovalp, nval, _cmpxchg) \ +({ \ + typeof(pcp) __val, __old = *(ovalp); \ + __val = _cmpxchg(pcp, __old, nval); \ + if (__val != __old) \ + *(ovalp) = __val; \ + __val == __old; \ +}) + +#define raw_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ ({ \ typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __ret; \ - __ret = *__p; \ - if (__ret == (oval)) \ + typeof(pcp) __val = *__p, __old = *(ovalp); \ + bool __ret; \ + if (__val == __old) { \ *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ __ret; \ }) +#define raw_cpu_generic_cmpxchg (pcp, oval, nval) \ +({ \ + typeof(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \ + __old; \ +}) +#define raw_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \@@ -170,6 +191,16 @@ do { \ __ret; \ }) +#define this_cpu_generic_try_cmpxchg (pcp, ovalp, nval) \ +({ \ + bool __ret; \ + unsigned long __flags; \ + raw_local_irq_save(__flags); \ + __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \ + raw_local_irq_restore(__flags); \ + __ret; \ +}) +#define this_cpu_generic_cmpxchg (pcp, oval, nval) \ ({ \ typeof(pcp) __ret; \@@ -282,6 +313,43 @@ do { \#define raw_cpu_xchg_8 (pcp, nval) raw_cpu_generic_xchg(pcp, nval)#endif +#ifndef raw_cpu_try_cmpxchg_1 +#ifdef raw_cpu_cmpxchg_1 +#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1) +#else +#define raw_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_2 +#ifdef raw_cpu_cmpxchg_2 +#define raw_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2) +#else +#define raw_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_4 +#ifdef raw_cpu_cmpxchg_4 +#define raw_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4) +#else +#define raw_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_8 +#ifdef raw_cpu_cmpxchg_8 +#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8) +#else +#define raw_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_cmpxchg_1#define raw_cpu_cmpxchg_1 (pcp, oval, nval) \ raw_cpu_generic_cmpxchg(pcp, oval, nval)@@ -407,6 +475,43 @@ do { \#define this_cpu_xchg_8 (pcp, nval) this_cpu_generic_xchg(pcp, nval)#endif +#ifndef this_cpu_try_cmpxchg_1 +#ifdef this_cpu_cmpxchg_1 +#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1) +#else +#define this_cpu_try_cmpxchg_1 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_2 +#ifdef this_cpu_cmpxchg_2 +#define this_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2) +#else +#define this_cpu_try_cmpxchg_2 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_4 +#ifdef this_cpu_cmpxchg_4 +#define this_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4) +#else +#define this_cpu_try_cmpxchg_4 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_8 +#ifdef this_cpu_cmpxchg_8 +#define this_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8) +#else +#define this_cpu_try_cmpxchg_8 (pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_cmpxchg_1#define this_cpu_cmpxchg_1 (pcp, oval, nval) \ this_cpu_generic_cmpxchg(pcp, oval, nval)include/linux/percpu-defs.h +19
-0@@ -343,6 +343,21 @@ static __always_inline void __this_cpu_p pscr2_ret__; \ }) +#define __pcpu_size_call_return2bool (stem, variable, ...) \ +({ \ + bool pscr2_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ + case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ + case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ + case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pscr2_ret__; \ +}) +/* * Special handling for cmpxchg_double. cmpxchg_double is passed two * percpu variables. The first has to be aligned to a double word @@ -426,6 +441,8 @@ do { \#define raw_cpu_xchg (pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)#define raw_cpu_cmpxchg (pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) +#define raw_cpu_try_cmpxchg (pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)#define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)@@ -513,6 +530,8 @@ do { \#define this_cpu_xchg (pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)#define this_cpu_cmpxchg (pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) +#define this_cpu_try_cmpxchg (pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)#define this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
Re: [PATCH v2 5/12] percpu: Add {raw,this}_cpu_try_cmpxchg()
Hi Konrad,
Quoted 55 lines · Konrad Dybcio
Quoted 4 lines · Peter Zijlstra
Add the
try_cmpxchg()form to the per-cpu ops.Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
+CC Nathan, llvm list
Hi all, this patch seems to break booting on Qualcomm ARM64 platforms when compiled with clang (GCC works fine) for some reason..:
next-20230605 - works
next-20230606 - doesn'tgrev -m 1
dc4e51fd9846on next-20230606 - works again b4 shazam <this_msgid> -P 1-4 - still works b4 shazam <this_msgid> -P 5 - breaksConfirmed on at least Qualcomm QCM2290, SM8250.
Checking the serial console, it hits a
BUG_ON:[ 0.000000] ------------[ cut here ]------------ [ 0.000000] kernel BUG at mm/vmalloc.c:1638! [ 0.000000] Internal error: Oops - BUG: 00000000f2000800 [#1] SMP [ 0.000000] Modules linked in: [ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted [snip] [ 0.000000] Hardware name: Qualcomm Technologies, Inc. Robotics RB1 (DT) [ 0.000000] pstate: 000000c5 (nzcv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.000000] pc : alloc_vmap_area+0xafc/0xb08 [ 0.000000] lr : alloc_vmap_area+0x9e4/0xb08 [ 0.000000] sp : ffffa50137f53c20 [ 0.000000] x29: ffffa50137f53c60 x28: ffffa50137f30c18 x27: 0000000000000000 [ 0.000000] x26: 0000000000007fff x25: ffff800080000000 x24: 000000000000cfff [ 0.000000] x23: ffffffffffff8000 x22: ffffa50137fef970 x21: fffffbfff0000000 [ 0.000000] x20: ffff022982003208 x19: ffff0229820031f8 x18: ffffa50137f64f70 [ 0.000000] x17: ffffa50137fef980 x16: ffffa501375e6d08 x15: 0000000000000001 [ 0.000000] x14: ffffa5013831e1a0 x13: ffffa50137f30c18 x12: 0000000000402dc2 [ 0.000000] x11: 0000000000000000 x10: ffff022982003018 x9 : ffffa5013831e188 [ 0.000000] x8 : ffffcb55ff003228 x7 : 0000000000000000 x6 : 0000000000000048 [ 0.000000] x5 : 0000000000000000 x4 : ffffa50137f53bd0 x3 : ffffa50136490000 [ 0.000000] x2 : 0000000000000001 x1 : ffffa5013831e190 x0 : ffff022982003208 [ 0.000000] Call trace: [ 0.000000] alloc_vmap_area+0xafc/0xb08 [ 0.000000] __get_vm_area_node+0x108/0x1e8 [ 0.000000] __vmalloc_node_range+0x1fc/0x728 [ 0.000000] __vmalloc_node+0x5c/0x70 [ 0.000000] init_IRQ+0x90/0x11c [ 0.000000] start_kernel+0x1ac/0x3bc [ 0.000000] __primary_switched+0xc4/0xcc [ 0.000000] Code: f000e300 91062000 943bd9ba 17ffff8f (d4210000) [ 0.000000] ---[ end trace 0000000000000000 ]--- [ 0.000000] Kernel panic - not syncing: Attempted to kill the idle task!
Quoted 2 lines
Compiled with clang 15.0.7 from Arch repos, with
make ARCH=arm64 LLVM=1
Thanks a lot for testing with LLVM, submitting this report, and doing a bisect. I sent a patch to fix this a couple of days ago and Peter pushed it to -tip today, so it should be in the next -next release:
https://git.kernel.org/tip/093d9b240a1fa261ff8aeb7c7cc484dedacfda53
Cheers,
Nathan
Re: [PATCH v2 5/12] percpu: Add {raw,this}_cpu_try_cmpxchg()
Quoted 58 lines · Nathan Chancellor
Hi Konrad,
Quoted 55 lines · Konrad Dybcio
Quoted 4 lines · Peter Zijlstra
Add the
try_cmpxchg()form to the per-cpu ops.Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
+CC Nathan, llvm list
Hi all, this patch seems to break booting on Qualcomm ARM64 platforms when compiled with clang (GCC works fine) for some reason..:
next-20230605 - works
next-20230606 - doesn'tgrev -m 1
dc4e51fd9846on next-20230606 - works again b4 shazam <this_msgid> -P 1-4 - still works b4 shazam <this_msgid> -P 5 - breaksConfirmed on at least Qualcomm QCM2290, SM8250.
Checking the serial console, it hits a
BUG_ON:[ 0.000000] ------------[ cut here ]------------ [ 0.000000] kernel BUG at mm/vmalloc.c:1638! [ 0.000000] Internal error: Oops - BUG: 00000000f2000800 [#1] SMP [ 0.000000] Modules linked in: [ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted [snip] [ 0.000000] Hardware name: Qualcomm Technologies, Inc. Robotics RB1 (DT) [ 0.000000] pstate: 000000c5 (nzcv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.000000] pc : alloc_vmap_area+0xafc/0xb08 [ 0.000000] lr : alloc_vmap_area+0x9e4/0xb08 [ 0.000000] sp : ffffa50137f53c20 [ 0.000000] x29: ffffa50137f53c60 x28: ffffa50137f30c18 x27: 0000000000000000 [ 0.000000] x26: 0000000000007fff x25: ffff800080000000 x24: 000000000000cfff [ 0.000000] x23: ffffffffffff8000 x22: ffffa50137fef970 x21: fffffbfff0000000 [ 0.000000] x20: ffff022982003208 x19: ffff0229820031f8 x18: ffffa50137f64f70 [ 0.000000] x17: ffffa50137fef980 x16: ffffa501375e6d08 x15: 0000000000000001 [ 0.000000] x14: ffffa5013831e1a0 x13: ffffa50137f30c18 x12: 0000000000402dc2 [ 0.000000] x11: 0000000000000000 x10: ffff022982003018 x9 : ffffa5013831e188 [ 0.000000] x8 : ffffcb55ff003228 x7 : 0000000000000000 x6 : 0000000000000048 [ 0.000000] x5 : 0000000000000000 x4 : ffffa50137f53bd0 x3 : ffffa50136490000 [ 0.000000] x2 : 0000000000000001 x1 : ffffa5013831e190 x0 : ffff022982003208 [ 0.000000] Call trace: [ 0.000000] alloc_vmap_area+0xafc/0xb08 [ 0.000000] __get_vm_area_node+0x108/0x1e8 [ 0.000000] __vmalloc_node_range+0x1fc/0x728 [ 0.000000] __vmalloc_node+0x5c/0x70 [ 0.000000] init_IRQ+0x90/0x11c [ 0.000000] start_kernel+0x1ac/0x3bc [ 0.000000] __primary_switched+0xc4/0xcc [ 0.000000] Code: f000e300 91062000 943bd9ba 17ffff8f (d4210000) [ 0.000000] ---[ end trace 0000000000000000 ]--- [ 0.000000] Kernel panic - not syncing: Attempted to kill the idle task!
Quoted 5 lines
Quoted 2 lines
Compiled with clang 15.0.7 from Arch repos, with
make ARCH=arm64 LLVM=1Thanks a lot for testing with LLVM, submitting this report, and doing a bisect.
No, thank *you* for making it even possible ;)
I sent a patch to fix this a couple of days ago and Peter pushed
Quoted 3 lines
it to -tip today, so it should be in the next -next release:
https://git.kernel.org/tip/093d9b240a1fa261ff8aeb7c7cc484dedacfda53
Amazing, I can boot the most recent next-20230609 with it again!
Konrad
Quoted 3 lines
Cheers,
Nathan