x86/KVM/VMX: Introduce and use try_cmpxchg64()
Subsystems (5)
cover x86/KVM/VMX: Introduce and use try_cmpxchg64()
This patch series introduces try_cmpxchg64() atomic locking function.
try_cmpxchg64() provides the same interface for 64 bit and 32 bit targets, emits CMPXCHGQ for 64 bit targets and CMPXCHG8B for 32 bit targets, and provides appropriate fallbacks when CMPXCHG8B is unavailable.
try_cmpxchg64() reuses flags from CMPXCHGQ/CMPXCHG8B instructions and avoids unneeded CMP for 64 bit targets or XOR/XOR/OR sequence for 32 bit targets.
Uros Bizjak (3):
asm-generic/atomic: Add try_cmpxchg64() instrumentation locking/atomic/x86: Introduce arch_try_cmpxchg64() KVM/VMX: Use try_cmpxchg64() in posted_intr.c
arch/x86/include/asm/cmpxchg_32.h | 62 +++++++++++++++++++---- arch/x86/include/asm/cmpxchg_64.h | 6 +++ arch/x86/kvm/vmx/posted_intr.c | 9 ++-- include/asm-generic/atomic-instrumented.h | 46 ++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 5 files changed, 108 insertions(+), 17 deletions(-)
Signature · 1 line
2.26.2
PATCH 1/3 asm-generic/atomic: Add try_cmpxchg64() instrumentation
Instrument try_cmpxchg64() similar to try_cmpxchg().
--- include/asm-generic/atomic-instrumented.h | 46 ++++++++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-)
include/asm-generic/atomic-instrumented.h +45 -1
@@ -1793,6 +1793,50 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif +#if !defined(arch_try_cmpxchg64_relaxed) || defined(arch_try_cmpxchg64) +#define try_cmpxchg64 (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg64_acquire) +#define try_cmpxchg64_acquire (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg64_release) +#define try_cmpxchg64_release (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif + +#if defined(arch_try_cmpxchg64_relaxed) +#define try_cmpxchg64_relaxed (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#endif +#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \
@@ -1830,4 +1874,4 @@ atomic64_dec_if_positive(atomic64_t *v) })#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ -// 4bec382e44520f4d8267e42620054db26a659ea3 +// 28b0549ca119ed58a9b61da6e277ee30de18bdfc
scripts/atomic/gen-atomic-instrumented.sh +1 -1
@@ -186,7 +186,7 @@ grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do +for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_optional_xchg "${xchg}" "${order}" done
Signature · 1 line
2.26.2
PATCH 2/3 locking/atomic/x86: Introduce arch_try_cmpxchg64()
In reply to Uros BizjakAdd arch_try_cmpxchg64(), similar to arch_try_cmpxchg(), that operates with 64 bit operands. This function provides the same interface for 32 bit and 64 bit targets.
--- arch/x86/include/asm/cmpxchg_32.h | 62 ++++++++++++++++++++++++++----- arch/x86/include/asm/cmpxchg_64.h | 6 +++ 2 files changed, 59 insertions(+), 9 deletions(-)
arch/x86/include/asm/cmpxchg_32.h +53 -9
@@ -35,15 +35,6 @@ static inline void set_64bit(volatile u64 *ptr, u64 value) : "memory"); } -#ifdef CONFIG_X86_CMPXCHG64 -#define arch_cmpxchg64 (ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ - (unsigned long long)(n))) -#define arch_cmpxchg64_local (ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ - (unsigned long long)(n))) -#endif -static inline u64 __cmpxchg64 (volatile u64 *ptr,u64 old,u64 new) {u64 prev;
@@ -71,6 +62,39 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) }#ifndef CONFIG_X86_CMPXCHG64 +#define arch_cmpxchg64 (ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ + (unsigned long long)(n))) +#define arch_cmpxchg64_local (ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ + +#define __raw_try_cmpxchg64 (_ptr, _pold, _new, lock) \ +({ \ + bool success; \ + __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ + __typeof__(*(_ptr)) __old = *_old; \ + __typeof__(*(_ptr)) __new = (_new); \ + asm volatile(lock "cmpxchg8b %1" \ + CC_SET(z) \ + : CC_OUT(z) (success), \ + "+m" (*_ptr), \ + "+A" (__old) \ + : "b" ((unsigned int)__new), \ + "c" ((unsigned int)(__new>>32)) \ + : "memory"); \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); \ +}) + +#define __try_cmpxchg64 (ptr, pold, new) \ + __raw_try_cmpxchg64((ptr), (pold), (new), LOCK_PREFIX) + +#define arch_try_cmpxchg64 (ptr, pold, new) \ + __try_cmpxchg64((ptr), (pold), (new)) + +#else +/* * Building a kernel capable running on 80386 and 80486. It may be necessary * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
@@ -108,6 +132,26 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) : "memory"); \ __ret; }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + bool success; \ + __typeof__(ptr) _old = (__typeof__(ptr))(po); \ + __typeof__(*(ptr)) __old = *_old; \ + __typeof__(*(ptr)) __new = (n); \ + alternative_io(LOCK_PREFIX_HERE \ + "call cmpxchg8b_emu", \ + "lock; cmpxchg8b (%%esi)" , \ + X86_FEATURE_CX8, \ + "+A" (__old), \ + "S" ((ptr)), \ + "b" ((unsigned int)__new), \ + "c" ((unsigned int)(__new>>32)) \ + : "memory"); \ + success = (__old == *_old); \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); \ +})#endif #define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8)
arch/x86/include/asm/cmpxchg_64.h +6 -0
@@ -19,6 +19,12 @@ static inline void set_64bit(volatile u64 *ptr, u64 val)arch_cmpxchg_local ((ptr), (o), (n)); \ }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_try_cmpxchg((ptr), (po), (n)); \ +}) +#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX16)#endif /* _ASM_X86_CMPXCHG_64_H */
Signature · 1 line
2.26.2
Re: [PATCH 2/3] locking/atomic/x86: Introduce arch_try_cmpxchg64()
Quoted 20 lines · Uros Bizjak
Add
arch_try_cmpxchg64(), similar toarch_try_cmpxchg(), that operates with 64 bit operands. This function provides the same interface for 32 bit and 64 bit targets.Signed-off-by Uros Bizjak <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc Borislav Petkov <[email protected]>Cc "H. Peter Anvin" <[email protected]>--- arch/x86/include/asm/cmpxchg_32.h | 62 ++++++++++++++++++++++++++----- arch/x86/include/asm/cmpxchg_64.h | 6 +++ 2 files changed, 59 insertions(+), 9 deletions(-)arch/x86/include/asm/cmpxchg_32.h +0
-0
arch/x86/include/asm/cmpxchg_32.h static inline void set_64bit(volatile u64 *ptr, u64 value) · line 38
Quoted 4 lines
arch/x86/include/asm/cmpxchg_32.h +0
-1@@ -35,15 +35,6 @@ static inline void set_64bit(volatile u64 *ptr, u64 value) : "memory"); } -#ifdef CONFIG_X86_CMPXCHG64
Oops, I didn't notice that I had left a reversed #ifdef condition in the patch (to test 32 bit target without X86_CMPXCHG64).
Obviously, CONFIG_X86_CMPXCHG64 has to be defined to use CMPXCHG8B, so please use #ifdef here.
Uros.
Quoted 98 lines
arch/x86/include/asm/cmpxchg_32.h +53
-8-#define arch_cmpxchg64 (ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ - (unsigned long long)(n))) -#define arch_cmpxchg64_local (ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ - (unsigned long long)(n))) -#endif -static inline u64 __cmpxchg64 (volatile u64 *ptr,u64 old,u64 new) {u64 prev;@@ -71,6 +62,39 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) }#ifndef CONFIG_X86_CMPXCHG64 +#define arch_cmpxchg64 (ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ + (unsigned long long)(n))) +#define arch_cmpxchg64_local (ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ + +#define __raw_try_cmpxchg64 (_ptr, _pold, _new, lock) \ +({ \ + bool success; \ + __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ + __typeof__(*(_ptr)) __old = *_old; \ + __typeof__(*(_ptr)) __new = (_new); \ + asm volatile(lock "cmpxchg8b %1" \ + CC_SET(z) \ + : CC_OUT(z) (success), \ + "+m" (*_ptr), \ + "+A" (__old) \ + : "b" ((unsigned int)__new), \ + "c" ((unsigned int)(__new>>32)) \ + : "memory"); \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); \ +}) + +#define __try_cmpxchg64 (ptr, pold, new) \ + __raw_try_cmpxchg64((ptr), (pold), (new), LOCK_PREFIX) + +#define arch_try_cmpxchg64 (ptr, pold, new) \ + __try_cmpxchg64((ptr), (pold), (new)) + +#else +/* * Building a kernel capable running on 80386 and 80486. It may be necessary * to simulate the cmpxchg8b on the 80386 and 80486 CPU. @@ -108,6 +132,26 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) : "memory"); \ __ret; }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + bool success; \ + __typeof__(ptr) _old = (__typeof__(ptr))(po); \ + __typeof__(*(ptr)) __old = *_old; \ + __typeof__(*(ptr)) __new = (n); \ + alternative_io(LOCK_PREFIX_HERE \ + "call cmpxchg8b_emu", \ + "lock; cmpxchg8b (%%esi)" , \ + X86_FEATURE_CX8, \ + "+A" (__old), \ + "S" ((ptr)), \ + "b" ((unsigned int)__new), \ + "c" ((unsigned int)(__new>>32)) \ + : "memory"); \ + success = (__old == *_old); \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); \ +})#endif #define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8)arch/x86/include/asm/cmpxchg_64.h +6
-1@@ -19,6 +19,12 @@ static inline void set_64bit(volatile u64 *ptr, u64 val)arch_cmpxchg_local ((ptr), (o), (n)); \ }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_try_cmpxchg((ptr), (po), (n)); \ +}) +#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX16)#endif /* _ASM_X86_CMPXCHG_64_H */ --2.26.2
kernel test robot
build botRe: [PATCH 2/3] locking/atomic/x86: Introduce arch_try_cmpxchg64()
In reply to Uros BizjakThank you for the patch! Yet something to improve:
Hi Uros,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on tip/master]
[also build test ERROR on next-20201215]
[cannot apply to tip/x86/core kvm/linux-next v5.10]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uros-Bizjak/x86-KVM-VMX-Introduce-and-use-try_cmpxchg64/20201216-024049 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git d1c29f5debd4633eb0e9ea1bc00aaad48b077a9b config: i386-tinyconfig (attached as .config) compiler: gcc-9 (Debian 9.3.0-15) 9.3.0 reproduce (this is a W=1 build): # https://github.com/0day-ci/linux/commit/60a11e7e63e120b5fd41b5346cf5a05ea71c7cb2 git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Uros-Bizjak/x86-KVM-VMX-Introduce-and-use-try_cmpxchg64/20201216-024049 git checkout 60a11e7e63e120b5fd41b5346cf5a05ea71c7cb2 # save the attached .config to linux build tree make W=1 ARCH=i386
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
ld: arch/x86/events/core.o: in function `x86_perf_event_update':
>> core.c:(.text+0x847): undefined reference to `cmpxchg8b_emu' ld: kernel/sched/clock.o: in function `sched_clock_local.constprop.0': >> clock.c:(.text+0x1b4): undefined reference to `cmpxchg8b_emu' ld: kernel/sched/clock.o: in function `sched_clock_cpu': clock.c:(.text+0x293): undefined reference to `cmpxchg8b_emu' >> ld: clock.c:(.text+0x2e3): undefined reference to `cmpxchg8b_emu'
ld: kernel/events/core.o: in function `perf_swevent_set_period':
core.c:(.text+0x8cbb): undefined reference to `cmpxchg8b_emu'
ld: fs/inode.o:inode.c:(.text+0x10ca): more undefined references to `cmpxchg8b_emu' follow
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
PATCH 3/3 KVM/VMX: Use try_cmpxchg64() in posted_intr.c
In reply to Uros BizjakUse try_cmpxchg64() instead of cmpxchg64() to reuse flags from cmpxchg/cmpxchg8b instruction. For 64 bit targets flags reuse avoids a CMP instruction, while for 32 bit targets flags reuse avoids XOR/XOR/OR instruction sequence.
--- arch/x86/kvm/vmx/posted_intr.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
arch/x86/kvm/vmx/posted_intr.c +3 -6
@@ -60,8 +60,7 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) new.ndst = (dest <<8 ) &0xFF00 ; new.sn =0 ; - }while (cmpxchg64 (&pi_desc->control, old.control, - new.control) != old.control); + }while (!try_cmpxchg64 (&pi_desc->control, &old.control, new.control)); after_clear_sn:
@@ -111,8 +110,7 @@ static void __pi_post_block(struct kvm_vcpu *vcpu)/* set 'NV' to 'notification vector' */ new.nv =POSTED_INTR_VECTOR ; - }while (cmpxchg64 (&pi_desc->control, old.control, - new.control) != old.control); + }while (!try_cmpxchg64 (&pi_desc->control, &old.control, new.control));if (!WARN_ON_ONCE (vcpu->pre_pcpu ==-1 )) {spin_lock (&per_cpu (blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
@@ -181,8 +179,7 @@ int pi_pre_block(struct kvm_vcpu *vcpu)/* set 'NV' to 'wakeup vector' */ new.nv =POSTED_INTR_WAKEUP_VECTOR ; - }while (cmpxchg64 (&pi_desc->control, old.control, - new.control) != old.control); + }while (!try_cmpxchg64 (&pi_desc->control, &old.control, new.control));/* We should not block the vCPU if an interrupt is posted for it. */ if (pi_test_on (pi_desc) ==1 )
Signature · 1 line
2.26.2
Re: [PATCH 3/3] KVM/VMX: Use try_cmpxchg64() in posted_intr.c
Quoted 3 lines · Uros Bizjak
Use
try_cmpxchg64()instead ofcmpxchg64()to reuse flags from cmpxchg/cmpxchg8b instruction. For 64 bit targets flags reuse avoids a CMP instruction,
It ends up doing way more (in a good way) than eliminate the CMP, at least with gcc-10. There's a ripple effect and the compiler ends up generating the loop in-line, whereas without the "try" version the loop is put out-of-line.
Quoted 3 lines
while for 32 bit targets flags reuse avoids XOR/XOR/OR instruction sequence.
Signed-off-by Uros Bizjak <[email protected]>
Sean Christopherson
maintainerQuoted 24 lines · Uros Bizjak
This patch series introduces
try_cmpxchg64()atomic locking function.
try_cmpxchg64()provides the same interface for 64 bit and 32 bit targets, emits CMPXCHGQ for 64 bit targets and CMPXCHG8B for 32 bit targets, and provides appropriate fallbacks when CMPXCHG8B is unavailable.
try_cmpxchg64()reuses flags from CMPXCHGQ/CMPXCHG8B instructions and avoids unneeded CMP for 64 bit targets or XOR/XOR/OR sequence for 32 bit targets.Cc Will Deacon <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Boqun Feng <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc Borislav Petkov <[email protected]>Cc "H. Peter Anvin" <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Sean Christopherson <[email protected]>Cc Vitaly Kuznetsov <[email protected]>Cc Wanpeng Li <[email protected]>Cc Jim Mattson <[email protected]>Cc Joerg Roedel <[email protected]>
Quoted 4 lines
Uros Bizjak (3):
asm-generic/atomic: Addtry_cmpxchg64()instrumentation locking/atomic/x86: Introducearch_try_cmpxchg64()KVM/VMX: Usetry_cmpxchg64()inposted_intr.c
For anyone else trying to apply this, it depends on v5.11-rc1 (commit 29f006fdefe6, "asm-generic/atomic: Add try_cmpxchg() fallbacks"), which hasn't yet been merged into Paolo's tree.
Quoted 10 lines
arch/x86/include/asm/cmpxchg_32.h | 62 +++++++++++++++++++---- arch/x86/include/asm/cmpxchg_64.h | 6 +++ arch/x86/kvm/vmx/posted_intr.c | 9 ++-- include/asm-generic/atomic-instrumented.h | 46 ++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 5 files changed, 108 insertions(+), 17 deletions(-)Signature · 1 line
2.26.2
Quoted 29 lines · Uros Bizjak
This patch series introduces
try_cmpxchg64()atomic locking function.
try_cmpxchg64()provides the same interface for 64 bit and 32 bit targets, emits CMPXCHGQ for 64 bit targets and CMPXCHG8B for 32 bit targets, and provides appropriate fallbacks when CMPXCHG8B is unavailable.
try_cmpxchg64()reuses flags from CMPXCHGQ/CMPXCHG8B instructions and avoids unneeded CMP for 64 bit targets or XOR/XOR/OR sequence for 32 bit targets.Cc Will Deacon <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Boqun Feng <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc Borislav Petkov <[email protected]>Cc "H. Peter Anvin" <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Sean Christopherson <[email protected]>Cc Vitaly Kuznetsov <[email protected]>Cc Wanpeng Li <[email protected]>Cc Jim Mattson <[email protected]>Cc Joerg Roedel <[email protected]>Uros Bizjak (3):
asm-generic/atomic: Addtry_cmpxchg64()instrumentation locking/atomic/x86: Introducearch_try_cmpxchg64()KVM/VMX: Usetry_cmpxchg64()inposted_intr.c
Quoted 7 lines
arch/x86/include/asm/cmpxchg_32.h | 62 +++++++++++++++++++---- arch/x86/include/asm/cmpxchg_64.h | 6 +++ arch/x86/kvm/vmx/posted_intr.c | 9 ++-- include/asm-generic/atomic-instrumented.h | 46 ++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 5 files changed, 108 insertions(+), 17 deletions(-)
Queued, thanks.
Paolo