locking/atomic/x86: Introduce try_cmpxchg64
Subsystems (3)
PATCH 1/1 locking/atomic/x86: Introduce try_cmpxchg64
This patch adds try_cmpxchg64 to improve code around cmpxchg8b. While the resulting code improvements on x86_64 are minor (a compare and a move saved), the improvements on x86_32 are quite noticeable. The code improves from:
84: 89 74 24 30 mov %esi,0x30(%esp) 88: 89 fe mov %edi,%esi 8a: 0f b7 0c 02 movzwl (%edx,%eax,1),%ecx 8e: c1 e1 08 shl $0x8,%ecx 91: 0f b7 c9 movzwl %cx,%ecx 94: 89 4c 24 34 mov %ecx,0x34(%esp) 98: 8b 96 24 1e 00 00 mov 0x1e24(%esi),%edx 9e: 8b 86 20 1e 00 00 mov 0x1e20(%esi),%eax a4: 8b 5c 24 34 mov 0x34(%esp),%ebx a8: 8b 7c 24 30 mov 0x30(%esp),%edi ac: 89 44 24 38 mov %eax,0x38(%esp) b0: 0f b6 44 24 38 movzbl 0x38(%esp),%eax b5: 8b 4c 24 38 mov 0x38(%esp),%ecx b9: 89 54 24 3c mov %edx,0x3c(%esp) bd: 83 e0 fd and $0xfffffffd,%eax c0: 89 5c 24 64 mov %ebx,0x64(%esp) c4: 8b 54 24 3c mov 0x3c(%esp),%edx c8: 89 4c 24 60 mov %ecx,0x60(%esp) cc: 8b 4c 24 34 mov 0x34(%esp),%ecx d0: 88 44 24 60 mov %al,0x60(%esp) d4: 8b 44 24 38 mov 0x38(%esp),%eax d8: c6 44 24 62 f2 movb $0xf2,0x62(%esp) dd: 8b 5c 24 60 mov 0x60(%esp),%ebx e1: f0 0f c7 0f lock cmpxchg8b (%edi) e5: 89 d1 mov %edx,%ecx e7: 8b 54 24 3c mov 0x3c(%esp),%edx eb: 33 44 24 38 xor 0x38(%esp),%eax ef: 31 ca xor %ecx,%edx f1: 09 c2 or %eax,%edx f3: 75 a3 jne 98 <t+0x98>
to:
84: 0f b7 0c 02 movzwl (%edx,%eax,1),%ecx 88: c1 e1 08 shl $0x8,%ecx 8b: 0f b7 c9 movzwl %cx,%ecx 8e: 8b 86 20 1e 00 00 mov 0x1e20(%esi),%eax 94: 8b 96 24 1e 00 00 mov 0x1e24(%esi),%edx 9a: 89 4c 24 64 mov %ecx,0x64(%esp) 9e: 89 c3 mov %eax,%ebx a0: 89 44 24 60 mov %eax,0x60(%esp) a4: 83 e3 fd and $0xfffffffd,%ebx a7: c6 44 24 62 f2 movb $0xf2,0x62(%esp) ac: 88 5c 24 60 mov %bl,0x60(%esp) b0: 8b 5c 24 60 mov 0x60(%esp),%ebx b4: f0 0f c7 0f lock cmpxchg8b (%edi) b8: 75 d4 jne 8e <t+0x8e>
The implementation extends the implementation of existing cmpxchg64.
--- arch/x86/include/asm/cmpxchg_32.h | 43 ++++++++++++++++++++++ arch/x86/include/asm/cmpxchg_64.h | 6 +++ include/linux/atomic/atomic-instrumented.h | 40 +++++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 4 files changed, 89 insertions(+), 2 deletions(-)
arch/x86/include/asm/cmpxchg_32.h +43 -0
@@ -42,6 +42,9 @@ static inline void set_64bit(volatile u64 *ptr, u64 value)#define arch_cmpxchg64_local (ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ (unsigned long long)(n))) +#define arch_try_cmpxchg64 (ptr, po, n) \ + ((__typeof__(*(ptr)))__try_cmpxchg64((ptr), (unsigned long long *)(po), \ + (unsigned long long)(n)))#endif static inline u64 __cmpxchg64 (volatile u64 *ptr,u64 old,u64 new)
@@ -70,6 +73,25 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)return prev; } +static inline bool __try_cmpxchg64 (volatile u64 *ptr,u64 *pold,u64 new) +{ +bool success; +u64 prev; + asm volatile(LOCK_PREFIX "cmpxchg8b %2" + CC_SET(z) + : CC_OUT(z) (success), + "=A" (prev), + "+m" (*ptr) + : "b" ((u32)new), + "c" ((u32)(new >> 32)), + "1" (*pold) + : "memory"); + +if (unlikely (!success)) + *pold = prev; +return success; +} +#ifndef CONFIG_X86_CMPXCHG64 /* * Building a kernel capable running on 80386 and 80486. It may be necessary
@@ -108,6 +130,27 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) : "memory"); \ __ret; }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + bool success; \ + __typeof__(*(ptr)) __prev; \ + __typeof__(ptr) _old = (__typeof__(ptr))(po); \ + __typeof__(*(ptr)) __old = *_old; \ + __typeof__(*(ptr)) __new = (n); \ + alternative_io(LOCK_PREFIX_HERE \ + "call cmpxchg8b_emu", \ + "lock; cmpxchg8b (%%esi)" , \ + X86_FEATURE_CX8, \ + "=A" (__prev), \ + "S" ((ptr)), "0" (__old), \ + "b" ((unsigned int)__new), \ + "c" ((unsigned int)(__new>>32)) \ + : "memory"); \ + success = (__prev == __old); \ + if (unlikely(!success)) \ + *_old = __prev; \ + likely(success); \ +})#endif #define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX8)
arch/x86/include/asm/cmpxchg_64.h +6 -0
@@ -19,6 +19,12 @@ static inline void set_64bit(volatile u64 *ptr, u64 val)arch_cmpxchg_local ((ptr), (o), (n)); \ }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_try_cmpxchg((ptr), (po), (n)); \ +}) +#define system_has_cmpxchg_double () boot_cpu_has(X86_FEATURE_CX16)#endif /* _ASM_X86_CMPXCHG_64_H */
include/linux/atomic/atomic-instrumented.h +39 -1
@@ -2006,6 +2006,44 @@ atomic_long_dec_if_positive(atomic_long_t *v)arch_try_cmpxchg_relaxed (__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) +#define try_cmpxchg64 (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + kcsan_mb(); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_acquire (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_release (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + kcsan_release(); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_relaxed (ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) +#define cmpxchg_local (ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \
@@ -2045,4 +2083,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) })#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 87c974b93032afd42143613434d1a7788fa598f9 +// 764f741eb77a7ad565dc8d99ce2837d5542e8aee
scripts/atomic/gen-atomic-instrumented.sh +1 -1
@@ -166,7 +166,7 @@ grep '^[a-z]' "$1" | while read name meta args; do done -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do +for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do for order in "" "_acquire" "_release" "_relaxed"; do gen_xchg "${xchg}" "${order}" "" printf "\n"
Signature · 1 line
2.35.1
Peter Zijlstra
maintainerQuoted 3 lines · Uros Bizjak
This patch adds
try_cmpxchg64to improve code around cmpxchg8b. While the resulting code improvements onx86_64are minor (a compare and a move saved), the improvements onx86_32are quite noticeable. The code improves from:
What user of cmpxchg64 is this?
Quoted 7 lines · Peter Zijlstra
Quoted 3 lines · Uros Bizjak
This patch adds
try_cmpxchg64to improve code around cmpxchg8b. While the resulting code improvements onx86_64are minor (a compare and a move saved), the improvements onx86_32are quite noticeable. The code improves from:What user of cmpxchg64 is this?
This is cmpxchg64 in pi_try_set_control from arch/x86/kvm/vmx/posted_intr.c, as shown in a RFC patch [1].
There are some more opportunities for try_cmpxchg64 in KVM, namely fast_pf_fix_direct_spte in arch/x86/kvm/mmu/mmu.c and tdp_mmu_set_spte_atomic in arch/x86/kvm/mmu/tdp_mmu.c
[1] https://www.spinics.net/lists/kvm/msg266042.html
Uros.
Peter Zijlstra
maintainerQuoted 11 lines · Uros Bizjak
Quoted 7 lines · Peter Zijlstra
Quoted 3 lines · Uros Bizjak
This patch adds
try_cmpxchg64to improve code around cmpxchg8b. While the resulting code improvements onx86_64are minor (a compare and a move saved), the improvements onx86_32are quite noticeable. The code improves from:What user of cmpxchg64 is this?
This is cmpxchg64 in
pi_try_set_controlfromarch/x86/kvm/vmx/posted_intr.c, as shown in a RFC patch [1].
I can't read that code, my brain is hard wired to read pi as priority inheritance/inversion.
Still, does 32bit actually support that stuff?
Quoted 3 lines
There are some more opportunities for
try_cmpxchg64in KVM, namelyfast_pf_fix_direct_spteinarch/x86/kvm/mmu/mmu.candtdp_mmu_set_spte_atomicinarch/x86/kvm/mmu/tdp_mmu.c
tdp_mmu is definitely 64bit only and as such shouldn't need to use cmpxchg64.
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Quoted 11 lines · Peter Zijlstra
Quoted 9 lines · Uros Bizjak
Quoted 7 lines · Peter Zijlstra
Quoted 3 lines · Uros Bizjak
This patch adds
try_cmpxchg64to improve code around cmpxchg8b. While the resulting code improvements onx86_64are minor (a compare and a move saved), the improvements onx86_32are quite noticeable. The code improves from:What user of cmpxchg64 is this?
Quoted 7 lines
Quoted 2 lines
This is cmpxchg64 in
pi_try_set_controlfromarch/x86/kvm/vmx/posted_intr.c, as shown in a RFC patch [1].I can't read that code, my brain is hard wired to read pi as priority inheritance/inversion.
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
vmx/evmcs.o vmx/nested.o vmx/posted_intr.o
And when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
error: call to ‘__cmpxchg_wrong_size’ declared with attribute error:
Bad argument size for cmpxchg
So, the majority of the patch deals with 32bits and tries to implement the inlined insn correctly for all cases. The 64bit part is simply a call to arch_try_cmpxchg, and the rest is auto-generated from scripts.
Quoted 7 lines
Quoted 4 lines
There are some more opportunities for
try_cmpxchg64in KVM, namelyfast_pf_fix_direct_spteinarch/x86/kvm/mmu/mmu.candtdp_mmu_set_spte_atomicinarch/x86/kvm/mmu/tdp_mmu.c
tdp_mmuis definitely 64bit only and as such shouldn't need to use cmpxchg64.
Indeed.
Quoted 3 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
Uros.
Quoted 11 lines · Peter Zijlstra
Quoted 9 lines · Uros Bizjak
Quoted 7 lines · Peter Zijlstra
Quoted 3 lines · Uros Bizjak
This patch adds
try_cmpxchg64to improve code around cmpxchg8b. While the resulting code improvements onx86_64are minor (a compare and a move saved), the improvements onx86_32are quite noticeable. The code improves from:What user of cmpxchg64 is this?
Quoted 7 lines
Quoted 2 lines
This is cmpxchg64 in
pi_try_set_controlfromarch/x86/kvm/vmx/posted_intr.c, as shown in a RFC patch [1].I can't read that code, my brain is hard wired to read pi as priority inheritance/inversion.
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
vmx/evmcs.o vmx/nested.o vmx/posted_intr.o
And when cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
error: call to ‘__cmpxchg_wrong_size’ declared with attribute error:
Bad argument size for cmpxchg
So, the majority of the patch deals with 32bits and tries to implement the inlined insn correctly for all cases. The 64bit part is simply a call to arch_try_cmpxchg, and the rest is auto-generated from scripts.
Quoted 7 lines
Quoted 4 lines
There are some more opportunities for
try_cmpxchg64in KVM, namelyfast_pf_fix_direct_spteinarch/x86/kvm/mmu/mmu.candtdp_mmu_set_spte_atomicinarch/x86/kvm/mmu/tdp_mmu.c
tdp_mmuis definitely 64bit only and as such shouldn't need to use cmpxchg64.
Indeed.
Quoted 3 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease a similar situation elsewhere in the sources.
Uros.
Sean Christopherson
reviewerQuoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 5 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
32-bit KVM is mostly useful for testing the mess that is nested NPT; an L1 hypervsior can use 32-bit paging for NPT, so KVM needs to at least make sure it doesn't blow up if such a hypervisor is encountered. But in terms of the performance of 32-bit KVM, I doubt there is a person in the world that cares.
Quoted 18 lines · Sean Christopherson
Quoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 3 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Quoted 7 lines
Quoted 2 lines
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
Yes, the idea was to improve cmpxchg64 with the implementation of try_cmpxchg64 for 64bit targets. However, the issue with 32bit targets stood in the way, so the effort with 32-bit implementation was mainly to unblock progression for 64-bit targets.
Uros.
Quoted 26 lines · Uros Bizjak
Quoted 17 lines · Sean Christopherson
Quoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 3 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Quoted 7 lines
Quoted 2 lines
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
Quoted 4 lines
Yes, the idea was to improve cmpxchg64 with the implementation of
try_cmpxchg64for 64bit targets. However, the issue with 32bit targets stood in the way, so the effort with 32-bit implementation was mainly to unblock progression for 64-bit targets.
Would that allow tdp mmu to work on 32 bit?
Best regards,
Maxim Levitsky
Quoted 3 lines
Uros.
Sean Christopherson
reviewerQuoted 27 lines · Maxim Levitsky
Quoted 26 lines · Uros Bizjak
Quoted 24 lines · Sean Christopherson
Quoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 5 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
Quoted 6 lines
Quoted 4 lines
Yes, the idea was to improve cmpxchg64 with the implementation of
try_cmpxchg64for 64bit targets. However, the issue with 32bit targets stood in the way, so the effort with 32-bit implementation was mainly to unblock progression for 64-bit targets.Would that allow tdp mmu to work on 32 bit?
From a purely technical perspective, there's nothing that prevents enabling the TDP MMU on 32-bit kernels. The TDP MMU is 64-bit only to simplify the implementation and to reduce the maintenance and validation costs.
Quoted 33 lines · Sean Christopherson
Quoted 32 lines · Maxim Levitsky
Quoted 30 lines · Uros Bizjak
Quoted 17 lines · Sean Christopherson
Quoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 3 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Quoted 7 lines
Quoted 2 lines
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
Yes, the idea was to improve cmpxchg64 with the implementation of
try_cmpxchg64for 64bit targets. However, the issue with 32bit targets stood in the way, so the effort with 32-bit implementation was mainly to unblock progression for 64-bit targets.
Quoted 6 lines
Quoted 1 line
Would that allow tdp mmu to work on 32 bit?
From a purely technical perspective, there's nothing that prevents enabling the TDP MMU on 32-bit kernels. The TDP MMU is 64-bit only to simplify the implementation and to reduce the maintenance and validation costs.
I understand exactly that, so the question, will this patch help make the tdp mmu work transparently on 32 bit kernels? I heard that 64 bit cmpxchg was one of the main reasons that it is 64 bit only.
I am asking because there was some talk to eliminate the direct mode from the legacy non tdp mmu, which would simplify its code by a lot, but then it will make 32 bit kernel fail back to shadowing mmu.
I know that nobody needs 32 bit KVM host support, but it is useful to be sure that nesting still works, and doesn't crash the host and such.
Best regards,
Maxim Levitsky
Sean Christopherson
reviewerQuoted 36 lines · Maxim Levitsky
Quoted 35 lines · Sean Christopherson
Quoted 33 lines · Maxim Levitsky
Quoted 26 lines · Uros Bizjak
Quoted 17 lines · Sean Christopherson
Quoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 3 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Quoted 7 lines
Quoted 2 lines
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
Quoted 4 lines
Yes, the idea was to improve cmpxchg64 with the implementation of
try_cmpxchg64for 64bit targets. However, the issue with 32bit targets stood in the way, so the effort with 32-bit implementation was mainly to unblock progression for 64-bit targets.Would that allow tdp mmu to work on 32 bit?
Quoted 7 lines
Quoted 3 lines
From a purely technical perspective, there's nothing that prevents enabling the TDP MMU on 32-bit kernels. The TDP MMU is 64-bit only to simplify the implementation and to reduce the maintenance and validation costs.
I understand exactly that, so the question, will this patch help make the tdp mmu work transparently on 32 bit kernels? I heard that 64 bit cmpxchg was one of the main reasons that it is 64 bit only.
I don't think it moves the needled much, e.g. non-atomic 64-bit accesses are still problematic, and we'd have to update the TDP MMU to deal with PAE paging (thanks NPT). All those problems are solvable, it's purely a matter of the ongoing costs to solve them.
Quoted 3 lines
I am asking because there was some talk to eliminate the direct mode from the legacy non tdp mmu, which would simplify its code by a lot, but then it will make 32 bit kernel fail back to shadowing mmu.
Simplify which code? Between the nonpaging code and direct shadow pages in indirect MMUs, the vast majority of the "direct" support in the legacy MMU needs to be kept even if TDP support is dropped. And the really nasty stuff, e.g. PAE roots, would need to be carried over to the TDP MMU.
Quoted 50 lines · Sean Christopherson
Quoted 36 lines · Maxim Levitsky
Quoted 35 lines · Sean Christopherson
Quoted 33 lines · Maxim Levitsky
Quoted 26 lines · Uros Bizjak
Quoted 17 lines · Sean Christopherson
Quoted 10 lines · Uros Bizjak
Quoted 1 line · Peter Zijlstra
Still, does 32bit actually support that stuff?
Unfortunately, it does:
kvm-intel-y +=
vmx/vmx.ovmx/vmenter.ovmx/pmu_intel.ovmx/vmcs12.o\
vmx/evmcs.ovmx/nested.ovmx/posted_intr.oAnd when existing cmpxchg64 is substituted with cmpxchg, the compilation dies for 32bits with:
...
Quoted 3 lines
Quoted 2 lines
Anyway, your patch looks about right, but I find it *really* hard to care about 32bit code these days.
Quoted 7 lines
Quoted 2 lines
Thanks, this is also my sentiment, but I hope the patch will enable better code and perhaps ease similar situation I have had elsewhere.
IMO, if we merge this it should be solely on the benefits to 64-bit code. Yes, KVM still supports 32-bit kernels, but I'm fairly certain the only people that run 32-bit KVM are KVM developers. 32-bit KVM has been completely broken for multiple releases at least once, maybe twice, and no one ever complained.
Quoted 4 lines
Yes, the idea was to improve cmpxchg64 with the implementation of
try_cmpxchg64for 64bit targets. However, the issue with 32bit targets stood in the way, so the effort with 32-bit implementation was mainly to unblock progression for 64-bit targets.Would that allow tdp mmu to work on 32 bit?
Quoted 7 lines
Quoted 3 lines
From a purely technical perspective, there's nothing that prevents enabling the TDP MMU on 32-bit kernels. The TDP MMU is 64-bit only to simplify the implementation and to reduce the maintenance and validation costs.
I understand exactly that, so the question, will this patch help make the tdp mmu work transparently on 32 bit kernels? I heard that 64 bit cmpxchg was one of the main reasons that it is 64 bit only.
I don't think it moves the needled much, e.g. non-atomic 64-bit accesses are still problematic, and we'd have to update the TDP MMU to deal with PAE paging (thanks NPT). All those problems are solvable, it's purely a matter of the ongoing costs to solve them.
Quoted 9 lines
Quoted 3 lines
I am asking because there was some talk to eliminate the direct mode from the legacy non tdp mmu, which would simplify its code by a lot, but then it will make 32 bit kernel fail back to shadowing mmu.
Simplify which code? Between the nonpaging code and direct shadow pages in indirect MMUs, the vast majority of the "direct" support in the legacy MMU needs to be kept even if TDP support is dropped. And the really nasty stuff, e.g. PAE roots, would need to be carried over to the TDP MMU.
I guess this makes sense. I haven't researched the code well enough to know the exact answer. I was just curious if this patch makes any difference :)
Thanks!
Best regards,
Maxim Levitsky
Peter Zijlstra
maintainerOn Tue, May 10, 2022 at 05:42:17PM +0200, Uros Bizjak wrote:
For the Changelog I would focus on the 64bit improvement and leave 32bit as a side-note.
Quoted 31 lines
--- arch/x86/include/asm/cmpxchg_32.h | 43 ++++++++++++++++++++++ arch/x86/include/asm/cmpxchg_64.h | 6 +++ include/linux/atomic/atomic-instrumented.h | 40 +++++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 4 files changed, 89 insertions(+), 2 deletions(-)arch/x86/include/asm/cmpxchg_32.h +9
-0@@ -42,6 +42,9 @@ static inline void set_64bit(volatile u64 *ptr, u64 value)#define arch_cmpxchg64_local (ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ (unsigned long long)(n))) +#define arch_try_cmpxchg64 (ptr, po, n) \ + ((__typeof__(*(ptr)))__try_cmpxchg64((ptr), (unsigned long long *)(po), \ + (unsigned long long)(n)))#endif static inline u64 __cmpxchg64 (volatile u64 *ptr,u64 old,u64 new)@@ -70,6 +73,25 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)return prev; } +static inline bool __try_cmpxchg64 (volatile u64 *ptr,u64 *pold,u64 new) +{ +bool success; +u64 prev; + asmvolatile (LOCK_PREFIX "cmpxchg8b %2" + CC_SET(z)
arch/x86/include/asm/cmpxchg_32.h static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) · line 91
Quoted 10 lines
arch/x86/include/asm/cmpxchg_32.h +10
-0+ : CC_OUT(z) (success), + "=A" (prev), + "+m" (*ptr) + : "b" ((u32)new), + "c" ((u32)(new >> 32)), + "1" (*pold) + : "memory"); + +if (unlikely (!success)) + *pold = prev;
I would prefer this be more like the existing try_cmpxchg code, perhaps:
u64 old = *pold; asm volatile (LOCK_PREFIX "cmpxchg8b %[ptr]" CC_SET(z) : CC_OUT(z) (success), [ptr] "+m" (*ptr) "+A" (old) : "b" ((u32)new) "c" ((u32)(new >> 32)) : "memory"); if (unlikely(!success)) *pold = old;
The existing 32bit cmpxchg code is a 'bit' crusty.
Quoted 21 lines
arch/x86/include/asm/cmpxchg_32.h +14
-0+return success; +} +#ifndef CONFIG_X86_CMPXCHG64 /* * Building a kernel capable running on 80386 and 80486. It may be necessary @@ -108,6 +130,27 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) : "memory"); \ __ret; }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + bool success; \ + __typeof__(*(ptr)) __prev; \ + __typeof__(ptr) _old = (__typeof__(ptr))(po); \ + __typeof__(*(ptr)) __old = *_old; \ + __typeof__(*(ptr)) __new = (n); \ + alternative_io(LOCK_PREFIX_HERE \ + "call cmpxchg8b_emu", \ + "lock; cmpxchg8b (%%esi)" , \ + X86_FEATURE_CX8, \
arch/x86/include/asm/cmpxchg_32.h static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) · line 153
Quoted 10 lines
arch/x86/include/asm/cmpxchg_32.h +10
-0+"=A" (__prev), \ +"S" ((ptr)),"0" (__old), \ +"b" ((unsigned int )__new), \ +"c" ((unsigned int )(__new>>32 )) \ + : "memory"); \ + success = (__prev == __old); \ +if (unlikely (!success)) \ + *_old = __prev; \ +likely (success); \ +})
Wouldn't this be better written like the normal fallback wrapper?
static __always_inline boolarch_try_cmpxchg64(u64 *v, u64 *old, u64 new)
{
u64 r, o = *old;
r = arch_cmpxchg64(v, o, new);
if (unlikely(r != o))
*old = r;
return likely(r == o);
}Less magical, same exact code.
Quoted 5 lines · Peter Zijlstra
On Tue, May 10, 2022 at 05:42:17PM +0200, Uros Bizjak wrote:
For the Changelog I would focus on the 64bit improvement and leave 32bit as a side-note.
Thanks, I will rephrase the ChangeLog.
Quoted 60 lines
Quoted 32 lines
--- arch/x86/include/asm/cmpxchg_32.h | 43 ++++++++++++++++++++++ arch/x86/include/asm/cmpxchg_64.h | 6 +++ include/linux/atomic/atomic-instrumented.h | 40 +++++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 4 files changed, 89 insertions(+), 2 deletions(-)arch/x86/include/asm/cmpxchg_32.h +9
-0@@ -42,6 +42,9 @@ static inline void set_64bit(volatile u64 *ptr, u64 value)#define arch_cmpxchg64_local (ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ (unsigned long long)(n))) +#define arch_try_cmpxchg64 (ptr, po, n) \ + ((__typeof__(*(ptr)))__try_cmpxchg64((ptr), (unsigned long long *)(po), \ + (unsigned long long)(n)))#endif static inline u64 __cmpxchg64 (volatile u64 *ptr,u64 old,u64 new)@@ -70,6 +73,25 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)return prev; } +static inline bool __try_cmpxchg64 (volatile u64 *ptr,u64 *pold,u64 new) +{ +bool success; +u64 prev; + asmvolatile (LOCK_PREFIX "cmpxchg8b %2" + CC_SET(z)arch/x86/include/asm/cmpxchg_32.h static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) · line 91
Quoted 10 lines
arch/x86/include/asm/cmpxchg_32.h +10
-0+ : CC_OUT(z) (success), + "=A" (prev), + "+m" (*ptr) + : "b" ((u32)new), + "c" ((u32)(new >> 32)), + "1" (*pold) + : "memory"); + +if (unlikely (!success)) + *pold = prev;I would prefer this be more like the existing
try_cmpxchgcode, perhaps:u64 old = *pold; asm volatile (LOCK_PREFIX "cmpxchg8b %[ptr]" CC_SET(z) : CC_OUT(z) (success), [ptr] "+m" (*ptr) "+A" (old) : "b" ((u32)new) "c" ((u32)(new >> 32)) : "memory"); if (unlikely(!success)) *pold = old;
Quoted 1 line
The existing 32bit cmpxchg code is a 'bit' crusty.
I was trying to follow the existing __cmpxchg64 as much as possible, with the intention of a follow-up patch that would modernize everything in cmpxchg_32.h. I can surely go the other way and submit modernized new code.
Quoted 44 lines
Quoted 21 lines
arch/x86/include/asm/cmpxchg_32.h +14
-0+return success; +} +#ifndef CONFIG_X86_CMPXCHG64 /* * Building a kernel capable running on 80386 and 80486. It may be necessary @@ -108,6 +130,27 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) : "memory"); \ __ret; }) +#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + bool success; \ + __typeof__(*(ptr)) __prev; \ + __typeof__(ptr) _old = (__typeof__(ptr))(po); \ + __typeof__(*(ptr)) __old = *_old; \ + __typeof__(*(ptr)) __new = (n); \ + alternative_io(LOCK_PREFIX_HERE \ + "call cmpxchg8b_emu", \ + "lock; cmpxchg8b (%%esi)" , \ + X86_FEATURE_CX8, \arch/x86/include/asm/cmpxchg_32.h static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) · line 153
Quoted 10 lines
arch/x86/include/asm/cmpxchg_32.h +10
-0+"=A" (__prev), \ +"S" ((ptr)),"0" (__old), \ +"b" ((unsigned int )__new), \ +"c" ((unsigned int )(__new>>32 )) \ + : "memory"); \ + success = (__prev == __old); \ +if (unlikely (!success)) \ + *_old = __prev; \ +likely (success); \ +})Wouldn't this be better written like the normal fallback wrapper?
static
__always_inlineboolarch_try_cmpxchg64(u64 *v, u64 *old, u64 new){ u64 r, o = *old; r = arch_cmpxchg64(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); }
Quoted 1 line
Less magical, same exact code.
Also, I tried to follow up the existing #defines. Will improve the code according to your suggestion here.
Thanks,
Uros.
Quoted 34 lines · Uros Bizjak
Quoted 34 lines
Quoted 11 lines
arch/x86/include/asm/cmpxchg_32.h +11
-0+#define arch_try_cmpxchg64 (ptr, po, n) \ +({ \ + bool success; \ + __typeof__(*(ptr)) __prev; \ + __typeof__(ptr) _old = (__typeof__(ptr))(po); \ + __typeof__(*(ptr)) __old = *_old; \ + __typeof__(*(ptr)) __new = (n); \ + alternative_io(LOCK_PREFIX_HERE \ + "call cmpxchg8b_emu", \ + "lock; cmpxchg8b (%%esi)" , \ + X86_FEATURE_CX8, \arch/x86/include/asm/cmpxchg_32.h static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) · line 153
Quoted 10 lines
arch/x86/include/asm/cmpxchg_32.h +10
-0+"=A" (__prev), \ +"S" ((ptr)),"0" (__old), \ +"b" ((unsigned int )__new), \ +"c" ((unsigned int )(__new>>32 )) \ + : "memory"); \ + success = (__prev == __old); \ +if (unlikely (!success)) \ + *_old = __prev; \ +likely (success); \ +})Wouldn't this be better written like the normal fallback wrapper?
static
__always_inlineboolarch_try_cmpxchg64(u64 *v, u64 *old, u64 new){ u64 r, o = *old; r = arch_cmpxchg64(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); }
Quoted 4 lines
Quoted 1 line
Less magical, same exact code.
Also, I tried to follow up the existing #defines. Will improve the code according to your suggestion here.
In the v2 patch, generic fallbacks were introduced, so that arch_try_cmpxchg64 can be used when only arch_cmpxchg64 is defined.
Uros.