Skip to contentSkip to conversation

Add support for cmpxchg16b emulation

PATCH Superseded Sairaj Kodilkar 3 participants 9 messages Jul 6 – Aug 25

Subsystems (3)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover Add support for cmpxchg16b emulation

This series adds 128-bit compare-exchange support needed for KVM to emulate guest cmpxchg16b instructions.

The AMD IOMMU driver updates 256-bit device table entries with two 128-bit cmpxchg operations. For hardware-accelerated vIOMMU, QEMU traps those DTE accesses to install nested page tables [1]. KVM must emulate guest cmpxchg16b on the trapped MMIO regions; without that, DTE access emulation fails.

Patch 1: extends the x86 user CMPXCHG helpers with __try_cmpxchg128_user_asm() (cmpxchg16b on x86-64), wired into unsafe_try_cmpxchg_user().

Patch 2: extends KVM CMPXCHG8B emulation to 16-byte operands (REX.W=1) and uses the new helper for atomic 16-byte compare-exchange on guest RAM via emulator_cmpxchg_emulated().

You can find the KUT to test this series at [2].

[1] https://github.com/AMDESE/qemu-iommu/blob/wip/for_iommufd_hw_queue-v8_amd_viommu_20260106/hw/i386/amd_viommu.c#L517
[2] https://lore.kernel.org/kvm/[email protected]/T/#u

---

Base: 8cd9520d35a6c38db6567e97dd93b1f11f185dc6

---

Sairaj Kodilkar (2):
x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands
KVM: x86: Add support for cmpxchg16b emulation

 arch/x86/include/asm/uaccess.h | 56 +++++++++++++++++++++++++++++++++-
 arch/x86/kvm/emulate.c         | 50 ++++++++++++++++++++----------
 arch/x86/kvm/kvm_emulate.h     |  6 ++++
 arch/x86/kvm/x86.c             |  7 ++++-
 4 files changed, 101 insertions(+), 18 deletions(-)
Signature · 1 line
2.34.1
#1 lore raw

PATCH v2 1/2 x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Extend the existing user CMPXCHG helpers to support 16-byte operands on x86-64, using LOCK_PREFIX "cmpxchg16b". This mirrors the existing __try_cmpxchg64_user_asm() / cmpxchg8b path provided for 32-bit kernels, where KVM needs an atomic compare-exchange wider than the generic cmpxchg helper can provide.

On 32-bit kernels, stub the helper to always return failure because cmpxchg16b requires 64-bit GPRs and is not available.

KVM uses this to atomically emulate guest cmpxchg16b on guest RAM mapped via userspace addresses.

---
 arch/x86/include/asm/uaccess.h | 56 +++++++++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 1 deletion(-)
arch/x86/include/asm/uaccess.h +55 -1
index 367297b188c3..123755d47109 100644
@@ -407,6 +407,25 @@ do {									\
 	if (unlikely(!success))						\
 		*_old = __old;						\
 	likely(success);					})
+#else // !CONFIG_X86_32
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label)	({	\
+	bool success;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm_goto_output("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg16b %[ptr]\n"		\
+		     _ASM_EXTABLE_UA(1b, %l[label])			\
+		     : "=@ccz" (success),				\
+		       "+A" (__old),					\
+		       [ptr] "+m" (*_ptr)				\
+		     : "b" ((u64)__new),				\
+		       "c" ((u64)((u128)__new >> 64))			\
+		     : "memory"						\
+		     : label);						\
+	if (unlikely(!success))						\
+		*_old = __old;						\
+	likely(success);					})
 #endif // CONFIG_X86_32
 #else  // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 #define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label)	({ \
@@ -463,6 +482,30 @@ do {									\
 	if (unlikely(!__result))					\
 		*_old = __old;						\
 	likely(__result);					})
+#else //!CONFIG_X86_32
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label)	({	\
+	int __result;							\
+	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
+	__typeof__(*(_ptr)) __old = *_old;				\
+	__typeof__(*(_ptr)) __new = (_new);				\
+	asm volatile("\n"						\
+		     "1: " LOCK_PREFIX "cmpxchg16b %[ptr]\n"		\
+		     "mov $0, %[result]\n\t"				\
+		     "setz %b[result]\n"				\
+		     "2:\n"						\
+		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
+					   %[result])			\
+		     : [result] "=q" (__result),			\
+		       "+A" (__old),					\
+		       [ptr] "+m" (*_ptr)				\
+		     : "b" ((u64)__new),				\
+		       "c" ((u64)((u128)__new >> 64))			\
+		     : "memory", "cc");					\
+	if (unlikely(__result < 0))					\
+		goto label;						\
+	if (unlikely(!__result))					\
+		*_old = __old;						\
+	likely(__result);					})
 #endif // CONFIG_X86_32
 #endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
 
@@ -551,11 +594,18 @@ do {										\
 
 extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
+#else
 #define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)		\
 	__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
+
 #endif
 
+
 /*
  * Force the pointer to u<size> to match the size expected by the asm helper.
  * clang/LLVM compiles all cases and only discards the unused paths after
@@ -580,6 +630,10 @@ extern void __try_cmpxchg_user_wrong_size(void);
 	case 8:	__ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
 						 (_nval), _label);		\
 		break;								\
+	case 16:								\
+		__ret = __try_cmpxchg128_user_asm((__force u128 *)(_ptr),	\
+						  (_oldp), (_nval), _label);	\
+		break;								\
 	default: __try_cmpxchg_user_wrong_size();				\
 	}									\
 	__ret;						})
Signature · 1 line
2.34.1
#2 lore raw

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider: - [Medium] The 32-bit stub for `__try_cmpxchg128_user_asm` incorrectly evaluates to 1 (success) instead of…

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] The 32-bit stub for __try_cmpxchg128_user_asm incorrectly evaluates to 1 (success) instead of 0 (failure), silently skipping emulation while falsely signaling a successful memory update. --

commit 11b4c82317b376374b32df0435151f876d6239e5
Author: Sairaj Kodilkar <[email protected]>

x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

This commit extends the user CMPXCHG helpers to support 16-byte operands on x86-64 using the cmpxchg16b instruction. It also adds a 32-bit stub that is intended to always return failure since cmpxchg16b requires 64-bit GPRs.

Quoted 4 lines
arch/x86/include/asm/uaccess.h +0 -0
index 367297b188c39..123755d471099 100644

[ ... ]

arch/x86/include/asm/uaccess.h do { \ · line 601

Quoted 10 lines
arch/x86/include/asm/uaccess.h +5 -1
@@ -551,11 +594,18 @@ do {										\
 
 extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user macro treat this as success (true) rather than failure (false)?

This could cause the wrapper to negate the value and return 0 (success) to the caller, falsely signaling a successful memory exchange without actually updating memory or the old value. Even though there are no current 32-bit callers using a 16-byte operand, should this evaluate to 0 instead of 1 to ensure theoretical future callers don't encounter a silent atomicity break?

Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
#3 lore raw

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Quoted 38 lines · On 7/6/2026 12:22 PM, [email protected] wrote:

[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] The 32-bit stub for __try_cmpxchg128_user_asm incorrectly evaluates to 1 (success) instead of 0 (failure), silently skipping emulation while falsely signaling a successful memory update. --

commit 11b4c82317b376374b32df0435151f876d6239e5
Author: Sairaj Kodilkar <[email protected]>

x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

This commit extends the user CMPXCHG helpers to support 16-byte operands on x86-64 using the cmpxchg16b instruction. It also adds a 32-bit stub that is intended to always return failure since cmpxchg16b requires 64-bit GPRs.

Quoted 4 lines
arch/x86/include/asm/uaccess.h +0 -0
index 367297b188c39..123755d471099 100644

[ ... ]

arch/x86/include/asm/uaccess.h do { \ · line 601

Quoted 10 lines
arch/x86/include/asm/uaccess.h +5 -1
@@ -551,11 +594,18 @@ do {                                                                            \
 
  extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user macro treat this as success (true) rather than failure (false)?

Quoted 8 lines

This could cause the wrapper to negate the value and return 0 (success) to the caller, falsely signaling a successful memory exchange without actually updating memory or the old value. Even though there are no current 32-bit callers using a 16-byte operand, should this evaluate to 0 instead of 1 to ensure theoretical future callers don't encounter a silent atomicity break?

--
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1

Hi,

Right, It should return 0 instead of 1, will rectify this in the next patch.

Thanks
Sairaj

#4 lore raw

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Quoted 29 lines · Sairaj Kodilkar
Quoted 28 lines · On 7/6/2026 12:22 PM, [email protected] wrote:
Quoted 4 lines
arch/x86/include/asm/uaccess.h +0 -0
index 367297b188c39..123755d471099 100644

[ ... ]

arch/x86/include/asm/uaccess.h do { \ · line 601

Quoted 10 lines
arch/x86/include/asm/uaccess.h +5 -1
@@ -551,11 +594,18 @@ do {                                                                            \
 
  extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user macro treat this as success (true) rather than failure (false)?

This could cause the wrapper to negate the value and return 0 (success) to the caller, falsely signaling a successful memory exchange without actually updating memory or the old value. Even though there are no current 32-bit callers using a 16-byte operand, should this evaluate to 0 instead of 1 to ensure theoretical future callers don't encounter a silent atomicity break?

Quoted 5 lines
Quoted 2 lines

--
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1

Hi,

Right, It should return 0 instead of 1, will rectify this in the next patch.

Is it possible to fail the build instead? Attempting to generate CMPXCHG16B on 32-bit should straigt up fail.

#5 lore raw

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Quoted 30 lines · Sean Christopherson
Quoted 29 lines · Sairaj Kodilkar
Quoted 28 lines · On 7/6/2026 12:22 PM, [email protected] wrote:
Quoted 4 lines
arch/x86/include/asm/uaccess.h +0 -0
index 367297b188c39..123755d471099 100644

[ ... ]

arch/x86/include/asm/uaccess.h do { \ · line 601

Quoted 10 lines
arch/x86/include/asm/uaccess.h +5 -1
@@ -551,11 +594,18 @@ do {                                                                            \
 
  extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user macro treat this as success (true) rather than failure (false)?

This could cause the wrapper to negate the value and return 0 (success) to the caller, falsely signaling a successful memory exchange without actually updating memory or the old value. Even though there are no current 32-bit callers using a 16-byte operand, should this evaluate to 0 instead of 1 to ensure theoretical future callers don't encounter a silent atomicity break?

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

--
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1

Hi,

Right, It should return 0 instead of 1, will rectify this in the next patch.

Is it possible to fail the build instead? Attempting to generate CMPXCHG16B on 32-bit should straigt up fail.

Hi Sean,

A compile-time failure in __try_cmpxchg128_user_asm would unfortunately break i386 builds even for callers that never use a 16-byte operand.

unsafe_try_cmpxchg_user dispatches on sizeof(*(_ptr)) via a switch, and the 128-bit path is one of those cases. As noted in the comment above the macro, Clang/LLVM tends to compile all switch arms when expanding this macro, and only discards unused paths afterward.

One way to solve this is to create two different macros of unsafe_try_cmpxchg_user for 32 and 64 bit, right now I am in favour of returning failure. Let me know what do you think about it.

Thanks
Sairaj

#6 lore raw

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Quoted 42 lines · Sairaj Kodilkar
Quoted 30 lines · Sean Christopherson
Quoted 29 lines · Sairaj Kodilkar
Quoted 28 lines · On 7/6/2026 12:22 PM, [email protected] wrote:
Quoted 4 lines
arch/x86/include/asm/uaccess.h +0 -0
index 367297b188c39..123755d471099 100644

[ ... ]

arch/x86/include/asm/uaccess.h do { \ · line 601

Quoted 10 lines
arch/x86/include/asm/uaccess.h +5 -1
@@ -551,11 +594,18 @@ do {                                                                            \
 
  extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user macro treat this as success (true) rather than failure (false)?

This could cause the wrapper to negate the value and return 0 (success) to the caller, falsely signaling a successful memory exchange without actually updating memory or the old value. Even though there are no current 32-bit callers using a 16-byte operand, should this evaluate to 0 instead of 1 to ensure theoretical future callers don't encounter a silent atomicity break?

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

--
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1

Hi,

Right, It should return 0 instead of 1, will rectify this in the next patch.

Is it possible to fail the build instead? Attempting to generate CMPXCHG16B on 32-bit should straigt up fail.

Hi Sean,

Quoted 7 lines

A compile-time failure in __try_cmpxchg128_user_asm would unfortunately break i386 builds even for callers that never use a 16-byte operand.

unsafe_try_cmpxchg_user dispatches on sizeof(*(_ptr)) via a switch, and the 128-bit path is one of those cases. As noted in the comment above the macro, Clang/LLVM tends to compile all switch arms when expanding this macro, and only discards unused paths afterward.

If that's actually happening, then this code is already broken, because u128 exists only on (some) 64-bit kernels.

A straight static_assert() will fail, but BUILD_BUG_ON() (or more precisely, __compiletime_assert()) and all of the associated magic exists for these exact scenarios. Unlike static_assert(), __compiletime_assert() relies on dead-code elimination, i.e. does exactly what we want.

arch/x86/include/asm/uaccess.h +1 -4
index a58d50d7a219..710a0b42eeef 100644
@@ -595,10 +595,7 @@ do {										\
 extern void __try_cmpxchg_user_wrong_size(void);
 
 #ifdef CONFIG_X86_32
-/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
- * instruction).
- */
-#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) ({ BUILD_BUG_ON(1); 1; })
 #else
 #define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)		\
 	__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
#7 lore raw

Re: [PATCH v2 1/2] x86/uaccess: Extend CMPXCHG user helpers to 128-bit operands

Quoted 51 lines · Sean Christopherson
Quoted 42 lines · Sairaj Kodilkar
Quoted 30 lines · Sean Christopherson
Quoted 29 lines · Sairaj Kodilkar
Quoted 28 lines · On 7/6/2026 12:22 PM, [email protected] wrote:
Quoted 4 lines
arch/x86/include/asm/uaccess.h +0 -0
index 367297b188c39..123755d471099 100644

[ ... ]

arch/x86/include/asm/uaccess.h do { \ · line 601

Quoted 10 lines
arch/x86/include/asm/uaccess.h +5 -1
@@ -551,11 +594,18 @@ do {                                                                            \
 
  extern void __try_cmpxchg_user_wrong_size(void);
 
-#ifndef CONFIG_X86_32
+#ifdef CONFIG_X86_32
+/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
+ * instruction).
+ */
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)

[Severity: Medium]
Does this 32-bit stub correctly signal failure?

Since this macro evaluates to 1, won't the surrounding unsafe_try_cmpxchg_user macro treat this as success (true) rather than failure (false)?

This could cause the wrapper to negate the value and return 0 (success) to the caller, falsely signaling a successful memory exchange without actually updating memory or the old value. Even though there are no current 32-bit callers using a 16-byte operand, should this evaluate to 0 instead of 1 to ensure theoretical future callers don't encounter a silent atomicity break?

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

--
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1

Hi,

Right, It should return 0 instead of 1, will rectify this in the next patch.

Is it possible to fail the build instead? Attempting to generate CMPXCHG16B on 32-bit should straigt up fail.

Hi Sean,

Quoted 7 lines

A compile-time failure in __try_cmpxchg128_user_asm would unfortunately break i386 builds even for callers that never use a 16-byte operand.

unsafe_try_cmpxchg_user dispatches on sizeof(*(_ptr)) via a switch, and the 128-bit path is one of those cases. As noted in the comment above the macro, Clang/LLVM tends to compile all switch arms when expanding this macro, and only discards unused paths afterward.

Quoted 7 lines

If that's actually happening, then this code is already broken, because u128 exists only on (some) 64-bit kernels.

A straight static_assert() will fail, but BUILD_BUG_ON() (or more precisely, __compiletime_assert()) and all of the associated magic exists for these exact scenarios. Unlike static_assert(), __compiletime_assert() relies on dead-code elimination, i.e. does exactly what we want.

Sorry, I got confused between static assertion and compile time assertion. BUILD_BUG_ON() should work. Will update and repost the patches.

Thanks
Sairaj

Quoted 18 lines
arch/x86/include/asm/uaccess.h +1 -4
index a58d50d7a219..710a0b42eeef 100644
@@ -595,10 +595,7 @@ do {										\
 extern void __try_cmpxchg_user_wrong_size(void);
 
 #ifdef CONFIG_X86_32
-/* Always fail on 32 bit arch as it do not support 128 cmpxchg (i.e. cmpxchg16b
- * instruction).
- */
-#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) (1)
+#define __try_cmpxchg128_user_asm(_ptr, _pold, _new, label) ({ BUILD_BUG_ON(1); 1; })
 #else
 #define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label)		\
 	__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
#8 lore raw

PATCH v2 2/2 KVM: x86: Add support for cmpxchg16b emulation

In reply to Sairaj Kodilkar

AMD and Intel both provides support for 128 bit cmpxchg operands using cmpxchg8b/cmpxchg16b instructions (opcode 0FC7). However, kvm does not support emulating cmpxchg16b (i.e when destination memory is 128 bit and REX.W = 1) which causes emulation failure when QEMU guest performs a cmpxchg16b on a memory region setup as a IO.

This failure is seen on the AMD IOMMU driver which writes 256-bit device table entries with two 128-bit cmpxchg operations. For guests using hardware-accelerated vIOMMU, QEMU traps device table accesses to set up nested page tables (see [1]). Without 128-bit cmpxchg emulation, KVM cannot handle these traps and DTE access emulation fails.

Hence extend cmpxchg8b to perform cmpxchg16b when the destination memory is 128 bit.

[1] https://github.com/AMDESE/qemu-iommu/blob/wip/for_iommufd_hw_queue-v8_amd_viommu_20260106/hw/i386/amd_viommu.c#L517

---
 arch/x86/kvm/emulate.c     | 50 ++++++++++++++++++++++++++------------
 arch/x86/kvm/kvm_emulate.h |  6 +++++
 arch/x86/kvm/x86.c         |  7 +++++-
 3 files changed, 46 insertions(+), 17 deletions(-)
arch/x86/kvm/emulate.c +34 -16
index c8e292e9a24d..9df55b3c5627 100644
@@ -2188,24 +2188,36 @@ static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
 	return rc;
 }
 
+#define em_cmpxchg8b_16b(__c, rbits, mbits)\
+do {												\
+	u##mbits old = __c->dst.orig_val##mbits;						\
+												\
+	BUILD_BUG_ON(rbits * 2 != mbits);							\
+												\
+	if (((u##rbits) (old >> 0) != (u##rbits) reg_read(ctxt, VCPU_REGS_RAX)) ||		\
+	    ((u##rbits) (old >> rbits) != (u##rbits) reg_read(ctxt, VCPU_REGS_RDX))) {		\
+		*reg_write(ctxt, VCPU_REGS_RAX) = (u##rbits) (old >> 0);			\
+		*reg_write(ctxt, VCPU_REGS_RDX) = (u##rbits) (old >> rbits);			\
+		ctxt->eflags &= ~X86_EFLAGS_ZF;							\
+	} else {										\
+		ctxt->dst.val##mbits = ((u##mbits)reg_read(ctxt, VCPU_REGS_RCX) << rbits) |	\
+					(u##rbits) reg_read(ctxt, VCPU_REGS_RBX);		\
+												\
+		ctxt->eflags |= X86_EFLAGS_ZF;							\
+	}											\
+} while(0)
+
 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
 {
-	u64 old = ctxt->dst.orig_val64;
-
-	if (ctxt->dst.bytes == 16)
+	if (WARN_ON_ONCE(8 + !!(ctxt->rex_bits & REX_W) * 8 != ctxt->dst.bytes))
 		return X86EMUL_UNHANDLEABLE;
 
-	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
-	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
-		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
-		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
-		ctxt->eflags &= ~X86_EFLAGS_ZF;
-	} else {
-		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
-			(u32) reg_read(ctxt, VCPU_REGS_RBX);
-
-		ctxt->eflags |= X86_EFLAGS_ZF;
-	}
+	if (!(ctxt->rex_bits & REX_W))
+		em_cmpxchg8b_16b(ctxt, 32, 64);
+#ifdef CONFIG_X86_64
+	else
+		em_cmpxchg8b_16b(ctxt, 64, 128);
+#endif
 	return X86EMUL_CONTINUE;
 }
 
@@ -5405,8 +5417,14 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, bool check_intercepts)
 			goto done;
 		}
 	}
-	/* Copy full 64-bit value for CMPXCHG8B.  */
-	ctxt->dst.orig_val64 = ctxt->dst.val64;
+	/* Copy full 64/128-bit value for CMPXCHG8B.  */
+
+#ifdef CONFIG_X86_64
+	if (ctxt->dst.bytes == 16)
+		ctxt->dst.orig_val128 = ctxt->dst.val128;
+	else
+#endif
+		ctxt->dst.orig_val64 = ctxt->dst.val64;
 
 special_insn:
 
arch/x86/kvm/kvm_emulate.h +6 -0
index fb3dab4b5a53..a51677217ada 100644
@@ -255,6 +255,9 @@ struct operand {
 	union {
 		unsigned long orig_val;
 		u64 orig_val64;
+#ifdef CONFIG_X86_64
+		u128 orig_val128;
+#endif
 	};
 	union {
 		unsigned long *reg;
@@ -268,6 +271,9 @@ struct operand {
 	union {
 		unsigned long val;
 		u64 val64;
+#ifdef CONFIG_X86_64
+		u128 val128;
+#endif
 		char valptr[sizeof(avx256_t)];
 		sse128_t vec_val;
 		avx256_t vec_val2;
arch/x86/kvm/x86.c +6 -1
index 3fb64905d190..40c84f8c4912 100644
@@ -8322,7 +8322,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
 	int r;
 
 	/* guests cmpxchg8b have to be emulated atomically */
-	if (bytes > 8 || (bytes & (bytes - 1)))
+	if (bytes > 2 * sizeof(unsigned long) || (bytes & (bytes - 1)))
 		goto emul_write;
 
 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
@@ -8362,6 +8362,11 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
 	case 8:
 		r = emulator_try_cmpxchg_user(u64, hva, old, new);
 		break;
+#ifdef CONFIG_X86_64
+	case 16:
+		r = emulator_try_cmpxchg_user(u128, hva, old, new);
+		break;
+#endif
 	default:
 		BUG();
 	}
Signature · 1 line
2.34.1