Skip to contentSkip to conversation

KVM: x86: XSAVE/XRSTOR live migration support

PATCH Superseded Sheng Yang 3 participants 6 messages Jun 11, 2010 – Jun 14, 2010

Subsystems (4)
Documentation x86 KVM KVM x86
lore
Share & tools
lore

Sheng Yang

author
#0 lore raw

PATCH v3 1/1 KVM: x86: XSAVE/XRSTOR live migration support

This patch enable save/restore of xsave state.

Signed-off-by Sheng Yang <[email protected]>
---
 Documentation/kvm/api.txt    |   74 +++++++++++++++++++++
 arch/x86/include/asm/kvm.h   |   22 ++++++
 arch/x86/include/asm/xsave.h |    7 ++-
 arch/x86/kvm/x86.c           |  145 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/kvm.h          |   12 ++++
 5 files changed, 258 insertions(+), 2 deletions(-)
Documentation/kvm/api.txt +74 -0
index 159b4ef..ffba03f 100644
@@ -922,6 +922,80 @@ Define which vcpu is the Bootstrap Processor (BSP).  Values are the same
 as the vcpu id in KVM_CREATE_VCPU.  If this ioctl is not called, the default
 is vcpu 0.
 
+4.41 KVM_GET_XSAVE
+
+Capability: KVM_CAP_XSAVE
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xsave (out)
+Returns: 0 on success, -1 on error
+
+struct kvm_xsave {
+	__u32 region[1024];
+};
+
+This ioctl would copy current vcpu's xsave struct to the userspace.
+
+4.42 KVM_SET_XSAVE
+
+Capability: KVM_CAP_XSAVE
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xsave (in)
+Returns: 0 on success, -1 on error
+
+struct kvm_xsave {
+	__u32 region[1024];
+};
+
+This ioctl would copy userspace's xsave struct to the kernel.
+
+4.43 KVM_GET_XCRS
+
+Capability: KVM_CAP_XCRS
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xcrs (out)
+Returns: 0 on success, -1 on error
+
+struct kvm_xcr {
+	__u32 xcr;
+	__u32 reserved;
+	__u64 value;
+};
+
+struct kvm_xcrs {
+	__u32 nr_xcrs;
+	__u32 flags;
+	struct kvm_xcr xcrs[KVM_MAX_XCRS];
+	__u64 padding[16];
+};
+
+This ioctl would copy current vcpu's xcrs to the userspace.
+
+4.44 KVM_SET_XCRS
+
+Capability: KVM_CAP_XCRS
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xcrs (in)
+Returns: 0 on success, -1 on error
+
+struct kvm_xcr {
+	__u32 xcr;
+	__u32 reserved;
+	__u64 value;
+};
+
+struct kvm_xcrs {
+	__u32 nr_xcrs;
+	__u32 flags;
+	struct kvm_xcr xcrs[KVM_MAX_XCRS];
+	__u64 padding[16];
+};
+
+This ioctl would set vcpu's xcr to the value userspace specified.
+
 5. The kvm_run structure
 
 Application code obtains a pointer to the kvm_run structure by
arch/x86/include/asm/kvm.h +22 -0
index ff90055..4d8dcbd 100644
@@ -22,6 +22,8 @@
 #define __KVM_HAVE_XEN_HVM
 #define __KVM_HAVE_VCPU_EVENTS
 #define __KVM_HAVE_DEBUGREGS
+#define __KVM_HAVE_XSAVE
+#define __KVM_HAVE_XCRS
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256

@@ -299,4 +301,24 @@ struct kvm_debugregs {
 	__u64 reserved[9];
 };
 
+/* for KVM_CAP_XSAVE */
+struct kvm_xsave {
+	__u32 region[1024];
+};
+
+#define KVM_MAX_XCRS	16
+
+struct kvm_xcr {
+	__u32 xcr;
+	__u32 reserved;
+	__u64 value;
+};
+
+struct kvm_xcrs {
+	__u32 nr_xcrs;
+	__u32 flags;
+	struct kvm_xcr xcrs[KVM_MAX_XCRS];
+	__u64 padding[16];
+};
+
 #endif /* _ASM_X86_KVM_H */
arch/x86/include/asm/xsave.h +5 -2
index 29ee4e4..32c3666 100644
@@ -13,8 +13,11 @@
 
 #define FXSAVE_SIZE	512
 
-#define XSTATE_YMM_SIZE 256
-#define XSTATE_YMM_OFFSET (512 + 64)
+#define XSAVE_HDR_SIZE	    64
+#define XSAVE_HDR_OFFSET    FXSAVE_SIZE
+
+#define XSAVE_YMM_SIZE	    256
+#define XSAVE_YMM_OFFSET    (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
 
 /*
  * These are the features that the OS can handle currently.
arch/x86/kvm/x86.c +145 -0

169 lines not on this page · raw message

include/linux/kvm.h +12 -0
index 23ea022..6fd40f5 100644
@@ -524,6 +524,12 @@ struct kvm_enable_cap {
 #define KVM_CAP_PPC_OSI 52
 #define KVM_CAP_PPC_UNSET_IRQ 53
 #define KVM_CAP_ENABLE_CAP 54
+#ifdef __KVM_HAVE_XSAVE
+#define KVM_CAP_XSAVE 55
+#endif
+#ifdef __KVM_HAVE_XCRS
+#define KVM_CAP_XCRS 56
+#endif
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -714,6 +720,12 @@ struct kvm_clock_data {
 #define KVM_GET_DEBUGREGS         _IOR(KVMIO,  0xa1, struct kvm_debugregs)
 #define KVM_SET_DEBUGREGS         _IOW(KVMIO,  0xa2, struct kvm_debugregs)
 #define KVM_ENABLE_CAP            _IOW(KVMIO,  0xa3, struct kvm_enable_cap)
+/* Available with KVM_CAP_XSAVE */
+#define KVM_GET_XSAVE		  _IOR(KVMIO,  0xa4, struct kvm_xsave)
+#define KVM_SET_XSAVE		  _IOW(KVMIO,  0xa5, struct kvm_xsave)
+/* Available with KVM_CAP_XCRS */
+#define KVM_GET_XCRS		  _IOR(KVMIO,  0xa6, struct kvm_xcrs)
+#define KVM_SET_XCRS		  _IOW(KVMIO,  0xa7, struct kvm_xcrs)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
Signature · 1 line
1.7.0.1
#1 lore raw
Quoted 7 lines · Sheng Yang

This patch enable save/restore of xsave state.

arch/x86/kvm/x86.c +4 -0
+static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
+					struct kvm_xsave *guest_xsave)
+{
+	u64 xstate_bv =

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2409

Quoted 10 lines
arch/x86/kvm/x86.c +10 -0
+		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
+	int size;
+
+	if (cpu_has_xsave) {
+		if (xstate_bv&  XSTATE_YMM)
+			size = XSAVE_YMM_OFFSET + XSAVE_YMM_SIZE;
+		else
+			size = XSAVE_HDR_OFFSET + XSAVE_HDR_SIZE;
+		memcpy(&vcpu->arch.guest_fpu.state->xsave,
+			guest_xsave->region, size);

This allows userspace to overflow host memory by specifying XSTATE_YMM on a host that doesn't support it.

Better to just use the host's size of the structure.

Quoted 10 lines
arch/x86/kvm/x86.c +9 -0
+	} else {
+		if (xstate_bv&  ~XSTATE_FPSSE)
+			return -EINVAL;
+		size = sizeof(struct i387_fxsave_struct);
+		memcpy(&vcpu->arch.guest_fpu.state->fxsave,
+			guest_xsave->region, size);
+	}
+	return 0;
+}

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2440

Quoted 10 lines
arch/x86/kvm/x86.c +9 -0
+
 
+
+static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
+				       struct kvm_xcrs *guest_xcrs)
+{
+	int i, r = 0;
+
+	if (!cpu_has_xsave)
+		return -EINVAL;

Too strict?

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2443

Quoted 4 lines
arch/x86/kvm/x86.c +3 -0
+
+	if (guest_xcrs->nr_xcrs>  KVM_MAX_XCRS)
+		return -EFAULT;

EFAULT is for faults during access to userspace. EINVAL or E2BIG.

Need to ensure flags is 0 for forward compatibility.

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2453

Quoted 11 lines
arch/x86/kvm/x86.c +10 -0
+
+	for (i = 0; i<  guest_xcrs->nr_xcrs; i++)
+		/* Only support XCR0 currently */
+		if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) {
+			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
+				guest_xcrs->xcrs[0].value);
+			break;
+		}
+	if (r)
+		r = -EFAULT;

EINVAL

Quoted 4 lines
arch/x86/kvm/x86.c +3 -0
+	return r;
+}
+
Signature · 1 line
error compiling committee.c: too many arguments to function
author
#2 lore raw
Quoted 21 lines · On Sunday 13 June 2010 16:26:18 Avi Kivity wrote:
Quoted 6 lines · Sheng Yang

This patch enable save/restore of xsave state.

arch/x86/kvm/x86.c +4 -0
+static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
+					struct kvm_xsave *guest_xsave)
+{
+	u64 xstate_bv =

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2409

Quoted 10 lines
arch/x86/kvm/x86.c +10 -0
+		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
+	int size;
+
+	if (cpu_has_xsave) {
+		if (xstate_bv&  XSTATE_YMM)
+			size = XSAVE_YMM_OFFSET + XSAVE_YMM_SIZE;
+		else
+			size = XSAVE_HDR_OFFSET + XSAVE_HDR_SIZE;
+		memcpy(&vcpu->arch.guest_fpu.state->xsave,
+			guest_xsave->region, size);

This allows userspace to overflow host memory by specifying XSTATE_YMM on a host that doesn't support it.

Quoted 1 line

Better to just use the host's size of the structure.

Yes, should good enough.

Quoted 12 lines
Quoted 12 lines
arch/x86/kvm/x86.c +10 -0
+	} else {
+		if (xstate_bv&  ~XSTATE_FPSSE)
+			return -EINVAL;
+		size = sizeof(struct i387_fxsave_struct);
+		memcpy(&vcpu->arch.guest_fpu.state->fxsave,
+			guest_xsave->region, size);
+	}
+	return 0;
+}
+
Quoted 10 lines

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2440

Quoted 8 lines
arch/x86/kvm/x86.c +8 -0
+
+static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
+				       struct kvm_xcrs *guest_xcrs)
+{
+	int i, r = 0;
+
+	if (!cpu_has_xsave)
+		return -EINVAL;

Too strict?

For no cpu_has_xsave, the KVM_CAP_XCRS would return 0, so this ioctl shouldn't be called.

Quoted 8 lines

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2443

Quoted 4 lines
arch/x86/kvm/x86.c +3 -0
+
+	if (guest_xcrs->nr_xcrs>  KVM_MAX_XCRS)
+		return -EFAULT;

EFAULT is for faults during access to userspace. EINVAL or E2BIG.

Need to ensure flags is 0 for forward compatibility.

OK.

Quoted 13 lines

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2453

Quoted 11 lines
arch/x86/kvm/x86.c +10 -0
+
+	for (i = 0; i<  guest_xcrs->nr_xcrs; i++)
+		/* Only support XCR0 currently */
+		if (guest_xcrs->xcrs[0].xcr == XCR_XFEATURE_ENABLED_MASK) {
+			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
+				guest_xcrs->xcrs[0].value);
+			break;
+		}
+	if (r)
+		r = -EFAULT;

EINVAL

OK

Quoted 4 lines

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2456

Quoted 4 lines
arch/x86/kvm/x86.c +3 -0
+	return r;
+}
+

--
regards
Yang, Sheng

#3 lore raw
Quoted 12 lines · Sheng Yang
Quoted 12 lines

arch/x86/kvm/x86.c static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · line 2440

Quoted 10 lines
arch/x86/kvm/x86.c +8 -0
+
+static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
+				       struct kvm_xcrs *guest_xcrs)
+{
+	int i, r = 0;
+
+	if (!cpu_has_xsave)
+		return -EINVAL;

Too strict?

Quoted 3 lines

For no cpu_has_xsave, the KVM_CAP_XCRS would return 0, so this ioctl shouldn't be called.

Right.

Signature · 1 line
error compiling committee.c: too many arguments to function
author
#4 lore raw

PATCH v4 1/1 KVM: x86: XSAVE/XRSTOR live migration support

This patch enable save/restore of xsave state.

Signed-off-by Sheng Yang <[email protected]>
---
 Documentation/kvm/api.txt    |   74 ++++++++++++++++++++++
 arch/x86/include/asm/kvm.h   |   22 +++++++
 arch/x86/include/asm/xsave.h |    7 ++-
 arch/x86/kvm/x86.c           |  139 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/kvm.h          |   12 ++++
 5 files changed, 252 insertions(+), 2 deletions(-)
Documentation/kvm/api.txt +74 -0
index 159b4ef..ffba03f 100644
@@ -922,6 +922,80 @@ Define which vcpu is the Bootstrap Processor (BSP).  Values are the same
 as the vcpu id in KVM_CREATE_VCPU.  If this ioctl is not called, the default
 is vcpu 0.
 
+4.41 KVM_GET_XSAVE
+
+Capability: KVM_CAP_XSAVE
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xsave (out)
+Returns: 0 on success, -1 on error
+
+struct kvm_xsave {
+	__u32 region[1024];
+};
+
+This ioctl would copy current vcpu's xsave struct to the userspace.
+
+4.42 KVM_SET_XSAVE
+
+Capability: KVM_CAP_XSAVE
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xsave (in)
+Returns: 0 on success, -1 on error
+
+struct kvm_xsave {
+	__u32 region[1024];
+};
+
+This ioctl would copy userspace's xsave struct to the kernel.
+
+4.43 KVM_GET_XCRS
+
+Capability: KVM_CAP_XCRS
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xcrs (out)
+Returns: 0 on success, -1 on error
+
+struct kvm_xcr {
+	__u32 xcr;
+	__u32 reserved;
+	__u64 value;
+};
+
+struct kvm_xcrs {
+	__u32 nr_xcrs;
+	__u32 flags;
+	struct kvm_xcr xcrs[KVM_MAX_XCRS];
+	__u64 padding[16];
+};
+
+This ioctl would copy current vcpu's xcrs to the userspace.
+
+4.44 KVM_SET_XCRS
+
+Capability: KVM_CAP_XCRS
+Architectures: x86
+Type: vcpu ioctl
+Parameters: struct kvm_xcrs (in)
+Returns: 0 on success, -1 on error
+
+struct kvm_xcr {
+	__u32 xcr;
+	__u32 reserved;
+	__u64 value;
+};
+
+struct kvm_xcrs {
+	__u32 nr_xcrs;
+	__u32 flags;
+	struct kvm_xcr xcrs[KVM_MAX_XCRS];
+	__u64 padding[16];
+};
+
+This ioctl would set vcpu's xcr to the value userspace specified.
+
 5. The kvm_run structure
 
 Application code obtains a pointer to the kvm_run structure by
arch/x86/include/asm/kvm.h +22 -0
index ff90055..4d8dcbd 100644
@@ -22,6 +22,8 @@
 #define __KVM_HAVE_XEN_HVM
 #define __KVM_HAVE_VCPU_EVENTS
 #define __KVM_HAVE_DEBUGREGS
+#define __KVM_HAVE_XSAVE
+#define __KVM_HAVE_XCRS
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256

@@ -299,4 +301,24 @@ struct kvm_debugregs {
 	__u64 reserved[9];
 };
 
+/* for KVM_CAP_XSAVE */
+struct kvm_xsave {
+	__u32 region[1024];
+};
+
+#define KVM_MAX_XCRS	16
+
+struct kvm_xcr {
+	__u32 xcr;
+	__u32 reserved;
+	__u64 value;
+};
+
+struct kvm_xcrs {
+	__u32 nr_xcrs;
+	__u32 flags;
+	struct kvm_xcr xcrs[KVM_MAX_XCRS];
+	__u64 padding[16];
+};
+
 #endif /* _ASM_X86_KVM_H */
arch/x86/include/asm/xsave.h +5 -2
index 29ee4e4..32c3666 100644
@@ -13,8 +13,11 @@
 
 #define FXSAVE_SIZE	512
 
-#define XSTATE_YMM_SIZE 256
-#define XSTATE_YMM_OFFSET (512 + 64)
+#define XSAVE_HDR_SIZE	    64
+#define XSAVE_HDR_OFFSET    FXSAVE_SIZE
+
+#define XSAVE_YMM_SIZE	    256
+#define XSAVE_YMM_OFFSET    (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
 
 /*
  * These are the features that the OS can handle currently.
arch/x86/kvm/x86.c +139 -0

163 lines not on this page · raw message

include/linux/kvm.h +12 -0
index 23ea022..6fd40f5 100644
@@ -524,6 +524,12 @@ struct kvm_enable_cap {
 #define KVM_CAP_PPC_OSI 52
 #define KVM_CAP_PPC_UNSET_IRQ 53
 #define KVM_CAP_ENABLE_CAP 54
+#ifdef __KVM_HAVE_XSAVE
+#define KVM_CAP_XSAVE 55
+#endif
+#ifdef __KVM_HAVE_XCRS
+#define KVM_CAP_XCRS 56
+#endif
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -714,6 +720,12 @@ struct kvm_clock_data {
 #define KVM_GET_DEBUGREGS         _IOR(KVMIO,  0xa1, struct kvm_debugregs)
 #define KVM_SET_DEBUGREGS         _IOW(KVMIO,  0xa2, struct kvm_debugregs)
 #define KVM_ENABLE_CAP            _IOW(KVMIO,  0xa3, struct kvm_enable_cap)
+/* Available with KVM_CAP_XSAVE */
+#define KVM_GET_XSAVE		  _IOR(KVMIO,  0xa4, struct kvm_xsave)
+#define KVM_SET_XSAVE		  _IOW(KVMIO,  0xa5, struct kvm_xsave)
+/* Available with KVM_CAP_XCRS */
+#define KVM_GET_XCRS		  _IOR(KVMIO,  0xa6, struct kvm_xcrs)
+#define KVM_SET_XCRS		  _IOW(KVMIO,  0xa7, struct kvm_xcrs)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
Signature · 1 line
1.7.0.1