Skip to contentSkip to conversation

KVM: VMX: Enable XSAVE/XRSTORE for guest

PATCH In review Sheng Yang 3 participants 4 messages May 26, 2010

Subsystems (3)
lore
Share & tools
lore

Sheng Yang

author
#0 lore raw

PATCH v5 1/1 KVM: VMX: Enable XSAVE/XRSTORE for guest

From: Dexuan Cui <[email protected]>

This patch enable guest to use XSAVE/XRSTORE instructions.

We assume that host_xcr0 would use all possible bits that OS supported.

And we loaded xcr0 in the same way we handled fpu - do it as late as we can.

---

I've done a prototype of LM support, would send out tomorrow. But the test case in QEmu side seems got something wrong. I always got an segfault at:
qemu-kvm/hw/fw_cfg.c:223
223 s->entries[arch][key].data = data;

Haven't looked into it yet. But maybe someone knows the reason...

 arch/x86/include/asm/kvm_host.h |    3 +
 arch/x86/include/asm/vmx.h      |    1 +
 arch/x86/kvm/kvm_cache_regs.h   |    6 ++
 arch/x86/kvm/vmx.c              |   26 ++++++++++
 arch/x86/kvm/x86.c              |  101 ++++++++++++++++++++++++++++++++++++---
 include/linux/kvm_host.h        |    2 +-
 6 files changed, 131 insertions(+), 8 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -0
index d08bb4a..532b220 100644
@@ -302,6 +302,7 @@ struct kvm_vcpu_arch {
 	} update_pte;
 
 	struct fpu guest_fpu;
+	u64 xcr0;
 
 	gva_t mmio_fault_cr2;
 	struct kvm_pio_request pio;
@@ -564,6 +565,8 @@ u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn);
 
 extern bool tdp_enabled;
 
+extern u64 host_xcr0;
+
 enum emulation_result {
 	EMULATE_DONE,       /* no further processing */
 	EMULATE_DO_MMIO,      /* kvm_run filled with mmio request */
arch/x86/include/asm/vmx.h +1 -0
index 9e6779f..346ea66 100644
@@ -266,6 +266,7 @@ enum vmcs_field {
 #define EXIT_REASON_EPT_VIOLATION       48
 #define EXIT_REASON_EPT_MISCONFIG       49
 #define EXIT_REASON_WBINVD		54
+#define EXIT_REASON_XSETBV		55
 
 /*
  * Interruption-information format
arch/x86/kvm/kvm_cache_regs.h +6 -0
index d2a98f8..6491ac8 100644
@@ -71,4 +71,10 @@ static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu)
 	return kvm_read_cr4_bits(vcpu, ~0UL);
 }
 
+static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu)
+{
+	return (kvm_register_read(vcpu, VCPU_REGS_RAX) & -1u)
+		| ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32);
+}
+
 #endif
arch/x86/kvm/vmx.c +26 -0
index 99ae513..a946841 100644
@@ -36,6 +36,8 @@
 #include <asm/vmx.h>
 #include <asm/virtext.h>
 #include <asm/mce.h>
+#include <asm/i387.h>
+#include <asm/xcr.h>
 
 #include "trace.h"
 
@@ -3354,6 +3356,29 @@ static int handle_wbinvd(struct kvm_vcpu *vcpu)
 	return 1;
 }
 
+static int handle_xsetbv(struct kvm_vcpu *vcpu)
+{
+	u64 new_bv = kvm_read_edx_eax(vcpu);
+
+	if (kvm_register_read(vcpu, VCPU_REGS_RCX) != 0)
+		goto err;
+	if (vmx_get_cpl(vcpu) != 0)
+		goto err;
+	if (!(new_bv & XSTATE_FP))
+		goto err;
+	if ((new_bv & XSTATE_YMM) && !(new_bv & XSTATE_SSE))
+		goto err;
+	if (new_bv & ~host_xcr0)
+		goto err;
+	vcpu->arch.xcr0 = new_bv;
+	xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
+	skip_emulated_instruction(vcpu);
+	return 1;
+err:
+	kvm_inject_gp(vcpu, 0);
+	return 1;
+}
+
 static int handle_apic_access(struct kvm_vcpu *vcpu)
 {
 	return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
@@ -3632,6 +3657,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
 	[EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
 	[EXIT_REASON_WBINVD]                  = handle_wbinvd,
+	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
 	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
 	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
 	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
arch/x86/kvm/x86.c +94 -7

191 lines not on this page · raw message

include/linux/kvm_host.h +1 -1
index 4e8fdbf..3784d58 100644
@@ -88,7 +88,7 @@ struct kvm_vcpu {
 	int srcu_idx;
 
 	int fpu_active;
-	int guest_fpu_loaded;
+	int guest_fpu_loaded, guest_xcr0_loaded;
 	wait_queue_head_t wq;
 	int sigset_active;
 	sigset_t sigset;
Signature · 1 line
1.7.0.1
#1 lore raw
Quoted 10 lines · Sheng Yang

From: Dexuan Cui<[email protected]>

This patch enable guest to use XSAVE/XRSTORE instructions.

We assume that host_xcr0 would use all possible bits that OS supported.

And we loaded xcr0 in the same way we handled fpu - do it as late as we can.

Looks really good now, only a couple of minor comments and I think we're done.

Quoted 7 lines

I've done a prototype of LM support, would send out tomorrow. But the test case in QEmu side seems got something wrong. I always got an segfault at:
qemu-kvm/hw/fw_cfg.c:223
223 s->entries[arch][key].data = data;

Haven't looked into it yet. But maybe someone knows the reason...

I saw this too, then it disappeared, can't remember why. Perhaps a clean build is needed?

Quoted 8 lines
arch/x86/kvm/vmx.c +6 -0
+static int handle_xsetbv(struct kvm_vcpu *vcpu)
+{
+	u64 new_bv = kvm_read_edx_eax(vcpu);
+
+	if (kvm_register_read(vcpu, VCPU_REGS_RCX) != 0)
+		goto err;

arch/x86/kvm/vmx.c static int handle_wbinvd(struct kvm_vcpu *vcpu) · line 3374

Quoted 10 lines
arch/x86/kvm/vmx.c +10 -0
+	if (vmx_get_cpl(vcpu) != 0)
+		goto err;
+	if (!(new_bv&  XSTATE_FP))
+		goto err;
+	if ((new_bv&  XSTATE_YMM)&&  !(new_bv&  XSTATE_SSE))
+		goto err;
+	if (new_bv&  ~host_xcr0)
+		goto err;
+	vcpu->arch.xcr0 = new_bv;
+	xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);

Please move all the code above to kvm_set_xcr0() in x86.c, since it's not vendor specific. This would also allow you to make host_xcr0 local to x86.c.

Quoted 13 lines
arch/x86/kvm/vmx.c +6 -0
+	skip_emulated_instruction(vcpu);
+	return 1;
+err:
+	kvm_inject_gp(vcpu, 0);
+	return 1;
+}
 
  /*
   * List of msr numbers which we expose to userspace through KVM_GET_MSRS
   * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
@@ -1813,6 +1847,14 @@ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
  	r = 0;
  	kvm_apic_set_version(vcpu);

arch/x86/kvm/vmx.c static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, · line 1858

Quoted 10 lines
arch/x86/kvm/vmx.c +8 -0
  	kvm_x86_ops->cpuid_update(vcpu);
+	update_cpuid(vcpu);
+
+	/*
+	 * Ensure guest xcr0 is valid for loading, also using as
+	 * the indicator of if guest cpuid has XSAVE
+	 */
+	if (guest_cpuid_has_xsave(vcpu))
+		vcpu->arch.xcr0 = XSTATE_FP;
 

This is problematic because it enforces an ordering between KVM_SET_XCR and KVM_SET_CPUID. So I think you can use kvm_read_cr4_bits(OSXSAVE) instead of checking vcpu->arch.xcr0. Sorry for the bad advice earlier.

Quoted 4 lines
arch/x86/kvm/x86.c +0 -0
@@ -5134,12 +5207,26 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 
  	vcpu->guest_fpu_loaded = 1;
  	unlazy_fpu(current);

arch/x86/kvm/x86.c void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) · line 5219

Quoted 10 lines
arch/x86/kvm/x86.c +9 -0
+	/*
+	 * Restore all possible states in the guest,
+	 * and assume host would use all available bits.
+	 * Guest xcr0 would be loaded later.
+	 */
+	if (cpu_has_xsave&&  vcpu->arch.xcr0) {
+		xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
+		vcpu->guest_xcr0_loaded = 0;
+	}
 

Has to be before unlazy_fpu(), so host fpu uses host xcr0.

It's sufficient to check for guest cr4.osxsave, no need to check for cpu_has_xsave. But you need to check for guest_xcr0_loaded!

arch/x86/kvm/x86.c void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) · line 5232

Quoted 12 lines
arch/x86/kvm/x86.c +5 -0
  	fpu_restore_checking(&vcpu->arch.guest_fpu);
  	trace_kvm_fpu(1);
  }
 
  void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
  {
+	if (vcpu->guest_xcr0_loaded) {
+		vcpu->guest_xcr0_loaded = 0;
+		xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
+	}
+
 

This duplicates the above. So better to have kvm_load_guest_xcr0()/kvm_put_guest_xcr0().

Signature · 1 line
error compiling committee.c: too many arguments to function
#2 lore raw
Quoted 7 lines · Avi Kivity
Quoted 6 lines · Sheng Yang

I've done a prototype of LM support, would send out tomorrow. But the test
case in QEmu side seems got something wrong. I always got an segfault at:
qemu-kvm/hw/fw_cfg.c:223
223 s->entries[arch][key].data = data;

Quoted 5 lines
Quoted 2 lines

Haven't looked into it yet. But maybe someone knows the reason...

I saw this too, then it disappeared, can't remember why. Perhaps a clean build is needed?

qemu-kvm apparently lacks upstream commit a71cd2a523f9b35ffeba8de3c536e494e255e6ea which should resolve at least some of those mysterious crashes.

Jan

Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux
#3 lore raw
Quoted 13 lines · Jan Kiszka
Quoted 11 lines · Avi Kivity
Quoted 9 lines · Sheng Yang

I've done a prototype of LM support, would send out tomorrow. But the test
case in QEmu side seems got something wrong. I always got an segfault at:
qemu-kvm/hw/fw_cfg.c:223
223 s->entries[arch][key].data = data;

Haven't looked into it yet. But maybe someone knows the reason...

Quoted 8 lines
Quoted 3 lines

I saw this too, then it disappeared, can't remember why. Perhaps a clean build is needed?

qemu-kvm apparently lacks upstream commit a71cd2a523f9b35ffeba8de3c536e494e255e6ea which should resolve at least some of those mysterious crashes.

That explains the fact that rebuild fixed it. I'll merge qemu.git shortly.

Signature · 1 line
error compiling committee.c: too many arguments to function