KVM: nVMX: Fully support of nested VMX preemption timer
PATCH v5 1/1 KVM: nVMX: Fully support of nested VMX preemption timer
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.
With this patch, nested VMX preemption timer features are fully supported.
---
ChangeLog to v4:
Format changes and remove a flag in nested_vmx.
arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +1 -0
@@ -536,6 +536,7 @@/* MSR_IA32_VMX_MISC bits */ #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29) +#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE 0x1F/* AMD-V MSRs */ #define MSR_VM_CR 0xc0010114
arch/x86/kvm/vmx.c +42 -2
80 lines not on this page · raw message
Signature · 1 line
1.7.9.5
Quoted 11 lines · Arthur Chunqi Li
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Signed-off-by Arthur Chunqi Li <[email protected]>
One more if() on vmentry path is unfortunate, but I do not see a way around it yet.
Quoted 111 lines
---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)arch/x86/include/uapi/asm/msr-index.h +1
-0@@ -536,6 +536,7 @@/* MSR_IA32_VMX_MISC bits */ #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29) +#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE 0x1F/* AMD-V MSRs */ #define MSR_VM_CR 0xc0010114arch/x86/kvm/vmx.c +42
-2@@ -2204,7 +2204,13 @@ static __init void nested_vmx_setup_ctls_msrs(void)#ifdef CONFIG_X86_64 VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif -VM_EXIT_LOAD_IA32_PAT |VM_EXIT_SAVE_IA32_PAT ; +VM_EXIT_LOAD_IA32_PAT |VM_EXIT_SAVE_IA32_PAT | +VM_EXIT_SAVE_VMX_PREEMPTION_TIMER ; +if (!(nested_vmx_pinbased_ctls_high &PIN_BASED_VMX_PREEMPTION_TIMER ) || + !(nested_vmx_exit_ctls_high &VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) { + nested_vmx_exit_ctls_high &= ~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER ; + nested_vmx_pinbased_ctls_high &= ~PIN_BASED_VMX_PREEMPTION_TIMER ; + } nested_vmx_exit_ctls_high |= (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |VM_EXIT_LOAD_IA32_EFER );@@ -6707,6 +6713,27 @@ static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) *info2 =vmcs_read32 (VM_EXIT_INTR_INFO ); } +static void nested_adjust_preemption_timer (struct kvm_vcpu *vcpu) +{ +u64 delta_tsc_l1; +u32 preempt_val_l1, preempt_val_l2, preempt_scale; + +if (!(get_vmcs12 (vcpu)->pin_based_vm_exec_control & +PIN_BASED_VMX_PREEMPTION_TIMER )) +return ; + preempt_scale =native_read_msr (MSR_IA32_VMX_MISC ) & +MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE ; + preempt_val_l2 =vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); + delta_tsc_l1 =vmx_read_l1_tsc (vcpu,native_read_tsc ()) + - vcpu->arch.last_guest_tsc; + preempt_val_l1 = delta_tsc_l1 >> preempt_scale; +if (preempt_val_l2 <= preempt_val_l1) + preempt_val_l2 =0 ; +else + preempt_val_l2 -= preempt_val_l1; +vmcs_write32 (VMX_PREEMPTION_TIMER_VALUE , preempt_val_l2); +} +/* * The guest has exited. See if we can fix it or if we need userspace * assistance. @@ -7131,6 +7158,8 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)atomic_switch_perf_msrs (vmx); debugctlmsr =get_debugctlmsr (); +if (is_guest_mode (vcpu) && !(vmx->nested.nested_run_pending)) +nested_adjust_preemption_timer (vcpu); vmx->__launched = vmx->loaded_vmcs->launched; asm(/* Store host registers */ @@ -7518,6 +7547,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) {struct vcpu_vmx *vmx =to_vmx (vcpu);u32 exec_control; +u32 exit_control;vmcs_write16 (GUEST_ES_SELECTOR , vmcs12->guest_es_selector);vmcs_write16 (GUEST_CS_SELECTOR , vmcs12->guest_cs_selector);@@ -7691,7 +7721,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)* we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER * bits are further modified by vmx_set_efer() below. */ -vmcs_write32 (VM_EXIT_CONTROLS , vmcs_config.vmexit_ctrl); + exit_control = vmcs_config.vmexit_ctrl; +if (vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER ) + exit_control |=VM_EXIT_SAVE_VMX_PREEMPTION_TIMER ; +vmcs_write32 (VM_EXIT_CONTROLS , exit_control);/* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are * emulated by vmx_set_efer(), below. @@ -8090,6 +8123,13 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_pending_dbg_exceptions =vmcs_readl (GUEST_PENDING_DBG_EXCEPTIONS ); +if ((vmcs12->pin_based_vm_exec_control & +PIN_BASED_VMX_PREEMPTION_TIMER ) && + (vmcs12->vm_exit_controls & +VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) + vmcs12->vmx_preemption_timer_value = +vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); +/* * In some cases (usually, nested EPT), L2 is allowed to change its * own CR3 without exiting. If it has changed it, we must keep it. Signature · 1 line
1.7.9.5
--
Gleb.
Paolo Bonzini
maintainerIl 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 111 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)arch/x86/include/uapi/asm/msr-index.h +1
-0@@ -536,6 +536,7 @@/* MSR_IA32_VMX_MISC bits */ #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29) +#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE 0x1F/* AMD-V MSRs */ #define MSR_VM_CR 0xc0010114arch/x86/kvm/vmx.c +35
-2@@ -2204,7 +2204,13 @@ static __init void nested_vmx_setup_ctls_msrs(void)#ifdef CONFIG_X86_64 VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif -VM_EXIT_LOAD_IA32_PAT |VM_EXIT_SAVE_IA32_PAT ; +VM_EXIT_LOAD_IA32_PAT |VM_EXIT_SAVE_IA32_PAT | +VM_EXIT_SAVE_VMX_PREEMPTION_TIMER ; +if (!(nested_vmx_pinbased_ctls_high &PIN_BASED_VMX_PREEMPTION_TIMER ) || + !(nested_vmx_exit_ctls_high &VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) { + nested_vmx_exit_ctls_high &= ~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER ; + nested_vmx_pinbased_ctls_high &= ~PIN_BASED_VMX_PREEMPTION_TIMER ; + } nested_vmx_exit_ctls_high |= (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |VM_EXIT_LOAD_IA32_EFER );@@ -6707,6 +6713,27 @@ static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) *info2 =vmcs_read32 (VM_EXIT_INTR_INFO ); } +static void nested_adjust_preemption_timer (struct kvm_vcpu *vcpu) +{ +u64 delta_tsc_l1; +u32 preempt_val_l1, preempt_val_l2, preempt_scale; + +if (!(get_vmcs12 (vcpu)->pin_based_vm_exec_control & +PIN_BASED_VMX_PREEMPTION_TIMER )) +return ; + preempt_scale =native_read_msr (MSR_IA32_VMX_MISC ) & +MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE ; + preempt_val_l2 =vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); + delta_tsc_l1 =vmx_read_l1_tsc (vcpu,native_read_tsc ()) + - vcpu->arch.last_guest_tsc; + preempt_val_l1 = delta_tsc_l1 >> preempt_scale; +if (preempt_val_l2 <= preempt_val_l1) + preempt_val_l2 =0 ; +else + preempt_val_l2 -= preempt_val_l1; +vmcs_write32 (VMX_PREEMPTION_TIMER_VALUE , preempt_val_l2); +} +/* * The guest has exited. See if we can fix it or if we need userspace * assistance. @@ -7131,6 +7158,8 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)atomic_switch_perf_msrs (vmx); debugctlmsr =get_debugctlmsr (); +if (is_guest_mode (vcpu) && !(vmx->nested.nested_run_pending)) +nested_adjust_preemption_timer (vcpu); vmx->__launched = vmx->loaded_vmcs->launched; asm(/* Store host registers */ @@ -7518,6 +7547,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) {struct vcpu_vmx *vmx =to_vmx (vcpu);u32 exec_control; +u32 exit_control;vmcs_write16 (GUEST_ES_SELECTOR , vmcs12->guest_es_selector);vmcs_write16 (GUEST_CS_SELECTOR , vmcs12->guest_cs_selector);@@ -7691,7 +7721,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)* we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER * bits are further modified by vmx_set_efer() below. */ -vmcs_write32 (VM_EXIT_CONTROLS , vmcs_config.vmexit_ctrl); + exit_control = vmcs_config.vmexit_ctrl; +if (vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER ) + exit_control |=VM_EXIT_SAVE_VMX_PREEMPTION_TIMER ; +vmcs_write32 (VM_EXIT_CONTROLS , exit_control);/* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are * emulated by vmx_set_efer(), below. @@ -8090,6 +8123,13 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_pending_dbg_exceptions =vmcs_readl (GUEST_PENDING_DBG_EXCEPTIONS );
arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135
Quoted 10 lines
arch/x86/kvm/vmx.c +7
-0+if ((vmcs12->pin_based_vm_exec_control & +PIN_BASED_VMX_PREEMPTION_TIMER ) && + (vmcs12->vm_exit_controls & +VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) + vmcs12->vmx_preemption_timer_value = +vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); +/* * In some cases (usually, nested EPT), L2 is allowed to change its * own CR3 without exiting. If it has changed it, we must keep it.
Applying to kvm/queue, thanks.
Paolo
Paolo Bonzini
maintainerIl 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
I tried redoing the patch with a completely different logic, and I get exactly the same behavior and the same wrong values for the TSC. My patch (below is the diff from Arthur's) is a bit simpler; like Arthur I run L2 with the save-timer control set, but I do not need any adjustment upon entry to L2 because I just use the value left by the processor in the VMCS. Also, in your patch L1 is also running with the preemption timer, if I understand the code correctly.
arch/x86/kvm/vmx.c +12 -31
@@ -6730,27 +6731,6 @@ static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *in *info2 =vmcs_read32 (VM_EXIT_INTR_INFO ); } -static void nested_adjust_preemption_timer (struct kvm_vcpu *vcpu) -{ -u64 delta_tsc_l1; -u32 preempt_val_l1, preempt_val_l2, preempt_scale; - -if (!(get_vmcs12 (vcpu)->pin_based_vm_exec_control & -PIN_BASED_VMX_PREEMPTION_TIMER )) -return ; - preempt_scale =native_read_msr (MSR_IA32_VMX_MISC ) & -MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE ; - preempt_val_l2 =vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); - delta_tsc_l1 =vmx_read_l1_tsc (vcpu,native_read_tsc ()) - - vcpu->arch.last_guest_tsc; - preempt_val_l1 = delta_tsc_l1 >> preempt_scale; -if (preempt_val_l2 <= preempt_val_l1) - preempt_val_l2 =0 ; -else - preempt_val_l2 -= preempt_val_l1; -vmcs_write32 (VMX_PREEMPTION_TIMER_VALUE , preempt_val_l2); -} -/* * The guest has exited. See if we can fix it or if we need userspace * assistance.
@@ -7161,8 +7141,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)atomic_switch_perf_msrs (vmx); debugctlmsr =get_debugctlmsr (); -if (is_guest_mode (vcpu) && !(vmx->nested.nested_run_pending)) -nested_adjust_preemption_timer (vcpu); vmx->__launched = vmx->loaded_vmcs->launched; asm(/* Store host registers */
@@ -7294,6 +7272,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_CR3)); vcpu->arch.regs_dirty =0 ; +if (is_guest_mode (vcpu)) { +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); +if ((vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER ) && + (vmcs12->vm_exit_controls &VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) { + vmcs12->vmx_preemption_timer_value = +vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); + } + } + vmx->idt_vectoring_info =vmcs_read32 (IDT_VECTORING_INFO_FIELD ); vmx->loaded_vmcs->launched =1 ;
@@ -7633,7 +7633,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)vmcs_write64 (VMCS_LINK_POINTER ,-1ull );vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL , - (vmcs_config.pin_based_exec_ctrl | + (vmx_pin_based_exec_ctrl (vmx) | vmcs12->pin_based_vm_exec_control));if (vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER )
@@ -8159,13 +8146,6 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_pending_dbg_exceptions =vmcs_readl (GUEST_PENDING_DBG_EXCEPTIONS ); -if ((vmcs12->pin_based_vm_exec_control & -PIN_BASED_VMX_PREEMPTION_TIMER ) && - (vmcs12->vm_exit_controls & -VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) - vmcs12->vmx_preemption_timer_value = -vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); -/* * In some cases (usually, nested EPT), L2 is allowed to change its * own CR3 without exiting. If it has changed it, we must keep it.
@@ -8396,6 +8375,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)load_vmcs12_host_state (vcpu, vmcs12); +vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL ,vmx_pin_based_exec_ctrl (vmx)); +/* Update TSC_OFFSET if TSC was changed while L2 ran */ vmcs_write64 (TSC_OFFSET , vmx->nested.vmcs01_tsc_offset);
Also note that I used vmx_pin_based_exec_ctrl instead of vmcs_config.pin_based_exec_ctrl. Can anyone spot anything wrong in this code?
Paolo
Quoted 19 lines · Paolo Bonzini
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)
Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
So this behaviour is a regression of this patch (and your own version as well)?
Quoted 49 lines
I tried redoing the patch with a completely different logic, and I get exactly the same behavior and the same wrong values for the TSC. My patch (below is the diff from Arthur's) is a bit simpler; like Arthur I run L2 with the save-timer control set, but I do not need any adjustment upon entry to L2 because I just use the value left by the processor in the VMCS. Also, in your patch L1 is also running with the preemption timer, if I understand the code correctly.
arch/x86/kvm/vmx.c +0
-23@@ -6730,27 +6731,6 @@ static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *in *info2 =vmcs_read32 (VM_EXIT_INTR_INFO ); } -static void nested_adjust_preemption_timer (struct kvm_vcpu *vcpu) -{ -u64 delta_tsc_l1; -u32 preempt_val_l1, preempt_val_l2, preempt_scale; - -if (!(get_vmcs12 (vcpu)->pin_based_vm_exec_control & -PIN_BASED_VMX_PREEMPTION_TIMER )) -return ; - preempt_scale =native_read_msr (MSR_IA32_VMX_MISC ) & -MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE ; - preempt_val_l2 =vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); - delta_tsc_l1 =vmx_read_l1_tsc (vcpu,native_read_tsc ()) - - vcpu->arch.last_guest_tsc; - preempt_val_l1 = delta_tsc_l1 >> preempt_scale; -if (preempt_val_l2 <= preempt_val_l1) - preempt_val_l2 =0 ; -else - preempt_val_l2 -= preempt_val_l1; -vmcs_write32 (VMX_PREEMPTION_TIMER_VALUE , preempt_val_l2); -} -/* * The guest has exited. See if we can fix it or if we need userspace * assistance. @@ -7161,8 +7141,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)atomic_switch_perf_msrs (vmx); debugctlmsr =get_debugctlmsr (); -if (is_guest_mode (vcpu) && !(vmx->nested.nested_run_pending)) -nested_adjust_preemption_timer (vcpu); vmx->__launched = vmx->loaded_vmcs->launched; asm(/* Store host registers */
arch/x86/kvm/vmx.c static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) · line 7280
Quoted 9 lines
arch/x86/kvm/vmx.c +6
-0@@ -7294,6 +7272,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_CR3)); vcpu->arch.regs_dirty =0 ; +if (is_guest_mode (vcpu)) { +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); +if ((vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER ) && + (vmcs12->vm_exit_controls &VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) { + vmcs12->vmx_preemption_timer_value = +vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE );
Is the preemption timer guaranteed to continue to tick while in L0? I think this is what you are relying on here, right?
Jan
Quoted 45 lines
arch/x86/kvm/vmx.c +6
-8+ } + } + vmx->idt_vectoring_info =vmcs_read32 (IDT_VECTORING_INFO_FIELD ); vmx->loaded_vmcs->launched =1 ;@@ -7633,7 +7633,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)vmcs_write64 (VMCS_LINK_POINTER ,-1ull );vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL , - (vmcs_config.pin_based_exec_ctrl | + (vmx_pin_based_exec_ctrl (vmx) | vmcs12->pin_based_vm_exec_control));if (vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER )@@ -8159,13 +8146,6 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_pending_dbg_exceptions =vmcs_readl (GUEST_PENDING_DBG_EXCEPTIONS ); -if ((vmcs12->pin_based_vm_exec_control & -PIN_BASED_VMX_PREEMPTION_TIMER ) && - (vmcs12->vm_exit_controls & -VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) - vmcs12->vmx_preemption_timer_value = -vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE ); -/* * In some cases (usually, nested EPT), L2 is allowed to change its * own CR3 without exiting. If it has changed it, we must keep it. @@ -8396,6 +8375,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)load_vmcs12_host_state (vcpu, vmcs12); +vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL ,vmx_pin_based_exec_ctrl (vmx)); +/* Update TSC_OFFSET if TSC was changed while L2 ran */ vmcs_write64 (TSC_OFFSET , vmx->nested.vmcs01_tsc_offset);Also note that I used
vmx_pin_based_exec_ctrlinstead of vmcs_config.pin_based_exec_ctrl. Can anyone spot anything wrong in this code?Paolo
Paolo Bonzini
maintainer-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Il 26/09/2013 19:19, Jan Kiszka ha scritto:
Quoted 29 lines · Paolo Bonzini
Quoted 19 lines
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
Quoted 2 lines
So this behaviour is a regression of this patch (and your own version as well)?
Without this patch Arthur's preemption timer test doesn't work at all.
If I only apply this hunk, which disables the preemption timer while in L1:
@@ -8396,6 +8375,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)
load_vmcs12_host_state(vcpu, vmcs12);
+ vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
+
/* Update TSC_OFFSET if TSC was changed while L2 ran */
vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset);then the testcase works for somewhat larger values of the preemption timer (up to ~1500000 TSC cycles), but then fails.
If I apply the full patch below, the save-timer-value control works but the timer stops working except for very small values of the preemption timer.
Quoted 4 lines
Quoted 4 lines
arch/x86/kvm/vmx.c +0
-0@@ -7294,6 +7272,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_CR3)); vcpu->arch.regs_dirty =0 ;
Quoted 9 lines
arch/x86/kvm/vmx.c static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) · line 7280
Quoted 6 lines
arch/x86/kvm/vmx.c +6
-0+if (is_guest_mode (vcpu)) { +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); +if ((vmcs12->pin_based_vm_exec_control &PIN_BASED_VMX_PREEMPTION_TIMER ) && + (vmcs12->vm_exit_controls &VM_EXIT_SAVE_VMX_PREEMPTION_TIMER )) { + vmcs12->vmx_preemption_timer_value = +vmcs_read32 (VMX_PREEMPTION_TIMER_VALUE );Is the preemption timer guaranteed to continue to tick while in L0? I think this is what you are relying on here, right?
No, it is definitely not ticking. But I was trying to simplify Arthur's code as much as possible, while fixing the worst bug (i.e. that after L2->L0->L2 the timer value restarts ticking from the initial value). Support for save-timer-value comes for free, and doesn't affect the result of the testcase.
But this would cause the preemption timer to be late. It doesn't explain why the preemption timer expires faster than the (scaled) TSC.
Paolo
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.19 (GNU/Linux)
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/
iQIcBAEBAgAGBQJSRHM1AAoJEBvWZb6bTYbyMHEP+gJro4KyKB9cAPE6+e/305mV /px/FWoY1Y5GlQ9juVV2DGQRipKVeIcN3pf3vZYZMSWzPCmbprmnx5nAy3NIpDi7 UM8gjZ+8dyzIfU/BfaEQWrI5o1mzcOhQTUVHw3bz09NxUWgoq7bp+eODR/kRyrw3 5EVOHjzaffZOug//z+boPRnaLQjvd4cFKv9HuKoo4hGjgQLYC6HfYWou2wvRzw9V La63p8w3gkVn2Fx31ppszLyS1qDIAJ0C5b3RwQp2jl5Kzp407VK7QGe9xoPiD8ZO bTsJs8nTbbITezxk8z9TUbSWWoVCd0M8aAfKhLB4xzubIWC/qQDA0Mwcu2SiNo3v m+vZsX3A358efmmjnkA3FUlGwwFBp5xWk9XuHkLhxh2AYWBl3+/IAZXg2yJP8SSq ShD4mcImWZTLFI5IK/02erDNdGYLTlnGC06537Fv9fzGAOtgowaOc8ScSxJDQn/n MbBBGvGa3Ps6RnF+eY4Xi8N9rsfbcVAGbVSdVMR0SXanuUT+h8R1kd7xtnYV2LpH twAreBVO+zxKhYYfnLj6Gu/oDDWAENDFMf1t6xumlflchR2gdcztmgp/OMkTK4QS IhWiaCzjxl/lD9vxyxstIhJNvMzsKk0o+K4X0h80oKizoDJybPcNZIaU2wNXs3iJ Q5nhpbTsl+kZLUlAWvlp
=vW1q
-----END PGP SIGNATURE-----
Paolo Bonzini
maintainerIl 26/09/2013 19:47, Paolo Bonzini ha scritto:
Quoted 13 lines
If I only apply this hunk, which disables the preemption timer while in L1:
arch/x86/kvm/vmx.c +2
-0@@ -8396,6 +8375,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)load_vmcs12_host_state (vcpu, vmcs12); +vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL ,vmx_pin_based_exec_ctrl (vmx)); +/* Update TSC_OFFSET if TSC was changed while L2 ran */ vmcs_write64 (TSC_OFFSET , vmx->nested.vmcs01_tsc_offset);
Quoted 2 lines
then the testcase works for somewhat larger values of the preemption timer (up to ~1500000 TSC cycles), but then fails.
I mean if I apply it on top of current kvm/next, without Arthur's patch.
If I apply the hunk on top of Arthur's patch nothing changes and the timer testcase starts breaking around ~65000 TSC cycles.
It is a bit problematic that adding printks changes something, so that the test starts passing. I haven't tried tracepoints yet.
Jan, which L1 is using the preemption timer? Any reason why you added it? I wonder if it isn't better to revert it, since it is quite broken.
Paolo
Quoted 9 lines · Paolo Bonzini
Il 26/09/2013 19:47, Paolo Bonzini ha scritto:
Quoted 8 lines
If I only apply this hunk, which disables the preemption timer while in L1:
arch/x86/kvm/vmx.c +0
-0@@ -8396,6 +8375,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)load_vmcs12_host_state (vcpu, vmcs12);
Quoted 7 lines
Quoted 7 lines
arch/x86/kvm/vmx.c +2
-0+vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL ,vmx_pin_based_exec_ctrl (vmx)); +/* Update TSC_OFFSET if TSC was changed while L2 ran */ vmcs_write64 (TSC_OFFSET , vmx->nested.vmcs01_tsc_offset);then the testcase works for somewhat larger values of the preemption timer (up to ~1500000 TSC cycles), but then fails.
Err, does this mean we run L1 with PIN_BASED_VM_EXEC_CONTROL of L2? Ouch. Should be fixed independently.
Quoted 5 lines
I mean if I apply it on top of current kvm/next, without Arthur's patch.
If I apply the hunk on top of Arthur's patch nothing changes and the timer testcase starts breaking around ~65000 TSC cycles.
So there are several overlapping issues.
Quoted 3 lines
It is a bit problematic that adding printks changes something, so that the test starts passing. I haven't tried tracepoints yet.
Yeah, just convert to trace_printk.
Quoted 3 lines
Jan, which L1 is using the preemption timer? Any reason why you added it?
The one I'm going to present in Edinburgh. Will publish code soon, but it's unspectacular regard preemption timer usage: I'm only using it to enforce immediate exits on next vmentry, i.e. set the timer value to 0.
Quoted 1 line
I wonder if it isn't better to revert it, since it is quite broken.
It's in upstream since 3.10, so reverting doesn't buy us anything, we need to fix it anyway for long-term stable.
Jan
Quoted 15 lines · Jan Kiszka
Quoted 14 lines · Paolo Bonzini
Il 26/09/2013 19:47, Paolo Bonzini ha scritto:
Quoted 13 lines
If I only apply this hunk, which disables the preemption timer while in L1:
arch/x86/kvm/vmx.c +2
-0@@ -8396,6 +8375,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)load_vmcs12_host_state (vcpu, vmcs12); +vmcs_write32 (PIN_BASED_VM_EXEC_CONTROL ,vmx_pin_based_exec_ctrl (vmx)); +/* Update TSC_OFFSET if TSC was changed while L2 ran */ vmcs_write64 (TSC_OFFSET , vmx->nested.vmcs01_tsc_offset);
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
then the testcase works for somewhat larger values of the preemption timer (up to ~1500000 TSC cycles), but then fails.
Err, does this mean we run L1 with
PIN_BASED_VM_EXEC_CONTROLof L2? Ouch. Should be fixed independently.
No, it doesn't mean this. L1 and L2 run on different VMCS, thus should be able to set their PIN_BASED_VM_EXEC_CONTROL independently. I have no clue ATM what that hunk can make a difference for you. Will have a closer look.
BTW, aren't many VMCS fields written redundantly in prepare_vmcs02? I mean, those that weren't changed by L1 in the shadow VMCS or require updates for other reasons. There seems to be some room for saving a few cycles.
Jan
Quoted 33 lines · Paolo Bonzini
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1Il 26/09/2013 19:19, Jan Kiszka ha scritto:
Quoted 29 lines · Paolo Bonzini
Quoted 27 lines
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
Quoted 5 lines
Quoted 2 lines
So this behaviour is a regression of this patch (and your own version as well)?
Without this patch Arthur's preemption timer test doesn't work at all.
Have you ruled out test bugs?
I see things like:
+ int i, j;
...
+ // Consume enough time to let L2->L0->L2 occurs
+ for(i = 0; i < 100000; i++)
+ for (j = 0; j < 10000; j++);
which can be optimized out. Or use of rdtsc() which can be problematic if host tsc is not synchronised.
--
Gleb.
Quoted 19 lines · Paolo Bonzini
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)
Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
I've done some measurements with the help of ftrace on the time it takes to let the preemption timer trigger (no adjustments via Arthur's patch were involved): On my Core i7-620M, the preemption timer seems to tick almost 10 times faster than spec and scale value (5) suggests. I've loaded a value of 100000, and it took about 130 µs until I got a vmexit with reason PREEMPTION_TIMER (no other exists in between).
qemu-system-x86-13765 [003] 298562.966079: bprint: prepare_vmcs02: preempt val 100000 qemu-system-x86-13765 [003] 298562.966083: kvm_entry: vcpu 0 qemu-system-x86-13765 [003] 298562.966212: kvm_exit: reason PREEMPTION_TIMER rip 0x401fea info 0 0
That's a frequency of ~769 MHz. The TSC ticks at 2.66 GHz. But 769 MHz * 2^5 is 24.6 GHz. I've read the spec several times, but it seems pretty clear on this. It just doesn't match reality. Very strange.
Jan
Quoted 36 lines · Jan Kiszka
Quoted 19 lines · Paolo Bonzini
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
I've done some measurements with the help of ftrace on the time it takes to let the preemption timer trigger (no adjustments via Arthur's patch were involved): On my Core i7-620M, the preemption timer seems to tick almost 10 times faster than spec and scale value (5) suggests. I've loaded a value of 100000, and it took about 130 µs until I got a vmexit with reason
PREEMPTION_TIMER(no other exists in between).
Quoted 7 lines
qemu-system-x86-13765 [003] 298562.966079: bprint: prepare_vmcs02: preempt val 100000 qemu-system-x86-13765 [003] 298562.966083: kvm_entry: vcpu 0 qemu-system-x86-13765 [003] 298562.966212: kvm_exit: reason PREEMPTION_TIMER rip 0x401fea info 0 0That's a frequency of ~769 MHz. The TSC ticks at 2.66 GHz. But 769 MHz * 2^5 is 24.6 GHz. I've read the spec several times, but it seems pretty clear on this. It just doesn't match reality. Very strange.
...but documented: I found an related errata for my processor (AAT59) and also for Xeon 5500 (AAK139). At least current Haswell generation is no affected. I can test the patch on a Haswell board I have at work later this week.
Jan
Quoted 41 lines · Jan Kiszka
Quoted 40 lines · Jan Kiszka
Quoted 19 lines · Paolo Bonzini
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
I've done some measurements with the help of ftrace on the time it takes to let the preemption timer trigger (no adjustments via Arthur's patch were involved): On my Core i7-620M, the preemption timer seems to tick almost 10 times faster than spec and scale value (5) suggests. I've loaded a value of 100000, and it took about 130 µs until I got a vmexit with reason
PREEMPTION_TIMER(no other exists in between).qemu-system-x86-13765 [003] 298562.966079: bprint: prepare_vmcs02: preempt val 100000 qemu-system-x86-13765 [003] 298562.966083: kvm_entry: vcpu 0 qemu-system-x86-13765 [003] 298562.966212: kvm_exit: reason PREEMPTION_TIMER rip 0x401fea info 0 0
Quoted 8 lines
Quoted 3 lines
That's a frequency of ~769 MHz. The TSC ticks at 2.66 GHz. But 769 MHz * 2^5 is 24.6 GHz. I've read the spec several times, but it seems pretty clear on this. It just doesn't match reality. Very strange.
...but documented: I found an related errata for my processor (AAT59) and also for Xeon 5500 (AAK139). At least current Haswell generation is no affected. I can test the patch on a Haswell board I have at work later this week.
To complete this story: Arthur's patch works fine on a non-broken CPU (here: i7-4770S).
Arthur, find some fix-ups for your test case below. It avoids printing from within L2 as this could deadlock when the timer fires and L1 then tries to print something. Also, it disables the preemption timer on leave so that it cannot fire later on again. If you want to fold this into your patch, feel free. Otherwise I can post a separate patch on top.
Jan
x86/vmx_tests.c +16 -10
@@ -141,6 +141,9 @@ void preemption_timer_init() preempt_val =10000000 ;vmcs_write (PREEMPT_TIMER_VALUE , preempt_val); preempt_scale =rdmsr (MSR_IA32_VMX_MISC ) &0x1F ; + +if (!(ctrl_exit_rev.clr &EXI_SAVE_PREEMPT )) +printf ("\tSave preemption value is not supported\n" ); }void preemption_timer_main ()
@@ -150,9 +153,7 @@ void preemption_timer_main()printf ("\tPreemption timer is not supported\n" );return ; } -if (!(ctrl_exit_rev.clr &EXI_SAVE_PREEMPT )) -printf ("\tSave preemption value is not supported\n" ); -else { +if (ctrl_exit_rev.clr &EXI_SAVE_PREEMPT ) {set_stage (0 );vmcall ();if (get_stage () ==1 )
@@ -161,8 +162,8 @@ void preemption_timer_main()while (1 ) {if (((rdtsc () - tsc_val) >> preempt_scale) >10 * preempt_val) { -report ("Preemption timer" ,0 ); -break ; +set_stage (2 ); +vmcall (); } } }
@@ -183,7 +184,7 @@ int preemption_timer_exit_handler()report ("Preemption timer" ,0 ); else report("Preemption timer", 1); -return VMX_TEST_VMEXIT ; +break ;case VMX_VMCALL :switch (get_stage ()) {case 0 :
@@ -195,24 +196,29 @@ int preemption_timer_exit_handler() EXI_SAVE_PREEMPT) & ctrl_exit_rev.clr;vmcs_write (EXI_CONTROLS , ctrl_exit); } -break ; +vmcs_write (GUEST_RIP , guest_rip + insn_len); +return VMX_TEST_RESUME ;case 1 :if (vmcs_read (PREEMPT_TIMER_VALUE ) >= preempt_val)report ("Save preemption value" ,0 );else report ("Save preemption value" ,1 ); +vmcs_write (GUEST_RIP , guest_rip + insn_len); +return VMX_TEST_RESUME ; +case 2 : +report ("Preemption timer" ,0 );break ;default :printf ("Invalid stage.\n" );print_vmexit_info (); -return VMX_TEST_VMEXIT ; +break ; } -vmcs_write (GUEST_RIP , guest_rip + insn_len); -return VMX_TEST_RESUME ; +break ;default :printf ("Unknown exit reason, %d\n" , reason);print_vmexit_info (); } +vmcs_write (PIN_CONTROLS ,vmcs_read (PIN_CONTROLS ) & ~PIN_PREEMPT );return VMX_TEST_VMEXIT ; }
Signature · 2 lines
Siemens AG, Corporate Technology, CT RTC ITP SES-DE Corporate Competence Center Embedded Linux
Paolo Bonzini
maintainerIl 10/10/2013 18:12, Jan Kiszka ha scritto:
Quoted 54 lines · Jan Kiszka
Quoted 41 lines · Jan Kiszka
Quoted 40 lines · Paolo Bonzini
Quoted 19 lines
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
I've done some measurements with the help of ftrace on the time it takes to let the preemption timer trigger (no adjustments via Arthur's patch were involved): On my Core i7-620M, the preemption timer seems to tick almost 10 times faster than spec and scale value (5) suggests. I've loaded a value of 100000, and it took about 130 µs until I got a vmexit with reason
PREEMPTION_TIMER(no other exists in between).qemu-system-x86-13765 [003] 298562.966079: bprint: prepare_vmcs02: preempt val 100000 qemu-system-x86-13765 [003] 298562.966083: kvm_entry: vcpu 0 qemu-system-x86-13765 [003] 298562.966212: kvm_exit: reason PREEMPTION_TIMER rip 0x401fea info 0 0Quoted 8 lines
Quoted 3 lines
That's a frequency of ~769 MHz. The TSC ticks at 2.66 GHz. But 769 MHz * 2^5 is 24.6 GHz. I've read the spec several times, but it seems pretty clear on this. It just doesn't match reality. Very strange.
...but documented: I found an related errata for my processor (AAT59) and also for Xeon 5500 (AAK139). At least current Haswell generation is no affected. I can test the patch on a Haswell board I have at work later this week.
To complete this story: Arthur's patch works fine on a non-broken CPU (here: i7-4770S).
Quoted 6 lines
Arthur, find some fix-ups for your test case below. It avoids printing from within L2 as this could deadlock when the timer fires and L1 then tries to print something. Also, it disables the preemption timer on leave so that it cannot fire later on again. If you want to fold this into your patch, feel free. Otherwise I can post a separate patch on top.
Is that a Signed-off-by? :)
BTW, VirtualBox has a test for this erratum. It would be nice to skip the test when the processor is found to be buggy.
I'll put Arthur's patch back. Thanks for testing!
Paolo
static bool hmR0InitIntelIsSubjectToVmxPreemptionTimerErratum(void)
{
uint32_t u = ASMCpuId_EAX(1);
u &= ~(RT_BIT_32(14) | RT_BIT_32(15) | RT_BIT_32(28) | RT_BIT_32(29) | RT_BIT_32(30) | RT_BIT_32(31));
if ( u == UINT32_C(0x000206E6) /* 323344.pdf - BA86 - D0 - Intel Xeon Processor 7500 Series */
|| u == UINT32_C(0x00020652) /* 323056.pdf - AAX65 - C2 - Intel Xeon Processor L3406 */
|| u == UINT32_C(0x00020652) /* 322814.pdf - AAT59 - C2 - Intel CoreTM i7-600, i5-500, i5-400 and i3-300 Mobile Processor Series */
|| u == UINT32_C(0x00020652) /* 322911.pdf - AAU65 - C2 - Intel CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
|| u == UINT32_C(0x00020655) /* 322911.pdf - AAU65 - K0 - Intel CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
|| u == UINT32_C(0x000106E5) /* 322373.pdf - AAO95 - B1 - Intel Xeon Processor 3400 Series */
|| u == UINT32_C(0x000106E5) /* 322166.pdf - AAN92 - B1 - Intel CoreTM i7-800 and i5-700 Desktop Processor Series */
|| u == UINT32_C(0x000106E5) /* 320767.pdf - AAP86 - B1 - Intel Core i7-900 Mobile Processor Extreme Edition Series, Intel Core i7-800 and i7-700 Mobile Processor Series */
|| u == UINT32_C(0x000106A0) /*?321333.pdf - AAM126 - C0 - Intel Xeon Processor 3500 Series Specification */
|| u == UINT32_C(0x000106A1) /*?321333.pdf - AAM126 - C1 - Intel Xeon Processor 3500 Series Specification */
|| u == UINT32_C(0x000106A4) /* 320836.pdf - AAJ124 - C0 - Intel Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
|| u == UINT32_C(0x000106A5) /* 321333.pdf - AAM126 - D0 - Intel Xeon Processor 3500 Series Specification */
|| u == UINT32_C(0x000106A5) /* 321324.pdf - AAK139 - D0 - Intel Xeon Processor 5500 Series Specification */
|| u == UINT32_C(0x000106A5) /* 320836.pdf - AAJ124 - D0 - Intel Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
)
return true;
return false;
}Quoted 82 lines
Jan
x86/vmx_tests.c +16
-10@@ -141,6 +141,9 @@ void preemption_timer_init() preempt_val =10000000 ;vmcs_write (PREEMPT_TIMER_VALUE , preempt_val); preempt_scale =rdmsr (MSR_IA32_VMX_MISC ) &0x1F ; + +if (!(ctrl_exit_rev.clr &EXI_SAVE_PREEMPT )) +printf ("\tSave preemption value is not supported\n" ); }void preemption_timer_main ()@@ -150,9 +153,7 @@ void preemption_timer_main()printf ("\tPreemption timer is not supported\n" );return ; } -if (!(ctrl_exit_rev.clr &EXI_SAVE_PREEMPT )) -printf ("\tSave preemption value is not supported\n" ); -else { +if (ctrl_exit_rev.clr &EXI_SAVE_PREEMPT ) {set_stage (0 );vmcall ();if (get_stage () ==1 )@@ -161,8 +162,8 @@ void preemption_timer_main()while (1 ) {if (((rdtsc () - tsc_val) >> preempt_scale) >10 * preempt_val) { -report ("Preemption timer" ,0 ); -break ; +set_stage (2 ); +vmcall (); } } }@@ -183,7 +184,7 @@ int preemption_timer_exit_handler()report ("Preemption timer" ,0 ); else report("Preemption timer", 1); -return VMX_TEST_VMEXIT ; +break ;case VMX_VMCALL :switch (get_stage ()) {case 0 :@@ -195,24 +196,29 @@ int preemption_timer_exit_handler() EXI_SAVE_PREEMPT) & ctrl_exit_rev.clr;vmcs_write (EXI_CONTROLS , ctrl_exit); } -break ; +vmcs_write (GUEST_RIP , guest_rip + insn_len); +return VMX_TEST_RESUME ;case 1 :if (vmcs_read (PREEMPT_TIMER_VALUE ) >= preempt_val)report ("Save preemption value" ,0 );else report ("Save preemption value" ,1 ); +vmcs_write (GUEST_RIP , guest_rip + insn_len); +return VMX_TEST_RESUME ; +case 2 : +report ("Preemption timer" ,0 );break ;default :printf ("Invalid stage.\n" );print_vmexit_info (); -return VMX_TEST_VMEXIT ; +break ; } -vmcs_write (GUEST_RIP , guest_rip + insn_len); -return VMX_TEST_RESUME ; +break ;default :printf ("Unknown exit reason, %d\n" , reason);print_vmexit_info (); } +vmcs_write (PIN_CONTROLS ,vmcs_read (PIN_CONTROLS ) & ~PIN_PREEMPT );return VMX_TEST_VMEXIT ; }
Paolo Bonzini
maintainerIl 10/10/2013 17:20, Paolo Bonzini ha scritto:
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Arthur, find some fix-ups for your test case below. It avoids printing from within L2 as this could deadlock when the timer fires and L1 then tries to print something. Also, it disables the preemption timer on leave so that it cannot fire later on again. If you want to fold this into your patch, feel free. Otherwise I can post a separate patch on top.
Is that a Signed-off-by? :)
Ping?
Paolo
Quoted 10 lines · Paolo Bonzini
Il 10/10/2013 17:20, Paolo Bonzini ha scritto:
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Arthur, find some fix-ups for your test case below. It avoids printing from within L2 as this could deadlock when the timer fires and L1 then tries to print something. Also, it disables the preemption timer on leave so that it cannot fire later on again. If you want to fold this into your patch, feel free. Otherwise I can post a separate patch on top.
Is that a Signed-off-by? :)
Ping?
Sent as separate patch two days ago.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT RTC ITP SES-DE Corporate Competence Center Embedded Linux
Hi Jan,
Quoted 54 lines · Jan Kiszka
Quoted 41 lines · Jan Kiszka
Quoted 40 lines · Jan Kiszka
Quoted 19 lines · Paolo Bonzini
Il 16/09/2013 10:11, Arthur Chunqi Li ha scritto:
Quoted 10 lines
This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexitL2->L0with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.With this patch, nested VMX preemption timer features are fully supported.
Quoted 7 lines
Signed-off-by Arthur Chunqi Li <[email protected]>---
ChangeLog to v4:
Format changes and remove a flag innested_vmx.arch/x86/include/uapi/asm/msr-index.h | 1 + arch/x86/kvm/vmx.c | 44 +++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-)Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
I've done some measurements with the help of ftrace on the time it takes to let the preemption timer trigger (no adjustments via Arthur's patch were involved): On my Core i7-620M, the preemption timer seems to tick almost 10 times faster than spec and scale value (5) suggests. I've loaded a value of 100000, and it took about 130 盜 until I got a vmexit with reason
PREEMPTION_TIMER(no other exists in between).qemu-system-x86-13765 [003] 298562.966079: bprint: prepare_vmcs02: preempt val 100000 qemu-system-x86-13765 [003] 298562.966083: kvm_entry: vcpu 0 qemu-system-x86-13765 [003] 298562.966212: kvm_exit: reason PREEMPTION_TIMER rip 0x401fea info 0 0Quoted 8 lines
Quoted 3 lines
That's a frequency of ~769 MHz. The TSC ticks at 2.66 GHz. But 769 MHz * 2^5 is 24.6 GHz. I've read the spec several times, but it seems pretty clear on this. It just doesn't match reality. Very strange.
...but documented: I found an related errata for my processor (AAT59) and also for Xeon 5500 (AAK139). At least current Haswell generation is no affected. I can test the patch on a Haswell board I have at work later this week.
To complete this story: Arthur's patch works fine on a non-broken CPU (here: i7-4770S).
Quoted 6 lines
Arthur, find some fix-ups for your test case below. It avoids printing from within L2 as this could deadlock when the timer fires and L1 then tries to print something. Also, it disables the preemption timer on leave so that it cannot fire later on again. If you want to fold this into your patch, feel free. Otherwise I can post a separate patch on top.
I think this can be treated as a separate patch to our test suite. You can post it on top.
I have tested it and it works fine.
Arthur
Quoted 85 lines
Jan
x86/vmx_tests.c +16
-11@@ -141,6 +141,9 @@ void preemption_timer_init() preempt_val =10000000 ;vmcs_write (PREEMPT_TIMER_VALUE , preempt_val); preempt_scale =rdmsr (MSR_IA32_VMX_MISC ) &0x1F ; + +if (!(ctrl_exit_rev.clr &EXI_SAVE_PREEMPT )) +printf ("\tSave preemption value is not supported\n" ); }void preemption_timer_main ()@@ -150,9 +153,7 @@ void preemption_timer_main()printf ("\tPreemption timer is not supported\n" );return ; } -if (!(ctrl_exit_rev.clr &EXI_SAVE_PREEMPT )) -printf ("\tSave preemption value is not supported\n" ); -else { +if (ctrl_exit_rev.clr &EXI_SAVE_PREEMPT ) {set_stage (0 );vmcall ();if (get_stage () ==1 )@@ -161,8 +162,8 @@ void preemption_timer_main()while (1 ) {if (((rdtsc () - tsc_val) >> preempt_scale) >10 * preempt_val) { -report ("Preemption timer" ,0 ); -break ; +set_stage (2 ); +vmcall (); } } }@@ -183,7 +184,7 @@ int preemption_timer_exit_handler()report ("Preemption timer" ,0 ); else report("Preemption timer", 1); -return VMX_TEST_VMEXIT ; +break ;case VMX_VMCALL :switch (get_stage ()) {case 0 :@@ -195,24 +196,29 @@ int preemption_timer_exit_handler() EXI_SAVE_PREEMPT) & ctrl_exit_rev.clr;vmcs_write (EXI_CONTROLS , ctrl_exit); } -break ; +vmcs_write (GUEST_RIP , guest_rip + insn_len); +return VMX_TEST_RESUME ;case 1 :if (vmcs_read (PREEMPT_TIMER_VALUE ) >= preempt_val)report ("Save preemption value" ,0 );else report ("Save preemption value" ,1 ); +vmcs_write (GUEST_RIP , guest_rip + insn_len); +return VMX_TEST_RESUME ; +case 2 : +report ("Preemption timer" ,0 );break ;default :printf ("Invalid stage.\n" );print_vmexit_info (); -return VMX_TEST_VMEXIT ; +break ; } -vmcs_write (GUEST_RIP , guest_rip + insn_len); -return VMX_TEST_RESUME ; +break ;default :printf ("Unknown exit reason, %d\n" , reason);print_vmexit_info (); } +vmcs_write (PIN_CONTROLS ,vmcs_read (PIN_CONTROLS ) & ~PIN_PREEMPT );return VMX_TEST_VMEXIT ; } --Siemens AG, Corporate Technology, CT RTC ITP SES-DE Corporate Competence Center Embedded Linux
Signature · 5 lines
Arthur Chunqi Li Department of Computer Science School of EECS Peking University Beijing, China
Paolo Bonzini
maintainer-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Il 30/09/2013 11:08, Jan Kiszka ha scritto:
Quoted 10 lines · Paolo Bonzini
Quoted 8 lines
Hi all,
the test fails for me if the preemption timer value is set to a value that is above ~2000 (which means ~65000 TSC cycles on this machine). The preemption timer seems to count faster than what is expected, for example only up to 4 million cycles if you set it to one million. So, I am leaving the patch out of kvm/queue for now, until I can test it on more processors.
Quoted 7 lines
I've done some measurements with the help of ftrace on the time it takes to let the preemption timer trigger (no adjustments via Arthur's patch were involved): On my Core i7-620M, the preemption timer seems to tick almost 10 times faster than spec and scale value (5) suggests. I've loaded a value of 100000, and it took about 130 µs until I got a vmexit with reason
PREEMPTION_TIMER(no other exists in between).
10x is similar to what I was observing.
Paolo
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.21 (GNU/Linux)
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/
iQIcBAEBAgAGBQJSTSY2AAoJEBvWZb6bTYbyZmMQAJ7i2irCo/CDAK/zS//BHI/E OE2jsqKnUVkcSee5XJ+bdyclqwLmp7J/vEySKlXpdrzr2lcZ/FNP2muXQ1HXeK8L I3eoSbuYTY3DPHKTjR09GVNfzIZIC6H8TwKb8RdbtZgwci1r9kjmwcpAJt3Qh0UR xM/6a5gnOubdCGx6SdFBVPL+OfZ3zu1Si6Aw+3mnNYO9KvLpbtA3lO4u4HteTchM KOszK0rmR3Y1LoWQdUhuTgrP1DMFZyZKhW4nHOIq/DWK+/al+0knJ5z5QaeZgP3z GkvwzDbzJKGTGRJpAiAixQ40uKIycHkv6IKjvJn2iT6/XhI78W2+X5OVjhU4kt8p aJ3h+KacopaYQpxaHBZo8jSvT0U+vX5mxvoBbn6okaKMw/iZ7eXVR6n0Pi3zHF35 g8jUalHKbunY8q0bFjZcvVfmAIVf+oBEQ67nkxZSUw3vW4zvx+Vmtx7j0MwzmLc7 2IWSAWehSaYCsKzHh7Qf+j8/8qCDaPZDmaAIrzE9ODglMHd/fZk/zA3rhJ7+5MnZ FKVNV/ABcnvm/TP3s0bIL7hWHUawZ/PlxJ0IxKvD25TUm70kpG9FIbrbRZ25ltsh mmI+aL5HK5ztJR4lSj0jk51YBYlLWoef7yWzDSnVgnebceY5KP7o1WVzSPt+AJlD Z6dK0g1rJk39kQV5nrXh
=BPPg
-----END PGP SIGNATURE-----