Optimize STI; HLT while an interrupt is pending
cover Optimize STI; HLT while an interrupt is pending
If the guest executes the following code sequence
CLI
...
(interrupt becomes pending)
STI
HLT
then we will exit on HLT, enter the guest, and exit immediately again on virtual interrupt pending. As this is wasteful, this patch set short circuits the whole operation to inject the interrupt immediately.
(as I've seen this occur very rarely in ftrace, compared to what I saw before I wrote the patch, please don't apply patches 2 and 3 until I've had a chance to benchmark it)
Avi Kivity (3):
KVM: VMX: Cache CPU_BASED_VM_EXEC_CONTROL VMCS field
KVM: VMX: Short circuit STI; HLT while an interrupt is pending
KVM: SVM: Short circuit STI; HLT while an interrupt is pending
arch/x86/kvm/svm.c | 9 +++++++ arch/x86/kvm/vmx.c | 69 ++++++++++++++++++++++++++++----------------------- 2 files changed, 47 insertions(+), 31 deletions(-)
PATCH 1/3 KVM: VMX: Cache CPU_BASED_VM_EXEC_CONTROL VMCS field
The CPU_BASED_VM_EXEC_CONTROL VMCS field is write-only, so we can cache it in the vcpu structure and avoid a costly vmcs_read32() every time we want to change a bit.
--- arch/x86/kvm/vmx.c | 60 +++++++++++++++++++++++++-------------------------- 1 files changed, 29 insertions(+), 31 deletions(-)
arch/x86/kvm/vmx.c +29 -31
@@ -130,6 +130,7 @@ struct vcpu_vmx {u8 fail;u32 exit_intr_info;u32 idt_vectoring_info; +u32 cpu_based_vm_exec_control;struct shared_msr_entry *guest_msrs;int nmsrs;int save_nmsrs;
@@ -643,6 +644,18 @@ static void vmcs_set_bits(unsigned long field, u32 mask)vmcs_writel (field,vmcs_readl (field) | mask); } +static void set_cpu_based_vm_exec_ctrl_bits (struct vcpu_vmx *vmx,u32 mask) +{ + vmx->cpu_based_vm_exec_control |= mask; +vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , vmx->cpu_based_vm_exec_control); +} + +static void clear_cpu_based_vm_exec_ctrl_bits (struct vcpu_vmx *vmx,u32 mask) +{ + vmx->cpu_based_vm_exec_control &= ~mask; +vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , vmx->cpu_based_vm_exec_control); +} +static void update_exception_bitmap (struct kvm_vcpu *vcpu) {u32 eb;
@@ -1932,18 +1945,16 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,vmx_decache_cr3 (vcpu);if (!(cr0 &X86_CR0_PG )) {/* From paging/starting to nonpaging */ -vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , -vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ) | - (CPU_BASED_CR3_LOAD_EXITING | -CPU_BASED_CR3_STORE_EXITING )); +set_cpu_based_vm_exec_ctrl_bits (to_vmx (vcpu), +CPU_BASED_CR3_LOAD_EXITING | +CPU_BASED_CR3_STORE_EXITING ); vcpu->arch.cr0 = cr0;vmx_set_cr4 (vcpu,kvm_read_cr4 (vcpu)); }else if (!is_paging (vcpu)) {/* From nonpaging to paging */ -vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , -vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ) & - ~(CPU_BASED_CR3_LOAD_EXITING | -CPU_BASED_CR3_STORE_EXITING )); +clear_cpu_based_vm_exec_ctrl_bits (to_vmx (vcpu), +CPU_BASED_CR3_LOAD_EXITING | +CPU_BASED_CR3_STORE_EXITING ); vcpu->arch.cr0 = cr0;vmx_set_cr4 (vcpu,kvm_read_cr4 (vcpu)); }
@@ -2622,6 +2633,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) exec_control |=CPU_BASED_CR3_STORE_EXITING |CPU_BASED_CR3_LOAD_EXITING |CPU_BASED_INVLPG_EXITING ; + vmx->cpu_based_vm_exec_control = exec_control;vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , exec_control);if (cpu_has_secondary_exec_ctrls ()) {
@@ -2873,17 +2885,12 @@ out:static void enable_irq_window (struct kvm_vcpu *vcpu) { -u32 cpu_based_vm_exec_control; - - cpu_based_vm_exec_control =vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ); - cpu_based_vm_exec_control |=CPU_BASED_VIRTUAL_INTR_PENDING ; -vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , cpu_based_vm_exec_control); +set_cpu_based_vm_exec_ctrl_bits (to_vmx (vcpu), +CPU_BASED_VIRTUAL_INTR_PENDING ); }static void enable_nmi_window (struct kvm_vcpu *vcpu) { -u32 cpu_based_vm_exec_control; -if (!cpu_has_virtual_nmis ()) {enable_irq_window (vcpu);return ;
@@ -2893,9 +2900,8 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)enable_irq_window (vcpu);return ; } - cpu_based_vm_exec_control =vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ); - cpu_based_vm_exec_control |=CPU_BASED_VIRTUAL_NMI_PENDING ; -vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , cpu_based_vm_exec_control); +set_cpu_based_vm_exec_ctrl_bits (to_vmx (vcpu), +CPU_BASED_VIRTUAL_NMI_PENDING ); }static void vmx_inject_irq (struct kvm_vcpu *vcpu)
@@ -3408,13 +3414,9 @@ static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)static int handle_interrupt_window (struct kvm_vcpu *vcpu) { -u32 cpu_based_vm_exec_control; -/* clear pending irq */ - cpu_based_vm_exec_control =vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ); - cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING ; -vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , cpu_based_vm_exec_control); - +clear_cpu_based_vm_exec_ctrl_bits (to_vmx (vcpu), +CPU_BASED_VIRTUAL_INTR_PENDING );kvm_make_request (KVM_REQ_EVENT , vcpu); ++vcpu->stat.irq_window_exits;
@@ -3671,12 +3673,8 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)static int handle_nmi_window (struct kvm_vcpu *vcpu) { -u32 cpu_based_vm_exec_control; - -/* clear pending NMI */ - cpu_based_vm_exec_control =vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ); - cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING ; -vmcs_write32 (CPU_BASED_VM_EXEC_CONTROL , cpu_based_vm_exec_control); +clear_cpu_based_vm_exec_ctrl_bits (to_vmx (vcpu), +CPU_BASED_VIRTUAL_NMI_PENDING ); ++vcpu->stat.nmi_window_exits;kvm_make_request (KVM_REQ_EVENT , vcpu);
@@ -3691,7 +3689,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)u32 cpu_exec_ctrl;bool intr_window_requested; - cpu_exec_ctrl =vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ); + cpu_exec_ctrl = vmx->cpu_based_vm_exec_control; intr_window_requested = cpu_exec_ctrl &CPU_BASED_VIRTUAL_INTR_PENDING ;while (!guest_state_valid (vcpu)) {
Signature · 1 line
1.7.1
PATCH 2/3 KVM: VMX: Short circuit STI; HLT while an interrupt is pending
In reply to Avi KivityShort-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.
Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.
--- arch/x86/kvm/vmx.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)
arch/x86/kvm/vmx.c +9 -0
@@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)static int handle_halt (struct kvm_vcpu *vcpu) {skip_emulated_instruction (vcpu); +/* +* Short-circuit an STI; HLT sequence while an interrupt is pending: +* instead of halting, re-entering the guest, and exiting immediately +* on an interrupt window exit, go directly to the last step. +*/ +if ((to_vmx (vcpu)->cpu_based_vm_exec_control + &CPU_BASED_VIRTUAL_INTR_PENDING ) + && (kvm_get_rflags (vcpu) &X86_EFLAGS_IF )) +return handle_interrupt_window (vcpu);return kvm_emulate_halt (vcpu); }
Signature · 1 line
1.7.1
Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending
Quoted 22 lines · Avi Kivity
Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.
Signed-off-by Avi Kivity <[email protected]>--- arch/x86/kvm/vmx.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)arch/x86/kvm/vmx.c +1
-0@@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)static int handle_halt (struct kvm_vcpu *vcpu) {skip_emulated_instruction (vcpu); +/*
arch/x86/kvm/vmx.c static int handle_interrupt_window(struct kvm_vcpu *vcpu) · line 3450
Quoted 10 lines
arch/x86/kvm/vmx.c +8
-0+* Short-circuit an STI; HLT sequence while an interrupt is pending: +* instead of halting, re-entering the guest, and exiting immediately +* on an interrupt window exit, go directly to the last step. +*/ +if ((to_vmx (vcpu)->cpu_based_vm_exec_control + &CPU_BASED_VIRTUAL_INTR_PENDING ) + && (kvm_get_rflags (vcpu) &X86_EFLAGS_IF )) +return handle_interrupt_window (vcpu);return kvm_emulate_halt (vcpu); }
Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?
Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending
Quoted 34 lines · Marcelo Tosatti
Quoted 24 lines · Avi Kivity
Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.
Signed-off-by Avi Kivity<[email protected]>---
arch/x86/kvm/vmx.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)diff --git
a/arch/x86/kvm/vmx.cb/arch/x86/kvm/vmx.c
index ee1cd1a..541da0e 100644
---a/arch/x86/kvm/vmx.c
+++b/arch/x86/kvm/vmx.c
@@ -3437,6 +3437,15 @@ static inthandle_interrupt_window(struct kvm_vcpu *vcpu)
static inthandle_halt(struct kvm_vcpu *vcpu)
{
skip_emulated_instruction(vcpu);
+ /*
+ * Short-circuit an STI; HLT sequence while an interrupt is pending:
+ * instead of halting, re-entering the guest, and exiting immediatelyQuoted 8 lines
+ * on an interrupt window exit, go directly to the last step.
+ */
+ if ((to_vmx(vcpu)->cpu_based_vm_exec_control
+ &CPU_BASED_VIRTUAL_INTR_PENDING)
+ && (kvm_get_rflags(vcpu)&X86_EFLAGS_IF))
+ returnhandle_interrupt_window(vcpu);
returnkvm_emulate_halt(vcpu);
}
Quoted 2 lines
Why does the normal vcpu entry path fails to inject the interrupt? Because after halt,
KVM_REQ_EVENTis not set?
Yes. Also, we want to clear CPU_BASED_VIRTUAL_INTR_PENDING.
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending
Quoted 35 lines · Avi Kivity
Quoted 34 lines · Marcelo Tosatti
Quoted 32 lines · Avi Kivity
Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.
Signed-off-by Avi Kivity<[email protected]>---
arch/x86/kvm/vmx.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)diff --git
a/arch/x86/kvm/vmx.cb/arch/x86/kvm/vmx.c
index ee1cd1a..541da0e 100644
---a/arch/x86/kvm/vmx.c
+++b/arch/x86/kvm/vmx.c
@@ -3437,6 +3437,15 @@ static inthandle_interrupt_window(struct kvm_vcpu *vcpu)
static inthandle_halt(struct kvm_vcpu *vcpu)
{
skip_emulated_instruction(vcpu);
+ /*
+ * Short-circuit an STI; HLT sequence while an interrupt is pending:
+ * instead of halting, re-entering the guest, and exiting immediately
+ * on an interrupt window exit, go directly to the last step.
+ */
+ if ((to_vmx(vcpu)->cpu_based_vm_exec_control
+ &CPU_BASED_VIRTUAL_INTR_PENDING)
+ && (kvm_get_rflags(vcpu)&X86_EFLAGS_IF))
+ returnhandle_interrupt_window(vcpu);
returnkvm_emulate_halt(vcpu);
}
Quoted 4 lines
Quoted 2 lines
Why does the normal vcpu entry path fails to inject the interrupt? Because after halt,
KVM_REQ_EVENTis not set?Yes. Also, we want to clear
CPU_BASED_VIRTUAL_INTR_PENDING.
Is there a reason why it cannot be handled in the main loop?
Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending
Quoted 36 lines · Marcelo Tosatti
Quoted 35 lines · Avi Kivity
Quoted 34 lines · Marcelo Tosatti
Quoted 32 lines · Avi Kivity
Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.
Signed-off-by Avi Kivity<[email protected]>---
arch/x86/kvm/vmx.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index ee1cd1a..541da0e 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_halt(struct kvm_vcpu *vcpu) { skip_emulated_instruction(vcpu); + /* + * Short-circuit an STI; HLT sequence while an interrupt is pending: + * instead of halting, re-entering the guest, and exiting immediately + * on an interrupt window exit, go directly to the last step. + */ + if ((to_vmx(vcpu)->cpu_based_vm_exec_control + & CPU_BASED_VIRTUAL_INTR_PENDING) + && (kvm_get_rflags(vcpu)& X86_EFLAGS_IF)) + return handle_interrupt_window(vcpu); return kvm_emulate_halt(vcpu); }
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
Why does the normal vcpu entry path fails to inject the interrupt? Because after halt,
KVM_REQ_EVENTis not set?Yes. Also, we want to clear
CPU_BASED_VIRTUAL_INTR_PENDING.Is there a reason why it cannot be handled in the main loop?
Don't follow. What are you suggesting?
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending
Quoted 19 lines · Avi Kivity
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 18 lines
index ee1cd1a..541da0e 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_halt(struct kvm_vcpu *vcpu) { skip_emulated_instruction(vcpu); + /* + * Short-circuit an STI; HLT sequence while an interrupt is pending: + * instead of halting, re-entering the guest, and exiting immediately + * on an interrupt window exit, go directly to the last step. + */ + if ((to_vmx(vcpu)->cpu_based_vm_exec_control + & CPU_BASED_VIRTUAL_INTR_PENDING) + && (kvm_get_rflags(vcpu)& X86_EFLAGS_IF)) + return handle_interrupt_window(vcpu); return kvm_emulate_halt(vcpu); }
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
Why does the normal vcpu entry path fails to inject the interrupt? Because after halt,
KVM_REQ_EVENTis not set?Yes. Also, we want to clear
CPU_BASED_VIRTUAL_INTR_PENDING.Is there a reason why it cannot be handled in the main loop?
Don't follow. What are you suggesting?
That vcpu main loop (inject_pending_events etc) should be able to inject the interrupt and clear interrupt exiting, instead of a short circuit in specific exit handlers, as an improvement on top of the current patchset. Any numbers, BTW?
Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending
Quoted 25 lines · Marcelo Tosatti
Quoted 24 lines · Avi Kivity
Quoted 24 lines
Quoted 23 lines
Quoted 19 lines
Quoted 10 lines
index ee1cd1a..541da0e 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_halt(struct kvm_vcpu *vcpu) { skip_emulated_instruction(vcpu); + /* + * Short-circuit an STI; HLT sequence while an interrupt is pending: + * instead of halting, re-entering the guest, and exiting immediatelyQuoted 8 lines
+ * on an interrupt window exit, go directly to the last step. + */ + if ((to_vmx(vcpu)->cpu_based_vm_exec_control + & CPU_BASED_VIRTUAL_INTR_PENDING) + && (kvm_get_rflags(vcpu)& X86_EFLAGS_IF)) + return handle_interrupt_window(vcpu); return kvm_emulate_halt(vcpu); }Quoted 2 lines
Why does the normal vcpu entry path fails to inject the interrupt? Because after halt,
KVM_REQ_EVENTis not set?Yes. Also, we want to clear
CPU_BASED_VIRTUAL_INTR_PENDING.
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
Is there a reason why it cannot be handled in the main loop?
Don't follow. What are you suggesting?
That vcpu main loop (
inject_pending_eventsetc) should be able to inject the interrupt and clear interrupt exiting, instead of a short circuit in specific exit handlers, as an improvement on top of the current patchset.
Yes, it makes sense to make move virtual interrupt pending logic into x86.c. On the other hand, it may conflict with using the hardware interrupt queue on svm (which we don't make use of yet).
Quoted 1 line
Any numbers, BTW?
Not yet.
Signature · 1 line
error compiling committee.c: too many arguments to function
PATCH 3/3 KVM: SVM: Short circuit STI; HLT while an interrupt is pending
In reply to Avi KivityShort-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.
Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.
--- arch/x86/kvm/svm.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)
arch/x86/kvm/svm.c +9 -0
@@ -184,6 +184,7 @@ module_param(nested, int, S_IRUGO);static void svm_flush_tlb (struct kvm_vcpu *vcpu);static void svm_complete_interrupts (struct vcpu_svm *svm); +static int interrupt_window_interception (struct vcpu_svm *svm);static int nested_svm_exit_handled (struct vcpu_svm *svm);static int nested_svm_intercept (struct vcpu_svm *svm);
@@ -1745,6 +1746,14 @@ static int halt_interception(struct vcpu_svm *svm) { svm->next_rip =kvm_rip_read (&svm->vcpu) +1 ;skip_emulated_instruction (&svm->vcpu); +/* +* Short-circuit an STI; HLT sequence while an interrupt is pending: +* instead of halting, re-entering the guest, and exiting immediately +* on an interrupt window exit, go directly to the last step. +*/ +if ((svm->vmcb->control.intercept & (1ULL <<INTERCEPT_VINTR )) + && (kvm_get_rflags (&svm->vcpu) &X86_EFLAGS_IF )) +return interrupt_window_interception (svm);return kvm_emulate_halt (&svm->vcpu); }
Signature · 1 line
1.7.1