Skip to contentSkip to conversation

Optimize STI; HLT while an interrupt is pending

PATCH In review Avi Kivity 2 participants 10 messages Feb 14, 2011 – Feb 20, 2011

Subsystems (3)
lore
Share & tools
lore

Avi Kivity

author
#0 lore raw

cover Optimize STI; HLT while an interrupt is pending

If the guest executes the following code sequence

CLI
...
(interrupt becomes pending)
STI
HLT

then we will exit on HLT, enter the guest, and exit immediately again on virtual interrupt pending. As this is wasteful, this patch set short circuits the whole operation to inject the interrupt immediately.

(as I've seen this occur very rarely in ftrace, compared to what I saw before I wrote the patch, please don't apply patches 2 and 3 until I've had a chance to benchmark it)

Avi Kivity (3):
KVM: VMX: Cache CPU_BASED_VM_EXEC_CONTROL VMCS field
KVM: VMX: Short circuit STI; HLT while an interrupt is pending
KVM: SVM: Short circuit STI; HLT while an interrupt is pending

 arch/x86/kvm/svm.c |    9 +++++++
 arch/x86/kvm/vmx.c |   69 ++++++++++++++++++++++++++++-----------------------
 2 files changed, 47 insertions(+), 31 deletions(-)
author
#1 lore raw

PATCH 1/3 KVM: VMX: Cache CPU_BASED_VM_EXEC_CONTROL VMCS field

The CPU_BASED_VM_EXEC_CONTROL VMCS field is write-only, so we can cache it in the vcpu structure and avoid a costly vmcs_read32() every time we want to change a bit.

---
 arch/x86/kvm/vmx.c |   60 +++++++++++++++++++++++++--------------------------
 1 files changed, 29 insertions(+), 31 deletions(-)
arch/x86/kvm/vmx.c +29 -31
index f76137c..ee1cd1a 100644
@@ -130,6 +130,7 @@ struct vcpu_vmx {
 	u8                    fail;
 	u32                   exit_intr_info;
 	u32                   idt_vectoring_info;
+	u32                   cpu_based_vm_exec_control;
 	struct shared_msr_entry *guest_msrs;
 	int                   nmsrs;
 	int                   save_nmsrs;
@@ -643,6 +644,18 @@ static void vmcs_set_bits(unsigned long field, u32 mask)
 	vmcs_writel(field, vmcs_readl(field) | mask);
 }
 
+static void set_cpu_based_vm_exec_ctrl_bits(struct vcpu_vmx *vmx, u32 mask)
+{
+	vmx->cpu_based_vm_exec_control |= mask;
+	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx->cpu_based_vm_exec_control);
+}
+
+static void clear_cpu_based_vm_exec_ctrl_bits(struct vcpu_vmx *vmx, u32 mask)
+{
+	vmx->cpu_based_vm_exec_control &= ~mask;
+	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx->cpu_based_vm_exec_control);
+}
+
 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
 {
 	u32 eb;
@@ -1932,18 +1945,16 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
 	vmx_decache_cr3(vcpu);
 	if (!(cr0 & X86_CR0_PG)) {
 		/* From paging/starting to nonpaging */
-		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
-			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
-			     (CPU_BASED_CR3_LOAD_EXITING |
-			      CPU_BASED_CR3_STORE_EXITING));
+		set_cpu_based_vm_exec_ctrl_bits(to_vmx(vcpu),
+						CPU_BASED_CR3_LOAD_EXITING |
+						CPU_BASED_CR3_STORE_EXITING);
 		vcpu->arch.cr0 = cr0;
 		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
 	} else if (!is_paging(vcpu)) {
 		/* From nonpaging to paging */
-		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
-			     vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
-			     ~(CPU_BASED_CR3_LOAD_EXITING |
-			       CPU_BASED_CR3_STORE_EXITING));
+		clear_cpu_based_vm_exec_ctrl_bits(to_vmx(vcpu),
+						  CPU_BASED_CR3_LOAD_EXITING |
+						  CPU_BASED_CR3_STORE_EXITING);
 		vcpu->arch.cr0 = cr0;
 		vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
 	}
@@ -2622,6 +2633,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
 		exec_control |= CPU_BASED_CR3_STORE_EXITING |
 				CPU_BASED_CR3_LOAD_EXITING  |
 				CPU_BASED_INVLPG_EXITING;
+	vmx->cpu_based_vm_exec_control = exec_control;
 	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
 
 	if (cpu_has_secondary_exec_ctrls()) {
@@ -2873,17 +2885,12 @@ out:
 
 static void enable_irq_window(struct kvm_vcpu *vcpu)
 {
-	u32 cpu_based_vm_exec_control;
-
-	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
-	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
-	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
+	set_cpu_based_vm_exec_ctrl_bits(to_vmx(vcpu),
+					CPU_BASED_VIRTUAL_INTR_PENDING);
 }
 
 static void enable_nmi_window(struct kvm_vcpu *vcpu)
 {
-	u32 cpu_based_vm_exec_control;
-
 	if (!cpu_has_virtual_nmis()) {
 		enable_irq_window(vcpu);
 		return;
@@ -2893,9 +2900,8 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
 		enable_irq_window(vcpu);
 		return;
 	}
-	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
-	cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
-	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
+	set_cpu_based_vm_exec_ctrl_bits(to_vmx(vcpu),
+					CPU_BASED_VIRTUAL_NMI_PENDING);
 }
 
 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
@@ -3408,13 +3414,9 @@ static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
 
 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
 {
-	u32 cpu_based_vm_exec_control;
-
 	/* clear pending irq */
-	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
-	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
-	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
-
+	clear_cpu_based_vm_exec_ctrl_bits(to_vmx(vcpu),
+					  CPU_BASED_VIRTUAL_INTR_PENDING);
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 
 	++vcpu->stat.irq_window_exits;
@@ -3671,12 +3673,8 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
 
 static int handle_nmi_window(struct kvm_vcpu *vcpu)
 {
-	u32 cpu_based_vm_exec_control;
-
-	/* clear pending NMI */
-	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
-	cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
-	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
+	clear_cpu_based_vm_exec_ctrl_bits(to_vmx(vcpu),
+					  CPU_BASED_VIRTUAL_NMI_PENDING);
 	++vcpu->stat.nmi_window_exits;
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 
@@ -3691,7 +3689,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
 	u32 cpu_exec_ctrl;
 	bool intr_window_requested;
 
-	cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
+	cpu_exec_ctrl = vmx->cpu_based_vm_exec_control;
 	intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
 
 	while (!guest_state_valid(vcpu)) {
Signature · 1 line
1.7.1
author
#2 lore raw

PATCH 2/3 KVM: VMX: Short circuit STI; HLT while an interrupt is pending

In reply to Avi Kivity

Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.

Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.

---
 arch/x86/kvm/vmx.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)
arch/x86/kvm/vmx.c +9 -0
index ee1cd1a..541da0e 100644
@@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
 static int handle_halt(struct kvm_vcpu *vcpu)
 {
 	skip_emulated_instruction(vcpu);
+	/*
+	 * Short-circuit an STI; HLT sequence while an interrupt is pending:
+	 * instead of halting, re-entering the guest, and exiting immediately
+	 * on an interrupt window exit, go directly to the last step.
+	 */
+	if ((to_vmx(vcpu)->cpu_based_vm_exec_control
+	     & CPU_BASED_VIRTUAL_INTR_PENDING)
+	    && (kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
+		return handle_interrupt_window(vcpu);
 	return kvm_emulate_halt(vcpu);
 }
 
Signature · 1 line
1.7.1
#3 lore raw

Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending

Quoted 22 lines · Avi Kivity

Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.

Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.

---
 arch/x86/kvm/vmx.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)
arch/x86/kvm/vmx.c +1 -0
index ee1cd1a..541da0e 100644
@@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
 static int handle_halt(struct kvm_vcpu *vcpu)
 {
 	skip_emulated_instruction(vcpu);
+	/*

arch/x86/kvm/vmx.c static int handle_interrupt_window(struct kvm_vcpu *vcpu) · line 3450

Quoted 10 lines
arch/x86/kvm/vmx.c +8 -0
+	 * Short-circuit an STI; HLT sequence while an interrupt is pending:
+	 * instead of halting, re-entering the guest, and exiting immediately
+	 * on an interrupt window exit, go directly to the last step.
+	 */
+	if ((to_vmx(vcpu)->cpu_based_vm_exec_control
+	     & CPU_BASED_VIRTUAL_INTR_PENDING)
+	    && (kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
+		return handle_interrupt_window(vcpu);
 	return kvm_emulate_halt(vcpu);
 }

Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?

author
#4 lore raw

Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending

Quoted 34 lines · Marcelo Tosatti
Quoted 24 lines · Avi Kivity

Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.

Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.

---

  arch/x86/kvm/vmx.c |    9 +++++++++
  1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index ee1cd1a..541da0e 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
static int handle_halt(struct kvm_vcpu *vcpu)
{
skip_emulated_instruction(vcpu);
+ /*
+ * Short-circuit an STI; HLT sequence while an interrupt is pending:
+ * instead of halting, re-entering the guest, and exiting immediately

Quoted 8 lines

+ * on an interrupt window exit, go directly to the last step.
+ */
+ if ((to_vmx(vcpu)->cpu_based_vm_exec_control
+ & CPU_BASED_VIRTUAL_INTR_PENDING)
+ && (kvm_get_rflags(vcpu)& X86_EFLAGS_IF))
+ return handle_interrupt_window(vcpu);
return kvm_emulate_halt(vcpu);
}

Quoted 2 lines

Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?

Yes. Also, we want to clear CPU_BASED_VIRTUAL_INTR_PENDING.

Signature · 1 line
error compiling committee.c: too many arguments to function
#5 lore raw

Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending

Quoted 35 lines · Avi Kivity
Quoted 34 lines · Marcelo Tosatti
Quoted 32 lines · Avi Kivity

Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.

Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.

---

  arch/x86/kvm/vmx.c |    9 +++++++++
  1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index ee1cd1a..541da0e 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
static int handle_halt(struct kvm_vcpu *vcpu)
{
skip_emulated_instruction(vcpu);
+ /*
+ * Short-circuit an STI; HLT sequence while an interrupt is pending:
+ * instead of halting, re-entering the guest, and exiting immediately
+ * on an interrupt window exit, go directly to the last step.
+ */
+ if ((to_vmx(vcpu)->cpu_based_vm_exec_control
+ & CPU_BASED_VIRTUAL_INTR_PENDING)
+ && (kvm_get_rflags(vcpu)& X86_EFLAGS_IF))
+ return handle_interrupt_window(vcpu);
return kvm_emulate_halt(vcpu);
}

Quoted 4 lines
Quoted 2 lines

Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?

Yes. Also, we want to clear CPU_BASED_VIRTUAL_INTR_PENDING.

Is there a reason why it cannot be handled in the main loop?

author
#6 lore raw

Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending

Quoted 36 lines · Marcelo Tosatti
Quoted 35 lines · Avi Kivity
Quoted 34 lines · Marcelo Tosatti
Quoted 32 lines · Avi Kivity

Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.

Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.

---

   arch/x86/kvm/vmx.c |    9 +++++++++
   1 files changed, 9 insertions(+), 0 deletions(-)
  diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
  index ee1cd1a..541da0e 100644
  --- a/arch/x86/kvm/vmx.c
  +++ b/arch/x86/kvm/vmx.c
  @@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
   static int handle_halt(struct kvm_vcpu *vcpu)
   {
   	skip_emulated_instruction(vcpu);
  +	/*
  +	 * Short-circuit an STI; HLT sequence while an interrupt is pending:
  +	 * instead of halting, re-entering the guest, and exiting immediately
  +	 * on an interrupt window exit, go directly to the last step.
  +	 */
  +	if ((to_vmx(vcpu)->cpu_based_vm_exec_control
  +	&   CPU_BASED_VIRTUAL_INTR_PENDING)
  +	&&   (kvm_get_rflags(vcpu)&   X86_EFLAGS_IF))
  +		return handle_interrupt_window(vcpu);
   	return kvm_emulate_halt(vcpu);
   }
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?

Yes. Also, we want to clear CPU_BASED_VIRTUAL_INTR_PENDING.

Is there a reason why it cannot be handled in the main loop?

Don't follow. What are you suggesting?

Signature · 1 line
error compiling committee.c: too many arguments to function
#7 lore raw

Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending

Quoted 19 lines · Avi Kivity
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 18 lines
  index ee1cd1a..541da0e 100644
  --- a/arch/x86/kvm/vmx.c
  +++ b/arch/x86/kvm/vmx.c
  @@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
   static int handle_halt(struct kvm_vcpu *vcpu)
   {
   	skip_emulated_instruction(vcpu);
  +	/*
  +	 * Short-circuit an STI; HLT sequence while an interrupt is pending:
  +	 * instead of halting, re-entering the guest, and exiting immediately
  +	 * on an interrupt window exit, go directly to the last step.
  +	 */
  +	if ((to_vmx(vcpu)->cpu_based_vm_exec_control
  +	&   CPU_BASED_VIRTUAL_INTR_PENDING)
  +	&&   (kvm_get_rflags(vcpu)&   X86_EFLAGS_IF))
  +		return handle_interrupt_window(vcpu);
   	return kvm_emulate_halt(vcpu);
   }
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?

Yes. Also, we want to clear CPU_BASED_VIRTUAL_INTR_PENDING.

Is there a reason why it cannot be handled in the main loop?

Don't follow. What are you suggesting?

That vcpu main loop (inject_pending_events etc) should be able to inject the interrupt and clear interrupt exiting, instead of a short circuit in specific exit handlers, as an improvement on top of the current patchset. Any numbers, BTW?

author
#8 lore raw

Re: [PATCH 2/3] KVM: VMX: Short circuit STI; HLT while an interrupt is pending

Quoted 25 lines · Marcelo Tosatti
Quoted 24 lines · Avi Kivity
Quoted 24 lines
Quoted 23 lines
Quoted 19 lines
Quoted 10 lines
   index ee1cd1a..541da0e 100644
   --- a/arch/x86/kvm/vmx.c
   +++ b/arch/x86/kvm/vmx.c
   @@ -3437,6 +3437,15 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
    static int handle_halt(struct kvm_vcpu *vcpu)
    {
    	skip_emulated_instruction(vcpu);
   +	/*
   +	 * Short-circuit an STI; HLT sequence while an interrupt is pending:
   +	 * instead of halting, re-entering the guest, and exiting immediately
Quoted 8 lines
   +	 * on an interrupt window exit, go directly to the last step.
   +	 */
   +	if ((to_vmx(vcpu)->cpu_based_vm_exec_control
   +	&    CPU_BASED_VIRTUAL_INTR_PENDING)
   +	&&    (kvm_get_rflags(vcpu)&    X86_EFLAGS_IF))
   +		return handle_interrupt_window(vcpu);
    	return kvm_emulate_halt(vcpu);
    }
Quoted 2 lines

Why does the normal vcpu entry path fails to inject the interrupt? Because after halt, KVM_REQ_EVENT is not set?

Yes. Also, we want to clear CPU_BASED_VIRTUAL_INTR_PENDING.

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

Is there a reason why it cannot be handled in the main loop?

Don't follow. What are you suggesting?

That vcpu main loop (inject_pending_events etc) should be able to inject the interrupt and clear interrupt exiting, instead of a short circuit in specific exit handlers, as an improvement on top of the current patchset.

Yes, it makes sense to make move virtual interrupt pending logic into x86.c. On the other hand, it may conflict with using the hardware interrupt queue on svm (which we don't make use of yet).

Quoted 1 line

Any numbers, BTW?

Not yet.

Signature · 1 line
error compiling committee.c: too many arguments to function
author
#9 lore raw

PATCH 3/3 KVM: SVM: Short circuit STI; HLT while an interrupt is pending

In reply to Avi Kivity

Short-circuit an STI; HLT sequence while an interrupt is pending:
instead of halting, re-entering the guest, and exiting immediately on an interrupt window exit, go directly to the last step.

Saves a vmexit on workloads where interrupts are received synchronously; an example is a disk backed by the host page cache where there is no latency (from the guest's point of view) between the request and fulfilment.

---
 arch/x86/kvm/svm.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)
arch/x86/kvm/svm.c +9 -0
index 53c5d8a..b70af8b 100644
@@ -184,6 +184,7 @@ module_param(nested, int, S_IRUGO);
 
 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 static void svm_complete_interrupts(struct vcpu_svm *svm);
+static int interrupt_window_interception(struct vcpu_svm *svm);
 
 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 static int nested_svm_intercept(struct vcpu_svm *svm);
@@ -1745,6 +1746,14 @@ static int halt_interception(struct vcpu_svm *svm)
 {
 	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
 	skip_emulated_instruction(&svm->vcpu);
+	/*
+	 * Short-circuit an STI; HLT sequence while an interrupt is pending:
+	 * instead of halting, re-entering the guest, and exiting immediately
+	 * on an interrupt window exit, go directly to the last step.
+	 */
+	if ((svm->vmcb->control.intercept & (1ULL << INTERCEPT_VINTR))
+	    && (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF))
+		return interrupt_window_interception(svm);
 	return kvm_emulate_halt(&svm->vcpu);
 }
 
Signature · 1 line
1.7.1