Skip to contentSkip to conversation

deal with interrupt shadow state for emulated instruction

PATCH In review Glauber Costa 2 participants 3 messages Apr 28, 2009 – May 5, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 deal with interrupt shadow state for emulated instruction

we currently unblock shadow interrupt state when we skip an instruction, but failing to do so when we actually emulate one. This blocks interrupts in key instruction blocks, in particular sti; hlt; sequences

If the instruction emulated is an sti, we have to block shadow interrupts. The same goes for mov ss. pop ss also needs it, but we don't currently emulate it.

Without this patch, I cannot boot gpxe option roms at vmx machines. This is described at https://bugzilla.redhat.com/show_bug.cgi?id=494469

CC H. Peter Anvin <[email protected]>
CC Avi Kivity <[email protected]>
---
 arch/x86/include/asm/kvm_host.h        |    2 +
 arch/x86/include/asm/kvm_x86_emulate.h |    6 +++
 arch/x86/kvm/svm.c                     |   31 ++++++++++++++++--
 arch/x86/kvm/vmx.c                     |   55 ++++++++++++++++++++++++++------
 arch/x86/kvm/x86.c                     |    7 +++-
 arch/x86/kvm/x86_emulate.c             |   25 ++++++++++++++-
 6 files changed, 111 insertions(+), 15 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -0
index cb306cf..9455a30 100644
@@ -510,6 +510,8 @@ struct kvm_x86_ops {
 	void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run);
 	int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu);
 	void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
+	void (*interrupt_shadow_mask)(struct kvm_vcpu *vcpu, int mask);
+	u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
 	void (*patch_hypercall)(struct kvm_vcpu *vcpu,
 				unsigned char *hypercall_addr);
 	int (*get_irq)(struct kvm_vcpu *vcpu);
arch/x86/include/asm/kvm_x86_emulate.h +6 -0
index 6a15973..b7ed2c4 100644
@@ -143,6 +143,9 @@ struct decode_cache {
 	struct fetch_cache fetch;
 };
 
+#define X86_SHADOW_INT_MOV_SS  1
+#define X86_SHADOW_INT_STI     2
+
 struct x86_emulate_ctxt {
 	/* Register state before/after emulation. */
 	struct kvm_vcpu *vcpu;
@@ -152,6 +155,9 @@ struct x86_emulate_ctxt {
 	int mode;
 	u32 cs_base;
 
+	/* interruptibility state, as a result of execution of STI or MOV SS */
+	int interruptibility;
+
 	/* decode cache */
 	struct decode_cache decode;
 };
arch/x86/kvm/svm.c +28 -3
index 053f3c5..a6af946 100644
@@ -210,6 +210,31 @@ static int is_external_interrupt(u32 info)
 	return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 }
 
+static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	u32 ret = 0;
+
+	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
+		ret |= (X86_SHADOW_INT_STI && X86_SHADOW_INT_MOV_SS);
+	return ret;
+}
+
+static void svm_interrupt_shadow_mask(struct kvm_vcpu *vcpu, int mask)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+
+	if (mask == 0)
+		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
+	else
+		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
+
+	svm->vcpu.arch.interrupt_window_open =
+	       (!(svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
+		(svm->vmcb->save.rflags & X86_EFLAGS_IF) &&
+		(svm->vcpu.arch.hflags & HF_GIF_MASK));
+}
+
 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -223,9 +248,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 		       __func__, kvm_rip_read(vcpu), svm->next_rip);
 
 	kvm_rip_write(vcpu, svm->next_rip);
-	svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
-
-	vcpu->arch.interrupt_window_open = (svm->vcpu.arch.hflags & HF_GIF_MASK);
+	svm_interrupt_shadow_mask(vcpu, 0);
 }
 
 static int has_svm(void)
@@ -2667,6 +2690,8 @@ static struct kvm_x86_ops svm_x86_ops = {
 	.run = svm_vcpu_run,
 	.handle_exit = handle_exit,
 	.skip_emulated_instruction = skip_emulated_instruction,
+	.interrupt_shadow_mask = svm_interrupt_shadow_mask,
+	.get_interrupt_shadow = svm_get_interrupt_shadow,
 	.patch_hypercall = svm_patch_hypercall,
 	.get_irq = svm_get_irq,
 	.set_irq = svm_set_irq,
arch/x86/kvm/vmx.c +45 -10

75 lines not on this page · raw message

arch/x86/kvm/x86.c +6 -1
index ab1fdac..fd58ccc 100644
@@ -2362,7 +2362,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
 			u16 error_code,
 			int emulation_type)
 {
-	int r;
+	int r, shadow_mask;
 	struct decode_cache *c;
 
 	kvm_clear_exception_queue(vcpu);
@@ -2415,7 +2415,12 @@ int emulate_instruction(struct kvm_vcpu *vcpu,
 		return EMULATE_DONE;
 	}
 
+	vcpu->arch.emulate_ctxt.interruptibility = 0;
 	r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
+	shadow_mask = vcpu->arch.emulate_ctxt.interruptibility;
+
+	if (r == 0)
+		kvm_x86_ops->interrupt_shadow_mask(vcpu, shadow_mask);
 
 	if (vcpu->arch.pio.string)
 		return EMULATE_DO_MMIO;
arch/x86/kvm/x86_emulate.c +24 -1
index d2664fc..797d41f 100644
@@ -1618,6 +1618,16 @@ special_insn:
 		int err;
 
 		sel = c->src.val;
+		if (c->modrm_reg == VCPU_SREG_SS) {
+			u32 int_shadow =
+				kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu);
+			/* See sti emulation for an explanation of this */
+			if ((int_shadow & X86_SHADOW_INT_MOV_SS))
+				ctxt->interruptibility &= ~X86_SHADOW_INT_MOV_SS;
+			else
+				ctxt->interruptibility |= X86_SHADOW_INT_MOV_SS;
+		}
+
 		if (c->modrm_reg <= 5) {
 			type_bits = (c->modrm_reg == 1) ? 9 : 1;
 			err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
@@ -1846,10 +1856,23 @@ special_insn:
 		ctxt->eflags &= ~X86_EFLAGS_IF;
 		c->dst.type = OP_NONE;	/* Disable writeback. */
 		break;
-	case 0xfb: /* sti */
+	case 0xfb: { /* sti */
+		u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu);
+		/*
+		 * an sti; sti; sequence only disable interrupts for the first
+		 * instruction. So, if the last instruction, be it emulated or
+		 * not, left the system with the INT_STI flag enabled, it
+		 * means that the last instruction is an sti. We should not
+		 * leave the flag on in this case
+		 */
+		if ((int_shadow & X86_SHADOW_INT_STI))
+			ctxt->interruptibility &= ~X86_SHADOW_INT_STI;
+		else
+			ctxt->interruptibility |= X86_SHADOW_INT_STI;
 		ctxt->eflags |= X86_EFLAGS_IF;
 		c->dst.type = OP_NONE;	/* Disable writeback. */
 		break;
+	}
 	case 0xfc: /* cld */
 		ctxt->eflags &= ~EFLG_DF;
 		c->dst.type = OP_NONE;	/* Disable writeback. */
Signature · 1 line
1.5.6.6
#1 lore raw
Quoted 16 lines · Glauber Costa

we currently unblock shadow interrupt state when we skip an instruction, but failing to do so when we actually emulate one. This blocks interrupts in key instruction blocks, in particular sti; hlt; sequences

If the instruction emulated is an sti, we have to block shadow interrupts. The same goes for mov ss. pop ss also needs it, but we don't currently emulate it.

Without this patch, I cannot boot gpxe option roms at vmx machines. This is described at https://bugzilla.redhat.com/show_bug.cgi?id=494469

arch/x86/include/asm/kvm_host.h +0 -0
index cb306cf..9455a30 100644

arch/x86/include/asm/kvm_host.h struct kvm_x86_ops { · line 514

Quoted 5 lines
arch/x86/include/asm/kvm_host.h +1 -0
@@ -510,6 +510,8 @@ struct kvm_x86_ops {
 	void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run);
 	int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu);
 	void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
+	void (*interrupt_shadow_mask)(struct kvm_vcpu *vcpu, int mask);
 

Can you verb this function? set_interrupt_shadow would make it nicely complement get_interrupt_shadow.

Quoted 5 lines
arch/x86/include/asm/kvm_host.h +1 -0
+	u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
 	void (*patch_hypercall)(struct kvm_vcpu *vcpu,
 				unsigned char *hypercall_addr);
arch/x86/include/asm/kvm_host.h +0 -0
 	int (*get_irq)(struct kvm_vcpu *vcpu);

arch/x86/include/asm/kvm_host.h

Quoted 10 lines
arch/x86/include/asm/kvm_host.h +9 -0
 
+static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	u32 ret = 0;
+
+	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
+		ret |= (X86_SHADOW_INT_STI && X86_SHADOW_INT_MOV_SS);
+	return ret;
+}

Hmm, if the guest runs an infinite emulated 'mov ss', it will keep toggling the MOV_SS bit, but STI will remain set, so we'll never allow an interrupt into the guest kernel.

Quoted 8 lines
arch/x86/kvm/x86_emulate.c +0 -0
index d2664fc..797d41f 100644
@@ -1618,6 +1618,16 @@ special_insn:
 		int err;
 
 		sel = c->src.val;

arch/x86/kvm/x86_emulate.c special_insn: · line 1630

Quoted 10 lines
arch/x86/kvm/x86_emulate.c +9 -0
+		if (c->modrm_reg == VCPU_SREG_SS) {
+			u32 int_shadow =
+				kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu);
+			/* See sti emulation for an explanation of this */
+			if ((int_shadow & X86_SHADOW_INT_MOV_SS))
+				ctxt->interruptibility &= ~X86_SHADOW_INT_MOV_SS;
+			else
+				ctxt->interruptibility |= X86_SHADOW_INT_MOV_SS;
+		}
 

^=

Quoted 9 lines
arch/x86/kvm/x86_emulate.c +4 -1
@@ -1846,10 +1856,23 @@ special_insn:
 		ctxt->eflags &= ~X86_EFLAGS_IF;
 		c->dst.type = OP_NONE;	/* Disable writeback. */
 		break;
-	case 0xfb: /* sti */
+	case 0xfb: { /* sti */
+		u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu);
+		/*
+		 * an sti; sti; sequence only disable interrupts for the first

arch/x86/kvm/x86_emulate.c special_insn: · line 1872

Quoted 10 lines
arch/x86/kvm/x86_emulate.c +9 -0
+		 * instruction. So, if the last instruction, be it emulated or
+		 * not, left the system with the INT_STI flag enabled, it
+		 * means that the last instruction is an sti. We should not
+		 * leave the flag on in this case
+		 */
+		if ((int_shadow & X86_SHADOW_INT_STI))
+			ctxt->interruptibility &= ~X86_SHADOW_INT_STI;
+		else
+			ctxt->interruptibility |= X86_SHADOW_INT_STI;
 

^=

Signature · 1 line
error compiling committee.c: too many arguments to function
#2 lore raw
Quoted 3 lines

Hmm, if the guest runs an infinite emulated 'mov ss', it will keep toggling the MOV_SS bit, but STI will remain set, so we'll never allow an interrupt into the guest kernel.

We have no choice but returning both flags, since svm does not differentiate between them.

But see below for an alternative path that makes it a non-issue.

Quoted 12 lines
Quoted 12 lines
arch/x86/kvm/x86_emulate.c +4 -0
index d2664fc..797d41f 100644
@@ -1618,6 +1618,16 @@ special_insn:
 		int err;
  		sel = c->src.val;
+		if (c->modrm_reg == VCPU_SREG_SS) {
+			u32 int_shadow =
+				kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu);
+			/* See sti emulation for an explanation of this */
Quoted 8 lines

arch/x86/kvm/x86_emulate.c special_insn:

Quoted 6 lines
arch/x86/kvm/x86_emulate.c +5 -0
+			if ((int_shadow & X86_SHADOW_INT_MOV_SS))
+				ctxt->interruptibility &= ~X86_SHADOW_INT_MOV_SS;
+			else
+				ctxt->interruptibility |= X86_SHADOW_INT_MOV_SS;
+		}
 

^=

=p \o/

After re-reading this, masking the flags in here makes no sense.

I am moving to an approach in which I do

                        if (!(int_shadow & X86_SHADOW_INT_MOV_SS))
                                ctxt->interruptibility = X86_SHADOW_INT_MOV_SS;

Since if the next instruction is an sti, it is certainly not an sti; sti instruction (the current is mov ss, after all). So we should mask it anyway. This also solves nicely the problem you raised at svm.c.

Quoted 14 lines
Quoted 14 lines
arch/x86/kvm/x86_emulate.c +8 -1
@@ -1846,10 +1856,23 @@ special_insn:
 		ctxt->eflags &= ~X86_EFLAGS_IF;
 		c->dst.type = OP_NONE;	/* Disable writeback. */
 		break;
-	case 0xfb: /* sti */
+	case 0xfb: { /* sti */
+		u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu);
+		/*
+		 * an sti; sti; sequence only disable interrupts for the first
+		 * instruction. So, if the last instruction, be it emulated or
+		 * not, left the system with the INT_STI flag enabled, it
+		 * means that the last instruction is an sti. We should not
+		 * leave the flag on in this case
Quoted 8 lines

arch/x86/kvm/x86_emulate.c special_insn:

Quoted 6 lines
arch/x86/kvm/x86_emulate.c +5 -0
+		 */
+		if ((int_shadow & X86_SHADOW_INT_STI))
+			ctxt->interruptibility &= ~X86_SHADOW_INT_STI;
+		else
+			ctxt->interruptibility |= X86_SHADOW_INT_STI;
 

^=

ditto