KVM: migrate PIT timer

PATCH Applied Marcelo Tosatti 2 participants 4 messages May 25, 2008 – May 28, 2008

Subsystems (3)

x86 KVM KVM x86

lore

Share & tools

lore

Marcelo Tosatti

author

2008-05-25 05:29 UTC #0 lore raw

Avi,

There was a discussion regarding PIT timer migration sometime ago, you said:

"I'm not sure hrtimer migration would work 100% reliably (suppose it fired just after a vcpu migration) so I think a queue_work is better."

I fail to see any problem with a timer firing either before or after the request bit is set in the vcpu's request mask during vcpu_load().

Whatever the case, nothing will stop __vcpu_run() from migrating the timer to vcpu0 (in PIT's case), and any instances that fired on a different CPU between vcpu_load() and hrtimer_cancel/start will be injected in kvm_inject_pending_timer_irqs(). Is there anything you can think of?

-------

Migrate the PIT timer to the physical CPU which vcpu0 is scheduled on, similarly to what is done for the LAPIC timers, otherwise PIT interrupts will be delayed until an unrelated event causes an exit.

Signed-off-by Marcelo Tosatti <[email protected]>

arch/x86/kvm/i8254.c +14 -1

Index: kvm.realtip/arch/x86/kvm/i8254.c
===================================================================
@@ -200,7 +200,6 @@ static int __pit_timer_fn(struct kvm_kpi
 
 	atomic_inc(&pt->pending);
 	smp_mb__after_atomic_inc();
-	/* FIXME: handle case where the guest is in guest mode */
 	if (vcpu0 && waitqueue_active(&vcpu0->wq)) {
 		vcpu0->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		wake_up_interruptible(&vcpu0->wq);

@@ -237,6 +236,20 @@ static enum hrtimer_restart pit_timer_fn
 		return HRTIMER_NORESTART;
 }
 
+void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
+{
+	struct kvm_pit *pit = vcpu->kvm->arch.vpit;
+	struct hrtimer *timer;
+
+	if (!pit)
+		return;
+
+	timer = &pit->pit_state.pit_timer.timer;
+	if (hrtimer_cancel(timer))
+		hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS);
+}
+
+
 static void destroy_pit_timer(struct kvm_kpit_timer *pt)
 {
 	pr_debug("pit: execute del timer!\n");

Index: kvm.realtip/arch/x86/kvm/irq.h
===================================================================
-- kvm.realtip.orig/arch/x86/kvm/irq.h
+++ kvm.realtip/arch/x86/kvm/irq.h
@@ -84,6 +84,7 @@ void kvm_timer_intr_post(struct kvm_vcpu
void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu);
+void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu);

int pit_has_pending_timer(struct kvm_vcpu *vcpu); int apic_has_pending_timer(struct kvm_vcpu *vcpu);

arch/x86/kvm/svm.c +1 -1

Index: kvm.realtip/arch/x86/kvm/svm.c
===================================================================
@@ -690,7 +690,7 @@ static void svm_vcpu_load(struct kvm_vcp
 		delta = vcpu->arch.host_tsc - tsc_this;
 		svm->vmcb->control.tsc_offset += delta;
 		vcpu->cpu = cpu;
-		kvm_migrate_apic_timer(vcpu);
+		kvm_migrate_timers(vcpu);
 	}
 
 	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)

arch/x86/kvm/vmx.c +1 -1

Index: kvm.realtip/arch/x86/kvm/vmx.c
===================================================================
@@ -616,7 +616,7 @@ static void vmx_vcpu_load(struct kvm_vcp
 
 	if (vcpu->cpu != cpu) {
 		vcpu_clear(vmx);
-		kvm_migrate_apic_timer(vcpu);
+		kvm_migrate_timers(vcpu);
 		vpid_sync_vcpu_all(vmx);
 		local_irq_disable();
 		list_add(&vmx->local_vcpus_link,

arch/x86/kvm/x86.c +3 -1

Index: kvm.realtip/arch/x86/kvm/x86.c
===================================================================
@@ -2744,8 +2744,10 @@ again:
 		goto out;
 
 	if (vcpu->requests) {
-		if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
+		if (test_and_clear_bit(KVM_REQ_MIGRATE_APIC, &vcpu->requests))
 			__kvm_migrate_apic_timer(vcpu);
+		if (test_and_clear_bit(KVM_REQ_MIGRATE_PIT, &vcpu->requests))
+			__kvm_migrate_pit_timer(vcpu);
 		if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
 				       &vcpu->requests)) {
 			kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS;

include/linux/kvm_host.h +6 -3

Index: kvm.realtip/include/linux/kvm_host.h
===================================================================
@@ -29,10 +29,11 @@
  * vcpu->requests bit members
  */
 #define KVM_REQ_TLB_FLUSH          0
-#define KVM_REQ_MIGRATE_TIMER      1
+#define KVM_REQ_MIGRATE_APIC       1
 #define KVM_REQ_REPORT_TPR_ACCESS  2
 #define KVM_REQ_MMU_RELOAD         3
 #define KVM_REQ_TRIPLE_FAULT       4
+#define KVM_REQ_MIGRATE_PIT        5
 
 struct kvm_vcpu;
 extern struct kmem_cache *kvm_vcpu_cache;

@@ -294,9 +295,11 @@ static inline gpa_t gfn_to_gpa(gfn_t gfn
 	return (gpa_t)gfn << PAGE_SHIFT;
 }
 
-static inline void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
+static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
 {
-	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
+	set_bit(KVM_REQ_MIGRATE_APIC, &vcpu->requests);
+	if (vcpu->vcpu_id == 0)
+		set_bit(KVM_REQ_MIGRATE_PIT, &vcpu->requests);
 }
 
 enum kvm_stat_kind {

Avi Kivity

2008-05-27 13:17 UTC #1 lore raw

Quoted 8 lines · Marcelo Tosatti

Avi,
There was a discussion regarding PIT timer migration sometime ago, you said:
"I'm not sure hrtimer migration would work 100% reliably (suppose it fired just after a vcpu migration) so I think a queue_work is better."

Quoted 9 lines

I fail to see any problem with a timer firing either before or after the request bit is set in the vcpu's request mask during vcpu_load().
Whatever the case, nothing will stop __vcpu_run() from migrating the timer to vcpu0 (in PIT's case), and any instances that fired on a different CPU between vcpu_load() and hrtimer_cancel/start will be injected in kvm_inject_pending_timer_irqs(). Is there anything you can think of?

No, it seems fine.

Quoted 31 lines

-------
Migrate the PIT timer to the physical CPU which vcpu0 is scheduled on, similarly to what is done for the LAPIC timers, otherwise PIT interrupts will be delayed until an unrelated event causes an exit.
Signed-off-by Marcelo Tosatti <[email protected]>
Index: kvm.realtip/arch/x86/kvm/i8254.c
===================================================================
arch/x86/kvm/i8254.c +5 -1
@@ -200,7 +200,6 @@ static int __pit_timer_fn(struct kvm_kpi
 
 	atomic_inc(&pt->pending);
 	smp_mb__after_atomic_inc();
-	/* FIXME: handle case where the guest is in guest mode */
 	if (vcpu0 && waitqueue_active(&vcpu0->wq)) {
 		vcpu0->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		wake_up_interruptible(&vcpu0->wq);
@@ -237,6 +236,20 @@ static enum hrtimer_restart pit_timer_fn
 		return HRTIMER_NORESTART;
 }
 
+void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
+{
+	struct kvm_pit *pit = vcpu->kvm->arch.vpit;
+	struct hrtimer *timer;
+

arch/x86/kvm/i8254.c static enum hrtimer_restart pit_timer_fn · line 253

Quoted 10 lines

arch/x86/kvm/i8254.c +9 -0

+	if (!pit)
+		return;
+
+	timer = &pit->pit_state.pit_timer.timer;
+	if (hrtimer_cancel(timer))
+		hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS);
+}
+
+

Extra newline.

Quoted 77 lines

arch/x86/kvm/i8254.c +0 -0

 static void destroy_pit_timer(struct kvm_kpit_timer *pt)
 {

pr_debug("pit: execute del timer!\n");
Index: kvm.realtip/arch/x86/kvm/irq.h
===================================================================
-- kvm.realtip.orig/arch/x86/kvm/irq.h
+++ kvm.realtip/arch/x86/kvm/irq.h

arch/x86/kvm/i8254.c +1 -0

@@ -84,6 +84,7 @@ void kvm_timer_intr_post(struct kvm_vcpu
 void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu);
 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu);
 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu);
+void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu);
 
 int pit_has_pending_timer(struct kvm_vcpu *vcpu);
 int apic_has_pending_timer(struct kvm_vcpu *vcpu);

Index: kvm.realtip/arch/x86/kvm/svm.c
===================================================================

arch/x86/kvm/svm.c +1 -1

@@ -690,7 +690,7 @@ static void svm_vcpu_load(struct kvm_vcp
 		delta = vcpu->arch.host_tsc - tsc_this;
 		svm->vmcb->control.tsc_offset += delta;
 		vcpu->cpu = cpu;
-		kvm_migrate_apic_timer(vcpu);
+		kvm_migrate_timers(vcpu);
 	}
 
 	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)

Index: kvm.realtip/arch/x86/kvm/vmx.c
===================================================================

arch/x86/kvm/vmx.c +1 -1

@@ -616,7 +616,7 @@ static void vmx_vcpu_load(struct kvm_vcp
 
 	if (vcpu->cpu != cpu) {
 		vcpu_clear(vmx);
-		kvm_migrate_apic_timer(vcpu);
+		kvm_migrate_timers(vcpu);
 		vpid_sync_vcpu_all(vmx);
 		local_irq_disable();
 		list_add(&vmx->local_vcpus_link,

Index: kvm.realtip/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +3 -1

@@ -2744,8 +2744,10 @@ again:
 		goto out;
 
 	if (vcpu->requests) {
-		if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
+		if (test_and_clear_bit(KVM_REQ_MIGRATE_APIC, &vcpu->requests))
 			__kvm_migrate_apic_timer(vcpu);
+		if (test_and_clear_bit(KVM_REQ_MIGRATE_PIT, &vcpu->requests))
+			__kvm_migrate_pit_timer(vcpu);
 		if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
 				       &vcpu->requests)) {
 			kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS;

Index: kvm.realtip/include/linux/kvm_host.h
===================================================================

include/linux/kvm_host.h +2 -1

@@ -29,10 +29,11 @@
  * vcpu->requests bit members
  */
 #define KVM_REQ_TLB_FLUSH          0
-#define KVM_REQ_MIGRATE_TIMER      1
+#define KVM_REQ_MIGRATE_APIC       1
 #define KVM_REQ_REPORT_TPR_ACCESS  2
 #define KVM_REQ_MMU_RELOAD         3
 #define KVM_REQ_TRIPLE_FAULT       4
+#define KVM_REQ_MIGRATE_PIT        5
 
 struct kvm_vcpu;
 extern struct kmem_cache *kvm_vcpu_cache;

@@ -294,9 +295,11 @@ static inline gpa_t gfn_to_gpa(gfn_t gfn
 	return (gpa_t)gfn << PAGE_SHIFT;
 }

include/linux/kvm_host.h static inline gpa_t gfn_to_gpa(gfn_t gfn · line 304

Quoted 10 lines

include/linux/kvm_host.h +4 -2

 
-static inline void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
+static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
 {
-	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
+	set_bit(KVM_REQ_MIGRATE_APIC, &vcpu->requests);
+	if (vcpu->vcpu_id == 0)
+		set_bit(KVM_REQ_MIGRATE_PIT, &vcpu->requests);
 }

Instead of having two timer bits, how about keeping just on MIGRATE_TIMER bit which migrates both, and having __kvm_migrate_pit_timer() return if not on vcpu 0?

This can result in shorter code, and in less proliferation of the "pit is bound to vcpu 0" logic.

Signature · 1 line

error compiling committee.c: too many arguments to function

Marcelo Tosatti

author

2008-05-27 15:10 UTC #2 lore raw

Quoted 6 lines · Avi Kivity

Instead of having two timer bits, how about keeping just on MIGRATE_TIMER bit which migrates both, and having __kvm_migrate_pit_timer() return if not on vcpu 0?
This can result in shorter code, and in less proliferation of the "pit is bound to vcpu 0" logic.

Sure.

Signed-off-by Marcelo Tosatti <[email protected]>

arch/x86/kvm/i8254.c +13 -1

index 6d6dc6c..9b33b27 100644
@@ -200,7 +200,6 @@ static int __pit_timer_fn(struct kvm_kpit_state *ps)
 
 	atomic_inc(&pt->pending);
 	smp_mb__after_atomic_inc();
-	/* FIXME: handle case where the guest is in guest mode */
 	if (vcpu0 && waitqueue_active(&vcpu0->wq)) {
 		vcpu0->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		wake_up_interruptible(&vcpu0->wq);

@@ -237,6 +236,19 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
 		return HRTIMER_NORESTART;
 }
 
+void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
+{
+	struct kvm_pit *pit = vcpu->kvm->arch.vpit;
+	struct hrtimer *timer;
+
+	if (vcpu->vcpu_id != 0 || !pit)
+		return;
+
+	timer = &pit->pit_state.pit_timer.timer;
+	if (hrtimer_cancel(timer))
+		hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS);
+}
+
 static void destroy_pit_timer(struct kvm_kpit_timer *pt)
 {
 	pr_debug("pit: execute del timer!\n");

arch/x86/kvm/irq.c +6 -0

index ce1f583..76d736b 100644
@@ -94,3 +94,9 @@ void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec)
 	/* TODO: PIT, RTC etc. */
 }
 EXPORT_SYMBOL_GPL(kvm_timer_intr_post);
+
+void __kvm_migrate_timers(struct kvm_vcpu *vcpu)
+{
+	__kvm_migrate_apic_timer(vcpu);
+	__kvm_migrate_pit_timer(vcpu);
+}

arch/x86/kvm/irq.h +2 -0

index 1802134..2a15be2 100644
@@ -84,6 +84,8 @@ void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec);
 void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu);
 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu);
 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu);
+void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu);
+void __kvm_migrate_timers(struct kvm_vcpu *vcpu);
 
 int pit_has_pending_timer(struct kvm_vcpu *vcpu);
 int apic_has_pending_timer(struct kvm_vcpu *vcpu);

arch/x86/kvm/svm.c +1 -1

index baf9607..238e8f3 100644
@@ -690,7 +690,7 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 		delta = vcpu->arch.host_tsc - tsc_this;
 		svm->vmcb->control.tsc_offset += delta;
 		vcpu->cpu = cpu;
-		kvm_migrate_apic_timer(vcpu);
+		kvm_migrate_timers(vcpu);
 	}
 
 	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)

arch/x86/kvm/vmx.c +1 -1

index aaa99ed..22f6c19 100644
@@ -616,7 +616,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
 	if (vcpu->cpu != cpu) {
 		vcpu_clear(vmx);
-		kvm_migrate_apic_timer(vcpu);
+		kvm_migrate_timers(vcpu);
 		vpid_sync_vcpu_all(vmx);
 		local_irq_disable();
 		list_add(&vmx->local_vcpus_link,

arch/x86/kvm/x86.c +1 -1

index 314f470..07bf9a7 100644
@@ -2744,7 +2744,7 @@ again:
 
 	if (vcpu->requests) {
 		if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
-			__kvm_migrate_apic_timer(vcpu);
+			__kvm_migrate_timers(vcpu);
 		if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
 				       &vcpu->requests)) {
 			kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS;

include/linux/kvm_host.h +1 -1

index d622309..4741063 100644
@@ -294,7 +294,7 @@ static inline gpa_t gfn_to_gpa(gfn_t gfn)
 	return (gpa_t)gfn << PAGE_SHIFT;
 }
 
-static inline void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
+static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
 {
 	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
 }

Avi Kivity

Applied

2008-05-28 06:34 UTC #3 lore raw

Quoted 4 lines · Marcelo Tosatti

Quoted 2 lines · Avi Kivity
Instead of having two timer bits, how about keeping just on MIGRATE_TIMER bit which migrates both, and having __kvm_migrate_pit_timer() return if not on vcpu 0?

Quoted 10 lines

Quoted 2 lines
This can result in shorter code, and in less proliferation of the "pit is bound to vcpu 0" logic.
Sure.
Migrate the PIT timer to the physical CPU which vcpu0 is scheduled on, similarly to what is done for the LAPIC timers, otherwise PIT interrupts will be delayed until an unrelated event causes an exit.

Thanks, applied.

Signature · 1 line

Do not meddle in the internals of kernels, for they are subtle and quick to panic.