x86/hvm: fix corrupt ACPI PM-Timer during live migration
PATCH 1/1 x86/hvm: fix corrupt ACPI PM-Timer during live migration
The value of ACPI PM-Timer may be broken on save unless the timer mode is delay_for_missed_ticks.
With other timer modes, vcpu->arch.hvm_vcpu.guest_time is always zero and the adjustment from its value is wrong.
This patch fixes the saved value of ACPI PM-Timer:
- don't adjust the PM-Timer if vcpu->arch.hvm_vcpu.guest_time is zero.
- consolidate calculations of PM-Timer to one function. more precise on save.
xen/arch/x86/hvm/pmtimer.c +38 -22
@@ -84,28 +84,31 @@ void hvm_acpi_sleep_button(struct domain }/* Set the correct value in the timer, accounting for time elapsed -* since the last time we did that. */ -static void pmt_update_time (PMTState *s) +* since the last time we did that. +* return true if the counter's MSB has changed. */ +static bool_t pmt_count_up_and_test_msb (PMTState *s,uint64_t gtime) { -uint64_t curr_gtime, tmp;uint32_t tmr_val = s->pm.tmr_val, msb = tmr_val &TMR_VAL_MSB ; - -ASSERT (spin_is_locked (&s->lock)); +uint64_t tmp = ((gtime - s->last_gtime) * s->scale) + s->not_accounted; -/* Update the timer */ - curr_gtime =hvm_get_guest_time (s->vcpu); - tmp = ((curr_gtime - s->last_gtime) * s->scale) + s->not_accounted; s->not_accounted = (uint32_t )tmp; tmr_val += tmp >>32 ; tmr_val &=TMR_VAL_MASK ; - s->last_gtime = curr_gtime; + s->last_gtime = gtime;/* Update timer value atomically wrt lock-free reads in handle_pmt_io(). */ *(volatile uint32_t *)&s->pm.tmr_val = tmr_val; -/* If the counter's MSB has changed, set the status bit */ -if ( (tmr_val &TMR_VAL_MSB ) != msb ) +return ((tmr_val &TMR_VAL_MSB ) != msb); +} + +static void pmt_update_time (PMTState *s) +{ +ASSERT (spin_is_locked (&s->lock)); + +if (pmt_count_up_and_test_msb (s,hvm_get_guest_time (s->vcpu)) ) { +/* MSB has changed, set the status bit */ s->pm.pm1a_sts |=TMR_STS ;pmt_update_sci (s); }
@@ -244,21 +247,34 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ; +uint64_t guest_time;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ - x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; -/* No point in setting the SCI here because we'll already have saved the -* IRQ and *PIC state; we'll fix it up when we restore the domain */ + guest_time = s->vcpu->arch.hvm_vcpu.guest_time; +/* Update the counter to the guest's current time only if the +* timer mode is delay_for_missed_ticks */ +if ( guest_time !=0 ) + { +/* We always save with the domain paused, so the saved time +* should be after the last_gtime, but just in case, make sure +* we only go forwards */ +if ( (s64 )guest_time - (s64 )s->last_gtime <0 ) + { +dprintk (XENLOG_WARNING , +"Last update of PMT is ahead of guest's time by %ld\n" , + s->last_gtime - guest_time); + } +else + { +if (pmt_count_up_and_test_msb (s, guest_time) ) + s->pm.pm1a_sts |=TMR_STS ; +/* No point in setting the SCI here because we'll already +* have saved the IRQ and *PIC state; +* we'll fix it up when we restore the domain */ + } + } rc =hvm_save_entry (PMTIMER ,0 , h, &s->pm);
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Quoted 24 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +4
-8@@ -84,28 +84,31 @@ void hvm_acpi_sleep_button(struct domain }/* Set the correct value in the timer, accounting for time elapsed -* since the last time we did that. */ -static void pmt_update_time (PMTState *s) +* since the last time we did that. +* return true if the counter's MSB has changed. */ +static bool_t pmt_count_up_and_test_msb (PMTState *s,uint64_t gtime) { -uint64_t curr_gtime, tmp;uint32_t tmr_val = s->pm.tmr_val, msb = tmr_val &TMR_VAL_MSB ; - -ASSERT (spin_is_locked (&s->lock)); +uint64_t tmp = ((gtime - s->last_gtime) * s->scale) + s->not_accounted; -/* Update the timer */ - curr_gtime =hvm_get_guest_time (s->vcpu); - tmp = ((curr_gtime - s->last_gtime) * s->scale) + s->not_accounted; s->not_accounted = (uint32_t )tmp; tmr_val += tmp >>32 ;
xen/arch/x86/hvm/pmtimer.c void hvm_acpi_sleep_button(struct domain · line 102
Quoted 10 lines
xen/arch/x86/hvm/pmtimer.c +2
-3tmr_val &=TMR_VAL_MASK ; - s->last_gtime = curr_gtime; + s->last_gtime = gtime;/* Update timer value atomically wrt lock-free reads in handle_pmt_io(). */ *(volatile uint32_t *)&s->pm.tmr_val = tmr_val; -/* If the counter's MSB has changed, set the status bit */ -if ( (tmr_val &TMR_VAL_MSB ) != msb ) +return ((tmr_val &TMR_VAL_MSB ) != msb);
Single space only please.
Quoted 26 lines
xen/arch/x86/hvm/pmtimer.c +9
-5+} + +static void pmt_update_time (PMTState *s) +{ +ASSERT (spin_is_locked (&s->lock)); + +if (pmt_count_up_and_test_msb (s,hvm_get_guest_time (s->vcpu)) ) { +/* MSB has changed, set the status bit */ s->pm.pm1a_sts |=TMR_STS ;pmt_update_sci (s); }@@ -244,21 +247,34 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ; +uint64_t guest_time;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ - x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ;
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 258
Quoted 10 lines
xen/arch/x86/hvm/pmtimer.c +4
-6-if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; -/* No point in setting the SCI here because we'll already have saved the -* IRQ and *PIC state; we'll fix it up when we restore the domain */ + guest_time = s->vcpu->arch.hvm_vcpu.guest_time; +/* Update the counter to the guest's current time only if the +* timer mode is delay_for_missed_ticks */ +if ( guest_time !=0 )
How is guest_time being (non-)zero an indication of mode being delay_for_missed_ticks? I think you should check for the mode explicitly, as the value here being zero can just be an effect of a large enough negative v->arch.hvm_vcpu.stime_offset.
And besides that you don't explain why the update being done here is unnecessary in other modes - all you explain is that the way it's being done in those modes is wrong.
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 266
Quoted 8 lines
xen/arch/x86/hvm/pmtimer.c +8
-0+ { +/* We always save with the domain paused, so the saved time +* should be after the last_gtime, but just in case, make sure +* we only go forwards */ +if ( (s64 )guest_time - (s64 )s->last_gtime <0 ) + { + dprintk(XENLOG_WARNING, + "Last update of PMT is ahead of guest's time by %ld\n",
While probably fine this way for -unstable, please nevertheless use the correct PRId64 here (both to be formally correct and to ease eventual backporting).
Jan
Quoted 14 lines
xen/arch/x86/hvm/pmtimer.c +11
-0+ s->last_gtime - guest_time); + } +else + { +if (pmt_count_up_and_test_msb (s, guest_time) ) + s->pm.pm1a_sts |=TMR_STS ; +/* No point in setting the SCI here because we'll already +* have saved the IRQ and *PIC state; +* we'll fix it up when we restore the domain */ + } + } rc =hvm_save_entry (PMTIMER ,0 , h, &s->pm);
Hi Jan,
Thanks for reviewing the code.
Quoted 33 lines · Jan Beulich
Quoted 17 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +1
-5@@ -244,21 +247,34 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ; +uint64_t guest_time;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ - x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ;xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 258
Quoted 10 lines
xen/arch/x86/hvm/pmtimer.c +4
-6-if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; -/* No point in setting the SCI here because we'll already have saved the -* IRQ and *PIC state; we'll fix it up when we restore the domain */ + guest_time = s->vcpu->arch.hvm_vcpu.guest_time; +/* Update the counter to the guest's current time only if the +* timer mode is delay_for_missed_ticks */ +if ( guest_time !=0 )How is
guest_timebeing (non-)zero an indication of mode beingdelay_for_missed_ticks? I think you should check for the mode explicitly, as the value here being zero can just be an effect of a large enough negativev->arch.hvm_vcpu.stime_offset.
Quoted 3 lines
And besides that you don't explain why the update being done here is unnecessary in other modes - all you explain is that the way it's being done in those modes is wrong.
After due consideration, the update of PM-timer is necessary for other modes. I misunderstood it's just an adjustment for the delay_for_missed_ticks mode.
The cause of this bug is to refer the vcpu->arch.hvm_vcpu.guest_time.
Attached is the revised patch.
Aside from this patch, I've found another problem about delay_for_missed_ticks. PM-timer might be broken after pmt_timer_callback is invoked.
I'll think it over.
Thanks,
Kouya
# HG changeset patch
# User Kouya Shimura <[email protected]>
# Date 1360820290 -32400
# Node ID 9744ac3f23dd198e997bf032385a573408420762 # Parent 5af4f2ab06f33ce441fa550333a9049c09a9ef28 x86/hvm: fix corrupt ACPI PM-Timer during live migration
ACPI PM-Timer value is broken on saving a VM.
Since c/s 16274:44dde35cb2a6, vcpu->arch.hvm_vcpu.guest_time is not the correct guest's time any more.
Instead, hvm_get_guest_time() should be used in pmtimer_save to calculate the timer.
xen/arch/x86/hvm/pmtimer.c +6 -13
@@ -85,7 +85,7 @@ void hvm_acpi_sleep_button(struct domain/* Set the correct value in the timer, accounting for time elapsed * since the last time we did that. */ -static void pmt_update_time (PMTState *s) +static void pmt_update_time (PMTState *s,bool_t update_sci) {uint64_t curr_gtime, tmp;uint32_t tmr_val = s->pm.tmr_val, msb = tmr_val &TMR_VAL_MSB ;
@@ -107,7 +107,8 @@ static void pmt_update_time(PMTState *s)if ( (tmr_val &TMR_VAL_MSB ) != msb ) { s->pm.pm1a_sts |=TMR_STS ; -pmt_update_sci (s); +if ( update_sci ) +pmt_update_sci (s); } }
@@ -123,7 +124,7 @@ static void pmt_timer_callback(void *opaspin_lock (&s->lock);/* Recalculate the timer and make sure we get an SCI if we need one */ -pmt_update_time (s); +pmt_update_time (s,1 );/* How close are we to the next MSB flip? */ pmt_cycles_until_flip =TMR_VAL_MSB - (s->pm.tmr_val & (TMR_VAL_MSB -1 ));
@@ -221,7 +222,7 @@ static int handle_pmt_io(if (spin_trylock (&s->lock) ) {/* We hold the lock: update timer value and return it. */ -pmt_update_time (s); +pmt_update_time (s,1 ); *val = s->pm.tmr_val;spin_unlock (&s->lock); }
@@ -244,21 +245,13 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ - x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ;/* No point in setting the SCI here because we'll already have saved the * IRQ and *PIC state; we'll fix it up when we restore the domain */ +pmt_update_time (s,0 ); rc =hvm_save_entry (PMTIMER ,0 , h, &s->pm);
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 13 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-4@@ -244,21 +245,13 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */
So you lose this property of guaranteeing no backward move.
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 8 lines
xen/arch/x86/hvm/pmtimer.c +1
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ;/* No point in setting the SCI here because we'll already have saved the * IRQ and *PIC state; we'll fix it up when we restore the domain */ +pmt_update_time (s,0 );
And using this function you also have the new side effect of s->last_gtime being updated.
Perhaps the new parameter should be renamed (to, say, "saving"), and then allow suppressing all these behavioral changes.
Also, in delay_for_missed_ticks mode you now use a slightly different time for updating s->pm - did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, or hvm_get_guest_time() could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, that pt_freeze_time() runs before pmtimer_save()).
Jan
Quoted 8 lines · Jan Beulich
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-1@@ -244,21 +245,13 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;
Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 5 lines
xen/arch/x86/hvm/pmtimer.c +0
-3spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ So you lose this property of guaranteeing no backward move.
Exactly.
But the backward move is impossible except delay_for_missed_ticks mode. Since the monotonicity of hvm_get_guest_time() is guaranteed for other modes. About the problem of delay_for_missed_ticks mode, I slightly mentioned in the previous mail.
The backward move can be happend as the following:
1) pt_freeze_time ... real-time:100 guest-time:100 last_gtime:90 2) pmt_timer_callback ... real-time:110 guest-time:110 last_gtime:110 3) pt_thaw_time ... real-time:120 guest-time:100 last_gtime:110
4) pmt_update_time ... real-time:125 guest-time:105 < last_gtime:110
So, it can be happend not only in pmtimer_save() but also in handle_pmt_io().
Maybe pmt_udpate_time() should check the backward move.
Quoted 10 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 9 lines
xen/arch/x86/hvm/pmtimer.c +1
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ;/* No point in setting the SCI here because we'll already have saved the * IRQ and *PIC state; we'll fix it up when we restore the domain */ +pmt_update_time (s,0 );
Quoted 6 lines
And using this function you also have the new side effect of
s->last_gtimebeing updated.Perhaps the new parameter should be renamed (to, say, "saving"), and then allow suppressing all these behavioral changes.
The new side effect is also a bug fix, I think.
Since updating s->pm.tmr_val and s->last_gtime should be done at the same time. Updating only s->pm.tmr_val is wrong. Or else, when the vm is resumed on the same machine (migration failure, check-pointing, etc), the PM-timer value is double counted.
Quoted 8 lines
Also, in
delay_for_missed_ticksmode you now use a slightly different time for updatings->pm- did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, orhvm_get_guest_time()could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, thatpt_freeze_time()runs beforepmtimer_save()).
Modifying hvm_get_guest_time() is my thought, too.
But I don't like the idea because the delay_for_missed_ticks mode is unusual. I wonder who uses it.
Let me think it over.
Thanks,
Kouya
Quoted 72 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
Quoted 8 lines · Jan Beulich
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-1@@ -244,21 +245,13 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 5 lines
xen/arch/x86/hvm/pmtimer.c +0
-3spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ So you lose this property of guaranteeing no backward move.
Exactly.
But the backward move is impossible exceptdelay_for_missed_ticksmode. Since the monotonicity ofhvm_get_guest_time()is guaranteed for other modes.
About the problem ofdelay_for_missed_ticksmode, I slightly mentioned in the previous mail.The backward move can be happend as the following:
1) pt_freeze_time ... real-time:100 guest-time:100 last_gtime:90 2) pmt_timer_callback ... real-time:110 guest-time:110 last_gtime:110 3) pt_thaw_time ... real-time:120 guest-time:100 last_gtime:1104)
pmt_update_time... real-time:125 guest-time:105 < last_gtime:110So, it can be happend not only in
pmtimer_save()but also inhandle_pmt_io().Maybe
pmt_udpate_time()should check the backward move.Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io(
Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c +0
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ;/* No point in setting the SCI here because we'll already have saved the
Quoted 9 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 2 lines
xen/arch/x86/hvm/pmtimer.c +1
-0* IRQ and *PIC state; we'll fix it up when we restore the domain */ +pmt_update_time (s,0 );And using this function you also have the new side effect of
s->last_gtimebeing updated.Perhaps the new parameter should be renamed (to, say, "saving"), and then allow suppressing all these behavioral changes.
The new side effect is also a bug fix, I think.
Since updatings->pm.tmr_valands->last_gtimeshould be done at the same time. Updating onlys->pm.tmr_valis wrong. Or else, when the vm is resumed on the same machine (migration failure, check-pointing, etc), the PM-timer value is double counted.Quoted 8 lines
Also, in
delay_for_missed_ticksmode you now use a slightly different time for updatings->pm- did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, orhvm_get_guest_time()could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, thatpt_freeze_time()runs beforepmtimer_save()).
Quoted 5 lines
Modifying
hvm_get_guest_time()is my thought, too.
But I don't like the idea because thedelay_for_missed_ticksmode is unusual. I wonder who uses it.Let me think it over.
Ping?
Jan
Quoted 1 line
Ping?
Sorry for late.
Almost done, but the test is not enough yet.
Maybe I can post it next week.
Thanks,
Kouya
Quoted 82 lines · Jan Beulich
Quoted 72 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
Quoted 8 lines · Jan Beulich
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-1@@ -244,21 +245,13 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 5 lines
xen/arch/x86/hvm/pmtimer.c +0
-3spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ So you lose this property of guaranteeing no backward move.
Exactly.
But the backward move is impossible exceptdelay_for_missed_ticksmode. Since the monotonicity ofhvm_get_guest_time()is guaranteed for other modes.
About the problem ofdelay_for_missed_ticksmode, I slightly mentioned in the previous mail.The backward move can be happend as the following:
1) pt_freeze_time ... real-time:100 guest-time:100 last_gtime:90 2) pmt_timer_callback ... real-time:110 guest-time:110 last_gtime:110 3) pt_thaw_time ... real-time:120 guest-time:100 last_gtime:1104)
pmt_update_time... real-time:125 guest-time:105 < last_gtime:110So, it can be happend not only in
pmtimer_save()but also inhandle_pmt_io().Maybe
pmt_udpate_time()should check the backward move.Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io(
Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c +0
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ;/* No point in setting the SCI here because we'll already have saved the
Quoted 9 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 2 lines
xen/arch/x86/hvm/pmtimer.c +1
-0* IRQ and *PIC state; we'll fix it up when we restore the domain */ +pmt_update_time (s,0 );And using this function you also have the new side effect of
s->last_gtimebeing updated.Perhaps the new parameter should be renamed (to, say, "saving"), and then allow suppressing all these behavioral changes.
The new side effect is also a bug fix, I think.
Since updatings->pm.tmr_valands->last_gtimeshould be done at the same time. Updating onlys->pm.tmr_valis wrong. Or else, when the vm is resumed on the same machine (migration failure, check-pointing, etc), the PM-timer value is double counted.Quoted 8 lines
Also, in
delay_for_missed_ticksmode you now use a slightly different time for updatings->pm- did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, orhvm_get_guest_time()could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, thatpt_freeze_time()runs beforepmtimer_save()).Quoted 5 lines
Modifying
hvm_get_guest_time()is my thought, too.
But I don't like the idea because thedelay_for_missed_ticksmode is unusual. I wonder who uses it.Let me think it over.
Ping?
Jan
Quoted 76 lines · Jan Beulich
Quoted 76 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
Quoted 8 lines · Jan Beulich
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-1@@ -244,21 +245,13 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 5 lines
xen/arch/x86/hvm/pmtimer.c +0
-3spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ So you lose this property of guaranteeing no backward move.
Exactly.
But the backward move is impossible exceptdelay_for_missed_ticksmode. Since the monotonicity ofhvm_get_guest_time()is guaranteed for other modes.
About the problem ofdelay_for_missed_ticksmode, I slightly mentioned in the previous mail.The backward move can be happend as the following:
1) pt_freeze_time ... real-time:100 guest-time:100 last_gtime:90 2) pmt_timer_callback ... real-time:110 guest-time:110 last_gtime:110 3) pt_thaw_time ... real-time:120 guest-time:100 last_gtime:1104)
pmt_update_time... real-time:125 guest-time:105 < last_gtime:110So, it can be happend not only in
pmtimer_save()but also inhandle_pmt_io().Maybe
pmt_udpate_time()should check the backward move.Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io(
Quoted 7 lines
xen/arch/x86/hvm/pmtimer.c +0
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ;/* No point in setting the SCI here because we'll already have saved the
Quoted 9 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 2 lines
xen/arch/x86/hvm/pmtimer.c +1
-0* IRQ and *PIC state; we'll fix it up when we restore the domain */ +pmt_update_time (s,0 );And using this function you also have the new side effect of
s->last_gtimebeing updated.Perhaps the new parameter should be renamed (to, say, "saving"), and then allow suppressing all these behavioral changes.
The new side effect is also a bug fix, I think.
Since updatings->pm.tmr_valands->last_gtimeshould be done at the same time. Updating onlys->pm.tmr_valis wrong. Or else, when the vm is resumed on the same machine (migration failure, check-pointing, etc), the PM-timer value is double counted.Quoted 8 lines
Also, in
delay_for_missed_ticksmode you now use a slightly different time for updatings->pm- did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, orhvm_get_guest_time()could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, thatpt_freeze_time()runs beforepmtimer_save()).Modifying
hvm_get_guest_time()is my thought, too.
But I don't like the idea because thedelay_for_missed_ticksmode is unusual. I wonder who uses it.
Quoted 6 lines
Quoted 1 line
Let me think it over.
Ping?
Jan
Sorry for the delay.
I'll send two patches:
This is a new added patch.
- 1/2 prevent guest's timers from going backwards when timer_mode=0
Nothing but a comment is changed from the previous patch.
- 2/2 fix corrupt ACPI PM-Timer during live migration
Actually, I don't think I'd like to support timer_mode=0. timer_mode=0 is meaningless for a recent tickless linux kernel.
Thanks,
Kouya
Quoted 3 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
Actually, I don't think I'd like to support timer_mode=0. timer_mode=0 is meaningless for a recent tickless linux kernel.
But this is not a question of your liking, nor is it related to what recent Linux wants/needs.
Jan
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/2 x86/hvm: prevent guest's timers from going backwards, when timer_mode=0
In the case of delay_for_missed_ticks mode (timer_mode=0), the guest virtual time goes backwards when the vcpu is rescheduled. Therefore guest's HW timer might go backwards, too.
Case 1. SMP guest:
1) vcpu#1 is de-scheduled and the guest time freezes. (TIME 0:0.010)
2) vcpu#2 access a timer (0:0.020)
3) vcpu#1 is re-scheduled and the guest time thaws. (0:0.030->0:0.010)
4) vcpu#2 access a timer (0:0.015) // Backwards!!!
Case 2. asynchronous callback:
1) vcpu#1 is de-scheduled and the guest time freezes. (0:0.010)
2) pmt_timer_callback() is invoked (0:0.025)
3) vcpu#1 is re-scheduled and the guest time thaws. (0:0.030->0:0.010)
4) vcpu#1 access the PM-TIMER (0:0.015) // Backwards!!!
This patch affects only delay_for_missed_ticks mode (timer_mode=0) and ensures the monotonicity of the following timers:
- PIT
- HPET
- ACPI PM-TIMER
The following timers are OK since a vcpu never access the other vcpu's timer.
- Local APIC ( has some callbacks but it's called from pt_intr_post )
- TSC
Just in case, these timers can use the new function hvm_get_base_time() too, but doesn't. It's a little bit less efficient than hvm_get_guest_time().
Also, tidy up virtual platform timer code.
--- xen/arch/x86/hvm/hpet.c | 2 +- xen/arch/x86/hvm/i8254.c | 2 +- xen/arch/x86/hvm/pmtimer.c | 2 +- xen/arch/x86/hvm/vpt.c | 93 ++++++++++++++++++++++++++++++++++------- xen/include/asm-x86/hvm/hvm.h | 2 +- 5 files changed, 82 insertions(+), 19 deletions(-)
xen/arch/x86/hvm/hpet.c +1 -1
@@ -39,7 +39,7 @@/* Frequency_of_Xen_systeme_time / frequency_of_HPET = 16 */ #define STIME_PER_HPET_TICK 16#define guest_time_hpet (hpet) \ - (hvm_get_guest_time(vhpet_vcpu(hpet)) / STIME_PER_HPET_TICK) + (hvm_get_base_time(vhpet_vcpu(hpet)) / STIME_PER_HPET_TICK)#define HPET_TN_INT_ROUTE_CAP_SHIFT 32#define HPET_TN_CFG_BITS_READONLY_OR_RESERVED (HPET_TN_RESERVED | \
xen/arch/x86/hvm/i8254.c +1 -1
@@ -54,7 +54,7 @@ static int handle_speaker_io( int dir, uint32_t port, uint32_t bytes, uint32_t *val);#define get_guest_time (v) \ - (is_hvm_vcpu(v) ? hvm_get_guest_time(v) : (u64)get_s_time()) + (is_hvm_vcpu(v) ? hvm_get_base_time(v) : (u64)get_s_time())static int pit_get_count (PITState *pit,int channel) {
xen/arch/x86/hvm/pmtimer.c +1 -1
@@ -93,7 +93,7 @@ static void pmt_update_time(PMTState *s)ASSERT (spin_is_locked (&s->lock));/* Update the timer */ - curr_gtime =hvm_get_guest_time (s->vcpu); + curr_gtime =hvm_get_base_time (s->vcpu); tmp = ((curr_gtime - s->last_gtime) * s->scale) + s->not_accounted; s->not_accounted = (uint32_t )tmp; tmr_val += tmp >>32 ;
xen/arch/x86/hvm/vpt.c +78 -15
136 lines not on this page · raw message
xen/include/asm-x86/hvm/hvm.h +1 -1
@@ -226,8 +226,8 @@ void hvm_set_guest_tsc(struct vcpu *v, u64 guest_tsc);u64 hvm_get_guest_tsc (struct vcpu *v);void hvm_init_guest_time (struct domain *d); -void hvm_set_guest_time (struct vcpu *v,u64 guest_time);u64 hvm_get_guest_time (struct vcpu *v); +u64 hvm_get_base_time (struct vcpu *v); int vmsi_deliver( struct domain *d, int vector,
Signature · 1 line
1.7.9.5
PATCH 2/2 x86/hvm: fix corrupt ACPI PM-Timer during live migration
In reply to a message not in the archiveThe ACPI PM-Timer value is broken when a vm is saved. The function pmtimer_save() calculates the value on its own, but vcpu->arch.hvm_vcpu.guest_time is not valid (usually zero).
Another problem is a double counting of the PM-Timer value in failure to save a vm.
This patch corrects the value, and also makes the calculation more precise.
--- xen/arch/x86/hvm/pmtimer.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-)
xen/arch/x86/hvm/pmtimer.c +6 -13
@@ -85,7 +85,7 @@ void hvm_acpi_sleep_button(struct domain *d)/* Set the correct value in the timer, accounting for time elapsed * since the last time we did that. */ -static void pmt_update_time (PMTState *s) +static void pmt_update_time (PMTState *s,bool_t update_sci) {uint64_t curr_gtime, tmp;uint32_t tmr_val = s->pm.tmr_val, msb = tmr_val &TMR_VAL_MSB ;
@@ -107,7 +107,8 @@ static void pmt_update_time(PMTState *s)if ( (tmr_val &TMR_VAL_MSB ) != msb ) { s->pm.pm1a_sts |=TMR_STS ; -pmt_update_sci (s); +if ( update_sci ) +pmt_update_sci (s); } }
@@ -123,7 +124,7 @@ static void pmt_timer_callback(void *opaque)spin_lock (&s->lock);/* Recalculate the timer and make sure we get an SCI if we need one */ -pmt_update_time (s); +pmt_update_time (s,1 );/* How close are we to the next MSB flip? */ pmt_cycles_until_flip =TMR_VAL_MSB - (s->pm.tmr_val & (TMR_VAL_MSB -1 ));
@@ -221,7 +222,7 @@ static int handle_pmt_io(if (spin_trylock (&s->lock) ) {/* We hold the lock: update timer value and return it. */ -pmt_update_time (s); +pmt_update_time (s,1 ); *val = s->pm.tmr_val;spin_unlock (&s->lock); }
@@ -244,19 +245,11 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ - x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; +pmt_update_time (s,0 );/* No point in setting the SCI here because we'll already have saved the * IRQ and *PIC state; we'll fix it up when we restore the domain */
Signature · 1 line
1.7.9.5
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 13 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-4@@ -244,19 +245,11 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;spin_lock (&s->lock); -/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */
So on the previous patch version (which you said this one is identical to) I stated that you lose this property of guaranteeing no backward move. Am I right in assuming that patch 1 is now supposed to take care of this?
In any case I'll have to defer to Keir or Tim for that first one.
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 252
Quoted 6 lines
xen/arch/x86/hvm/pmtimer.c +1
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; +pmt_update_time (s,0 );
Here I can only quote part of my previous reply, which I don't think you responded to:
"Also, in delay_for_missed_ticks mode you now use a slightly
different time for updating s->pm - did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, or hvm_get_guest_time() could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, that pt_freeze_time() runs before pmtimer_save())."
You said you'd think about it, but I don't recall seeing any other outcome from that than the two patches, and I can't relate the first patch to this aspect.
Jan
Quoted 10 lines · Jan Beulich
Quoted 10 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-1@@ -244,19 +245,11 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;spin_lock (&s->lock);
Quoted 8 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 3 lines
xen/arch/x86/hvm/pmtimer.c +0
-3-/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ So on the previous patch version (which you said this one is identical to) I stated that you lose this property of guaranteeing no backward move. Am I right in assuming that patch 1 is now supposed to take care of this?
Yes. Patch 1 guarantees that the timer counter only goes forwards.
Quoted 20 lines
In any case I'll have to defer to Keir or Tim for that first one.
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 252
Quoted 6 lines
xen/arch/x86/hvm/pmtimer.c +1
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; +pmt_update_time (s,0 );Here I can only quote part of my previous reply, which I don't think you responded to:
"Also, in
delay_for_missed_ticksmode you now use a slightly
different time for updatings->pm- did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, orhvm_get_guest_time()could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, thatpt_freeze_time()runs beforepmtimer_save())."
Quoted 3 lines
You said you'd think about it, but I don't recall seeing any other outcome from that than the two patches, and I can't relate the first patch to this aspect.
In patch 1, pmt_update_time() calls the new function hvm_get_base_time() which returns the frozen time when the vcpu is de-scheduled. And I confirmed pmtimer_save() is surely called after pt_freeze_time() from my test and review.
So, if both patches are applied, there is no difference in delay_for_missed_ticks mode.
Thanks,
Kouya
Quoted 47 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
Quoted 10 lines · Jan Beulich
Quoted 10 lines
Quoted 1 line
Quoted 1 line · Kouya Shimura
xen/arch/x86/hvm/pmtimer.c +0
-1@@ -244,19 +245,11 @@ static int handle_pmt_io(static int pmtimer_save (struct domain *d,hvm_domain_context_t *h) {PMTState *s = &d->arch.hvm_domain.pl_time.vpmt; -uint32_t x, msb = s->pm.tmr_val &TMR_VAL_MSB ;int rc;spin_lock (&s->lock);Quoted 8 lines
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 254
Quoted 3 lines
xen/arch/x86/hvm/pmtimer.c +0
-3-/* Update the counter to the guest's current time. We always save -* with the domain paused, so the saved time should be after the -* last_gtime, but just in case, make sure we only go forwards */ So on the previous patch version (which you said this one is identical to) I stated that you lose this property of guaranteeing no backward move. Am I right in assuming that patch 1 is now supposed to take care of this?
Yes. Patch 1 guarantees that the timer counter only goes forwards.
Quoted 20 lines
In any case I'll have to defer to Keir or Tim for that first one.
xen/arch/x86/hvm/pmtimer.c static int handle_pmt_io( · line 252
Quoted 6 lines
xen/arch/x86/hvm/pmtimer.c +1
-5- x = ((s->vcpu->arch.hvm_vcpu.guest_time - s->last_gtime) * s->scale) >>32 ; -if ( x <1UL <<31 ) - s->pm.tmr_val += x; -if ( (s->pm.tmr_val &TMR_VAL_MSB ) != msb ) - s->pm.pm1a_sts |=TMR_STS ; +pmt_update_time (s,0 );Here I can only quote part of my previous reply, which I don't think you responded to:
"Also, in
delay_for_missed_ticksmode you now use a slightly
different time for updatings->pm- did you double check that this is not going to be a problem? Or else, the flag above could similarly be used to circumvent this, orhvm_get_guest_time()could be made return the frozen time (I suppose, but didn't verify - as it appears to be an assumption already before your patch -, thatpt_freeze_time()runs beforepmtimer_save())."Quoted 3 lines
You said you'd think about it, but I don't recall seeing any other outcome from that than the two patches, and I can't relate the first patch to this aspect.
Quoted 6 lines
In patch 1,
pmt_update_time()calls the new functionhvm_get_base_time()which returns the frozen time when the vcpu is de-scheduled. And I confirmedpmtimer_save()is surely called afterpt_freeze_time()from my test and review.
So, if both patches are applied, there is no difference indelay_for_missed_ticksmode.
Okay, thanks for confirming!
Jan
Kouya, Jan,
Quoted 8 lines
Quoted 6 lines
In patch 1,
pmt_update_time()calls the new functionhvm_get_base_time()which returns the frozen time when the vcpu is de-scheduled. And I confirmedpmtimer_save()is surely called afterpt_freeze_time()from my test and review.
So, if both patches are applied, there is no difference indelay_for_missed_ticksmode.Okay, thanks for confirming!
Did this ever get committed? I can't immediately find a commit.
I am wondering if this is related to the ACPI timer issues we are seeing on migration in qemu-upstream DM.
Signature · 1 line
Alex Bligh
Quoted 10 lines
Quoted 1 line
Quoted 1 line · Alex Bligh
Kouya, Jan,
Quoted 7 lines
Quoted 6 lines
In patch 1,
pmt_update_time()calls the new functionhvm_get_base_time()which returns the frozen time when the vcpu is de-scheduled. And I confirmedpmtimer_save()is surely called afterpt_freeze_time()from my test and review.
So, if both patches are applied, there is no difference indelay_for_missed_ticksmode.
Quoted 3 lines
Quoted 1 line
Okay, thanks for confirming!
Did this ever get committed? I can't immediately find a commit.
No, it didn't - no-one knowing that code well enough ever acked patch 1, and without that we can't apply the patch here.
Jan
Jan,
Quoted 4 lines · --On 15 May 2013 15:31:19 +0100 Jan Beulich <[email protected]> wrote:
Quoted 1 line
Did this ever get committed? I can't immediately find a commit.
No, it didn't - no-one knowing that code well enough ever acked patch 1, and without that we can't apply the patch here.
I certainly am not someone who knows that code well, so can't help with that. But I (or more accurately Diana) can reliably replicate live migrate on HVM and qemu-upstream DM causing (a) ACPI entries to disappear from xenstore and (b) walltime to fail to advance in the migrated domain until the walltime is manually set (stuck clock).
Is this likely to be related?
Diana's threads are here (second and third being most relevant):
http://lists.xen.org/archives/html/xen-devel/2013-05/msg01475.html
http://lists.xen.org/archives/html/xen-devel/2013-05/msg01474.html
http://lists.xen.org/archives/html/xen-devel/2013-05/msg01472.html
Signature · 1 line
Alex Bligh
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Alex Bligh
Quoted 4 lines · --On 15 May 2013 15:31:19 +0100 Jan Beulich <[email protected]> wrote:
Quoted 1 line
Did this ever get committed? I can't immediately find a commit.
No, it didn't - no-one knowing that code well enough ever acked patch 1, and without that we can't apply the patch here.
Quoted 8 lines
I certainly am not someone who knows that code well, so can't help with that. But I (or more accurately Diana) can reliably replicate live migrate on HVM and qemu-upstream DM causing (a) ACPI entries to disappear from xenstore and (b) walltime to fail to advance in the migrated domain until the walltime is manually set (stuck clock).
Is this likely to be related?
I'd like to defer to Kouya to tell whether that matches the symptoms he saw.
Jan
Quoted 15 lines · Jan Beulich
Quoted 15 lines
Quoted 1 line
Quoted 1 line · Alex Bligh
Quoted 4 lines · --On 15 May 2013 15:31:19 +0100 Jan Beulich <[email protected]> wrote:
Quoted 1 line
Did this ever get committed? I can't immediately find a commit.
No, it didn't - no-one knowing that code well enough ever acked patch 1, and without that we can't apply the patch here.
I certainly am not someone who knows that code well, so can't help with that. But I (or more accurately Diana) can reliably replicate live migrate on HVM and qemu-upstream DM causing (a) ACPI entries to disappear from xenstore and (b) walltime to fail to advance in the migrated domain until the walltime is manually set (stuck clock).
Quoted 4 lines
Quoted 1 line
Is this likely to be related?
I'd like to defer to Kouya to tell whether that matches the symptoms he saw.
I don't think my fix is related to Alex's problem.
I observed that gettimeofday() sometimes goes backward *a few seconds* on migration.
In linux OSs, ACPI Timer value is masked by 24bit. The valid range:
(1 / 3.579545MHz) * 0xffffff = 4.7sec
So, the effect of the corrupt ACPI timer is at most 4.7 sec. The clock shouldn't be frozen for a long time.
Apparently (a) sounds like a problem of xl toolstack.
--
Kouya
Quoted 1 line · --On 16 May 2013 14:29:00 +0900 Kouya Shimura <[email protected]> wrote:
Apparently (a) sounds like a problem of xl toolstack.
It isn't just the xl toolstack. We are seeing this with libxl (indeed did originally and had to go replicate with xl).
Signature · 1 line
Alex Bligh
Hi Keir, Tim,
I attached the patch again.
Can I get an ACK or NACK?
Thanks,
kouya
Quoted 10 lines · Jan Beulich
Quoted 10 lines
Quoted 1 line
Quoted 1 line · Alex Bligh
Kouya, Jan,
Quoted 7 lines
Quoted 6 lines
In patch 1,
pmt_update_time()calls the new functionhvm_get_base_time()which returns the frozen time when the vcpu is de-scheduled. And I confirmedpmtimer_save()is surely called afterpt_freeze_time()from my test and review.
So, if both patches are applied, there is no difference indelay_for_missed_ticksmode.
Quoted 9 lines
Quoted 3 lines
Quoted 1 line
Okay, thanks for confirming!
Did this ever get committed? I can't immediately find a commit.
No, it didn't - no-one knowing that code well enough ever acked patch 1, and without that we can't apply the patch here.
Jan
In the case of delay_for_missed_ticks mode (timer_mode=0), the guest virtual time goes backwards when the vcpu is rescheduled. Therefore guest's HW timer might go backwards, too.
Case 1. SMP guest:
1) vcpu#1 is de-scheduled and the guest time freezes. (TIME 0:0.010)
2) vcpu#2 access a timer (0:0.020)
3) vcpu#1 is re-scheduled and the guest time thaws. (0:0.030->0:0.010)
4) vcpu#2 access a timer (0:0.015) // Backwards!!!
Case 2. asynchronous callback:
1) vcpu#1 is de-scheduled and the guest time freezes. (0:0.010)
2) pmt_timer_callback() is invoked (0:0.025)
3) vcpu#1 is re-scheduled and the guest time thaws. (0:0.030->0:0.010)
4) vcpu#1 access the PM-TIMER (0:0.015) // Backwards!!!
This patch affects only delay_for_missed_ticks mode (timer_mode=0) and ensures the monotonicity of the following timers:
- PIT
- HPET
- ACPI PM-TIMER
The following timers are OK since a vcpu never access the other vcpu's timer.
- Local APIC ( has some callbacks but it's called from pt_intr_post )
- TSC
Just in case, these timers can use the new function hvm_get_base_time() too, but doesn't. It's a little bit less efficient than hvm_get_guest_time().
Also, tidy up virtual platform timer code.
--- xen/arch/x86/hvm/hpet.c | 2 +- xen/arch/x86/hvm/i8254.c | 2 +- xen/arch/x86/hvm/pmtimer.c | 2 +- xen/arch/x86/hvm/vpt.c | 93 ++++++++++++++++++++++++++++++++++------- xen/include/asm-x86/hvm/hvm.h | 2 +- 5 files changed, 82 insertions(+), 19 deletions(-)
xen/arch/x86/hvm/hpet.c +1 -1
@@ -39,7 +39,7 @@/* Frequency_of_Xen_systeme_time / frequency_of_HPET = 16 */ #define STIME_PER_HPET_TICK 16#define guest_time_hpet (hpet) \ - (hvm_get_guest_time(vhpet_vcpu(hpet)) / STIME_PER_HPET_TICK) + (hvm_get_base_time(vhpet_vcpu(hpet)) / STIME_PER_HPET_TICK)#define HPET_TN_INT_ROUTE_CAP_SHIFT 32#define HPET_TN_CFG_BITS_READONLY_OR_RESERVED (HPET_TN_RESERVED | \
xen/arch/x86/hvm/i8254.c +1 -1
@@ -54,7 +54,7 @@ static int handle_speaker_io( int dir, uint32_t port, uint32_t bytes, uint32_t *val);#define get_guest_time (v) \ - (is_hvm_vcpu(v) ? hvm_get_guest_time(v) : (u64)get_s_time()) + (is_hvm_vcpu(v) ? hvm_get_base_time(v) : (u64)get_s_time())static int pit_get_count (PITState *pit,int channel) {
xen/arch/x86/hvm/pmtimer.c +1 -1
@@ -93,7 +93,7 @@ static void pmt_update_time(PMTState *s)ASSERT (spin_is_locked (&s->lock));/* Update the timer */ - curr_gtime =hvm_get_guest_time (s->vcpu); + curr_gtime =hvm_get_base_time (s->vcpu); tmp = ((curr_gtime - s->last_gtime) * s->scale) + s->not_accounted; s->not_accounted = (uint32_t )tmp; tmr_val += tmp >>32 ;
xen/arch/x86/hvm/vpt.c +78 -15
136 lines not on this page · raw message
xen/include/asm-x86/hvm/hvm.h +1 -1
@@ -226,8 +226,8 @@ void hvm_set_guest_tsc(struct vcpu *v, u64 guest_tsc);u64 hvm_get_guest_tsc (struct vcpu *v);void hvm_init_guest_time (struct domain *d); -void hvm_set_guest_time (struct vcpu *v,u64 guest_time);u64 hvm_get_guest_time (struct vcpu *v); +u64 hvm_get_base_time (struct vcpu *v); int vmsi_deliver( struct domain *d, int vector,
Signature · 1 line
1.7.9.5
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Tim Deegan
Reviewed-byQuoted 4 lines · Kouya Shimura
Hi Keir, Tim,
I attached the patch again.
Can I get an ACK or NACK?
Well, I don't think this is suitable for committing:
xen/arch/x86/hvm/vpt.c static void pt_process_missed_ticks(struct periodic_time *pt) · line 195
Quoted 4 lines
xen/arch/x86/hvm/vpt.c +2
-0+#if 0 /* redundant */ if ( !mode_is (v->domain, delay_for_missed_ticks) )return ; +#endif
but apart from that it seems OK to me. It's a bit odd to have the platform timers (almost) stop ticking when their controlling vcpu is descheduled but I guess no worse than having them go backwards!
So if you fix the '#if 0' (either cut out the code properly or leave it alone, I don't mind which), you can add
You'll need an ack from George as well for the release; I suspect that will depend on how much testing you've done.
Tim.