ppc/xive: Trigger the pending interrupt for a guest after migration
PATCH 1/1 ppc/xive: Trigger the pending interrupt for a guest after migration
When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:
$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI CPU[0000]: OS 00 ff 00 ff ff 00 ff ff 80000400 CPU[0001]: OS 00 ff 00 ff ff 00 ff ff 80000401 CPU[0002]: OS 00 ff 00 ff ff 00 ff ff 80000402 CPU[0003]: OS 00 ff 00 ff ff 00 ff ff 80000403 CPU[0004]: OS 00 ff 00 ff ff 00 ff ff 80000404 CPU[0005]: OS 00 ff 00 ff ff 00 ff ff 80000405 CPU[0006]: OS 80 ff 02 ff ff 00 ff 06 80000406 CPU[0007]: OS 00 ff 00 ff ff 00 ff ff 80000407 00000000 MSI -- 00000010 0/6 3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000001 MSI -- 00000010 1/6 1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ] 00000002 MSI -- 00000010 2/6 4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ] 00000003 MSI -- 00000010 3/6 1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ] 00000004 MSI -- 00000010 4/6 2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000005 MSI -- 00000010 5/6 2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000006 MSI P- 00000010 6/6 5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ] 00000007 MSI -- 00000010 7/6 2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]
Fix this by calling xive_tctx_pipr_set() in vmstate_xive_tctx_post_load(), which ends up calling qemu_irq_raise() to present the interrupt.
--- hw/intc/xive.c | 3 +++ 1 file changed, 3 insertions(+)
hw/intc/xive.c +3 -0
@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)error_report_err (local_err);return ret; } + }else { +uint8_t pipr = tctx->regs[TM_QW1_OS +TM_PIPR ]; +xive_tctx_pipr_set (tctx,TM_QW1_OS , pipr,0 ); }return 0 ;
Signature · 1 line
2.53.0
Quoted 39 lines · Gautam Menghani
When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:
$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI CPU[0000]: OS 00 ff 00 ff ff 00 ff ff 80000400 CPU[0001]: OS 00 ff 00 ff ff 00 ff ff 80000401 CPU[0002]: OS 00 ff 00 ff ff 00 ff ff 80000402 CPU[0003]: OS 00 ff 00 ff ff 00 ff ff 80000403 CPU[0004]: OS 00 ff 00 ff ff 00 ff ff 80000404 CPU[0005]: OS 00 ff 00 ff ff 00 ff ff 80000405 CPU[0006]: OS 80 ff 02 ff ff 00 ff 06 80000406 CPU[0007]: OS 00 ff 00 ff ff 00 ff ff 80000407 00000000 MSI -- 00000010 0/6 3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000001 MSI -- 00000010 1/6 1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ] 00000002 MSI -- 00000010 2/6 4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ] 00000003 MSI -- 00000010 3/6 1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ] 00000004 MSI -- 00000010 4/6 2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000005 MSI -- 00000010 5/6 2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000006 MSI P- 00000010 6/6 5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ] 00000007 MSI -- 00000010 7/6 2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]Fix this by calling
xive_tctx_pipr_set()invmstate_xive_tctx_post_load(), which ends up callingqemu_irq_raise()to present the interrupt.Signed-off-by Gautam Menghani <[email protected]>--- hw/intc/xive.c | 3 +++ 1 file changed, 3 insertions(+)hw/intc/xive.c +0
-0
hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1088
Quoted 9 lines
hw/intc/xive.c +3
-0@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)error_report_err (local_err);return ret; } + }else { +uint8_t pipr = tctx->regs[TM_QW1_OS +TM_PIPR ]; +xive_tctx_pipr_set (tctx,TM_QW1_OS , pipr,0 ); }return 0 ;
Hi Gautam,
I have tested this patch and it has been fixing the issue which I reported. Here is my analysis:
a) Without applying the patch :
1) Start the guest and run stress-ng as below for sometime localhost:~ # stress-ng --cpu 4 --vm 2 --vm-bytes 1G --hdd 2 --hdd-bytes 1G --sched other --timeout 3600000s
stress-ng: info: [1464] setting to a 41 days, 16 hours, 0 secs run per stressor
stress-ng: info: [1464] dispatching hogs: 4 cpu, 2 vm, 2 hdd
2) Start the migration from H1 to H2:
ltc-lp7:~ # virsh migrate --live --domain sles16_anu qemu+ssh://10.xx.xx.xx/system --verbose --undefinesource --persistent --auto-converge --postcopy
Migration: [100.00 %]
3) Migration got completed but guest is not getting recovered from continuous softlockups even while having ample number of vcpus and amount of memory free on host and guest both -
[ 1336.003836][ C1] watchdog: BUG: soft lockup - CPU#1 stuck for 977s! [htxd_monitor:1337] [ 1336.006834][ C4] watchdog: BUG: soft lockup - CPU#4 stuck for 1002s! [rcu_exp_par_gp_:19] [ 1346.015839][ C0] BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 1090s! [ 1346.016355][ C0] BUG: workqueue lockup - pool cpus=3 node=0 flags=0x0 nice=0 stuck for 1107s! [ 1346.016874][ C0] BUG: workqueue lockup - pool cpus=7 node=0 flags=0x0 nice=0 stuck for 1093s! [ 1356.007835][ C6] watchdog: BUG: soft lockup - CPU#6 stuck for 912s! [systemd:1353] [ 1356.008835][ C7] watchdog: BUG: soft lockup - CPU#7 stuck for 998s! [systemd-journal:570] [ 1360.003836][ C1] watchdog: BUG: soft lockup - CPU#1 stuck for 999s! [htxd_monitor:1337] [ 1360.006834][ C4] watchdog: BUG: soft lockup - CPU#4 stuck for 1024s! [rcu_exp_par_gp_:19] [ 1368.933835][ C4] rcu: INFO: rcu_preempt self-detected stall on CPU [ 1368.933973][ C4] rcu: 4-....: (1129830 ticks this GP) idle=afc4/1/0x4000000000000002 softirq=3694/428556 fqs=259639 [ 1368.934106][ C4] rcu: hardirqs softirqs csw/system [ 1368.934188][ C4] rcu: number: 1 444039 0 [ 1368.934271][ C4] rcu: cputime: 3 8 1096165 ==> 1110021(ms) [ 1368.934373][ C4] rcu: (t=1140022 jiffies g=6177 q=1684 ncpus=8) [ 1376.224839][ C0] BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 1120s! [ 1376.225307][ C0] BUG: workqueue lockup - pool cpus=3 node=0 flags=0x0 nice=0 stuck for 1138s! [ 1376.225428][ C0] BUG: workqueue lockup - pool cpus=5 node=0 flags=0x0 nice=0 stuck for 715s! [ 1376.225548][ C0] BUG: workqueue lockup - pool cpus=6 node=0 flags=0x0 nice=0 stuck for 1027s! [ 1376.225667][ C0] BUG: workqueue lockup - pool cpus=7 node=0 flags=0x0 nice=0 stuck for 1123s! [ 1444.006835][ C4] watchdog: BUG: soft lockup - CPU#4 stuck for 1100s!
[rcu_exp_par_gp_:19]
b) After using the patch applied qemu this issue has been fixed. I tried multiple migration instances between both the hosts and this issue is not getting seen. Even I did try this same scenario with threads as 1/2/4/8 and for everything it worked fine. Thanks for working on the fix.
Please feel free to add my tested-by:
and please add my reported-by too:
Thank you,
Anushree Mathur
On 24/08/26 12:33, Gautam Menghani wrote:
Hi Gautam,
I was going through the implementation and had a small doubt regarding the scope of the fix for the TIMA rings. I can see that we are explicitly reading and setting the PIPR for TM_QW1_OS:
hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1086
Quoted 4 lines
hw/intc/xive.c +3
-0+ }else { +uint8_t pipr = tctx->regs[TM_QW1_OS +TM_PIPR ]; +xive_tctx_pipr_set (tctx,TM_QW1_OS , pipr,0 ); }
I guess vmstate_xive_tctx_post_load is a shared device code, I was wondering what happens in the case of the other rings? Should we also force a re-evaluation for TM_QW3_HV_PHYS or TM_QW2_HV_POOL to avoid similar lockups in those configurations? Or is this issue strictly coming only in the OS ring in practical scenarios? Please let me know your thoughts on this.
Also, one minor suggestion. Since this logic is going inside the else block of if (kvm_irqchip_in_kernel()), I feel it would be better if we can add a line in the commit message mentioning that this specifically fixes state restoration for environments relying on QEMU's emulated XIVE.
Thanks
~ Nikhil
Quoted 16 lines · Nikhil Kumar Singh
On 24/08/26 12:33, Gautam Menghani wrote:
Hi Gautam,
I was going through the implementation and had a small doubt regarding the scope of the fix for the TIMA rings. I can see that we are explicitly reading and setting the PIPR for
TM_QW1_OS:hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1086
Quoted 4 lines
hw/intc/xive.c +3
-0+ }else { +uint8_t pipr = tctx->regs[TM_QW1_OS +TM_PIPR ]; +xive_tctx_pipr_set (tctx,TM_QW1_OS , pipr,0 ); }
Quoted 5 lines
I guess
vmstate_xive_tctx_post_loadis a shared device code, I was wondering what happens in the case of the other rings? Should we also force a re-evaluation forTM_QW3_HV_PHYSorTM_QW2_HV_POOLto avoid similar lockups in those configurations? Or is this issue strictly coming only in the OS ring in practical scenarios? Please let me know your thoughts on this.
The OS ring is used by LPARs and KVM guests. The TM_QW3_HV_PHYS and TM_QW2_HV_POOL are used by hypervisors (PHYP and PowerNV linux). Since migration concerns only KVM guests and LPARs, handling the OS ring is enough here.
Quoted 6 lines
Also, one minor suggestion. Since this logic is going inside the else block of if (
kvm_irqchip_in_kernel()), I feel it would be better if we can add a line in the commit message mentioning that this specifically fixes state restoration for environments relying on QEMU's emulated XIVE.
Yes agree, will add this in the commit message. I'll wait to see if there are any other comments.
Thanks,
Gautam
Nikhil Kumar Singh
Reviewed-byHi Gautam,
Thanks for the clarification. That makes complete sense, since live migration is practically applicable only for KVM guests and LPARs, handling just the OS ring perfectly covers the requirement here.
Thanks for agreeing to update the commit message as well. With that added in v2, please feel free to add:
Regards
~ Nikhil
Mike Kowal, can you please review this?
Thanks,
Glenn
Quoted 48 lines · Gautam Menghani
When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:
$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI CPU[0000]: OS 00 ff 00 ff ff 00 ff ff 80000400 CPU[0001]: OS 00 ff 00 ff ff 00 ff ff 80000401 CPU[0002]: OS 00 ff 00 ff ff 00 ff ff 80000402 CPU[0003]: OS 00 ff 00 ff ff 00 ff ff 80000403 CPU[0004]: OS 00 ff 00 ff ff 00 ff ff 80000404 CPU[0005]: OS 00 ff 00 ff ff 00 ff ff 80000405 CPU[0006]: OS 80 ff 02 ff ff 00 ff 06 80000406 CPU[0007]: OS 00 ff 00 ff ff 00 ff ff 80000407 00000000 MSI -- 00000010 0/6 3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000001 MSI -- 00000010 1/6 1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ] 00000002 MSI -- 00000010 2/6 4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ] 00000003 MSI -- 00000010 3/6 1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ] 00000004 MSI -- 00000010 4/6 2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000005 MSI -- 00000010 5/6 2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000006 MSI P- 00000010 6/6 5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ] 00000007 MSI -- 00000010 7/6 2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]Fix this by calling
xive_tctx_pipr_set()invmstate_xive_tctx_post_load(), which ends up callingqemu_irq_raise()to present the interrupt.Signed-off-by Gautam Menghani <[email protected]>--- hw/intc/xive.c | 3 +++ 1 file changed, 3 insertions(+)hw/intc/xive.c +3
-0@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)error_report_err (local_err);return ret; } + }else { +uint8_t pipr = tctx->regs[TM_QW1_OS +TM_PIPR ]; +xive_tctx_pipr_set (tctx,TM_QW1_OS , pipr,0 ); }return 0 ;
Mike Kowal
Reviewed-byQuoted 54 lines · Miles Glenn
Mike Kowal, can you please review this?
Thanks,
Glenn
Quoted 47 lines · Gautam Menghani
When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:
$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI CPU[0000]: OS 00 ff 00 ff ff 00 ff ff 80000400 CPU[0001]: OS 00 ff 00 ff ff 00 ff ff 80000401 CPU[0002]: OS 00 ff 00 ff ff 00 ff ff 80000402 CPU[0003]: OS 00 ff 00 ff ff 00 ff ff 80000403 CPU[0004]: OS 00 ff 00 ff ff 00 ff ff 80000404 CPU[0005]: OS 00 ff 00 ff ff 00 ff ff 80000405 CPU[0006]: OS 80 ff 02 ff ff 00 ff 06 80000406 CPU[0007]: OS 00 ff 00 ff ff 00 ff ff 80000407 00000000 MSI -- 00000010 0/6 3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000001 MSI -- 00000010 1/6 1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ] 00000002 MSI -- 00000010 2/6 4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ] 00000003 MSI -- 00000010 3/6 1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ] 00000004 MSI -- 00000010 4/6 2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000005 MSI -- 00000010 5/6 2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ] 00000006 MSI P- 00000010 6/6 5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ] 00000007 MSI -- 00000010 7/6 2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]Fix this by calling
xive_tctx_pipr_set()invmstate_xive_tctx_post_load(), which ends up callingqemu_irq_raise()to present the interrupt.Signed-off-by Gautam Menghani<[email protected]>--- hw/intc/xive.c | 3 +++ 1 file changed, 3 insertions(+)hw/intc/xive.c +3
-0@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)error_report_err (local_err);return ret; } + }else { +uint8_t pipr = tctx->regs[TM_QW1_OS +TM_PIPR ]; +xive_tctx_pipr_set (tctx,TM_QW1_OS , pipr,0 ); }
Quoted 1 line
hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1088
Quoted 1 line
hw/intc/xive.c +0
-0return 0 ;