Skip to contentSkip to conversation

ppc/xive: Trigger the pending interrupt for a guest after migration

PATCH Superseded Gautam Menghani 5 participants 7 messages Aug 24 – Aug 25

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ppc/xive: Trigger the pending interrupt for a guest after migration

When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:

$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI
CPU[0000]:   OS    00   ff  00    ff   ff  00  ff   ff  80000400
CPU[0001]:   OS    00   ff  00    ff   ff  00  ff   ff  80000401
CPU[0002]:   OS    00   ff  00    ff   ff  00  ff   ff  80000402
CPU[0003]:   OS    00   ff  00    ff   ff  00  ff   ff  80000403
CPU[0004]:   OS    00   ff  00    ff   ff  00  ff   ff  80000404
CPU[0005]:   OS    00   ff  00    ff   ff  00  ff   ff  80000405
CPU[0006]:   OS    80   ff  02    ff   ff  00  ff   06  80000406
CPU[0007]:   OS    00   ff  00    ff   ff  00  ff   ff  80000407
  00000000 MSI --    00000010   0/6   3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
  00000001 MSI --    00000010   1/6   1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ]
  00000002 MSI --    00000010   2/6   4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ]
  00000003 MSI --    00000010   3/6   1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ]
  00000004 MSI --    00000010   4/6   2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
  00000005 MSI --    00000010   5/6   2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
  00000006 MSI P-    00000010   6/6   5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ]
  00000007 MSI --    00000010   7/6   2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]

Fix this by calling xive_tctx_pipr_set() in vmstate_xive_tctx_post_load(), which ends up calling qemu_irq_raise() to present the interrupt.

---
 hw/intc/xive.c | 3 +++
 1 file changed, 3 insertions(+)
hw/intc/xive.c +3 -0
index f473e6ac77..712498d4af 100644
@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)
             error_report_err(local_err);
             return ret;
         }
+    } else {
+        uint8_t pipr = tctx->regs[TM_QW1_OS + TM_PIPR];
+        xive_tctx_pipr_set(tctx, TM_QW1_OS, pipr, 0);
     }
 
     return 0;
Signature · 1 line
2.53.0
Tested-by on v1 1/1
#1 lore raw
Quoted 39 lines · Gautam Menghani

When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:

$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI
CPU[0000]:   OS    00   ff  00    ff   ff  00  ff   ff  80000400
CPU[0001]:   OS    00   ff  00    ff   ff  00  ff   ff  80000401
CPU[0002]:   OS    00   ff  00    ff   ff  00  ff   ff  80000402
CPU[0003]:   OS    00   ff  00    ff   ff  00  ff   ff  80000403
CPU[0004]:   OS    00   ff  00    ff   ff  00  ff   ff  80000404
CPU[0005]:   OS    00   ff  00    ff   ff  00  ff   ff  80000405
CPU[0006]:   OS    80   ff  02    ff   ff  00  ff   06  80000406
CPU[0007]:   OS    00   ff  00    ff   ff  00  ff   ff  80000407
   00000000 MSI --    00000010   0/6   3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
   00000001 MSI --    00000010   1/6   1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ]
   00000002 MSI --    00000010   2/6   4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ]
   00000003 MSI --    00000010   3/6   1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ]
   00000004 MSI --    00000010   4/6   2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
   00000005 MSI --    00000010   5/6   2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
   00000006 MSI P-    00000010   6/6   5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ]
   00000007 MSI --    00000010   7/6   2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]

Fix this by calling xive_tctx_pipr_set() in vmstate_xive_tctx_post_load(), which ends up calling qemu_irq_raise() to present the interrupt.

---
  hw/intc/xive.c | 3 +++
  1 file changed, 3 insertions(+)
hw/intc/xive.c +0 -0
index f473e6ac77..712498d4af 100644

hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1088

Quoted 9 lines
hw/intc/xive.c +3 -0
@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)
              error_report_err(local_err);
              return ret;
          }
+    } else {
+        uint8_t pipr = tctx->regs[TM_QW1_OS + TM_PIPR];
+        xive_tctx_pipr_set(tctx, TM_QW1_OS, pipr, 0);
      }
 
      return 0;

Hi Gautam,
I have tested this patch and it has been fixing the issue which I reported. Here is my analysis:

a) Without applying the patch :

1) Start the guest and run stress-ng as below for sometime localhost:~ # stress-ng --cpu 4 --vm 2 --vm-bytes 1G --hdd 2 --hdd-bytes 1G --sched other --timeout 3600000s
stress-ng: info:  [1464] setting to a 41 days, 16 hours, 0 secs run per stressor
stress-ng: info:  [1464] dispatching hogs: 4 cpu, 2 vm, 2 hdd

2) Start the migration from H1 to H2:

ltc-lp7:~ # virsh migrate --live --domain sles16_anu qemu+ssh://10.xx.xx.xx/system --verbose --undefinesource --persistent --auto-converge --postcopy
Migration: [100.00 %]

3) Migration got completed but guest is not getting recovered from continuous softlockups even while having ample number of vcpus and amount of memory free on host and guest both -

[ 1336.003836][    C1] watchdog: BUG: soft lockup - CPU#1 stuck for 977s!
[htxd_monitor:1337]
[ 1336.006834][    C4] watchdog: BUG: soft lockup - CPU#4 stuck for 1002s!
[rcu_exp_par_gp_:19]
[ 1346.015839][    C0] BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0
nice=0 stuck for 1090s!
[ 1346.016355][    C0] BUG: workqueue lockup - pool cpus=3 node=0 flags=0x0
nice=0 stuck for 1107s!
[ 1346.016874][    C0] BUG: workqueue lockup - pool cpus=7 node=0 flags=0x0
nice=0 stuck for 1093s!
[ 1356.007835][    C6] watchdog: BUG: soft lockup - CPU#6 stuck for 912s!
[systemd:1353]
[ 1356.008835][    C7] watchdog: BUG: soft lockup - CPU#7 stuck for 998s!
[systemd-journal:570]
[ 1360.003836][    C1] watchdog: BUG: soft lockup - CPU#1 stuck for 999s!
[htxd_monitor:1337]
[ 1360.006834][    C4] watchdog: BUG: soft lockup - CPU#4 stuck for 1024s!
[rcu_exp_par_gp_:19]
[ 1368.933835][    C4] rcu: INFO: rcu_preempt self-detected stall on CPU
[ 1368.933973][    C4] rcu:     4-....: (1129830 ticks this GP)
idle=afc4/1/0x4000000000000002 softirq=3694/428556 fqs=259639
[ 1368.934106][    C4] rcu:              hardirqs   softirqs  csw/system
[ 1368.934188][    C4] rcu:      number:        1     444039 0
[ 1368.934271][    C4] rcu:     cputime:        3          8 1096165   ==>
1110021(ms)
[ 1368.934373][    C4] rcu:     (t=1140022 jiffies g=6177 q=1684 ncpus=8)
[ 1376.224839][    C0] BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0
nice=0 stuck for 1120s!
[ 1376.225307][    C0] BUG: workqueue lockup - pool cpus=3 node=0 flags=0x0
nice=0 stuck for 1138s!
[ 1376.225428][    C0] BUG: workqueue lockup - pool cpus=5 node=0 flags=0x0
nice=0 stuck for 715s!
[ 1376.225548][    C0] BUG: workqueue lockup - pool cpus=6 node=0 flags=0x0
nice=0 stuck for 1027s!
[ 1376.225667][    C0] BUG: workqueue lockup - pool cpus=7 node=0 flags=0x0
nice=0 stuck for 1123s!
[ 1444.006835][    C4] watchdog: BUG: soft lockup - CPU#4 stuck for 1100s!

[rcu_exp_par_gp_:19]

b) After using the patch applied qemu this issue has been fixed. I tried multiple migration instances between both the hosts and this issue is not getting seen. Even I did try this same scenario with threads as 1/2/4/8 and for everything it worked fine. Thanks for working on the fix.

Please feel free to add my tested-by:

and please add my reported-by too:

Thank you,
Anushree Mathur

#2 lore raw
In reply to Gautam Menghani

On 24/08/26 12:33, Gautam Menghani wrote:

Hi Gautam,

I was going through the implementation and had a small doubt regarding the scope of the fix for the TIMA rings. I can see that we are explicitly reading and setting the PIPR for TM_QW1_OS:

hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1086

Quoted 4 lines
hw/intc/xive.c +3 -0
+    } else {
+        uint8_t pipr = tctx->regs[TM_QW1_OS + TM_PIPR];
+        xive_tctx_pipr_set(tctx, TM_QW1_OS, pipr, 0);
      }

I guess vmstate_xive_tctx_post_load is a shared device code, I was wondering what happens in the case of the other rings? Should we also force a re-evaluation for TM_QW3_HV_PHYS or TM_QW2_HV_POOL to avoid similar lockups in those configurations? Or is this issue strictly coming only in the OS ring in practical scenarios? Please let me know your thoughts on this.

Also, one minor suggestion. Since this logic is going inside the else block of if (kvm_irqchip_in_kernel()), I feel it would be better if we can add a line in the commit message mentioning that this specifically fixes state restoration for environments relying on QEMU's emulated XIVE.

Thanks
~ Nikhil

#3 lore raw
Quoted 16 lines · Nikhil Kumar Singh

On 24/08/26 12:33, Gautam Menghani wrote:

Hi Gautam,

I was going through the implementation and had a small doubt regarding the scope of the fix for the TIMA rings. I can see that we are explicitly reading and setting the PIPR for TM_QW1_OS:

hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1086

Quoted 4 lines
hw/intc/xive.c +3 -0
+    } else {
+        uint8_t pipr = tctx->regs[TM_QW1_OS + TM_PIPR];
+        xive_tctx_pipr_set(tctx, TM_QW1_OS, pipr, 0);
      }
Quoted 5 lines

I guess vmstate_xive_tctx_post_load is a shared device code, I was wondering what happens in the case of the other rings? Should we also force a re-evaluation for TM_QW3_HV_PHYS or TM_QW2_HV_POOL to avoid similar lockups in those configurations? Or is this issue strictly coming only in the OS ring in practical scenarios? Please let me know your thoughts on this.

The OS ring is used by LPARs and KVM guests. The TM_QW3_HV_PHYS and TM_QW2_HV_POOL are used by hypervisors (PHYP and PowerNV linux). Since migration concerns only KVM guests and LPARs, handling the OS ring is enough here.

Quoted 6 lines

Also, one minor suggestion. Since this logic is going inside the else block of if (kvm_irqchip_in_kernel()), I feel it would be better if we can add a line in the commit message mentioning that this specifically fixes state restoration for environments relying on QEMU's emulated XIVE.

Yes agree, will add this in the commit message. I'll wait to see if there are any other comments.

Thanks,
Gautam

Reviewed-by
#4 lore raw

Hi Gautam,

Thanks for the clarification. That makes complete sense, since live migration is practically applicable only for KVM guests and LPARs, handling just the OS ring perfectly covers the requirement here.

Thanks for agreeing to update the commit message as well. With that added in v2, please feel free to add:

Regards
~ Nikhil

#5 lore raw
In reply to Gautam Menghani

Mike Kowal, can you please review this?

Thanks,

Glenn

Quoted 48 lines · Gautam Menghani

When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:

$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI
CPU[0000]:   OS    00   ff  00    ff   ff  00  ff   ff  80000400
CPU[0001]:   OS    00   ff  00    ff   ff  00  ff   ff  80000401
CPU[0002]:   OS    00   ff  00    ff   ff  00  ff   ff  80000402
CPU[0003]:   OS    00   ff  00    ff   ff  00  ff   ff  80000403
CPU[0004]:   OS    00   ff  00    ff   ff  00  ff   ff  80000404
CPU[0005]:   OS    00   ff  00    ff   ff  00  ff   ff  80000405
CPU[0006]:   OS    80   ff  02    ff   ff  00  ff   06  80000406
CPU[0007]:   OS    00   ff  00    ff   ff  00  ff   ff  80000407
  00000000 MSI --    00000010   0/6   3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
  00000001 MSI --    00000010   1/6   1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ]
  00000002 MSI --    00000010   2/6   4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ]
  00000003 MSI --    00000010   3/6   1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ]
  00000004 MSI --    00000010   4/6   2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
  00000005 MSI --    00000010   5/6   2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
  00000006 MSI P-    00000010   6/6   5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ]
  00000007 MSI --    00000010   7/6   2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]

Fix this by calling xive_tctx_pipr_set() in vmstate_xive_tctx_post_load(), which ends up calling qemu_irq_raise() to present the interrupt.

---
 hw/intc/xive.c | 3 +++
 1 file changed, 3 insertions(+)
hw/intc/xive.c +3 -0
index f473e6ac77..712498d4af 100644
@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)
             error_report_err(local_err);
             return ret;
         }
+    } else {
+        uint8_t pipr = tctx->regs[TM_QW1_OS + TM_PIPR];
+        xive_tctx_pipr_set(tctx, TM_QW1_OS, pipr, 0);
     }
 
     return 0;
Reviewed-by
#6 lore raw
Quoted 54 lines · Miles Glenn

Mike Kowal, can you please review this?

Thanks,

Glenn

Quoted 47 lines · Gautam Menghani

When a KVM guest is migrated to another LPAR while a stress-ng workload is running, lockups are seen when the guest resumes on the destination LPAR. This root cause is that if there was a pending interrupt for a cpu on the source machine but it was not presented yet when the migration started, the interrupt is not retriggered on the destination LPAR. This is evident when running 'info pic' on the destination LPAR:

$ virsh qemu-monitor-command --hmp migrate 'info pic' | grep -e OS -e MSI
CPU[0000]:   OS    00   ff  00    ff   ff  00  ff   ff  80000400
CPU[0001]:   OS    00   ff  00    ff   ff  00  ff   ff  80000401
CPU[0002]:   OS    00   ff  00    ff   ff  00  ff   ff  80000402
CPU[0003]:   OS    00   ff  00    ff   ff  00  ff   ff  80000403
CPU[0004]:   OS    00   ff  00    ff   ff  00  ff   ff  80000404
CPU[0005]:   OS    00   ff  00    ff   ff  00  ff   ff  80000405
CPU[0006]:   OS    80   ff  02    ff   ff  00  ff   06  80000406
CPU[0007]:   OS    00   ff  00    ff   ff  00  ff   ff  80000407
   00000000 MSI --    00000010   0/6   3071/16384 @3710000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
   00000001 MSI --    00000010   1/6   1852/16384 @3980000 ^1 [ 80000010 80000033 80000033 80000033 80000033 ^00000000 ]
   00000002 MSI --    00000010   2/6   4424/16384 @40c0000 ^1 [ 80000020 80000020 80000020 80000010 80000010 ^00000000 ]
   00000003 MSI --    00000010   3/6   1913/16384 @4220000 ^1 [ 80000010 80000032 80000010 80000010 80000010 ^00000000 ]
   00000004 MSI --    00000010   4/6   2946/16384 @4360000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
   00000005 MSI --    00000010   5/6   2313/16384 @44a0000 ^1 [ 80000010 80000010 80000010 80000010 80000010 ^00000000 ]
   00000006 MSI P-    00000010   6/6   5825/16384 @45f0000 ^1 [ 80000024 80000024 80000024 80000010 80000016 ^00000000 ]
   00000007 MSI --    00000010   7/6   2346/16384 @4760000 ^1 [ 80000032 80000032 80000010 80000032 80000032 ^00000000 ]

Fix this by calling xive_tctx_pipr_set() in vmstate_xive_tctx_post_load(), which ends up calling qemu_irq_raise() to present the interrupt.

---
  hw/intc/xive.c | 3 +++
  1 file changed, 3 insertions(+)
hw/intc/xive.c +3 -0
index f473e6ac77..712498d4af 100644
@@ -1080,6 +1080,9 @@ static int vmstate_xive_tctx_post_load(void *opaque, int version_id)
              error_report_err(local_err);
              return ret;
          }
+    } else {
+        uint8_t pipr = tctx->regs[TM_QW1_OS + TM_PIPR];
+        xive_tctx_pipr_set(tctx, TM_QW1_OS, pipr, 0);
      }
 
Quoted 1 line

hw/intc/xive.c static int vmstate_xive_tctx_post_load(void *opaque, int version_id) · line 1088

Quoted 1 line
hw/intc/xive.c +0 -0
      return 0;