Skip to contentSkip to conversation

pvh/dom0: fix deadlock in GSI mapping

PATCH In review Roger Pau Monne 4 participants 6 messages Jan 28, 2019 – Jan 29, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 pvh/dom0: fix deadlock in GSI mapping

The current GSI mapping code can cause the following deadlock:

(XEN) *** Dumping CPU0 host state: ***
(XEN) ----[ Xen-4.12.0-rc x86_64 debug=y Tainted: C ]----
[...]
(XEN) Xen call trace:

(XEN)    [<ffff82d080239852>] vmac.c#_spin_lock_cb+0x32/0x70
(XEN)    [<ffff82d0802ed40f>] vmac.c#hvm_gsi_assert+0x2f/0x60 <- pick hvm.irq_lock
(XEN)    [<ffff82d080255cc9>] io.c#hvm_dirq_assist+0xd9/0x130 <- pick event_lock
(XEN)    [<ffff82d080255b4b>] io.c#dpci_softirq+0xdb/0x120
(XEN)    [<ffff82d080238ce6>] softirq.c#__do_softirq+0x46/0xa0
(XEN)    [<ffff82d08026f955>] domain.c#idle_loop+0x35/0x90

(XEN)
[...]
(XEN) *** Dumping CPU3 host state: ***
(XEN) ----[ Xen-4.12.0-rc x86_64 debug=y Tainted: C ]----
[...]
(XEN) Xen call trace:

(XEN)    [<ffff82d08023985d>] vmac.c#_spin_lock_cb+0x3d/0x70
(XEN)    [<ffff82d080281fc8>] vmac.c#allocate_and_map_gsi_pirq+0xc8/0x130 <- pick event_lock
(XEN)    [<ffff82d0802f44c0>] vioapic.c#vioapic_hwdom_map_gsi+0x80/0x130
(XEN)    [<ffff82d0802f4399>] vioapic.c#vioapic_write_redirent+0x119/0x1c0 <- pick hvm.irq_lock
(XEN)    [<ffff82d0802f4075>] vioapic.c#vioapic_write+0x35/0x40
(XEN)    [<ffff82d0802e96a2>] vmac.c#hvm_process_io_intercept+0xd2/0x230
(XEN)    [<ffff82d0802e9842>] vmac.c#hvm_io_intercept+0x22/0x50
(XEN)    [<ffff82d0802dbe9b>] emulate.c#hvmemul_do_io+0x21b/0x3c0
(XEN)    [<ffff82d0802db302>] emulate.c#hvmemul_do_io_buffer+0x32/0x70
(XEN)    [<ffff82d0802dcd29>] emulate.c#hvmemul_do_mmio_buffer+0x29/0x30
(XEN)    [<ffff82d0802dcc19>] emulate.c#hvmemul_phys_mmio_access+0xf9/0x1b0
(XEN)    [<ffff82d0802dc6d0>] emulate.c#hvmemul_linear_mmio_access+0xf0/0x180
(XEN)    [<ffff82d0802de971>] emulate.c#hvmemul_linear_mmio_write+0x21/0x30
(XEN)    [<ffff82d0802de742>] emulate.c#linear_write+0xa2/0x100
(XEN)    [<ffff82d0802dce15>] emulate.c#hvmemul_write+0xb5/0x120
(XEN)    [<ffff82d0802babba>] vmac.c#x86_emulate+0x132aa/0x149a0
(XEN)    [<ffff82d0802c04f9>] vmac.c#x86_emulate_wrapper+0x29/0x70
(XEN)    [<ffff82d0802db570>] emulate.c#_hvm_emulate_one+0x50/0x140
(XEN)    [<ffff82d0802e9e31>] vmac.c#hvm_emulate_one_insn+0x41/0x100
(XEN)    [<ffff82d080345066>] guest_4.o#sh_page_fault__guest_4+0x976/0xd30
(XEN)    [<ffff82d08030cc69>] vmac.c#vmx_vmexit_handler+0x949/0xea0
(XEN)    [<ffff82d08031411a>] vmac.c#vmx_asm_vmexit_handler+0xfa/0x270

In order to solve it move the vioapic_hwdom_map_gsi outside of the locked region in vioapic_write_redirent. vioapic_hwdom_map_gsi will not access any of the vioapic fields, so there's no need to call the function holding the hvm.irq_lock.

---
Cc: Jan Beulich <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: Juergen Gross <[email protected]>
---
This is a bugfix that only touches PVH dom0 code, hence I think it's safe to go in 4.12.

---
 xen/arch/x86/hvm/vioapic.c | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)
xen/arch/x86/hvm/vioapic.c +18 -14
index 2b74f92d51..2d71c33c1c 100644
@@ -236,20 +236,6 @@ static void vioapic_write_redirent(
 
     *pent = ent;
 
-    if ( is_hardware_domain(d) && unmasked )
-    {
-        int ret;
-
-        ret = vioapic_hwdom_map_gsi(gsi, ent.fields.trig_mode,
-                                    ent.fields.polarity);
-        if ( ret )
-        {
-            /* Mask the entry again. */
-            pent->fields.mask = 1;
-            unmasked = 0;
-        }
-    }
-
     if ( gsi == 0 )
     {
         vlapic_adjust_i8259_target(d);
@@ -266,6 +252,24 @@ static void vioapic_write_redirent(
 
     spin_unlock(&d->arch.hvm.irq_lock);
 
+    if ( is_hardware_domain(d) && unmasked )
+    {
+        /*
+         * NB: don't call vioapic_hwdom_map_gsi while holding hvm.irq_lock
+         * since it can cause deadlocks as event_lock is taken by
+         * allocate_and_map_gsi_pirq, and that will invert the locking order
+         * used by other parts of the code.
+         */
+        int ret = vioapic_hwdom_map_gsi(gsi, ent.fields.trig_mode,
+                                        ent.fields.polarity);
+        if ( ret )
+        {
+            gprintk(XENLOG_ERR,
+                    "unable to bind gsi %u to hardware domain: %d\n", gsi, ret);
+            unmasked = 0;
+        }
+    }
+
     if ( gsi == 0 || unmasked )
         pt_may_unmask_irq(d, NULL);
 }
Signature · 7 lines
2.20.1


_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Reviewed-by on v1 1/1
#1 lore raw
Quoted 41 lines · Roger Pau Monne

The current GSI mapping code can cause the following deadlock:

(XEN) *** Dumping CPU0 host state: ***
(XEN) ----[ Xen-4.12.0-rc x86_64 debug=y Tainted: C ]----
[...]
(XEN) Xen call trace:

(XEN)    [<ffff82d080239852>] vmac.c#_spin_lock_cb+0x32/0x70
(XEN)    [<ffff82d0802ed40f>] vmac.c#hvm_gsi_assert+0x2f/0x60 <- pick hvm.irq_lock
(XEN)    [<ffff82d080255cc9>] io.c#hvm_dirq_assist+0xd9/0x130 <- pick event_lock
(XEN)    [<ffff82d080255b4b>] io.c#dpci_softirq+0xdb/0x120
(XEN)    [<ffff82d080238ce6>] softirq.c#__do_softirq+0x46/0xa0
(XEN)    [<ffff82d08026f955>] domain.c#idle_loop+0x35/0x90

(XEN)
[...]
(XEN) *** Dumping CPU3 host state: ***
(XEN) ----[ Xen-4.12.0-rc x86_64 debug=y Tainted: C ]----
[...]
(XEN) Xen call trace:

(XEN)    [<ffff82d08023985d>] vmac.c#_spin_lock_cb+0x3d/0x70
(XEN)    [<ffff82d080281fc8>] vmac.c#allocate_and_map_gsi_pirq+0xc8/0x130 <- pick event_lock
(XEN)    [<ffff82d0802f44c0>] vioapic.c#vioapic_hwdom_map_gsi+0x80/0x130
(XEN)    [<ffff82d0802f4399>] vioapic.c#vioapic_write_redirent+0x119/0x1c0 <- pick hvm.irq_lock
(XEN)    [<ffff82d0802f4075>] vioapic.c#vioapic_write+0x35/0x40
(XEN)    [<ffff82d0802e96a2>] vmac.c#hvm_process_io_intercept+0xd2/0x230
(XEN)    [<ffff82d0802e9842>] vmac.c#hvm_io_intercept+0x22/0x50
(XEN)    [<ffff82d0802dbe9b>] emulate.c#hvmemul_do_io+0x21b/0x3c0
(XEN)    [<ffff82d0802db302>] emulate.c#hvmemul_do_io_buffer+0x32/0x70
(XEN)    [<ffff82d0802dcd29>] emulate.c#hvmemul_do_mmio_buffer+0x29/0x30
(XEN)    [<ffff82d0802dcc19>] emulate.c#hvmemul_phys_mmio_access+0xf9/0x1b0
(XEN)    [<ffff82d0802dc6d0>] emulate.c#hvmemul_linear_mmio_access+0xf0/0x180
(XEN)    [<ffff82d0802de971>] emulate.c#hvmemul_linear_mmio_write+0x21/0x30
(XEN)    [<ffff82d0802de742>] emulate.c#linear_write+0xa2/0x100
(XEN)    [<ffff82d0802dce15>] emulate.c#hvmemul_write+0xb5/0x120
(XEN)    [<ffff82d0802babba>] vmac.c#x86_emulate+0x132aa/0x149a0
(XEN)    [<ffff82d0802c04f9>] vmac.c#x86_emulate_wrapper+0x29/0x70
(XEN)    [<ffff82d0802db570>] emulate.c#_hvm_emulate_one+0x50/0x140
(XEN)    [<ffff82d0802e9e31>] vmac.c#hvm_emulate_one_insn+0x41/0x100
(XEN)    [<ffff82d080345066>] guest_4.o#sh_page_fault__guest_4+0x976/0xd30
(XEN)    [<ffff82d08030cc69>] vmac.c#vmx_vmexit_handler+0x949/0xea0
(XEN)    [<ffff82d08031411a>] vmac.c#vmx_asm_vmexit_handler+0xfa/0x270
Quoted 6 lines

In order to solve it move the vioapic_hwdom_map_gsi outside of the locked region in vioapic_write_redirent. vioapic_hwdom_map_gsi will not access any of the vioapic fields, so there's no need to call the function holding the hvm.irq_lock.

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

#2 lore raw
In reply to Roger Pau Monne
Quoted 5 lines
Quoted 1 line
Quoted 1 line · On 28.01.19 at 15:22, <[email protected]> wrote:

In order to solve it move the vioapic_hwdom_map_gsi outside of the locked region in vioapic_write_redirent. vioapic_hwdom_map_gsi will not access any of the vioapic fields, so there's no need to call the function holding the hvm.irq_lock.

True, but you also move the code across a vioapic_deliver() invocation. Is that delivery going to work when vioapic_hwdom_map_gsi() gets invoked only afterwards?

Jan

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

#3 lore raw
Quoted 9 lines · Jan Beulich
Quoted 5 lines
Quoted 1 line
Quoted 1 line · On 28.01.19 at 15:22, <[email protected]> wrote:

In order to solve it move the vioapic_hwdom_map_gsi outside of the locked region in vioapic_write_redirent. vioapic_hwdom_map_gsi will not access any of the vioapic fields, so there's no need to call the function holding the hvm.irq_lock.

True, but you also move the code across a vioapic_deliver() invocation. Is that delivery going to work when vioapic_hwdom_map_gsi() gets invoked only afterwards?

Yes, that vioapic_deliver will only get invoked when there's a pending gsi (hvm_irq->gsi_assert_count[idx] > 0), and that can only happen once the hardware gsi is bound to dom0 and the hvm.irq_lock has been released, so that the dpci softirq can increase the gsi assert count.

Thanks, Roger.

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

Reviewed-by
#4 lore raw
Quoted 8 lines
Quoted 1 line
Quoted 1 line · On 28.01.19 at 16:52, <[email protected]> wrote:
Quoted 6 lines · Jan Beulich
Quoted 5 lines
Quoted 1 line
Quoted 1 line · On 28.01.19 at 15:22, <[email protected]> wrote:

In order to solve it move the vioapic_hwdom_map_gsi outside of the locked region in vioapic_write_redirent. vioapic_hwdom_map_gsi will not access any of the vioapic fields, so there's no need to call the function holding the hvm.irq_lock.

Quoted 8 lines
Quoted 3 lines

True, but you also move the code across a vioapic_deliver() invocation. Is that delivery going to work when vioapic_hwdom_map_gsi() gets invoked only afterwards?

Yes, that vioapic_deliver will only get invoked when there's a pending gsi (hvm_irq->gsi_assert_count[idx] > 0), and that can only happen once the hardware gsi is bound to dom0 and the hvm.irq_lock has been released, so that the dpci softirq can increase the gsi assert count.

Oh, I had overlooked the -EEXIST early bail from vioapic_hwdom_map_gsi(), wrongly implying this could be a problem with other than the initial write of an RTE.

Jan

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

#5 lore raw
In reply to Roger Pau Monne
Quoted 41 lines · Roger Pau Monne

The current GSI mapping code can cause the following deadlock:

(XEN) *** Dumping CPU0 host state: ***
(XEN) ----[ Xen-4.12.0-rc x86_64 debug=y Tainted: C ]----
[...]
(XEN) Xen call trace:

(XEN)    [<ffff82d080239852>] vmac.c#_spin_lock_cb+0x32/0x70
(XEN)    [<ffff82d0802ed40f>] vmac.c#hvm_gsi_assert+0x2f/0x60 <- pick hvm.irq_lock
(XEN)    [<ffff82d080255cc9>] io.c#hvm_dirq_assist+0xd9/0x130 <- pick event_lock
(XEN)    [<ffff82d080255b4b>] io.c#dpci_softirq+0xdb/0x120
(XEN)    [<ffff82d080238ce6>] softirq.c#__do_softirq+0x46/0xa0
(XEN)    [<ffff82d08026f955>] domain.c#idle_loop+0x35/0x90

(XEN)
[...]
(XEN) *** Dumping CPU3 host state: ***
(XEN) ----[ Xen-4.12.0-rc x86_64 debug=y Tainted: C ]----
[...]
(XEN) Xen call trace:

(XEN)    [<ffff82d08023985d>] vmac.c#_spin_lock_cb+0x3d/0x70
(XEN)    [<ffff82d080281fc8>] vmac.c#allocate_and_map_gsi_pirq+0xc8/0x130 <- pick event_lock
(XEN)    [<ffff82d0802f44c0>] vioapic.c#vioapic_hwdom_map_gsi+0x80/0x130
(XEN)    [<ffff82d0802f4399>] vioapic.c#vioapic_write_redirent+0x119/0x1c0 <- pick hvm.irq_lock
(XEN)    [<ffff82d0802f4075>] vioapic.c#vioapic_write+0x35/0x40
(XEN)    [<ffff82d0802e96a2>] vmac.c#hvm_process_io_intercept+0xd2/0x230
(XEN)    [<ffff82d0802e9842>] vmac.c#hvm_io_intercept+0x22/0x50
(XEN)    [<ffff82d0802dbe9b>] emulate.c#hvmemul_do_io+0x21b/0x3c0
(XEN)    [<ffff82d0802db302>] emulate.c#hvmemul_do_io_buffer+0x32/0x70
(XEN)    [<ffff82d0802dcd29>] emulate.c#hvmemul_do_mmio_buffer+0x29/0x30
(XEN)    [<ffff82d0802dcc19>] emulate.c#hvmemul_phys_mmio_access+0xf9/0x1b0
(XEN)    [<ffff82d0802dc6d0>] emulate.c#hvmemul_linear_mmio_access+0xf0/0x180
(XEN)    [<ffff82d0802de971>] emulate.c#hvmemul_linear_mmio_write+0x21/0x30
(XEN)    [<ffff82d0802de742>] emulate.c#linear_write+0xa2/0x100
(XEN)    [<ffff82d0802dce15>] emulate.c#hvmemul_write+0xb5/0x120
(XEN)    [<ffff82d0802babba>] vmac.c#x86_emulate+0x132aa/0x149a0
(XEN)    [<ffff82d0802c04f9>] vmac.c#x86_emulate_wrapper+0x29/0x70
(XEN)    [<ffff82d0802db570>] emulate.c#_hvm_emulate_one+0x50/0x140
(XEN)    [<ffff82d0802e9e31>] vmac.c#hvm_emulate_one_insn+0x41/0x100
(XEN)    [<ffff82d080345066>] guest_4.o#sh_page_fault__guest_4+0x976/0xd30
(XEN)    [<ffff82d08030cc69>] vmac.c#vmx_vmexit_handler+0x949/0xea0
(XEN)    [<ffff82d08031411a>] vmac.c#vmx_asm_vmexit_handler+0xfa/0x270
Quoted 6 lines

In order to solve it move the vioapic_hwdom_map_gsi outside of the locked region in vioapic_write_redirent. vioapic_hwdom_map_gsi will not access any of the vioapic fields, so there's no need to call the function holding the hvm.irq_lock.

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel