iommu/vt-d: Don't queue_iova() if there is no flush queue
Subsystems (3)
PATCH 1/2 iommu/vt-d: Don't queue_iova() if there is no flush queue
Intel VT-d driver was reworked to use common deferred flushing implementation. Previously there was one global per-cpu flush queue, afterwards - one per domain.
Before deferring a flush, the queue should be allocated and initialized.
Currently only domains with IOMMU_DOMAIN_DMA type initialize their flush queue. It's probably worth to init it for static or unmanaged domains too, but it may be arguable - I'm leaving it to iommu folks.
Prevent queuing an iova flush if the domain doesn't have a queue. The defensive check seems to be worth to keep even if queue would be initialized for all kinds of domains. And is easy backportable.
On 4.19.43 stable kernel it has a user-visible effect: previously for devices in si domain there were crashes, on sata devices:
BUG: spinlock bad magic on CPU#6, swapper/0/1 lock: 0xffff88844f582008, .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0 CPU: 6 PID: 1 Comm: swapper/0 Not tainted 4.19.43 #1 Call Trace: <IRQ> dump_stack+0x61/0x7e spin_bug+0x9d/0xa3 do_raw_spin_lock+0x22/0x8e _raw_spin_lock_irqsave+0x32/0x3a queue_iova+0x45/0x115 intel_unmap+0x107/0x113 intel_unmap_sg+0x6b/0x76 __ata_qc_complete+0x7f/0x103 ata_qc_complete+0x9b/0x26a ata_qc_complete_multiple+0xd0/0xe3 ahci_handle_port_interrupt+0x3ee/0x48a ahci_handle_port_intr+0x73/0xa9 ahci_single_level_irq_intr+0x40/0x60 __handle_irq_event_percpu+0x7f/0x19a handle_irq_event_percpu+0x32/0x72 handle_irq_event+0x38/0x56 handle_edge_irq+0x102/0x121 handle_irq+0x147/0x15c do_IRQ+0x66/0xf2 common_interrupt+0xf/0xf RIP: 0010:__do_softirq+0x8c/0x2df
The same for usb devices that use ehci-pci:
BUG: spinlock bad magic on CPU#0, swapper/0/1 lock: 0xffff88844f402008, .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.19.43 #4 Call Trace: <IRQ> dump_stack+0x61/0x7e spin_bug+0x9d/0xa3 do_raw_spin_lock+0x22/0x8e _raw_spin_lock_irqsave+0x32/0x3a queue_iova+0x77/0x145 intel_unmap+0x107/0x113 intel_unmap_page+0xe/0x10 usb_hcd_unmap_urb_setup_for_dma+0x53/0x9d usb_hcd_unmap_urb_for_dma+0x17/0x100 unmap_urb_for_dma+0x22/0x24 __usb_hcd_giveback_urb+0x51/0xc3 usb_giveback_urb_bh+0x97/0xde tasklet_action_common.isra.4+0x5f/0xa1 tasklet_action+0x2d/0x30 __do_softirq+0x138/0x2df irq_exit+0x7d/0x8b smp_apic_timer_interrupt+0x10f/0x151 apic_timer_interrupt+0xf/0x20 </IRQ> RIP: 0010:_raw_spin_unlock_irqrestore+0x17/0x39
--- drivers/iommu/intel-iommu.c | 3 ++- drivers/iommu/iova.c | 18 ++++++++++++++---- include/linux/iova.h | 6 ++++++ 3 files changed, 22 insertions(+), 5 deletions(-)
drivers/iommu/intel-iommu.c +2 -1
@@ -3564,7 +3564,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) freelist =domain_unmap (domain, start_pfn, last_pfn); -if (intel_iommu_strict || (pdev && pdev->untrusted)) { +if (intel_iommu_strict || (pdev && pdev->untrusted) || + !has_iova_flush_queue (&domain->iovad)) {iommu_flush_iotlb_psi (iommu, domain, start_pfn, nrpages, !freelist,0 );/* free iova */
drivers/iommu/iova.c +14 -4
@@ -54,9 +54,14 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, }EXPORT_SYMBOL_GPL (init_iova_domain); +bool has_iova_flush_queue (struct iova_domain *iovad) +{ +return !!iovad->fq; +} +static void free_iova_flush_queue (struct iova_domain *iovad) { -if (!iovad->fq) +if (!has_iova_flush_queue (iovad))return ;if (timer_pending (&iovad->fq_timer))
@@ -74,13 +79,14 @@ static void free_iova_flush_queue(struct iova_domain *iovad)int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb,iova_entry_dtor entry_dtor) { +struct iova_fq __percpu *queue;int cpu;atomic64_set (&iovad->fq_flush_start_cnt,0 );atomic64_set (&iovad->fq_flush_finish_cnt,0 ); - iovad->fq = alloc_percpu(struct iova_fq); -if (!iovad->fq) + queue = alloc_percpu(struct iova_fq); +if (!queue)return -ENOMEM ; iovad->flush_cb = flush_cb;
@@ -89,13 +95,17 @@ int init_iova_flush_queue(struct iova_domain *iovad,for_each_possible_cpu (cpu) {struct iova_fq *fq; - fq =per_cpu_ptr (iovad->fq, cpu); + fq =per_cpu_ptr (queue, cpu); fq->head =0 ; fq->tail =0 ;spin_lock_init (&fq->lock); } +smp_wmb (); + + iovad->fq = queue; +timer_setup (&iovad->fq_timer, fq_flush_timeout,0 );atomic_set (&iovad->fq_timer_on,0 );
include/linux/iova.h +6 -0
@@ -155,6 +155,7 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,void copy_reserved_iova (struct iova_domain *from,struct iova_domain *to);void init_iova_domain (struct iova_domain *iovad,unsigned long granule,unsigned long start_pfn); +bool has_iova_flush_queue (struct iova_domain *iovad);int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb,iova_entry_dtor entry_dtor);struct iova *find_iova (struct iova_domain *iovad,unsigned long pfn);
@@ -235,6 +236,11 @@ static inline void init_iova_domain(struct iova_domain *iovad, { } +bool has_iova_flush_queue (struct iova_domain *iovad) +{ +return false; +} +static inline int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb,iova_entry_dtor entry_dtor)
Signature · 6 lines
2.22.0 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
PATCH 2/2 iommu/vt-d: Check if domain->pgd was allocated
There is a couple of places where on domain_init() failure domain_exit() is called. While currently domain_init() can fail only if alloc_pgtable_page() has failed.
Make domain_exit() check if domain->pgd present, before calling domain_unmap(), as it theoretically should crash on clearing pte entries in dma_pte_clear_level().
--- drivers/iommu/intel-iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
drivers/iommu/intel-iommu.c +5 -3
@@ -1835,7 +1835,6 @@ static inline int guestwidth_to_adjustwidth(int gaw)static void domain_exit (struct dmar_domain *domain) { -struct page *freelist;/* Remove associated devices and clear attached or cached domains */ domain_remove_dev_info (domain);
@@ -1843,9 +1842,12 @@ static void domain_exit(struct dmar_domain *domain)/* destroy iovas */ put_iova_domain (&domain->iovad); - freelist =domain_unmap (domain,0 ,DOMAIN_MAX_PFN (domain->gaw)); +if (domain->pgd) { +struct page *freelist; -dma_free_pagelist (freelist); + freelist =domain_unmap (domain,0 ,DOMAIN_MAX_PFN (domain->gaw)); +dma_free_pagelist (freelist); + }free_domain_mem (domain); }
Signature · 6 lines
2.22.0 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH 2/2] iommu/vt-d: Check if domain->pgd was allocated
Hi,
Quoted 11 lines · Dmitry Safonov
There is a couple of places where on
domain_init()failuredomain_exit()is called. While currentlydomain_init()can fail only ifalloc_pgtable_page()has failed.Make
domain_exit()check ifdomain->pgdpresent, before callingdomain_unmap(), as it theoretically should crash on clearing pte entries indma_pte_clear_level().Cc David Woodhouse <[email protected]>Cc Joerg Roedel <[email protected]>Cc Lu Baolu <[email protected]>
This looks good to me. Thank you!
Best regards,
Baolu
Quoted 34 lines
Signed-off-by Dmitry Safonov <[email protected]>--- drivers/iommu/intel-iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)drivers/iommu/intel-iommu.c +5
-3@@ -1835,7 +1835,6 @@ static inline int guestwidth_to_adjustwidth(int gaw)static void domain_exit (struct dmar_domain *domain) { -struct page *freelist;/* Remove associated devices and clear attached or cached domains */ domain_remove_dev_info (domain);@@ -1843,9 +1842,12 @@ static void domain_exit(struct dmar_domain *domain)/* destroy iovas */ put_iova_domain (&domain->iovad); - freelist =domain_unmap (domain,0 ,DOMAIN_MAX_PFN (domain->gaw)); +if (domain->pgd) { +struct page *freelist; -dma_free_pagelist (freelist); + freelist =domain_unmap (domain,0 ,DOMAIN_MAX_PFN (domain->gaw)); +dma_free_pagelist (freelist); + }free_domain_mem (domain); }
Sasha Levin
maintainerHi,
[This is an automated email]
This commit has been processed because it contains a "Fixes:" tag, fixing commit: 13cf01744608 iommu/vt-d: Make use of iova deferred flushing.
The bot has tested the following trees: v5.2.1, v5.1.18, v4.19.59, v4.14.133.
v5.2.1: Build OK!
v5.1.18: Build OK!
v4.19.59: Failed to apply! Possible dependencies:
02b4da5f84d1 ("intel-iommu: mark intel_dma_ops static") 0bbeb01a4faf ("iommu/vt-d: Manage scalalble mode PASID tables") 524a669bdd5f ("iommu/vt-d: remove the mapping_error dma_map_ops method") 932a6523ce39 ("iommu/vt-d: Use dev_printk() when possible") 964f2311a686 ("iommu/intel: small map_page cleanup") ef848b7e5a6a ("iommu/vt-d: Setup pasid entry for RID2PASID support") f7b0c4ce8cb3 ("iommu/vt-d: Flush IOTLB for untrusted device in time")
v4.14.133: Failed to apply! Possible dependencies:
0bbeb01a4faf ("iommu/vt-d: Manage scalalble mode PASID tables")
2e2e35d51279 ("iommu/vt-d: Missing checks for pasid tables if allocation fails")
2f13eb7c580f ("iommu/vt-d: Enable 5-level paging mode in the PASID entry")
3e781fcafedb ("iommu/vt-d: Remove unnecessary WARN_ON()")
4774cc524570 ("iommu/vt-d: Apply per pci device pasid table in SVA")
4fa064b26c2e ("iommu/vt-d: Clear pasid table entry when memory unbound")
524a669bdd5f ("iommu/vt-d: remove the mapping_error dma_map_ops method")
562831747f62 ("iommu/vt-d: Global PASID name space")
7ec916f82c48 ("Revert "iommu/intel-iommu: Enable CONFIG_DMA_DIRECT_OPS=y and clean up intel_{alloc,free}_coherent()"") 85319dcc8955 ("iommu/vt-d: Add for_each_device_domain() helper")
932a6523ce39 ("iommu/vt-d: Use dev_printk() when possible")
964f2311a686 ("iommu/intel: small map_page cleanup")
971401015d14 ("iommu/vt-d: Use real PASID for flush in caching mode")
9ddbfb42138d ("iommu/vt-d: Move device_domain_info to header")
a7fc93fed94b ("iommu/vt-d: Allocate and free pasid table")
af39507305fb ("iommu/vt-d: Apply global PASID in SVA")
be9e6598aeb0 ("iommu/vt-d: Handle memory shortage on pasid table allocation")
cc580e41260d ("iommu/vt-d: Per PCI device pasid table interfaces")
d657c5c73ca9 ("iommu/intel-iommu: Enable CONFIG_DMA_DIRECT_OPS=y and clean up intel_{alloc,free}_coherent()") ef848b7e5a6a ("iommu/vt-d: Setup pasid entry for RID2PASID support")
f7b0c4ce8cb3 ("iommu/vt-d: Flush IOTLB for untrusted device in time")
NOTE: The patch will not be queued to stable trees until it is upstream.
How should we proceed with this patch?
--
Thanks,
Sasha
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Hi,
Quoted 9 lines · Dmitry Safonov
Intel VT-d driver was reworked to use common deferred flushing implementation. Previously there was one global per-cpu flush queue, afterwards - one per domain.
Before deferring a flush, the queue should be allocated and initialized.
Currently only domains with
IOMMU_DOMAIN_DMAtype initialize their flush queue. It's probably worth to init it for static or unmanaged domains too, but it may be arguable - I'm leaving it to iommu folks.
We will submit per-device dma ops soon. Then we don't need to call intel_unmap() for the identity (static) domain. For unmanaged domains, the map/unmap happen only during VM startup/shutdown, I am not sure whether it's worth a flush queue.
This fix looks good to me anyway. We should always avoid deferring a flush if there's no flush queue there.
Best regards,
Baolu
Quoted 170 lines
Prevent queuing an iova flush if the domain doesn't have a queue. The defensive check seems to be worth to keep even if queue would be initialized for all kinds of domains. And is easy backportable.
On 4.19.43 stable kernel it has a user-visible effect: previously for devices in si domain there were crashes, on sata devices:
BUG: spinlock bad magic on CPU#6, swapper/0/1 lock: 0xffff88844f582008, .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0 CPU: 6 PID: 1 Comm: swapper/0 Not tainted 4.19.43 #1 Call Trace: <IRQ> dump_stack+0x61/0x7e spin_bug+0x9d/0xa3 do_raw_spin_lock+0x22/0x8e _raw_spin_lock_irqsave+0x32/0x3a queue_iova+0x45/0x115 intel_unmap+0x107/0x113 intel_unmap_sg+0x6b/0x76 __ata_qc_complete+0x7f/0x103 ata_qc_complete+0x9b/0x26a ata_qc_complete_multiple+0xd0/0xe3 ahci_handle_port_interrupt+0x3ee/0x48a ahci_handle_port_intr+0x73/0xa9 ahci_single_level_irq_intr+0x40/0x60 __handle_irq_event_percpu+0x7f/0x19a handle_irq_event_percpu+0x32/0x72 handle_irq_event+0x38/0x56 handle_edge_irq+0x102/0x121 handle_irq+0x147/0x15c do_IRQ+0x66/0xf2 common_interrupt+0xf/0xf RIP: 0010:__do_softirq+0x8c/0x2dfThe same for usb devices that use ehci-pci:
BUG: spinlock bad magic on CPU#0, swapper/0/1 lock: 0xffff88844f402008, .magic: 00000000, .owner: <none>/-1, .owner_cpu: 0 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.19.43 #4 Call Trace: <IRQ> dump_stack+0x61/0x7e spin_bug+0x9d/0xa3 do_raw_spin_lock+0x22/0x8e _raw_spin_lock_irqsave+0x32/0x3a queue_iova+0x77/0x145 intel_unmap+0x107/0x113 intel_unmap_page+0xe/0x10 usb_hcd_unmap_urb_setup_for_dma+0x53/0x9d usb_hcd_unmap_urb_for_dma+0x17/0x100 unmap_urb_for_dma+0x22/0x24 __usb_hcd_giveback_urb+0x51/0xc3 usb_giveback_urb_bh+0x97/0xde tasklet_action_common.isra.4+0x5f/0xa1 tasklet_action+0x2d/0x30 __do_softirq+0x138/0x2df irq_exit+0x7d/0x8b smp_apic_timer_interrupt+0x10f/0x151 apic_timer_interrupt+0xf/0x20 </IRQ> RIP: 0010:_raw_spin_unlock_irqrestore+0x17/0x39Cc David Woodhouse <[email protected]>Cc Joerg Roedel <[email protected]>Cc Lu Baolu <[email protected]>Cc <[email protected]> # 4.14+Fixes 13cf01744608 ("iommu/vt-d: Make use of iova deferred flushing")Signed-off-by Dmitry Safonov <[email protected]>--- drivers/iommu/intel-iommu.c | 3 ++- drivers/iommu/iova.c | 18 ++++++++++++++---- include/linux/iova.h | 6 ++++++ 3 files changed, 22 insertions(+), 5 deletions(-)drivers/iommu/intel-iommu.c +2
-1@@ -3564,7 +3564,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) freelist =domain_unmap (domain, start_pfn, last_pfn); -if (intel_iommu_strict || (pdev && pdev->untrusted)) { +if (intel_iommu_strict || (pdev && pdev->untrusted) || + !has_iova_flush_queue (&domain->iovad)) {iommu_flush_iotlb_psi (iommu, domain, start_pfn, nrpages, !freelist,0 );/* free iova */ drivers/iommu/iova.c +14
-4@@ -54,9 +54,14 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, }EXPORT_SYMBOL_GPL (init_iova_domain); +bool has_iova_flush_queue (struct iova_domain *iovad) +{ +return !!iovad->fq; +} +static void free_iova_flush_queue (struct iova_domain *iovad) { -if (!iovad->fq) +if (!has_iova_flush_queue (iovad))return ;if (timer_pending (&iovad->fq_timer))@@ -74,13 +79,14 @@ static void free_iova_flush_queue(struct iova_domain *iovad)int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb,iova_entry_dtor entry_dtor) { +struct iova_fq __percpu *queue;int cpu;atomic64_set (&iovad->fq_flush_start_cnt,0 );atomic64_set (&iovad->fq_flush_finish_cnt,0 ); - iovad->fq = alloc_percpu(struct iova_fq); -if (!iovad->fq) + queue = alloc_percpu(struct iova_fq); +if (!queue)return -ENOMEM ; iovad->flush_cb = flush_cb;@@ -89,13 +95,17 @@ int init_iova_flush_queue(struct iova_domain *iovad,for_each_possible_cpu (cpu) {struct iova_fq *fq; - fq =per_cpu_ptr (iovad->fq, cpu); + fq =per_cpu_ptr (queue, cpu); fq->head =0 ; fq->tail =0 ;spin_lock_init (&fq->lock); } +smp_wmb (); + + iovad->fq = queue; +timer_setup (&iovad->fq_timer, fq_flush_timeout,0 );atomic_set (&iovad->fq_timer_on,0 );include/linux/iova.h +6
-0@@ -155,6 +155,7 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,void copy_reserved_iova (struct iova_domain *from,struct iova_domain *to);void init_iova_domain (struct iova_domain *iovad,unsigned long granule,unsigned long start_pfn); +bool has_iova_flush_queue (struct iova_domain *iovad);int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb,iova_entry_dtor entry_dtor);struct iova *find_iova (struct iova_domain *iovad,unsigned long pfn);@@ -235,6 +236,11 @@ static inline void init_iova_domain(struct iova_domain *iovad, { } +bool has_iova_flush_queue (struct iova_domain *iovad) +{ +return false; +} +static inline int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb,iova_entry_dtor entry_dtor)
Quoted 1 line · Dmitry Safonov
BUG: spinlock bad magic on CPU#6, swapper/0/1
Applied both patches to iommu/fixes.
Joerg Roedel
maintainerinclude/linux/iova.h static inline void init_iova_domain(struct iova_domain *iovad, · line 243
Quoted 9 lines · Dmitry Safonov
include/linux/iova.h +5
-0@@ -235,6 +236,11 @@ static inline void init_iova_domain(struct iova_domain *iovad, { } +bool has_iova_flush_queue (struct iova_domain *iovad) +{ +return false; +} +
This needs to be 'static inline', I queued a patch on-top of my fixes branch.
Regards,
Joerg
Quoted 7 lines · On Tue 23 Jul 2019, 9:17 a.m. Joerg Roedel, <[email protected]> wrote:
Review static inline void init_iova_domain(struct
Quoted 1 line · Dmitry Safonov
(quoted diff) +0
-0@@ -235,6 +236,11 @@ static inline void init_iova_domain(struct
iova_domain*iovad,Quoted 3 lines
(quoted diff) +0
-0{ }
Quoted 8 lines
include/linux/iova.h static inline void init_iova_domain(struct iova_domain *iovad, · line 243
Quoted 5 lines
include/linux/iova.h +5
-0+bool has_iova_flush_queue (struct iova_domain *iovad) +{ +return false; +} +This needs to be 'static inline', I queued a patch on-top of my fixes branch.
Ugh, copy-n-paste declaration.
Thanks much,
Dmitry
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu