iommu/s390: Fix race with release_device ops
Subsystems (4)
Matthew Rosato
maintainerPATCH 1/1 iommu/s390: Fix race with release_device ops
With commit fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to the release_device op as it may occur at the same time as an opposing attach_dev or detach_dev since the group mutex is not held over release_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.
Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only release_device handles the re-init of the device DMA.
--- arch/s390/include/asm/pci.h | 1 + arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 68 ++++++++++++++++++++++++++++--------- 3 files changed, 54 insertions(+), 16 deletions(-)
arch/s390/include/asm/pci.h +1 -0
@@ -157,6 +157,7 @@ struct zpci_dev {/* DMA stuff */ unsigned long *dma_table;spinlock_t dma_table_lock; +spinlock_t dma_domain_lock;int tlb_refresh;spinlock_t iommu_bitmap_lock;
arch/s390/pci/pci.c +1 -0
@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)kref_init (&zdev->kref);mutex_init (&zdev->lock);mutex_init (&zdev->kzdev_lock); +spin_lock_init (&zdev->dma_domain_lock); rc =zpci_init_iommu (zdev);if (rc)
drivers/iommu/s390-iommu.c +52 -16
128 lines not on this page · raw message
Signature · 1 line
2.31.1
Quoted 81 lines · Matthew Rosato
With commit
fa7e9ecc5e1c("iommu/s390: Tolerate repeatattach_devcalls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to therelease_deviceop as it may occur at the same time as an opposingattach_devordetach_devsince the group mutex is not held overrelease_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only
release_devicehandles the re-init of the device DMA.Fixes fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls")Signed-off-by Matthew Rosato <[email protected]>--- arch/s390/include/asm/pci.h | 1 + arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 68 ++++++++++++++++++++++++++++--------- 3 files changed, 54 insertions(+), 16 deletions(-)arch/s390/include/asm/pci.h +1
-0@@ -157,6 +157,7 @@ struct zpci_dev {/* DMA stuff */ unsigned long *dma_table;spinlock_t dma_table_lock; +spinlock_t dma_domain_lock;int tlb_refresh;spinlock_t iommu_bitmap_lock;arch/s390/pci/pci.c +1
-0@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)kref_init (&zdev->kref);mutex_init (&zdev->lock);mutex_init (&zdev->kzdev_lock); +spin_lock_init (&zdev->dma_domain_lock); rc =zpci_init_iommu (zdev);if (rc)drivers/iommu/s390-iommu.c +18
-1@@ -90,15 +90,39 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,struct zpci_dev *zdev =to_zpci_dev (dev);struct s390_domain_device *domain_device;unsigned long flags; -int cc, rc; +int cc, rc =0 ;if (!zdev)return -ENODEV ; +/* First check compatibility */ +spin_lock_irqsave (&s390_domain->list_lock, flags); +/* First device defines the DMA range limits */ +if (list_empty (&s390_domain->devices)) { + domain->geometry.aperture_start = zdev->start_dma; + domain->geometry.aperture_end = zdev->end_dma; + domain->geometry.force_aperture = true; +/* Allow only devices with identical DMA range limits */ + }else if (domain->geometry.aperture_start != zdev->start_dma || + domain->geometry.aperture_end != zdev->end_dma) { + rc = -EINVAL ; + } +spin_unlock_irqrestore (&s390_domain->list_lock, flags); +if (rc) +return rc; + domain_device =kzalloc (sizeof (*domain_device),GFP_KERNEL );if (!domain_device)return -ENOMEM ; +/* Leave now if the device has already been released */
drivers/iommu/s390-iommu.c static int s390_iommu_attach_device(struct iommu_domain *domain, · line 128
Quoted 10 lines
drivers/iommu/s390-iommu.c +7
-0+spin_lock_irqsave (&zdev->dma_domain_lock, flags); +if (!dev_iommu_priv_get (dev)) { +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags); +kfree (domain_device); +return 0 ; + } +if (zdev->dma_table && !zdev->s390_domain) { cc =zpci_dma_exit_device (zdev);if (cc) {
Am I wrong? It seems to me that zpci_dma_exit_device here is called with the spin_lock locked but this function zpci_dma_exit_device calls vfree which may sleep.
Quoted 78 lines
drivers/iommu/s390-iommu.c +18
-15@@ -117,22 +141,11 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, rc = -EIO ; goto out_restore; } + zdev->s390_domain = s390_domain; +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags);spin_lock_irqsave (&s390_domain->list_lock, flags); -/* First device defines the DMA range limits */ -if (list_empty (&s390_domain->devices)) { - domain->geometry.aperture_start = zdev->start_dma; - domain->geometry.aperture_end = zdev->end_dma; - domain->geometry.force_aperture = true; -/* Allow only devices with identical DMA range limits */ - }else if (domain->geometry.aperture_start != zdev->start_dma || - domain->geometry.aperture_end != zdev->end_dma) { - rc = -EINVAL ; -spin_unlock_irqrestore (&s390_domain->list_lock, flags); - goto out_restore; - } domain_device->zdev = zdev; - zdev->s390_domain = s390_domain;list_add (&domain_device->list, &s390_domain->devices);spin_unlock_irqrestore (&s390_domain->list_lock, flags);@@ -147,6 +160,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, virt_to_phys(zdev->dma_table)); } out_free: +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags);kfree (domain_device);return rc;@@ -176,17 +190,22 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, }spin_unlock_irqrestore (&s390_domain->list_lock, flags); -if (found && (zdev->s390_domain == s390_domain)) { +spin_lock_irqsave (&zdev->dma_domain_lock, flags); +if (found && (zdev->s390_domain == s390_domain) && +dev_iommu_priv_get (dev)) { zdev->s390_domain =NULL ;zpci_unregister_ioat (zdev,0 );zpci_dma_init_device (zdev); } +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags); }static struct iommu_device *s390_iommu_probe_device (struct device *dev) {struct zpci_dev *zdev =to_zpci_dev (dev); +dev_iommu_priv_set (dev, zdev); +return &zdev->iommu_dev; }@@ -194,6 +213,7 @@ static void s390_iommu_release_device(struct device *dev) {struct zpci_dev *zdev =to_zpci_dev (dev);struct iommu_domain *domain; +unsigned long flags;/* * This is a workaround for a scenario where the IOMMU API common code @@ -206,10 +226,26 @@ static void s390_iommu_release_device(struct device *dev)* * So let's call detach_dev from here if it hasn't been called before. */ -if (zdev && zdev->s390_domain) { +if (zdev) { +/* +* Clear priv to block further attaches for this device, +* ensure detaches don't init DMA +*/ +spin_lock_irqsave (&zdev->dma_domain_lock, flags); +dev_iommu_priv_set (dev,NULL ); +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags);
drivers/iommu/s390-iommu.c static void s390_iommu_release_device(struct device *dev) · line 246
Quoted 10 lines
drivers/iommu/s390-iommu.c +7
-0+/* Make sure this device is removed from the domain list */ domain =iommu_get_domain_for_dev (dev);if (domain)s390_iommu_detach_device (domain, dev); +/* Now ensure DMA is initialized from here */ +spin_lock_irqsave (&zdev->dma_domain_lock, flags); +if (zdev->s390_domain) { + zdev->s390_domain =NULL ; +zpci_unregister_ioat (zdev,0 ); +zpci_dma_init_device (zdev);
If I do not make a mistake, zpci_dma_init_device() calls vzalloc() and dma_alloc_cpu_table() which both could sleep.
Quoted 6 lines
drivers/iommu/s390-iommu.c +2
-0+ } +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags); } }
Signature · 2 lines
Pierre Morel IBM Lab Boeblingen
Matthew Rosato
maintainerQuoted 83 lines · Pierre Morel
Quoted 80 lines · Matthew Rosato
With commit
fa7e9ecc5e1c("iommu/s390: Tolerate repeatattach_devcalls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to therelease_deviceop as it may occur at the same time as an opposingattach_devordetach_devsince the group mutex is not held over release_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only
release_devicehandles the re-init of the device DMA.Fixes fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls")Signed-off-by Matthew Rosato <[email protected]>---
arch/s390/include/asm/pci.h| 1 +
arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 68 ++++++++++++++++++++++++++++---------3 files changed, 54 insertions(+), 16 deletions(-)arch/s390/include/asm/pci.h +1
-0@@ -157,6 +157,7 @@ struct zpci_dev { /* DMA stuff */ unsigned long *dma_table; spinlock_t dma_table_lock; + spinlock_t dma_domain_lock; int tlb_refresh; spinlock_t iommu_bitmap_lock;arch/s390/pci/pci.c +1
-0@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) kref_init(&zdev->kref); mutex_init(&zdev->lock); mutex_init(&zdev->kzdev_lock); + spin_lock_init(&zdev->dma_domain_lock); rc = zpci_init_iommu(zdev); if (rc)drivers/iommu/s390-iommu.c +1
-1@@ -90,15 +90,39 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, struct zpci_dev *zdev = to_zpci_dev(dev); struct s390_domain_device *domain_device; unsigned long flags; - int cc, rc; + int cc, rc = 0; if (!zdev) return -ENODEV;+ /* First check compatibility */
+spin_lock_irqsave(&s390_domain->list_lock, flags);
+ /* First device defines the DMA range limits */
+ if (list_empty(&s390_domain->devices)) {
+domain->geometry.aperture_start=zdev->start_dma; +domain->geometry.aperture_end=zdev->end_dma;
+domain->geometry.force_aperture= true;
+ /* Allow only devices with identical DMA range limits */ + } else if (domain->geometry.aperture_start!=zdev->start_dma|| +domain->geometry.aperture_end!=zdev->end_dma) { + rc =-EINVAL;
+ }
+spin_unlock_irqrestore(&s390_domain->list_lock, flags); + if (rc)
+ return rc;
+
domain_device=kzalloc(sizeof(*domain_device), GFP_KERNEL); if (!domain_device)
return-ENOMEM;
+ /* Leave now if the device has already been released */ +spin_lock_irqsave(&zdev->dma_domain_lock, flags);
+ if (!dev_iommu_priv_get(dev)) {
+spin_unlock_irqrestore(&zdev->dma_domain_lock, flags); +kfree(domain_device);
Quoted 9 lines
Quoted 6 lines
+ return 0;
+ }
+
if (zdev->dma_table&& !zdev->s390_domain) { cc =zpci_dma_exit_device(zdev); if (cc) {Am I wrong? It seems to me that
zpci_dma_exit_devicehere is called with thespin_locklocked but this functionzpci_dma_exit_devicecalls vfree which may sleep.
Oh, good point, I just enabled lockdep to verify that.
I think we could just replace this with a mutex instead, it's not a performance path. I've been running tests successfully today with this patch modified to instead use a mutex for dma_domain_lock.
Alexander Gordeev
maintainerQuoted 42 lines · Matthew Rosato
Quoted 32 lines
Quoted 32 lines
drivers/iommu/s390-iommu.c +1
-1@@ -90,15 +90,39 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, struct zpci_dev *zdev = to_zpci_dev(dev); struct s390_domain_device *domain_device; unsigned long flags; - int cc, rc; + int cc, rc = 0; if (!zdev) return -ENODEV;+ /* First check compatibility */
+spin_lock_irqsave(&s390_domain->list_lock, flags);
+ /* First device defines the DMA range limits */
+ if (list_empty(&s390_domain->devices)) {
+domain->geometry.aperture_start=zdev->start_dma; +domain->geometry.aperture_end=zdev->end_dma;
+domain->geometry.force_aperture= true;
+ /* Allow only devices with identical DMA range limits */ + } else if (domain->geometry.aperture_start!=zdev->start_dma|| +domain->geometry.aperture_end!=zdev->end_dma) { + rc =-EINVAL;
+ }
+spin_unlock_irqrestore(&s390_domain->list_lock, flags); + if (rc)
+ return rc;
+
domain_device=kzalloc(sizeof(*domain_device), GFP_KERNEL); if (!domain_device)
return-ENOMEM;
+ /* Leave now if the device has already been released */ +spin_lock_irqsave(&zdev->dma_domain_lock, flags);
+ if (!dev_iommu_priv_get(dev)) {
+spin_unlock_irqrestore(&zdev->dma_domain_lock, flags); +kfree(domain_device);Quoted 9 lines
Quoted 6 lines
+ return 0;
+ }
+
if (zdev->dma_table&& !zdev->s390_domain) { cc =zpci_dma_exit_device(zdev); if (cc) {Am I wrong? It seems to me that
zpci_dma_exit_devicehere is called with thespin_locklocked but this functionzpci_dma_exit_devicecalls vfree which may sleep.
Quoted 3 lines
Oh, good point, I just enabled lockdep to verify that.
I think we could just replace this with a mutex instead, it's not a performance path. I've been running tests successfully today with this patch modified to instead use a mutex for
dma_domain_lock.
But your original version uses irq-savvy spinlocks. Are there data that need to be protected against interrupts?
Thanks!
Niklas Schnelle
maintainerQuoted 43 lines · Alexander Gordeev
Quoted 42 lines · Matthew Rosato
Quoted 41 lines
Quoted 28 lines
drivers/iommu/s390-iommu.c +16
-1@@ -90,15 +90,39 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,struct zpci_dev *zdev =to_zpci_dev (dev);struct s390_domain_device *domain_device;unsigned long flags; -int cc, rc; +int cc, rc =0 ;if (!zdev)return -ENODEV ; +/* First check compatibility */ +spin_lock_irqsave (&s390_domain->list_lock, flags); +/* First device defines the DMA range limits */ +if (list_empty (&s390_domain->devices)) { + domain->geometry.aperture_start = zdev->start_dma; + domain->geometry.aperture_end = zdev->end_dma; + domain->geometry.force_aperture = true; +/* Allow only devices with identical DMA range limits */ + }else if (domain->geometry.aperture_start != zdev->start_dma || + domain->geometry.aperture_end != zdev->end_dma) { + rc = -EINVAL ; + } +spin_unlock_irqrestore (&s390_domain->list_lock, flags); +if (rc) +return rc; + domain_device =kzalloc (sizeof (*domain_device),GFP_KERNEL );if (!domain_device)return -ENOMEM ; + /* Leave now if the device has already been released */drivers/iommu/s390-iommu.c static int s390_iommu_attach_device(struct iommu_domain *domain, · line 125
Quoted 10 lines
drivers/iommu/s390-iommu.c +7
-0+spin_lock_irqsave (&zdev->dma_domain_lock, flags); +if (!dev_iommu_priv_get (dev)) { +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags); +kfree (domain_device); +return 0 ; + } +if (zdev->dma_table && !zdev->s390_domain) { cc =zpci_dma_exit_device (zdev);if (cc) {Am I wrong? It seems to me that
zpci_dma_exit_devicehere is called with thespin_locklocked but this functionzpci_dma_exit_devicecalls vfree which may sleep.
Quoted 8 lines
Quoted 3 lines
Oh, good point, I just enabled lockdep to verify that.
I think we could just replace this with a mutex instead, it's not a performance path. I've been running tests successfully today with this patch modified to instead use a mutex for
dma_domain_lock.But your original version uses irq-savvy spinlocks. Are there data that need to be protected against interrupts?
Thanks!
I think that was a carry over from my original attempt that used the zdev->dma_domain_lock in some more places including in interrupt context. I think these are gone now so I think Matt is right in his version this can be a mutex.
Robin Murphy
reviewerQuoted 51 lines · Niklas Schnelle
Quoted 50 lines · Alexander Gordeev
Quoted 42 lines · Matthew Rosato
Quoted 32 lines
Quoted 32 lines
drivers/iommu/s390-iommu.c +20
-1@@ -90,15 +90,39 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,struct zpci_dev *zdev =to_zpci_dev (dev);struct s390_domain_device *domain_device;unsigned long flags; -int cc, rc; +int cc, rc =0 ;if (!zdev)return -ENODEV ; +/* First check compatibility */ +spin_lock_irqsave (&s390_domain->list_lock, flags); +/* First device defines the DMA range limits */ +if (list_empty (&s390_domain->devices)) { + domain->geometry.aperture_start = zdev->start_dma; + domain->geometry.aperture_end = zdev->end_dma; + domain->geometry.force_aperture = true; +/* Allow only devices with identical DMA range limits */ + }else if (domain->geometry.aperture_start != zdev->start_dma || + domain->geometry.aperture_end != zdev->end_dma) { + rc = -EINVAL ; + } +spin_unlock_irqrestore (&s390_domain->list_lock, flags); +if (rc) +return rc; + domain_device =kzalloc (sizeof (*domain_device),GFP_KERNEL );if (!domain_device)return -ENOMEM ; +/* Leave now if the device has already been released */ +spin_lock_irqsave (&zdev->dma_domain_lock, flags); +if (!dev_iommu_priv_get (dev)) { +spin_unlock_irqrestore (&zdev->dma_domain_lock, flags); +kfree (domain_device);Quoted 9 lines
drivers/iommu/s390-iommu.c static int s390_iommu_attach_device(struct iommu_domain *domain, · line 128
Quoted 6 lines
drivers/iommu/s390-iommu.c +3
-0+return 0 ; + } +if (zdev->dma_table && !zdev->s390_domain) { cc =zpci_dma_exit_device (zdev);if (cc) {Am I wrong? It seems to me that
zpci_dma_exit_devicehere is called with thespin_locklocked but this functionzpci_dma_exit_devicecalls vfree which may sleep.Quoted 3 lines
Oh, good point, I just enabled lockdep to verify that.
I think we could just replace this with a mutex instead, it's not a performance path. I've been running tests successfully today with this patch modified to instead use a mutex for
dma_domain_lock.But your original version uses irq-savvy spinlocks. Are there data that need to be protected against interrupts?
Quoted 6 lines
Quoted 1 line
Thanks!
I think that was a carry over from my original attempt that used the
zdev->dma_domain_lockin some more places including in interrupt context. I think these are gone now so I think Matt is right in his version this can be a mutex.
Yes, probe/release/attach/detach should absolutely not be happening from atomic/IRQ context. At the very least, the IOMMU core itself needs to take the group mutex in those paths.
Cheers,
Robin.