iommu/s390: Fix race with release_device ops
Subsystems (4)
Matthew Rosato
maintainerPATCH v3 1/1 iommu/s390: Fix race with release_device ops
With commit fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to the release_device op as it may occur at the same time as an opposing attach_dev or detach_dev since the group mutex is not held over release_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.
Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only release_device handles the re-init of the device DMA.
---
Changes since v2:
- Relocate the list_empty and aperture chekcs in attach_dev to the their original locations so they are all done under a single acquisition of the spinlock (Heiko)
--- arch/s390/include/asm/pci.h | 1 + arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 39 ++++++++++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 3 deletions(-)
arch/s390/include/asm/pci.h +1 -0
@@ -157,6 +157,7 @@ struct zpci_dev {/* DMA stuff */ unsigned long *dma_table;spinlock_t dma_table_lock; +struct mutex dma_domain_lock;/* protects s390_domain value */ int tlb_refresh;spinlock_t iommu_bitmap_lock;
arch/s390/pci/pci.c +1 -0
@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)kref_init (&zdev->kref);mutex_init (&zdev->lock);mutex_init (&zdev->kzdev_lock); +mutex_init (&zdev->dma_domain_lock); rc =zpci_init_iommu (zdev);if (rc)
drivers/iommu/s390-iommu.c +36 -3
@@ -99,6 +99,14 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,if (!domain_device)return -ENOMEM ; +/* Leave now if the device has already been released */ +mutex_lock (&zdev->dma_domain_lock); +if (!dev_iommu_priv_get (dev)) { +mutex_unlock (&zdev->dma_domain_lock); +kfree (domain_device); +return 0 ; + } +if (zdev->dma_table && !zdev->s390_domain) { cc =zpci_dma_exit_device (zdev);if (cc) {
@@ -132,9 +140,10 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, goto out_restore; } domain_device->zdev = zdev; - zdev->s390_domain = s390_domain;list_add (&domain_device->list, &s390_domain->devices);spin_unlock_irqrestore (&s390_domain->list_lock, flags); + zdev->s390_domain = s390_domain; +mutex_unlock (&zdev->dma_domain_lock);return 0 ;
@@ -147,6 +156,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, virt_to_phys(zdev->dma_table)); } out_free: +mutex_unlock (&zdev->dma_domain_lock);kfree (domain_device);return rc;
@@ -176,17 +186,22 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, }spin_unlock_irqrestore (&s390_domain->list_lock, flags); -if (found && (zdev->s390_domain == s390_domain)) { +mutex_lock (&zdev->dma_domain_lock); +if (found && (zdev->s390_domain == s390_domain) && +dev_iommu_priv_get (dev)) { zdev->s390_domain =NULL ;zpci_unregister_ioat (zdev,0 );zpci_dma_init_device (zdev); } +mutex_unlock (&zdev->dma_domain_lock); }static struct iommu_device *s390_iommu_probe_device (struct device *dev) {struct zpci_dev *zdev =to_zpci_dev (dev); +dev_iommu_priv_set (dev, zdev); +return &zdev->iommu_dev; }
@@ -206,10 +221,28 @@ static void s390_iommu_release_device(struct device *dev)* * So let's call detach_dev from here if it hasn't been called before. */ -if (zdev && zdev->s390_domain) { +if (zdev) { +/* +* Clear priv to block further attaches for this device, +* ensure detaches don't init DMA. Hold the domain lock +* to ensure that attach/detach get a consistent view of +* whether or not the device is released. +*/ +mutex_lock (&zdev->dma_domain_lock); +dev_iommu_priv_set (dev,NULL ); +mutex_unlock (&zdev->dma_domain_lock); +/* Make sure this device is removed from the domain list */ domain =iommu_get_domain_for_dev (dev);if (domain)s390_iommu_detach_device (domain, dev); +/* Now ensure DMA is initialized from here */ +mutex_lock (&zdev->dma_domain_lock); +if (zdev->s390_domain) { + zdev->s390_domain =NULL ; +zpci_unregister_ioat (zdev,0 ); +zpci_dma_init_device (zdev); + } +mutex_unlock (&zdev->dma_domain_lock); } }
Signature · 1 line
2.37.2
Quoted 121 lines · Matthew Rosato
With commit
fa7e9ecc5e1c("iommu/s390: Tolerate repeatattach_devcalls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to therelease_deviceop as it may occur at the same time as an opposingattach_devordetach_devsince the group mutex is not held overrelease_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only
release_devicehandles the re-init of the device DMA.Fixes fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls")Signed-off-by Matthew Rosato <[email protected]>---
Changes since v2:
- Relocate thelist_emptyand aperture chekcs inattach_devto the their original locations so they are all done under a single acquisition of the spinlock (Heiko)--- arch/s390/include/asm/pci.h | 1 + arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 39 ++++++++++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 3 deletions(-)arch/s390/include/asm/pci.h +1
-0@@ -157,6 +157,7 @@ struct zpci_dev {/* DMA stuff */ unsigned long *dma_table;spinlock_t dma_table_lock; +struct mutex dma_domain_lock;/* protects s390_domain value */ int tlb_refresh;spinlock_t iommu_bitmap_lock;arch/s390/pci/pci.c +1
-0@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)kref_init (&zdev->kref);mutex_init (&zdev->lock);mutex_init (&zdev->kzdev_lock); +mutex_init (&zdev->dma_domain_lock); rc =zpci_init_iommu (zdev);if (rc)drivers/iommu/s390-iommu.c +19
-3@@ -99,6 +99,14 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,if (!domain_device)return -ENOMEM ; +/* Leave now if the device has already been released */ +mutex_lock (&zdev->dma_domain_lock); +if (!dev_iommu_priv_get (dev)) { +mutex_unlock (&zdev->dma_domain_lock); +kfree (domain_device); +return 0 ; + } +if (zdev->dma_table && !zdev->s390_domain) { cc =zpci_dma_exit_device (zdev);if (cc) {@@ -132,9 +140,10 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, goto out_restore; } domain_device->zdev = zdev; - zdev->s390_domain = s390_domain;list_add (&domain_device->list, &s390_domain->devices);spin_unlock_irqrestore (&s390_domain->list_lock, flags); + zdev->s390_domain = s390_domain; +mutex_unlock (&zdev->dma_domain_lock);return 0 ;@@ -147,6 +156,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, virt_to_phys(zdev->dma_table)); } out_free: +mutex_unlock (&zdev->dma_domain_lock);kfree (domain_device);return rc;@@ -176,17 +186,22 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, }spin_unlock_irqrestore (&s390_domain->list_lock, flags); -if (found && (zdev->s390_domain == s390_domain)) { +mutex_lock (&zdev->dma_domain_lock); +if (found && (zdev->s390_domain == s390_domain) && +dev_iommu_priv_get (dev)) { zdev->s390_domain =NULL ;zpci_unregister_ioat (zdev,0 );zpci_dma_init_device (zdev); } +mutex_unlock (&zdev->dma_domain_lock); }static struct iommu_device *s390_iommu_probe_device (struct device *dev) {struct zpci_dev *zdev =to_zpci_dev (dev); +dev_iommu_priv_set (dev, zdev); +return &zdev->iommu_dev; }@@ -206,10 +221,28 @@ static void s390_iommu_release_device(struct device *dev)* * So let's call detach_dev from here if it hasn't been called before. */ -if (zdev && zdev->s390_domain) { +if (zdev) { +/*
drivers/iommu/s390-iommu.c static void s390_iommu_release_device(struct device *dev) · line 235
Quoted 10 lines
drivers/iommu/s390-iommu.c +9
-0+* Clear priv to block further attaches for this device, +* ensure detaches don't init DMA. Hold the domain lock +* to ensure that attach/detach get a consistent view of +* whether or not the device is released. +*/ +mutex_lock (&zdev->dma_domain_lock); +dev_iommu_priv_set (dev,NULL ); +mutex_unlock (&zdev->dma_domain_lock); +/* Make sure this device is removed from the domain list */ domain =iommu_get_domain_for_dev (dev);
Shouldn't you take the group_mutex before calling directly s390_iommu_detach_device from here?
Quoted 18 lines
drivers/iommu/s390-iommu.c +8
-0if (domain)s390_iommu_detach_device (domain, dev); +/* Now ensure DMA is initialized from here */ +mutex_lock (&zdev->dma_domain_lock); +if (zdev->s390_domain) { + zdev->s390_domain =NULL ; +zpci_unregister_ioat (zdev,0 ); +zpci_dma_init_device (zdev); + } +mutex_unlock (&zdev->dma_domain_lock); } }
Signature · 2 lines
Pierre Morel IBM Lab Boeblingen
Matthew Rosato
maintainerQuoted 119 lines · Pierre Morel
Quoted 116 lines · Matthew Rosato
With commit
fa7e9ecc5e1c("iommu/s390: Tolerate repeatattach_devcalls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to therelease_deviceop as it may occur at the same time as an opposingattach_devordetach_devsince the group mutex is not held over release_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only
release_devicehandles the re-init of the device DMA.Fixes fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls")Signed-off-by Matthew Rosato <[email protected]>---
Changes since v2:
- Relocate thelist_emptyand aperture chekcs inattach_devto the their original locations so they are all done under a single acquisition of the spinlock (Heiko)
---
arch/s390/include/asm/pci.h| 1 +
arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 39 ++++++++++++++++++++++++++++++++++---3 files changed, 38 insertions(+), 3 deletions(-)arch/s390/include/asm/pci.h +1
-0@@ -157,6 +157,7 @@ struct zpci_dev { /* DMA stuff */ unsigned long *dma_table; spinlock_t dma_table_lock; + struct mutex dma_domain_lock; /* protects s390_domain value */ int tlb_refresh; spinlock_t iommu_bitmap_lock;arch/s390/pci/pci.c +1
-0@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) kref_init(&zdev->kref); mutex_init(&zdev->lock); mutex_init(&zdev->kzdev_lock); + mutex_init(&zdev->dma_domain_lock); rc = zpci_init_iommu(zdev); if (rc)drivers/iommu/s390-iommu.c +0
-0@@ -99,6 +99,14 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, if (!domain_device) return -ENOMEM;+ /* Leave now if the device has already been released */ +
mutex_lock(&zdev->dma_domain_lock);
+ if (!dev_iommu_priv_get(dev)) {
+mutex_unlock(&zdev->dma_domain_lock);
+kfree(domain_device);
+ return 0;
+ }
+
if (zdev->dma_table&& !zdev->s390_domain) { cc =zpci_dma_exit_device(zdev);
if (cc) {drivers/iommu/s390-iommu.c +2
-1@@ -132,9 +140,10 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, goto out_restore; } domain_device->zdev = zdev; - zdev->s390_domain = s390_domain; list_add(&domain_device->list, &s390_domain->devices); spin_unlock_irqrestore(&s390_domain->list_lock, flags); + zdev->s390_domain = s390_domain; + mutex_unlock(&zdev->dma_domain_lock); return 0; @@ -147,6 +156,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, virt_to_phys(zdev->dma_table));}
out_free:
+mutex_unlock(&zdev->dma_domain_lock);
kfree(domain_device);
return rc;drivers/iommu/s390-iommu.c +5
-0@@ -176,17 +186,22 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, } spin_unlock_irqrestore(&s390_domain->list_lock, flags); - if (found && (zdev->s390_domain == s390_domain)) { + mutex_lock(&zdev->dma_domain_lock); + if (found && (zdev->s390_domain == s390_domain) && + dev_iommu_priv_get(dev)) { zdev->s390_domain = NULL; zpci_unregister_ioat(zdev, 0); zpci_dma_init_device(zdev); } + mutex_unlock(&zdev->dma_domain_lock); } static struct iommu_device *s390_iommu_probe_device(struct device *dev) { struct zpci_dev *zdev = to_zpci_dev(dev); + dev_iommu_priv_set(dev, zdev); + return &zdev->iommu_dev; } @@ -206,10 +221,28 @@ static void s390_iommu_release_device(struct device *dev) * * So let's call detach_dev from here if it hasn't been called before.*/
- if (zdev &&zdev->s390_domain) {
+ if (zdev) {
+ /*
+ * Clear priv to block further attaches for this device, + * ensure detaches don't init DMA. Hold the domain lock + * to ensure that attach/detach get a consistent view of + * whether or not the device is released.
+ */
Quoted 9 lines
Quoted 5 lines
+
mutex_lock(&zdev->dma_domain_lock);
+dev_iommu_priv_set(dev, NULL);
+mutex_unlock(&zdev->dma_domain_lock);
+ /* Make sure this device is removed from the domain list */ domain =iommu_get_domain_for_dev(dev);Shouldn't you take the
group_mutexbefore calling directlys390_iommu_detach_devicefrom here?
Well, we can't do that directly as the group mutex is only accessible to the iommu core. But perhaps the right answer is to call back into the core here by just replacing the existing call to s390_iommu_detach_device below with a call to iommu_detach_device (1 line change). Then we still ensure DMA is initialized next after this point. I've been running tests with such a change for the last few hours and this works well.
But this also made me notice another subtle issue re: a mismatch between the number of kallocs and kfrees for s390_domain_device. Need to investigate that further before a next version.
Quoted 19 lines
Quoted 18 lines
if (domain)
s390_iommu_detach_device(domain, dev);+ /* Now ensure DMA is initialized from here */ +
mutex_lock(&zdev->dma_domain_lock); + if (zdev->s390_domain) {
+zdev->s390_domain= NULL;
+zpci_unregister_ioat(zdev, 0);
+zpci_dma_init_device(zdev);
+ }
+mutex_unlock(&zdev->dma_domain_lock); }
}
Matthew Rosato
maintainerQuoted 130 lines · Matthew Rosato
Quoted 119 lines · Pierre Morel
Quoted 116 lines · Matthew Rosato
With commit
fa7e9ecc5e1c("iommu/s390: Tolerate repeatattach_devcalls") s390-iommu is supposed to handle dynamic switching between IOMMU domains and the DMA API handling. However, this commit does not sufficiently handle the case where the device is released via a call to therelease_deviceop as it may occur at the same time as an opposingattach_devordetach_devsince the group mutex is not held over release_device. This was observed if the device is deconfigured during a small window during vfio-pci initialization and can result in WARNs and potential kernel panics.Handle this by tracking when the device is probed/released via dev_iommu_priv_set/get(). Ensure that once the device is released only
release_devicehandles the re-init of the device DMA.Fixes fa7e9ecc5e1c ("iommu/s390: Tolerate repeat attach_dev calls")Signed-off-by Matthew Rosato <[email protected]>---
Changes since v2:
- Relocate thelist_emptyand aperture chekcs inattach_devto the their original locations so they are all done under a single acquisition of the spinlock (Heiko)
---
arch/s390/include/asm/pci.h| 1 +
arch/s390/pci/pci.c | 1 + drivers/iommu/s390-iommu.c | 39 ++++++++++++++++++++++++++++++++++---3 files changed, 38 insertions(+), 3 deletions(-)arch/s390/include/asm/pci.h +1
-0@@ -157,6 +157,7 @@ struct zpci_dev { /* DMA stuff */ unsigned long *dma_table; spinlock_t dma_table_lock; + struct mutex dma_domain_lock; /* protects s390_domain value */ int tlb_refresh; spinlock_t iommu_bitmap_lock;arch/s390/pci/pci.c +1
-0@@ -832,6 +832,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) kref_init(&zdev->kref); mutex_init(&zdev->lock); mutex_init(&zdev->kzdev_lock); + mutex_init(&zdev->dma_domain_lock); rc = zpci_init_iommu(zdev); if (rc)drivers/iommu/s390-iommu.c +0
-0@@ -99,6 +99,14 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, if (!domain_device) return -ENOMEM;+ /* Leave now if the device has already been released */ +
mutex_lock(&zdev->dma_domain_lock);
+ if (!dev_iommu_priv_get(dev)) {
+mutex_unlock(&zdev->dma_domain_lock);
+kfree(domain_device);
+ return 0;
+ }
+
if (zdev->dma_table&& !zdev->s390_domain) { cc =zpci_dma_exit_device(zdev);
if (cc) {drivers/iommu/s390-iommu.c +2
-1@@ -132,9 +140,10 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, goto out_restore; } domain_device->zdev = zdev; - zdev->s390_domain = s390_domain; list_add(&domain_device->list, &s390_domain->devices); spin_unlock_irqrestore(&s390_domain->list_lock, flags); + zdev->s390_domain = s390_domain; + mutex_unlock(&zdev->dma_domain_lock); return 0; @@ -147,6 +156,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, virt_to_phys(zdev->dma_table));}
out_free:
+mutex_unlock(&zdev->dma_domain_lock);
kfree(domain_device);
return rc;drivers/iommu/s390-iommu.c +5
-0@@ -176,17 +186,22 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, } spin_unlock_irqrestore(&s390_domain->list_lock, flags); - if (found && (zdev->s390_domain == s390_domain)) { + mutex_lock(&zdev->dma_domain_lock); + if (found && (zdev->s390_domain == s390_domain) && + dev_iommu_priv_get(dev)) { zdev->s390_domain = NULL; zpci_unregister_ioat(zdev, 0); zpci_dma_init_device(zdev); } + mutex_unlock(&zdev->dma_domain_lock); } static struct iommu_device *s390_iommu_probe_device(struct device *dev) { struct zpci_dev *zdev = to_zpci_dev(dev); + dev_iommu_priv_set(dev, zdev); + return &zdev->iommu_dev; } @@ -206,10 +221,28 @@ static void s390_iommu_release_device(struct device *dev) * * So let's call detach_dev from here if it hasn't been called before.*/
- if (zdev &&zdev->s390_domain) {
+ if (zdev) {
+ /*
+ * Clear priv to block further attaches for this device, + * ensure detaches don't init DMA. Hold the domain lock + * to ensure that attach/detach get a consistent view of + * whether or not the device is released.
+ */Quoted 9 lines
Quoted 5 lines
+
mutex_lock(&zdev->dma_domain_lock);
+dev_iommu_priv_set(dev, NULL);
+mutex_unlock(&zdev->dma_domain_lock);
+ /* Make sure this device is removed from the domain list */ domain =iommu_get_domain_for_dev(dev);Shouldn't you take the
group_mutexbefore calling directlys390_iommu_detach_devicefrom here?
Quoted 2 lines
Well, we can't do that directly as the group mutex is only accessible to the iommu core. But perhaps the right answer is to call back into the core here by just replacing the existing call to
s390_iommu_detach_devicebelow with a call toiommu_detach_device(1 line change). Then we still ensure DMA is initialized next after this point. I've been running tests with such a change for the last few hours and this works well.
Nope, looking closer that won't work -- There is a small window between calling iommu_get_domain_for_dev and then the subsequent call to iommu_detach_device() where the group->domain could still change, which would trigger a WARN_ON in iommu_detach_device (I managed to trigger this once).
But really, I think just leaving this as a call to s390_iommu_detach_device without the group_mutex held at this point is OK, all the detach will do at this point (since priv has already been nullified) is get the list_lock and clear the entry for the domain that happened to be the group domain at the time release_device started - and any attempts at further attaches will just quietly return 0 because priv is null. I guess technically we could also inline that list_del work into release_device and not even call s390_iommu_detach_device. But the reality is by the time we've reached this point the device is going away and so it's going to have to be removed from all associated domains very soon, not just the one that happens to be active -- So this combined with my 2nd patch (sending shortly as a v4 series) would ensure the device gets removed from each domain, either from a detach_dev or a domain_free call.
Quoted 22 lines
But this also made me notice another subtle issue re: a mismatch between the number of kallocs and kfrees for
s390_domain_device. Need to investigate that further before a next version.Quoted 19 lines
Quoted 18 lines
if (domain)
s390_iommu_detach_device(domain, dev);+ /* Now ensure DMA is initialized from here */ +
mutex_lock(&zdev->dma_domain_lock); + if (zdev->s390_domain) {
+zdev->s390_domain= NULL;
+zpci_unregister_ioat(zdev, 0);
+zpci_dma_init_device(zdev);
+ }
+mutex_unlock(&zdev->dma_domain_lock); }
}