Skip to contentSkip to conversation

vfio-ccw fixes for 5.20

PATCH Superseded Eric Farman 2 participants 7 messages Jul 28, 2022

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Eric Farman

author maintainer
#0 lore raw

v2 cover vfio-ccw fixes for 5.20

Matt, Alex,

Here is an updated series for the DMA UNMAP length problem. As before, it is built on Alex' vfio-next tree, and contains two small fixes identified while testing this.

Changelog:
v1->v2:
- [MR] Rework the boundary checking of page_array_iova_pinned, to compare against pfns instead of iovas.
- [MR] Add r-b to Patch 2 (Thank you!)
- [EF] (NEW) Noticed a missing return code check in the close routine
v1: https://lore.kernel.org/r/[email protected]/

Eric Farman (3):
vfio/ccw: Add length to DMA_UNMAP checks vfio/ccw: Remove FSM Close from remove handlers vfio/ccw: Check return code from subchannel quiesce

 drivers/s390/cio/vfio_ccw_cp.c  | 14 ++++++++++----
 drivers/s390/cio/vfio_ccw_cp.h  |  2 +-
 drivers/s390/cio/vfio_ccw_drv.c |  1 -
 drivers/s390/cio/vfio_ccw_fsm.c |  2 +-
 drivers/s390/cio/vfio_ccw_ops.c |  4 +---
 5 files changed, 13 insertions(+), 10 deletions(-)
Signature · 1 line
2.34.1
author maintainer
#1 lore raw

PATCH v2 1/3 vfio/ccw: Add length to DMA_UNMAP checks

As pointed out with the simplification of the VFIO_IOMMU_NOTIFY_DMA_UNMAP notifier [1], the length parameter was never used to check against the pinned pages.

Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.

[1] https://lore.kernel.org/kvm/[email protected]/

---
 drivers/s390/cio/vfio_ccw_cp.c  | 14 ++++++++++----
 drivers/s390/cio/vfio_ccw_cp.h  |  2 +-
 drivers/s390/cio/vfio_ccw_ops.c |  2 +-
 3 files changed, 12 insertions(+), 6 deletions(-)
drivers/s390/cio/vfio_ccw_cp.c +10 -4
index 8963f452f963..6202f1e3e792 100644
@@ -170,12 +170,17 @@ static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde
 	kfree(pa->pa_iova);
 }
 
-static bool page_array_iova_pinned(struct page_array *pa, unsigned long iova)
+static bool page_array_iova_pinned(struct page_array *pa, unsigned long iova,
+				   unsigned long length)
 {
+	unsigned long iova_pfn_start = iova >> PAGE_SHIFT;
+	unsigned long iova_pfn_end = (iova + length - 1) >> PAGE_SHIFT;
+	unsigned long pfn;
 	int i;
 
 	for (i = 0; i < pa->pa_nr; i++)
-		if (pa->pa_iova[i] == iova)
+		pfn = pa->pa_iova[i] >> PAGE_SHIFT;
+		if (pfn >= iova_pfn_start && pfn <= iova_pfn_end)
 			return true;
 
 	return false;
@@ -899,11 +904,12 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw)
  * cp_iova_pinned() - check if an iova is pinned for a ccw chain.
  * @cp: channel_program on which to perform the operation
  * @iova: the iova to check
+ * @length: the length to check from @iova
  *
  * If the @iova is currently pinned for the ccw chain, return true;
  * else return false.
  */
-bool cp_iova_pinned(struct channel_program *cp, u64 iova)
+bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length)
 {
 	struct ccwchain *chain;
 	int i;
@@ -913,7 +919,7 @@ bool cp_iova_pinned(struct channel_program *cp, u64 iova)
 
 	list_for_each_entry(chain, &cp->ccwchain_list, next) {
 		for (i = 0; i < chain->ch_len; i++)
-			if (page_array_iova_pinned(chain->ch_pa + i, iova))
+			if (page_array_iova_pinned(chain->ch_pa + i, iova, length))
 				return true;
 	}
 
drivers/s390/cio/vfio_ccw_cp.h +1 -1
index 3194d887e08e..54d26e242533 100644
@@ -46,6 +46,6 @@ void cp_free(struct channel_program *cp);
 int cp_prefetch(struct channel_program *cp);
 union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm);
 void cp_update_scsw(struct channel_program *cp, union scsw *scsw);
-bool cp_iova_pinned(struct channel_program *cp, u64 iova);
+bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length);
 
 #endif
drivers/s390/cio/vfio_ccw_ops.c +1 -1
index 0047fd88f938..3f67fa103c7f 100644
@@ -39,7 +39,7 @@ static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
 		container_of(vdev, struct vfio_ccw_private, vdev);
 
 	/* Drivers MUST unpin pages in response to an invalidation. */
-	if (!cp_iova_pinned(&private->cp, iova))
+	if (!cp_iova_pinned(&private->cp, iova, length))
 		return;
 
 	vfio_ccw_mdev_reset(private);
Signature · 1 line
2.34.1
maintainer Reviewed-by on v2 1/3
#2 lore raw

Re: [PATCH v2 1/3] vfio/ccw: Add length to DMA_UNMAP checks

Quoted 5 lines · Eric Farman

As pointed out with the simplification of the VFIO_IOMMU_NOTIFY_DMA_UNMAP notifier [1], the length parameter was never used to check against the pinned pages.

Quoted 7 lines

Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.

[1] https://lore.kernel.org/kvm/[email protected]/

With one nit comment, feel free to change or not...

Quoted 11 lines
---
  drivers/s390/cio/vfio_ccw_cp.c  | 14 ++++++++++----
  drivers/s390/cio/vfio_ccw_cp.h  |  2 +-
  drivers/s390/cio/vfio_ccw_ops.c |  2 +-
  3 files changed, 12 insertions(+), 6 deletions(-)
drivers/s390/cio/vfio_ccw_cp.c +0 -0
index 8963f452f963..6202f1e3e792 100644

drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 178

Quoted 10 lines
drivers/s390/cio/vfio_ccw_cp.c +5 -1
@@ -170,12 +170,17 @@ static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde
  	kfree(pa->pa_iova);
  }
 
-static bool page_array_iova_pinned(struct page_array *pa, unsigned long iova)
+static bool page_array_iova_pinned(struct page_array *pa, unsigned long iova,
+				   unsigned long length)
  {
+	unsigned long iova_pfn_start = iova >> PAGE_SHIFT;
+	unsigned long iova_pfn_end = (iova + length - 1) >> PAGE_SHIFT;
+	unsigned long pfn;

^ why do we switch to unsigned longs here when the callers use u64 for length (and iova)? Maybe just stick with u64s?

I thought maybe it was something introduced by Nicolin's series but it looks like the old pfn_array_iova_pinned did the same thing.

Quoted 57 lines
drivers/s390/cio/vfio_ccw_cp.c +5 -3
  	int i;
 
  	for (i = 0; i < pa->pa_nr; i++)
-		if (pa->pa_iova[i] == iova)
+		pfn = pa->pa_iova[i] >> PAGE_SHIFT;
+		if (pfn >= iova_pfn_start && pfn <= iova_pfn_end)
  			return true;
 
  	return false;
@@ -899,11 +904,12 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw)
   * cp_iova_pinned() - check if an iova is pinned for a ccw chain.
   * @cp: channel_program on which to perform the operation
   * @iova: the iova to check
+ * @length: the length to check from @iova
   *
   * If the @iova is currently pinned for the ccw chain, return true;
   * else return false.
   */
-bool cp_iova_pinned(struct channel_program *cp, u64 iova)
+bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length)
  {
  	struct ccwchain *chain;
  	int i;
@@ -913,7 +919,7 @@ bool cp_iova_pinned(struct channel_program *cp, u64 iova)
 
  	list_for_each_entry(chain, &cp->ccwchain_list, next) {
  		for (i = 0; i < chain->ch_len; i++)
-			if (page_array_iova_pinned(chain->ch_pa + i, iova))
+			if (page_array_iova_pinned(chain->ch_pa + i, iova, length))
  				return true;
  	}
 
drivers/s390/cio/vfio_ccw_cp.h +1 -1
index 3194d887e08e..54d26e242533 100644
@@ -46,6 +46,6 @@ void cp_free(struct channel_program *cp);
  int cp_prefetch(struct channel_program *cp);
  union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm);
  void cp_update_scsw(struct channel_program *cp, union scsw *scsw);
-bool cp_iova_pinned(struct channel_program *cp, u64 iova);
+bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length);
 
  #endif
drivers/s390/cio/vfio_ccw_ops.c +1 -1
index 0047fd88f938..3f67fa103c7f 100644
@@ -39,7 +39,7 @@ static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
  		container_of(vdev, struct vfio_ccw_private, vdev);
 
  	/* Drivers MUST unpin pages in response to an invalidation. */
-	if (!cp_iova_pinned(&private->cp, iova))
+	if (!cp_iova_pinned(&private->cp, iova, length))
  		return;
 
  	vfio_ccw_mdev_reset(private);
author maintainer
#3 lore raw

Re: [PATCH v2 1/3] vfio/ccw: Add length to DMA_UNMAP checks

Quoted 49 lines · Matthew Rosato
Quoted 5 lines · Eric Farman

As pointed out with the simplification of the VFIO_IOMMU_NOTIFY_DMA_UNMAP notifier [1], the length parameter was never used to check against the pinned pages.

Quoted 8 lines

Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.

[1]
https://lore.kernel.org/kvm/[email protected]/

With one nit comment, feel free to change or not...

Quoted 20 lines
---
  drivers/s390/cio/vfio_ccw_cp.c  | 14 ++++++++++----
  drivers/s390/cio/vfio_ccw_cp.h  |  2 +-
  drivers/s390/cio/vfio_ccw_ops.c |  2 +-
  3 files changed, 12 insertions(+), 6 deletions(-)
drivers/s390/cio/vfio_ccw_cp.c +0 -0
index 8963f452f963..6202f1e3e792 100644
@@ -170,12 +170,17 @@ static void page_array_unpin_free(struct

page_array *pa, struct vfio_device *vde
kfree(pa->pa_iova);
}

-static bool page_array_iova_pinned(struct page_array *pa, unsigned long iova)
+static bool page_array_iova_pinned(struct page_array *pa, unsigned long iova,

drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 178

Quoted 5 lines
drivers/s390/cio/vfio_ccw_cp.c +4 -0
+				   unsigned long length)
  {
+	unsigned long iova_pfn_start = iova >> PAGE_SHIFT;
+	unsigned long iova_pfn_end = (iova + length - 1) >> PAGE_SHIFT;
+	unsigned long pfn;

^ why do we switch to unsigned longs here when the callers use u64 for
length (and iova)? Maybe just stick with u64s?

Quoted 3 lines

I thought maybe it was something introduced by Nicolin's series but it
looks like the old pfn_array_iova_pinned did the same thing.

Heh, been that way since the first vfio-ccw commit. Other places that use iova (CCW fetching, etc.) carry the u64 label, so this does jump out now that you mention it.

I'll send a quick v3 because...

Quoted 8 lines

drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 184

Quoted 8 lines

int i;

for (i = 0; i < pa->pa_nr; i++)

drivers/s390/cio/vfio_ccw_cp.c +2 -1
-		if (pa->pa_iova[i] == iova)
+		pfn = pa->pa_iova[i] >> PAGE_SHIFT;
+		if (pfn >= iova_pfn_start && pfn <= iova_pfn_end)
  			return true;

...this loop needs to be bracketed, and got lost in my sync between systems.

Quoted 62 lines
Quoted 62 lines

return false;

drivers/s390/cio/vfio_ccw_cp.c +2 -1
@@ -899,11 +904,12 @@ void cp_update_scsw(struct channel_program
 *cp, union scsw *scsw)
   * cp_iova_pinned() - check if an iova is pinned for a ccw chain.
   * @cp: channel_program on which to perform the operation
   * @iova: the iova to check
+ * @length: the length to check from @iova
   *
   * If the @iova is currently pinned for the ccw chain, return
 true;
   * else return false.
   */
-bool cp_iova_pinned(struct channel_program *cp, u64 iova)
+bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64
 length)
  {
  	struct ccwchain *chain;
  	int i;
drivers/s390/cio/vfio_ccw_cp.c +1 -1
@@ -913,7 +919,7 @@ bool cp_iova_pinned(struct channel_program *cp,
 u64 iova)
 
  	list_for_each_entry(chain, &cp->ccwchain_list, next) {
  		for (i = 0; i < chain->ch_len; i++)
-			if (page_array_iova_pinned(chain->ch_pa + i,
 iova))
+			if (page_array_iova_pinned(chain->ch_pa + i,
 iova, length))
  				return true;
  	}
drivers/s390/cio/vfio_ccw_cp.h +1 -1
index 3194d887e08e..54d26e242533 100644
@@ -46,6 +46,6 @@ void cp_free(struct channel_program *cp);
  int cp_prefetch(struct channel_program *cp);
  union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8
 lpm);
  void cp_update_scsw(struct channel_program *cp, union scsw
 *scsw);
-bool cp_iova_pinned(struct channel_program *cp, u64 iova);
+bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64

length);

  #endif
drivers/s390/cio/vfio_ccw_ops.c +1 -1
index 0047fd88f938..3f67fa103c7f 100644
@@ -39,7 +39,7 @@ static void vfio_ccw_dma_unmap(struct vfio_device
 *vdev, u64 iova, u64 length)
  		container_of(vdev, struct vfio_ccw_private, vdev);
 
  	/* Drivers MUST unpin pages in response to an invalidation. */
-	if (!cp_iova_pinned(&private->cp, iova))
+	if (!cp_iova_pinned(&private->cp, iova, length))
  		return;
 
  	vfio_ccw_mdev_reset(private);
author maintainer
#4 lore raw

PATCH v2 2/3 vfio/ccw: Remove FSM Close from remove handlers

In reply to Eric Farman

Now that neither vfio_ccw_sch_probe() nor vfio_ccw_mdev_probe() affect the FSM state, it doesn't make sense for their _remove() counterparts try to revert things in this way. Since the FSM open and close are handled alongside MDEV open/close, these are unnecessary.

---
 drivers/s390/cio/vfio_ccw_drv.c | 1 -
 drivers/s390/cio/vfio_ccw_ops.c | 2 --
 2 files changed, 3 deletions(-)
drivers/s390/cio/vfio_ccw_drv.c +0 -1
index 4804101ccb0f..86d9e428357b 100644
@@ -241,7 +241,6 @@ static void vfio_ccw_sch_remove(struct subchannel *sch)
 {
 	struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
 
-	vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
 	mdev_unregister_device(&sch->dev);
 
 	dev_set_drvdata(&sch->dev, NULL);
drivers/s390/cio/vfio_ccw_ops.c +0 -2
index 3f67fa103c7f..4a806a2273b5 100644
@@ -130,8 +130,6 @@ static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
 
 	vfio_unregister_group_dev(&private->vdev);
 
-	vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
-
 	vfio_uninit_group_dev(&private->vdev);
 	atomic_inc(&private->avail);
 }
Signature · 1 line
2.34.1
author maintainer
#5 lore raw

PATCH v2 3/3 vfio/ccw: Check return code from subchannel quiesce

In reply to Eric Farman

If a subchannel is busy when a close is performed, the subchannel needs to be quiesced and left nice and tidy, so nothing unexpected (like a solicited interrupt) shows up while in the closed state. Unfortunately, the return code from this call isn't checked, so any busy subchannel is treated as a failing one.

Fix that, so that the close on a busy subchannel happens normally.

---
 drivers/s390/cio/vfio_ccw_fsm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/s390/cio/vfio_ccw_fsm.c +1 -1
index 4b8b623df24f..a59c758869f8 100644
@@ -407,7 +407,7 @@ static void fsm_close(struct vfio_ccw_private *private,
 
 	ret = cio_disable_subchannel(sch);
 	if (ret == -EBUSY)
-		vfio_ccw_sch_quiesce(sch);
+		ret = vfio_ccw_sch_quiesce(sch);
 	if (ret)
 		goto err_unlock;
 
Signature · 1 line
2.34.1
maintainer Reviewed-by on v2 3/3
#6 lore raw

Re: [PATCH v2 3/3] vfio/ccw: Check return code from subchannel quiesce

Quoted 9 lines · Eric Farman

If a subchannel is busy when a close is performed, the subchannel needs to be quiesced and left nice and tidy, so nothing unexpected (like a solicited interrupt) shows up while in the closed state. Unfortunately, the return code from this call isn't checked, so any busy subchannel is treated as a failing one.

Fix that, so that the close on a busy subchannel happens normally.

Quoted 17 lines
---
  drivers/s390/cio/vfio_ccw_fsm.c | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
drivers/s390/cio/vfio_ccw_fsm.c +1 -1
index 4b8b623df24f..a59c758869f8 100644
@@ -407,7 +407,7 @@ static void fsm_close(struct vfio_ccw_private *private,
 
  	ret = cio_disable_subchannel(sch);
  	if (ret == -EBUSY)
-		vfio_ccw_sch_quiesce(sch);
+		ret = vfio_ccw_sch_quiesce(sch);
  	if (ret)
  		goto err_unlock;