vfio-ccw fixes for 5.20
Subsystems (4)
Eric Farman
maintainercover vfio-ccw fixes for 5.20
Matt, Alex, Jason,
Here's two vfio-ccw patches for 5.20, built on Alex' vfio-next tree (now that Nicolin's series has landed). One addresses the DMA unmap problem that was identified last week, and another was something I stumbled across while testing it.
Hopefully no big surprises in here; look forward to hearing what you think.
Eric Farman (2):
vfio/ccw: Add length to DMA_UNMAP checks
vfio/ccw: Remove FSM Close from remove handlers
drivers/s390/cio/vfio_ccw_cp.c | 11 +++++++---- drivers/s390/cio/vfio_ccw_cp.h | 2 +- drivers/s390/cio/vfio_ccw_drv.c | 1 - drivers/s390/cio/vfio_ccw_ops.c | 4 +--- 4 files changed, 9 insertions(+), 9 deletions(-)
Signature · 1 line
2.34.1
Eric Farman
maintainerPATCH 1/2 vfio/ccw: Add length to DMA_UNMAP checks
As pointed out with the simplification of the VFIO_IOMMU_NOTIFY_DMA_UNMAP notifier [1], the length parameter was never used to check against the pinned pages.
Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.
[1] https://lore.kernel.org/kvm/[email protected]/
--- drivers/s390/cio/vfio_ccw_cp.c | 11 +++++++---- drivers/s390/cio/vfio_ccw_cp.h | 2 +- drivers/s390/cio/vfio_ccw_ops.c | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-)
drivers/s390/cio/vfio_ccw_cp.c +7 -4
@@ -170,12 +170,14 @@ static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdekfree (pa->pa_iova); } -static bool page_array_iova_pinned (struct page_array *pa,unsigned long iova) +static bool page_array_iova_pinned (struct page_array *pa,unsigned long iova, +unsigned long length) {int i;for (i =0 ; i < pa->pa_nr; i++) -if (pa->pa_iova[i] == iova) +if (pa->pa_iova[i] >= iova && + pa->pa_iova[i] <= iova + length)return true;return false;
@@ -899,11 +901,12 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw)* cp_iova_pinned() - check if an iova is pinned for a ccw chain. * @cp: channel_program on which to perform the operation * @iova: the iova to check +* @length: the length to check from @iova * * If the @iova is currently pinned for the ccw chain, return true; * else return false. */ -bool cp_iova_pinned (struct channel_program *cp,u64 iova) +bool cp_iova_pinned (struct channel_program *cp,u64 iova,u64 length) {struct ccwchain *chain;int i;
@@ -913,7 +916,7 @@ bool cp_iova_pinned(struct channel_program *cp, u64 iova)list_for_each_entry (chain, &cp->ccwchain_list, next) {for (i =0 ; i < chain->ch_len; i++) -if (page_array_iova_pinned (chain->ch_pa + i, iova)) +if (page_array_iova_pinned (chain->ch_pa + i, iova, length))return true; }
drivers/s390/cio/vfio_ccw_cp.h +1 -1
@@ -46,6 +46,6 @@ void cp_free(struct channel_program *cp);int cp_prefetch (struct channel_program *cp);union orb *cp_get_orb (struct channel_program *cp,u32 intparm,u8 lpm);void cp_update_scsw (struct channel_program *cp,union scsw *scsw); -bool cp_iova_pinned (struct channel_program *cp,u64 iova); +bool cp_iova_pinned (struct channel_program *cp,u64 iova,u64 length);#endif
drivers/s390/cio/vfio_ccw_ops.c +1 -1
@@ -39,7 +39,7 @@ static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) container_of(vdev, struct vfio_ccw_private, vdev);/* Drivers MUST unpin pages in response to an invalidation. */ -if (!cp_iova_pinned (&private->cp, iova)) +if (!cp_iova_pinned (&private->cp, iova, length))return ;vfio_ccw_mdev_reset (private);
Signature · 1 line
2.34.1
Matthew Rosato
maintainerRe: [PATCH 1/2] vfio/ccw: Add length to DMA_UNMAP checks
Quoted 26 lines · Eric Farman
As pointed out with the simplification of the
VFIO_IOMMU_NOTIFY_DMA_UNMAPnotifier [1], the length parameter was never used to check against the pinned pages.Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.
[1] https://lore.kernel.org/kvm/[email protected]/
Signed-off-by Eric Farman <[email protected]>--- drivers/s390/cio/vfio_ccw_cp.c | 11 +++++++---- drivers/s390/cio/vfio_ccw_cp.h | 2 +- drivers/s390/cio/vfio_ccw_ops.c | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-)drivers/s390/cio/vfio_ccw_cp.c +0
-0@@ -170,12 +170,14 @@ static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdekfree (pa->pa_iova); }
drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 180
Quoted 10 lines
drivers/s390/cio/vfio_ccw_cp.c +4
-2-static bool page_array_iova_pinned (struct page_array *pa,unsigned long iova) +static bool page_array_iova_pinned (struct page_array *pa,unsigned long iova, +unsigned long length) {int i;for (i =0 ; i < pa->pa_nr; i++) -if (pa->pa_iova[i] == iova) +if (pa->pa_iova[i] >= iova && + pa->pa_iova[i] <= iova + length)
For the sake of completeness, I think you want to be checking to make sure the end of the page is also within the range, not just the start?
if (pa->pa_iova[i] >= iova &&
pa->pa_iova[i] + PAGE_SIZE <= iova + length)
Eric Farman
maintainerRe: [PATCH 1/2] vfio/ccw: Add length to DMA_UNMAP checks
Quoted 48 lines · Matthew Rosato
Quoted 33 lines · Eric Farman
As pointed out with the simplification of the
VFIO_IOMMU_NOTIFY_DMA_UNMAPnotifier [1], the length parameter was never used to check against the pinned pages.Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.
[1]
https://lore.kernel.org/kvm/[email protected]/Signed-off-by Eric Farman <[email protected]>--- drivers/s390/cio/vfio_ccw_cp.c | 11 +++++++---- drivers/s390/cio/vfio_ccw_cp.h | 2 +- drivers/s390/cio/vfio_ccw_ops.c | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-)drivers/s390/cio/vfio_ccw_cp.c +0
-0@@ -170,12 +170,14 @@ static void page_array_unpin_free(struct
page_array*pa, structvfio_device*vde
kfree(pa->pa_iova);
}-static bool
page_array_iova_pinned(struct page_array *pa, unsigned long iova)
+static bool page_array_iova_pinned(structpage_array*pa, unsigned long iova,drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 180
Quoted 8 lines
drivers/s390/cio/vfio_ccw_cp.c +3
-1+ unsigned long length) {int i;for (i =0 ; i < pa->pa_nr; i++) -if (pa->pa_iova[i] == iova) +if (pa->pa_iova[i] >= iova && + pa->pa_iova[i] <= iova + length)For the sake of completeness, I think you want to be checking to make
sure the end of the page is also within the range, not just the start?
Quoted 2 lines
if (pa->pa_iova[i] >= iova &&
pa->pa_iova[i] + PAGE_SIZE <= iova + length)
Well +PAGE_SIZE would iterate to the next page, so that would be captured on the next iteration of the for(i) loop if the pages were contiguous (or not applicable, if the pages weren't).
But, since the comment is really about the end of the page (0xfff), I guess I'm not understanding what that gets us so perhaps you could help elaborate your question? From my chair, since the pa_iova argument passed to vfio_pin_pages() pins the whole page, checking the start address versus the end (or anywhere in between) should still capture its interaction with an affected range. That is to say, we don't care about the -whole- page being within the unmap range, but -any- part of it.
Matthew Rosato
maintainerRe: [PATCH 1/2] vfio/ccw: Add length to DMA_UNMAP checks
Quoted 49 lines · Eric Farman
Quoted 48 lines · Matthew Rosato
Quoted 33 lines · Eric Farman
As pointed out with the simplification of the
VFIO_IOMMU_NOTIFY_DMA_UNMAPnotifier [1], the length parameter was never used to check against the pinned pages.Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.
[1]
https://lore.kernel.org/kvm/[email protected]/Signed-off-by Eric Farman <[email protected]>--- drivers/s390/cio/vfio_ccw_cp.c | 11 +++++++---- drivers/s390/cio/vfio_ccw_cp.h | 2 +- drivers/s390/cio/vfio_ccw_ops.c | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-)drivers/s390/cio/vfio_ccw_cp.c +0
-0@@ -170,12 +170,14 @@ static void page_array_unpin_free(struct
page_array*pa, structvfio_device*vdekfree(pa->pa_iova); }-static bool
page_array_iova_pinned(struct page_array *pa, unsigned long iova)
+static bool page_array_iova_pinned(structpage_array*pa, unsigned long iova,drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 180
Quoted 8 lines
drivers/s390/cio/vfio_ccw_cp.c +3
-1+ unsigned long length) {int i;for (i =0 ; i < pa->pa_nr; i++) -if (pa->pa_iova[i] == iova) +if (pa->pa_iova[i] >= iova && + pa->pa_iova[i] <= iova + length)For the sake of completeness, I think you want to be checking to make
sure the end of the page is also within the range, not just the start?
Quoted 6 lines
Quoted 2 lines
if (pa->pa_iova[i] >= iova &&
pa->pa_iova[i] + PAGE_SIZE <= iova + length)Well +PAGE_SIZE would iterate to the next page, so that would be captured on the next iteration of the for(i) loop if the pages were contiguous (or not applicable, if the pages weren't).
FWIW, the '+ PAGE_SIZE' was to match the '+ length' in your comparison.
If you really only want to only look at the start of the pa_iova being within the range, then I think you want 'pa->pa_iova[i] < iova + length', not <=.
Quoted 10 lines
But, since the comment is really about the end of the page (
0xfff), I guess I'm not understanding what that gets us so perhaps you could help elaborate your question? From my chair, since thepa_iovaargument passed tovfio_pin_pages()pins the whole page, checking the start address versus the end (or anywhere in between) should still capture its interaction with an affected range. That is to say, we don't care about the -whole- page being within the unmap range, but -any- part of it.
As far as my suggestion to also look at the end of the pa_iova[i] -- This was particularly geared at ensuring the entire page fell within the range, not just a subset. But I think you're right, we don't really care about that. On the flip side, do we care if the iova somehow starts sometime between pa_iova[i] and pa_iova[i] + PAGE_SIZE - 1? That would still be a subset, though I'm not sure such a thing could happen today (e.g. an input 'iova' that is not on a page boundary)..
I wonder if the simplest thing would be to just copy what gvt does and convert to pfn as it takes all of this out of the equation and looks instead at whether the inputs overlaps at a page granularity (which is what we really care about), e.g. something like (untested):
u64 iov_pfn = iova >> PAGE_SHIFT;
u64 end_iov_pfn = iov_pfn + (length / PAGE_SIZE);
u64 pfn;
int i;
for (i = 0; i < pa->pa_nr; i++) {
pfn = pa->pa_iova[i] >> PAGE_SHIFT;
if (pfn >= iov_pfn && pfn < end_iov_pfn)
return true; }
Eric Farman
maintainerRe: [PATCH 1/2] vfio/ccw: Add length to DMA_UNMAP checks
Quoted 59 lines · Matthew Rosato
Quoted 51 lines · Eric Farman
Quoted 50 lines · Matthew Rosato
Quoted 35 lines · Eric Farman
As pointed out with the simplification of the
VFIO_IOMMU_NOTIFY_DMA_UNMAPnotifier [1], the length parameter was never used to check against the pinned pages.Let's correct that, and see if a page is within the affected range instead of simply the first page of the range.
[1]
https://lore.kernel.org/kvm/[email protected]/Signed-off-by Eric Farman <[email protected]>--- drivers/s390/cio/vfio_ccw_cp.c | 11 +++++++---- drivers/s390/cio/vfio_ccw_cp.h | 2 +- drivers/s390/cio/vfio_ccw_ops.c | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-)drivers/s390/cio/vfio_ccw_cp.c +0
-0@@ -170,12 +170,14 @@ static void page_array_unpin_free(struct
page_array*pa, structvfio_device*vdekfree(pa->pa_iova); }-static bool page_array_iova_pinned(struct
page_array*pa, unsigned
long iova)
+static bool page_array_iova_pinned(structpage_array*pa, unsigned
long iova,drivers/s390/cio/vfio_ccw_cp.c static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vde · line 180
Quoted 8 lines
drivers/s390/cio/vfio_ccw_cp.c +3
-1+ unsigned long length) {int i;for (i =0 ; i < pa->pa_nr; i++) -if (pa->pa_iova[i] == iova) +if (pa->pa_iova[i] >= iova && + pa->pa_iova[i] <= iova + length)For the sake of completeness, I think you want to be checking to make
sure the end of the page is also within the range, not just the start?Quoted 6 lines
Quoted 2 lines
if (pa->pa_iova[i] >= iova &&
pa->pa_iova[i] + PAGE_SIZE <= iova + length)Well +PAGE_SIZE would iterate to the next page, so that would be captured on the next iteration of the for(i) loop if the pages were contiguous (or not applicable, if the pages weren't).
Quoted 7 lines
FWIW, the '+
PAGE_SIZE' was to match the '+ length' in your comparison.If you really only want to only look at the start of the
pa_iovabeing
within the range, then I think you want 'pa->pa_iova[i] < iova + length', not <=.
Touche.
Quoted 16 lines
Quoted 15 lines
But, since the comment is really about the end of the page (
0xfff), I
guess I'm not understanding what that gets us so perhaps you could help
elaborate your question? From my chair, since thepa_iovaargument passed tovfio_pin_pages()pins the whole page, checking the start address versus the end (or anywhere in between) should still capture
its interaction with an affected range. That is to say, we don't care
about the -whole- page being within the unmap range, but -any- part of
it.
Quoted 6 lines
As far as my suggestion to also look at the end of the pa_iova[i] -- This was particularly geared at ensuring the entire page fell within the
range, not just a subset. But I think you're right, we don't really care about that. On the flip side, do we care if the iova somehow starts sometime between pa_iova[i] and pa_iova[i] +PAGE_SIZE- 1?
As I was mulling your original reply, I was having the same thought. I think the answer is it's probably unlikely, but was trying to figure out how to rope something in for that.
Quoted 11 lines
That
would still be a subset, though I'm not sure such a thing could happen
today (e.g. an input 'iova' that is not on a page boundary)..I wonder if the simplest thing would be to just copy what gvt does and
convert to pfn as it takes all of this out of the equation and looks instead at whether the inputs overlaps at a page granularity (which is
what we really care about)
Agreed.
Quoted 1 line
, e.g. something like (untested):
Ha. This is almost exactly what I had in place originally, before I applied Nicolin's series and the pfns were still available in the struct that pa points to. I should have left that all in place instead of cleaning it up this way. Let me get that refit and doublechecked, and I'll send a v2. Thanks!
Quoted 14 lines
u64
iov_pfn= iova >>PAGE_SHIFT;
u64end_iov_pfn=iov_pfn+ (length /PAGE_SIZE);
u64 pfn;
int i;for (i = 0; i < pa->pa_nr; i++) { pfn = pa->pa_iova[i] >> PAGE_SHIFT; if (pfn >= iov_pfn && pfn < end_iov_pfn) return true; }
Eric Farman
maintainerPATCH 2/2 vfio/ccw: Remove FSM Close from remove handlers
In reply to Eric FarmanNow that neither vfio_ccw_sch_probe() nor vfio_ccw_mdev_probe() affect the FSM state, it doesn't make sense for their _remove() counterparts try to revert things in this way. Since the FSM open and close are handled alongside MDEV open/close, these are unnecessary.
--- drivers/s390/cio/vfio_ccw_drv.c | 1 - drivers/s390/cio/vfio_ccw_ops.c | 2 -- 2 files changed, 3 deletions(-)
drivers/s390/cio/vfio_ccw_drv.c +0 -1
@@ -241,7 +241,6 @@ static void vfio_ccw_sch_remove(struct subchannel *sch) {struct vfio_ccw_private *private =dev_get_drvdata (&sch->dev); -vfio_ccw_fsm_event (private,VFIO_CCW_EVENT_CLOSE );mdev_unregister_device (&sch->dev);dev_set_drvdata (&sch->dev,NULL );
drivers/s390/cio/vfio_ccw_ops.c +0 -2
@@ -130,8 +130,6 @@ static void vfio_ccw_mdev_remove(struct mdev_device *mdev)vfio_unregister_group_dev (&private->vdev); -vfio_ccw_fsm_event (private,VFIO_CCW_EVENT_CLOSE ); -vfio_uninit_group_dev (&private->vdev);atomic_inc (&private->avail); }
Signature · 1 line
2.34.1
Re: [PATCH 2/2] vfio/ccw: Remove FSM Close from remove handlers
Quoted 7 lines · Eric Farman
Now that neither
vfio_ccw_sch_probe()norvfio_ccw_mdev_probe()affect the FSM state, it doesn't make sense for their_remove()counterparts try to revert things in this way. Since the FSM open and close are handled alongside MDEV open/close, these are unnecessary.Signed-off-by Eric Farman <[email protected]>
Quoted 30 lines
--- drivers/s390/cio/vfio_ccw_drv.c | 1 - drivers/s390/cio/vfio_ccw_ops.c | 2 -- 2 files changed, 3 deletions(-)drivers/s390/cio/vfio_ccw_drv.c +0
-1@@ -241,7 +241,6 @@ static void vfio_ccw_sch_remove(struct subchannel *sch) {struct vfio_ccw_private *private =dev_get_drvdata (&sch->dev); -vfio_ccw_fsm_event (private,VFIO_CCW_EVENT_CLOSE );mdev_unregister_device (&sch->dev);dev_set_drvdata (&sch->dev,NULL );drivers/s390/cio/vfio_ccw_ops.c +0
-2@@ -130,8 +130,6 @@ static void vfio_ccw_mdev_remove(struct mdev_device *mdev)vfio_unregister_group_dev (&private->vdev); -vfio_ccw_fsm_event (private,VFIO_CCW_EVENT_CLOSE ); -vfio_uninit_group_dev (&private->vdev);atomic_inc (&private->avail); }