vfio: handle DMA map/unmap up to the addressable limit
Subsystems (2)
v2 cover vfio: handle DMA map/unmap up to the addressable limit
This patch series aims to fix vfio_iommu_type.c to support VFIO_IOMMU_MAP_DMA and VFIO_IOMMU_UNMAP_DMA operations targeting IOVA ranges which lie against the addressable limit. i.e. ranges where iova_start + iova_size would overflow to exactly zero.
Today, the VFIO UAPI has an inconsistency: The VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE capability of VFIO_IOMMU_GET_INFO reports that ranges up to the end of the address space are available for use, but are not really due to bugs in handling boundary conditions.
For example:
vfio_find_dma_first_node is called to find the first dma node to unmap given an unmap range of [iova..iova+size). The check at the end of the function intends to test if the dma result lies beyond the end of the unmap range. The condition is incorrectly satisfied when iova+size overflows to zero, causing the function to return NULL.
The same issue happens inside vfio_dma_do_unmap's while loop.
Of primary concern are locations in the current code which perform comparisons against (iova + size) expressions, where overflow to zero is possible.
The initial list of candidate locations to audit was taken from the following:
$ rg 'iova.*\+.*size' -n drivers/vfio/vfio_iommu_type1.c | rg -v '\- 1'
173: else if (start >= dma->iova + dma->size)
192: if (start < dma->iova + dma->size) {
216: if (new->iova + new->size <= dma->iova)
1060: dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
1233: if (dma && dma->iova + dma->size != iova + size)
1380: if (dma && dma->iova + dma->size != iova + size)
1501: ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
1504: vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
1721: while (iova < dma->iova + dma->size) {
1743: i = iova + size;
1744: while (i < dma->iova + dma->size &&
1754: size_t n = dma->iova + dma->size - iova;
1785: iova += size;
1810: while (iova < dma->iova + dma->size) {
1823: i = iova + size;
1824: while (i < dma->iova + dma->size &&
2919: if (range.iova + range.size < range.iova)This series spend the first couple commits making mechanical preparations before the fix lands in the last commit.
---
Changes in v2:
- Change to patch series rather than single commit
- Expand scope to fix more than just the unmap discovery path
- Link to v1: https://lore.kernel.org/r/[email protected]
---
Alex Mastro (3):
vfio/type1: sanitize for overflow using check_*_overflow vfio/type1: move iova increment to unmap_unpin_* caller vfio/type1: handle DMA map/unmap up to the addressable limit
drivers/vfio/vfio_iommu_type1.c | 145 +++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 53 deletions(-)
---
base-commit: 407aa63018d15c35a34938633868e61174d2ef6e
change-id: 20251005-fix-unmap-c3f3e87dabfa
Best regards,
Signature · 1 line
Alex Mastro <[email protected]>
PATCH v2 1/3 vfio/type1: sanitize for overflow using check_*_overflow
Adopt check_*_overflow functions to clearly express overflow check intent.
--- drivers/vfio/vfio_iommu_type1.c | 54 ++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-)
drivers/vfio/vfio_iommu_type1.c +43 -11
@@ -37,6 +37,7 @@#include <linux/vfio.h> #include <linux/workqueue.h> #include <linux/notifier.h> +#include <linux/overflow.h> #include "vfio.h" #define DRIVER_VERSION "0.2"
@@ -825,14 +826,25 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,unsigned long remote_vaddr;struct vfio_dma *dma;bool do_accounting; +dma_addr_t iova_end; +size_t iova_size; -if (!iommu || !pages) +if (!iommu || !pages || npage <0 )return -EINVAL ;/* Supported for v2 version only */ if (!iommu->v2)return -EACCES ; +if (npage ==0 ) +return 0 ; + +if (check_mul_overflow (npage,PAGE_SIZE , &iova_size)) +return -EINVAL ; + +if (check_add_overflow (user_iova, iova_size -1 , &iova_end)) +return -EINVAL ; +mutex_lock (&iommu->lock); if (WARN_ONCE(iommu->vaddr_invalid_count,
@@ -938,12 +950,23 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, {struct vfio_iommu *iommu = iommu_data;bool do_accounting; +dma_addr_t iova_end; +size_t iova_size;int i;/* Supported for v2 version only */ if (WARN_ON (!iommu->v2))return ; +if (WARN_ON (npage <0 ) || npage ==0 ) +return ; + +if (WARN_ON (check_mul_overflow (npage,PAGE_SIZE , &iova_size))) +return ; + +if (WARN_ON (check_add_overflow (user_iova, iova_size -1 , &iova_end))) +return ; +mutex_lock (&iommu->lock); do_accounting =list_empty (&iommu->domain_list);
@@ -1304,6 +1327,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,int ret = -EINVAL , retries =0 ;unsigned long pgshift;dma_addr_t iova = unmap->iova; +dma_addr_t iova_end;u64 size = unmap->size;bool unmap_all = unmap->flags &VFIO_DMA_UNMAP_FLAG_ALL ;bool invalidate_vaddr = unmap->flags &VFIO_DMA_UNMAP_FLAG_VADDR ;
@@ -1328,7 +1352,8 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, goto unlock; size =U64_MAX ; }else if (!size || size & (pgsize -1 ) || - iova + size -1 < iova || size >SIZE_MAX ) { +check_add_overflow (iova, size -1 , &iova_end) || + size >SIZE_MAX ) { goto unlock; }
@@ -1376,7 +1401,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,if (dma && dma->iova != iova) goto unlock; - dma =vfio_find_dma (iommu, iova + size -1 ,0 ); + dma =vfio_find_dma (iommu, iova_end,0 );if (dma && dma->iova + dma->size != iova + size) goto unlock; }
@@ -1578,7 +1603,9 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, {bool set_vaddr = map->flags &VFIO_DMA_MAP_FLAG_VADDR ;dma_addr_t iova = map->iova; +dma_addr_t iova_end;unsigned long vaddr = map->vaddr; +unsigned long vaddr_end;size_t size = map->size;int ret =0 , prot =0 ;size_t pgsize;
@@ -1588,6 +1615,12 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,if (map->size != size || map->vaddr != vaddr || map->iova != iova)return -EINVAL ; +if (check_add_overflow (iova, size -1 , &iova_end)) +return -EINVAL ; + +if (check_add_overflow (vaddr, size -1 , &vaddr_end)) +return -EINVAL ; +/* READ/WRITE from device perspective */ if (map->flags &VFIO_DMA_MAP_FLAG_WRITE ) prot |=IOMMU_WRITE ;
@@ -1608,12 +1641,6 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, goto out_unlock; } -/* Don't allow IOVA or virtual address wrap */ -if (iova + size -1 < iova || vaddr + size -1 < vaddr) { - ret = -EINVAL ; - goto out_unlock; - } - dma =vfio_find_dma (iommu, iova, size);if (set_vaddr) {if (!dma) {
@@ -1640,7 +1667,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, goto out_unlock; } -if (!vfio_iommu_iova_dma_valid (iommu, iova, iova + size -1 )) { +if (!vfio_iommu_iova_dma_valid (iommu, iova, iova_end)) { ret = -EINVAL ; goto out_unlock; }
@@ -2908,6 +2935,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,unsigned long pgshift;size_t data_size = dirty.argsz - minsz;size_t iommu_pgsize; +dma_addr_t range_end;if (!data_size || data_size <sizeof (range))return -EINVAL ;
@@ -2916,8 +2944,12 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, sizeof(range)))return -EFAULT ; -if (range.iova + range.size < range.iova) +if (range.size ==0 ) +return 0 ; + +if (check_add_overflow (range.iova, range.size -1 , &range_end))return -EINVAL ; +if (!access_ok ((void __user *)range.bitmap.data, range.bitmap.size))return -EINVAL ;
Signature · 1 line
2.47.3
Re: [PATCH v2 1/3] vfio/type1: sanitize for overflow using check_*_overflow
Quoted 30 lines · Alex Mastro
Adopt check_*_overflow functions to clearly express overflow check intent.
Signed-off-by Alex Mastro <[email protected]>--- drivers/vfio/vfio_iommu_type1.c | 54 ++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-)drivers/vfio/vfio_iommu_type1.c +4
-1@@ -37,6 +37,7 @@#include <linux/vfio.h> #include <linux/workqueue.h> #include <linux/notifier.h> +#include <linux/overflow.h> #include "vfio.h" #define DRIVER_VERSION "0.2"@@ -825,14 +826,25 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,unsigned long remote_vaddr;struct vfio_dma *dma;bool do_accounting; +dma_addr_t iova_end; +size_t iova_size; -if (!iommu || !pages) +if (!iommu || !pages || npage <0 )return -EINVAL ;
drivers/vfio/vfio_iommu_type1.c static int vfio_iommu_type1_pin_pages(void *iommu_data, · line 843
Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +5
-0/* Supported for v2 version only */ if (!iommu->v2)return -EACCES ; +if (npage ==0 ) +return 0 ; + +if (check_mul_overflow (npage,PAGE_SIZE , &iova_size)) +return -EINVAL ;
-EOVERFLOW and everywhere else
drivers/vfio/vfio_iommu_type1.c static int vfio_iommu_type1_pin_pages(void *iommu_data, · line 846
Quoted 3 lines
drivers/vfio/vfio_iommu_type1.c +3
-0+ +if (check_add_overflow (user_iova, iova_size -1 , &iova_end)) +return -EINVAL ;
Let's be consistent with iommufd/etc, 'end' is start+size 'last' is start+size-1
Otherwise it is super confusing :(
Jason
Re: [PATCH v2 1/3] vfio/type1: sanitize for overflow using check_*_overflow
Quoted 6 lines · Jason Gunthorpe
Review
Quoted 2 lines · Alex Mastro
(quoted diff) +2
-0+ if (check_mul_overflow(npage, PAGE_SIZE, &iova_size)) + return -EINVAL;
-EOVERFLOWand everywhere else
Quoted 7 lines
drivers/vfio/vfio_iommu_type1.c static int vfio_iommu_type1_pin_pages(void *iommu_data, · line 846
Quoted 3 lines
drivers/vfio/vfio_iommu_type1.c +3
-0+ +if (check_add_overflow (user_iova, iova_size -1 , &iova_end)) +return -EINVAL ;Let's be consistent with iommufd/etc, 'end' is start+size 'last' is start+size-1
Otherwise it is super confusing :(
Both suggestions SGTM.
Re: [PATCH v2 1/3] vfio/type1: sanitize for overflow using check_*_overflow
Quoted 11 lines · Alex Mastro
Quoted 7 lines · Jason Gunthorpe
drivers/vfio/vfio_iommu_type1.c static int vfio_iommu_type1_pin_pages(void *iommu_data, · line 846
Quoted 2 lines · Alex Mastro
drivers/vfio/vfio_iommu_type1.c +2
-0+if (check_add_overflow (user_iova, iova_size -1 , &iova_end)) +return -EINVAL ;Let's be consistent with iommufd/etc, 'end' is start+size 'last' is start+size-1
Otherwise it is super confusing :(
Both suggestions SGTM.
I'm not sure about the latter anymore. There's somewhat pervasive precedent for using 'end' as the inclusive limit in vfio_iommu_type1.c. I am all for making things less confusing. I don't think I can introduce 'end' 'last' convention without preparing the existing code first.
Thoughts? Spend another commit renaming this to 'last'? Tolerate inconsistency between vfio and iommufd?
116 struct vfio_iova {
117 struct list_head list; 118 dma_addr_t start; 119 dma_addr_t end;
120 };
...
2037 end = resv->start + resv->length - 1;
2038
2039 list_for_each_entry_safe(n, next, iova, list) {
2040 int ret = 0;
2041
2042 /* No overlap */
2043 if (start > n->end || end < n->start)
2044 continue;
...
2052 if (start > n->start)
2053 ret = vfio_iommu_iova_insert(&n->list, n->start,
2054 start - 1);
2055 if (!ret && end < n->end)
2056 ret = vfio_iommu_iova_insert(&n->list, end + 1,
2057 n->end);
Re: [PATCH v2 1/3] vfio/type1: sanitize for overflow using check_*_overflow
Quoted 13 lines · Alex Mastro
Quoted 11 lines · Alex Mastro
Quoted 7 lines · Jason Gunthorpe
drivers/vfio/vfio_iommu_type1.c static int vfio_iommu_type1_pin_pages(void *iommu_data, · line 846
Quoted 2 lines · Alex Mastro
drivers/vfio/vfio_iommu_type1.c +2
-0+if (check_add_overflow (user_iova, iova_size -1 , &iova_end)) +return -EINVAL ;Let's be consistent with iommufd/etc, 'end' is start+size 'last' is start+size-1
Otherwise it is super confusing :(
Both suggestions SGTM.
Quoted 7 lines
I'm not sure about the latter anymore. There's somewhat pervasive precedent for using 'end' as the inclusive limit in
vfio_iommu_type1.c. I am all for making things less confusing. I don't think I can introduce 'end' 'last' convention without preparing the existing code first.Thoughts? Spend another commit renaming this to 'last'? Tolerate inconsistency between vfio and iommufd?
IDK, if it is actually internally consistent and not using end interchangably then it is probably Ok to keep doing it. If it is already inconsistent then use last for new code and leave the old as is?
Jason
Re: [PATCH v2 1/3] vfio/type1: sanitize for overflow using check_*_overflow
Quoted 19 lines · Jason Gunthorpe
Quoted 18 lines · Alex Mastro
Quoted 11 lines · Alex Mastro
Quoted 7 lines · Jason Gunthorpe
drivers/vfio/vfio_iommu_type1.c static int vfio_iommu_type1_pin_pages(void *iommu_data, · line 846
Quoted 2 lines · Alex Mastro
drivers/vfio/vfio_iommu_type1.c +2
-0+if (check_add_overflow (user_iova, iova_size -1 , &iova_end)) +return -EINVAL ;Let's be consistent with iommufd/etc, 'end' is start+size 'last' is start+size-1
Otherwise it is super confusing :(
Both suggestions SGTM.
I'm not sure about the latter anymore. There's somewhat pervasive precedent for using 'end' as the inclusive limit in
vfio_iommu_type1.c. I am all for making things less confusing. I don't think I can introduce 'end' 'last' convention without preparing the existing code first.
Quoted 7 lines
Quoted 2 lines
Thoughts? Spend another commit renaming this to 'last'? Tolerate inconsistency between vfio and iommufd?
IDK, if it is actually internally consistent and not using end interchangably then it is probably Ok to keep doing it. If it is already inconsistent then use last for new code and leave the old as is?
The only references to 'last' are for elements in a list, tree, unrelated to iova, and 'end' refers to (iova+size-1) for all cases that I saw.
For the sake of internal consistency, I'll keep 'end', and after this series, if it's worth it, we can take a pass to unify the terminology between iommufd and vfio.
PATCH v2 2/3 vfio/type1: move iova increment to unmap_unpin_* caller
In reply to Alex MastroMove incrementing iova to the caller of these functions as part of preparing to handle end of address space map/unmap.
--- drivers/vfio/vfio_iommu_type1.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
drivers/vfio/vfio_iommu_type1.c +10 -10
@@ -1020,7 +1020,7 @@ static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain,#define VFIO_IOMMU_TLB_SYNC_MAX 512 static size_t unmap_unpin_fast(struct vfio_domain *domain, - struct vfio_dma *dma, dma_addr_t *iova, + struct vfio_dma *dma, dma_addr_t iova, size_t len, phys_addr_t phys, long *unlocked, struct list_head *unmapped_list, int *unmapped_cnt,
@@ -1030,18 +1030,17 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain,struct vfio_regions *entry =kzalloc (sizeof (*entry),GFP_KERNEL );if (entry) { - unmapped =iommu_unmap_fast (domain->domain, *iova, len, + unmapped =iommu_unmap_fast (domain->domain, iova, len, iotlb_gather);if (!unmapped) {kfree (entry); }else { - entry->iova = *iova; + entry->iova = iova; entry->phys = phys; entry->len = unmapped;list_add_tail (&entry->list, unmapped_list); - *iova += unmapped; (*unmapped_cnt)++; } }
@@ -1060,18 +1059,17 @@ static size_t unmap_unpin_fast(struct vfio_domain *domain, }static size_t unmap_unpin_slow (struct vfio_domain *domain, -struct vfio_dma *dma,dma_addr_t *iova, +struct vfio_dma *dma,dma_addr_t iova,size_t len,phys_addr_t phys,long *unlocked) { -size_t unmapped =iommu_unmap (domain->domain, *iova, len); +size_t unmapped =iommu_unmap (domain->domain, iova, len);if (unmapped) { - *unlocked +=vfio_unpin_pages_remote (dma, *iova, + *unlocked +=vfio_unpin_pages_remote (dma, iova, phys >>PAGE_SHIFT , unmapped >>PAGE_SHIFT , false); - *iova += unmapped;cond_resched (); }return unmapped;
@@ -1134,16 +1132,18 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,* First, try to use fast unmap/unpin. In case of failure, * switch to slow unmap/unpin path. */ - unmapped =unmap_unpin_fast (domain, dma, &iova, len, phys, + unmapped =unmap_unpin_fast (domain, dma, iova, len, phys, &unlocked, &unmapped_region_list, &unmapped_region_cnt, &iotlb_gather);if (!unmapped) { - unmapped =unmap_unpin_slow (domain, dma, &iova, len, + unmapped =unmap_unpin_slow (domain, dma, iova, len, phys, &unlocked);if (WARN_ON (!unmapped))break ; } + + iova += unmapped; } dma->iommu_mapped = false;
Signature · 1 line
2.47.3
PATCH v2 3/3 vfio/type1: handle DMA map/unmap up to the addressable limit
In reply to Alex MastroHandle DMA map/unmap operations up to the addressable limit by comparing against inclusive end-of-range limits, and changing iteration to perform relative traversals across range sizes, rather than absolute traversals across addresses.
vfio_link_dma inserts a zero-sized vfio_dma into the rb-tree, and is only used for that purpose, so discard the size from consideration for the insertion point.
--- drivers/vfio/vfio_iommu_type1.c | 75 ++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 34 deletions(-)
drivers/vfio/vfio_iommu_type1.c +41 -34
@@ -166,12 +166,14 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, {struct rb_node *node = iommu->dma_list.rb_node; +WARN_ON (!size); +while (node) { struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); -if (start + size <= dma->iova) +if (start + size -1 < dma->iova) node = node->rb_left; -else if (start >= dma->iova + dma->size) +else if (start > dma->iova + dma->size -1 ) node = node->rb_right;else return dma;
@@ -181,16 +183,19 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, }static struct rb_node *vfio_find_dma_first_node (struct vfio_iommu *iommu, -dma_addr_t start,u64 size) +dma_addr_t start, +dma_addr_t end) {struct rb_node *res =NULL ;struct rb_node *node = iommu->dma_list.rb_node;struct vfio_dma *dma_res =NULL ; +WARN_ON (end < start); +while (node) { struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); -if (start < dma->iova + dma->size) { +if (start <= dma->iova + dma->size -1 ) { res = node; dma_res = dma;if (start >= dma->iova)
@@ -200,7 +205,7 @@ static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu, node = node->rb_right; } } -if (res && size && dma_res->iova >= start + size) +if (res && dma_res->iova > end) res =NULL ;return res; }
@@ -210,11 +215,13 @@ static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)struct rb_node **link = &iommu->dma_list.rb_node, *parent =NULL ;struct vfio_dma *dma; +WARN_ON (new->size !=0 ); +while (*link) { parent = *link; dma = rb_entry(parent, struct vfio_dma, node); -if (new->iova + new->size <= dma->iova) +if (new->iova <= dma->iova) link = &(*link)->rb_left;else link = &(*link)->rb_right;
@@ -1078,12 +1085,12 @@ static size_t unmap_unpin_slow(struct vfio_domain *domain,static long vfio_unmap_unpin (struct vfio_iommu *iommu,struct vfio_dma *dma,bool do_accounting) { -dma_addr_t iova = dma->iova, end = dma->iova + dma->size;struct vfio_domain *domain, *d;LIST_HEAD (unmapped_region_list);struct iommu_iotlb_gather iotlb_gather;int unmapped_region_cnt =0 ;long unlocked =0 ; +size_t pos =0 ;if (!dma->size)return 0 ;
@@ -1107,13 +1114,14 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, }iommu_iotlb_gather_init (&iotlb_gather); -while (iova < end) { +while (pos < dma->size) {size_t unmapped, len;phys_addr_t phys, next; +dma_addr_t iova = dma->iova + pos; phys =iommu_iova_to_phys (domain->domain, iova);if (WARN_ON (!phys)) { - iova +=PAGE_SIZE ; + pos +=PAGE_SIZE ;continue ; }
@@ -1122,7 +1130,7 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,* may require hardware cache flushing, try to find the * largest contiguous physical memory chunk to unmap. */ -for (len =PAGE_SIZE ; iova + len < end; len +=PAGE_SIZE ) { +for (len =PAGE_SIZE ; pos + len < dma->size; len +=PAGE_SIZE ) { next =iommu_iova_to_phys (domain->domain, iova + len);if (next != phys + len)break ;
@@ -1143,7 +1151,7 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,break ; } - iova += unmapped; + pos += unmapped; } dma->iommu_mapped = false;
@@ -1235,7 +1243,7 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, }static int vfio_iova_dirty_bitmap (u64 __user *bitmap,struct vfio_iommu *iommu, -dma_addr_t iova,size_t size,size_t pgsize) +dma_addr_t iova,dma_addr_t iova_end,size_t pgsize) {struct vfio_dma *dma;struct rb_node *n;
@@ -1252,8 +1260,8 @@ static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,if (dma && dma->iova != iova)return -EINVAL ; - dma =vfio_find_dma (iommu, iova + size -1 ,0 ); -if (dma && dma->iova + dma->size != iova + size) + dma =vfio_find_dma (iommu, iova_end,1 ); +if (dma && dma->iova + dma->size -1 != iova_end)return -EINVAL ;for (n =rb_first (&iommu->dma_list); n; n =rb_next (n)) {
@@ -1262,7 +1270,7 @@ static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,if (dma->iova < iova)continue ; -if (dma->iova > iova + size -1 ) +if (dma->iova > iova_end)break ; ret =update_user_bitmap (bitmap, iommu, dma, iova, pgsize);
@@ -1401,17 +1409,17 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,if (dma && dma->iova != iova) goto unlock; - dma =vfio_find_dma (iommu, iova_end,0 ); -if (dma && dma->iova + dma->size != iova + size) + dma =vfio_find_dma (iommu, iova_end,1 ); +if (dma && dma->iova + dma->size -1 != iova_end) goto unlock; } ret =0 ; - n = first_n =vfio_find_dma_first_node (iommu, iova, size); + n = first_n =vfio_find_dma_first_node (iommu, iova, iova_end);while (n) { dma = rb_entry(n, struct vfio_dma, node); -if (dma->iova >= iova + size) +if (dma->iova > iova_end)break ;if (!iommu->v2 && iova > dma->iova)
@@ -1740,12 +1748,12 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,for (; n; n =rb_next (n)) {struct vfio_dma *dma; -dma_addr_t iova; +size_t pos =0 ; dma = rb_entry(n, struct vfio_dma, node); - iova = dma->iova; -while (iova < dma->iova + dma->size) { +while (pos < dma->size) { +dma_addr_t iova = dma->iova + pos;phys_addr_t phys;size_t size;
@@ -1761,14 +1769,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, phys =iommu_iova_to_phys (d->domain, iova);if (WARN_ON (!phys)) { - iova +=PAGE_SIZE ; + pos +=PAGE_SIZE ;continue ; } size =PAGE_SIZE ; p = phys + size; i = iova + size; -while (i < dma->iova + dma->size && +while (pos + size < dma->size && p ==iommu_iova_to_phys (d->domain, i)) { size +=PAGE_SIZE ; p +=PAGE_SIZE ;
@@ -1776,9 +1784,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, } }else {unsigned long pfn; -unsigned long vaddr = dma->vaddr + - (iova - dma->iova); -size_t n = dma->iova + dma->size - iova; +unsigned long vaddr = dma->vaddr + pos; +size_t n = dma->size - pos;long npage; npage = vfio_pin_pages_remote(dma, vaddr,
@@ -1809,7 +1816,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, goto unwind; } - iova += size; + pos += size; } }
@@ -1826,29 +1833,29 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, unwind:for (; n; n =rb_prev (n)) { struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); -dma_addr_t iova; +size_t pos =0 ;if (dma->iommu_mapped) {iommu_unmap (domain->domain, dma->iova, dma->size);continue ; } - iova = dma->iova; -while (iova < dma->iova + dma->size) { +while (pos < dma->size) { +dma_addr_t iova = dma->iova + pos;phys_addr_t phys, p;size_t size;dma_addr_t i; phys =iommu_iova_to_phys (domain->domain, iova);if (!phys) { - iova +=PAGE_SIZE ; + pos +=PAGE_SIZE ;continue ; } size =PAGE_SIZE ; p = phys + size; i = iova + size; -while (i < dma->iova + dma->size && +while (pos + size < dma->size && p ==iommu_iova_to_phys (domain->domain, i)) { size +=PAGE_SIZE ; p +=PAGE_SIZE ;
@@ -2981,7 +2988,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,if (iommu->dirty_page_tracking) ret =vfio_iova_dirty_bitmap (range.bitmap.data, iommu, range.iova, - range.size, + range_end, range.bitmap.pgsize);else ret = -EINVAL ;
Signature · 1 line
2.47.3
Re: [PATCH v2 3/3] vfio/type1: handle DMA map/unmap up to the addressable limit
Quoted 4 lines · Alex Mastro
drivers/vfio/vfio_iommu_type1.c +0
-0@@ -1401,17 +1409,17 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,if (dma && dma->iova != iova) goto unlock;
drivers/vfio/vfio_iommu_type1.c static int vfio_dma_do_unmap(struct vfio_iommu *iommu, · line 1418
Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +3
-3- dma =vfio_find_dma (iommu, iova_end,0 ); -if (dma && dma->iova + dma->size != iova + size) + dma =vfio_find_dma (iommu, iova_end,1 ); +if (dma && dma->iova + dma->size -1 != iova_end) goto unlock; } ret =0 ; - n = first_n =vfio_find_dma_first_node (iommu, iova, size); + n = first_n =vfio_find_dma_first_node (iommu, iova, iova_end);
I missed updating iova_end to be consistent in the unmap_all case, which is broken with this change. Currently, iova_end is only assigned by the check_add_overflow call in the !unmap_all path. Will address in v3.