Improve ODP by using HMM API
Subsystems (2)
cover Improve ODP by using HMM API
From: Leon Romanovsky <[email protected]>
Based on:
https://lore.kernel.org/lkml/[email protected]/
---------------------------------------------------------------------------------------
From Yishai:
This series improves ODP performance by moving to use the HMM API as of below.
The get_user_pages_remote() functionality was replaced by HMM:
- No need anymore to allocate and free memory to hold its output per call.
- No need anymore to use the put_page() to unpin the pages.
- The logic to detect contiguous pages is done based on the returned order from HMM, no need to run per page, and evaluate.
Moving to use the HMM enables to reduce page faults in the system by using the snapshot mode. This mode allows existing pages in the CPU to become presented to the device without faulting.
This non-faulting mode may be used explicitly by an application with some new option of advice MR (i.e. PREFETCH_NO_FAULT) and is used upon ODP MR registration internally as part of initiating the device page table.
To achieve the above, internal changes in the ODP data structures were done and some flows were cleaned-up/adapted accordingly.
Thanks
Yishai Hadas (4):
IB/core: Improve ODP to use hmm_range_fault() IB/core: Enable ODP sync without faulting
RDMA/mlx5: Extend advice MR to support non faulting mode RDMA/mlx5: Sync device with CPU pages upon ODP MR registration
drivers/infiniband/Kconfig | 1 + drivers/infiniband/core/umem_odp.c | 286 ++++++++++-------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 6 + drivers/infiniband/hw/mlx5/mr.c | 14 +- drivers/infiniband/hw/mlx5/odp.c | 50 +++-- include/rdma/ib_umem_odp.h | 21 +- include/uapi/rdma/ib_user_ioctl_verbs.h | 1 + 7 files changed, 178 insertions(+), 201 deletions(-)
--
2.26.2
PATCH 1/4 IB/core: Improve ODP to use hmm_range_fault()
From: Yishai Hadas <[email protected]>
Move to use hmm_range_fault() which improve performance in few aspects comparing previous functionality of get_user_pages_remote().
This includes:
- Dropping the need to allocate and free memory to hold its output.
- No need any more to use the put_page() to unpin the pages.
- The logic to detect contiguous pages is done based on the returned order from HMM, no need to run per page and evaluate.
In addition,
Moving to use the HMM enables to reduce page faults in the system as it exposes the snapshot mode, this will be introduced in next patches from this series.
As part of this cleanup some flows and use the required data structures to work with HMM.
--- drivers/infiniband/Kconfig | 1 + drivers/infiniband/core/umem_odp.c | 273 +++++++++++------------------ drivers/infiniband/hw/mlx5/odp.c | 25 +-- include/rdma/ib_umem_odp.h | 21 +-- 4 files changed, 123 insertions(+), 197 deletions(-)
drivers/infiniband/Kconfig +1 -0
@@ -49,6 +49,7 @@ config INFINIBAND_ON_DEMAND_PAGING depends on INFINIBAND_USER_MEM select MMU_NOTIFIER select INTERVAL_TREE + select HMM_MIRROR default y help On demand paging support for the InfiniBand subsystem.
drivers/infiniband/core/umem_odp.c +106 -167
402 lines not on this page · raw message
drivers/infiniband/hw/mlx5/odp.c +8 -17
@@ -671,7 +671,6 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, {int page_shift, ret, np;bool downgrade = flags &MLX5_PF_FLAGS_DOWNGRADE ; -unsigned long current_seq;u64 access_mask;u64 start_idx;
@@ -682,25 +681,17 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,if (odp->umem.writable && !downgrade) access_mask |=ODP_WRITE_ALLOWED_BIT ; - current_seq =mmu_interval_read_begin (&odp->notifier); - - np =ib_umem_odp_map_dma_pages (odp, user_va, bcnt, access_mask, - current_seq); + np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask);if (np <0 )return np; -mutex_lock (&odp->umem_mutex); -if (!mmu_interval_read_retry (&odp->notifier, current_seq)) { -/* -* No need to check whether the MTTs really belong to -* this MR, since ib_umem_odp_map_dma_pages already -* checks this. -*/ - ret =mlx5_ib_update_xlt (mr, start_idx, np, - page_shift,MLX5_IB_UPD_XLT_ATOMIC ); - }else { - ret = -EAGAIN ; - } +/* +* No need to check whether the MTTs really belong to +* this MR, since ib_umem_odp_map_dma_and_lock already +* checks this. +*/ + ret =mlx5_ib_update_xlt (mr, start_idx, np, + page_shift,MLX5_IB_UPD_XLT_ATOMIC );mutex_unlock (&odp->umem_mutex);if (ret <0 ) {
include/rdma/ib_umem_odp.h +8 -13
@@ -14,17 +14,13 @@ struct ib_umem_odp {struct mmu_interval_notifier notifier;struct pid *tgid; +/* An array of the pfns included in the on-demand paging umem. */ +unsigned long *pfn_list; +/* -* An array of the pages included in the on-demand paging umem. -* Indices of pages that are currently not mapped into the device will -* contain NULL. -*/ -struct page **page_list; -/* -* An array of the same size as page_list, with DMA addresses mapped -* for pages the pages in page_list. The lower two bits designate -* access permissions. See ODP_READ_ALLOWED_BIT and -* ODP_WRITE_ALLOWED_BIT. +* An array with DMA addresses mapped for pfns in pfn_list. +* The lower two bits designate access permissions. +* See ODP_READ_ALLOWED_BIT and ODP_WRITE_ALLOWED_BIT. */ dma_addr_t *dma_list;/*
@@ -97,9 +93,8 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr, const struct mmu_interval_notifier_ops *ops);void ib_umem_odp_release (struct ib_umem_odp *umem_odp); -int ib_umem_odp_map_dma_pages (struct ib_umem_odp *umem_odp,u64 start_offset, -u64 bcnt,u64 access_mask, -unsigned long current_seq); +int ib_umem_odp_map_dma_and_lock (struct ib_umem_odp *umem_odp,u64 start_offset, +u64 bcnt,u64 access_mask);void ib_umem_odp_unmap_dma_pages (struct ib_umem_odp *umem_odp,u64 start_offset,u64 bound);
Signature · 1 line
2.26.2
Re: [PATCH rdma-next 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 7 lines
In addition,
Moving to use the HMM enables to reduce page faults in the system as it exposes the snapshot mode, this will be introduced in next patches from this series.As part of this cleanup some flows and use the required data structures to work with HMM.
Just saying HMM here seems weird. The function is hmm_range_fault. And it really needs to grow a better name eventually..
Quoted 23 lines
drivers/infiniband/core/umem_odp.c +6
-6unsigned long start;unsigned long end; -size_t pages; +size_t ndmas, npfns; start =ALIGN_DOWN (umem_odp->umem.address, page_size); if (check_add_overflow(umem_odp->umem.address,@@ -71,20 +72,21 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,if (unlikely (end < page_size))return -EOVERFLOW ; - pages = (end - start) >> umem_odp->page_shift; -if (!pages) + ndmas = (end - start) >> umem_odp->page_shift; +if (!ndmas)return -EINVAL ; - umem_odp->page_list =kvcalloc ( - pages,sizeof (*umem_odp->page_list),GFP_KERNEL ); -if (!umem_odp->page_list) + npfns = (end - start) >>PAGE_SHIFT ; + umem_odp->pfn_list =kvcalloc ( + npfns,sizeof (*umem_odp->pfn_list),GFP_KERNEL );
drivers/infiniband/core/umem_odp.c static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp, · line 89
Quoted 10 lines
drivers/infiniband/core/umem_odp.c +3
-2+if (!umem_odp->pfn_list)return -ENOMEM ; umem_odp->dma_list =kvcalloc ( - pages,sizeof (*umem_odp->dma_list),GFP_KERNEL ); + ndmas,sizeof (*umem_odp->dma_list),GFP_KERNEL );if (!umem_odp->dma_list) { ret = -ENOMEM ; - goto out_page_list; + goto out_pfn_list;
Why do you rename these variables? We're still mapping pages.
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 311
Quoted 10 lines
drivers/infiniband/core/umem_odp.c +3
-0static int ib_umem_odp_map_dma_single_page (struct ib_umem_odp *umem_odp, +unsigned int dma_index,struct page *page, +u64 access_mask) {struct ib_device *dev = umem_odp->umem.ibdev;dma_addr_t dma_addr; +if (umem_odp->dma_list[dma_index]) {
Note that 0 is a valid DMA address. I think due the access bit this works, but it is a little subtle..
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 313
Quoted 2 lines
drivers/infiniband/core/umem_odp.c +2
-0+ umem_odp->dma_list[dma_index] &=ODP_DMA_ADDR_MASK ; + umem_odp->dma_list[dma_index] |= access_mask;
This looks a little weird. Instead of &= ODP_DMA_ADDR_MASK I'd do a &= ~ACCESS_MASK as that makes the code a lot more logical.
But more importantly except for (dma_addr_t)-1 (DMA_MAPPING_ERROR) all dma_addr_t values are valid, so taking more than a single bit from a dma_addr_t is not strictly speaking correct.
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 319
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +4
-0+return 0 ; } + dma_addr = +ib_dma_map_page (dev, page,0 ,BIT (umem_odp->page_shift), +DMA_BIDIRECTIONAL );
The use of the BIT macro which is already obsfucating in its intended place is really out of place here.
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 325
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +6
-0+if (ib_dma_mapping_error (dev, dma_addr)) +return -EFAULT ; + + umem_odp->dma_list[dma_index] = dma_addr | access_mask; + umem_odp->npages++; +return 0 ;
I'd change the calling conventions and write the whole thing as:
static int ib_umem_odp_map_dma_single_page(struct ib_umem_odp *umem_odp
unsigned int dma_index, struct page *page, u64 access_mask)
{
struct ib_device *dev = umem_odp->umem.ibdev;
dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index];
if (!dma_addr) {
*dma_addr = ib_dma_map_page(dev, page, 0,
1 << umem_odp->page_shift,
DMA_BIDIRECTIONAL);
if (ib_dma_mapping_error(dev, *dma_addr))
return -EFAULT;
umem_odp->npages++;
}
*dma_addr &= ~ODP_ACCESS_MASK;
*dma_addr |= access_mask;
return 0;
}drivers/infiniband/hw/mlx5/odp.c static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, · line 692
Quoted 5 lines
drivers/infiniband/hw/mlx5/odp.c +5
-0+/* +* No need to check whether the MTTs really belong to +* this MR, since ib_umem_odp_map_dma_and_lock already +* checks this. +*/
You could easily squeeze the three lines into two while still staying under 80 characters for each line.
Jason Gunthorpe
maintainerRe: [PATCH rdma-next 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 2 lines · Christoph Hellwig
Note that 0 is a valid DMA address. I think due the access bit this works, but it is a little subtle..
It should be checked again carefully.. This looks a bit questionable if the flags are what makes it work:
+ dma_addr = dma & ODP_DMA_ADDR_MASK;
+ if (dma_addr) {
Quoted 3 lines
But more importantly except for (dma_addr_t)-1 (
DMA_MAPPING_ERROR) alldma_addr_tvalues are valid, so taking more than a single bit from adma_addr_tis not strictly speaking correct.
This is the result of dma_map_page(). The HW requires that dma_map_page(page_size) returns a DMA address that is page_size aligned, even for huge pages.
So at least the last 12 bits of the DMA address are always 0, they can be used for flags. This is also the HW representation in the page table.
Last time you pointed at this code you agreed this alignment was met, is it still OK?
Jason
Re: [PATCH rdma-next 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 11 lines · Jason Gunthorpe
drivers/infiniband/core/umem_odp.c +2
-0+ dma_addr = dma &ODP_DMA_ADDR_MASK ; +if (dma_addr) {Quoted 4 lines
But more importantly except for (dma_addr_t)-1 (
DMA_MAPPING_ERROR) alldma_addr_tvalues are valid, so taking more than a single bit from adma_addr_tis not strictly speaking correct.This is the result of
dma_map_page(). The HW requires thatdma_map_page(page_size)returns a DMA address that ispage_sizealigned, even for huge pages.
Quoted 6 lines
So at least the last 12 bits of the DMA address are always 0, they can be used for flags. This is also the HW representation in the page table.
Last time you pointed at this code you agreed this alignment was met, is it still OK?
True, the alignment actually makes this work. I'd still suggest to throw in a comment about the actual usage.
Re: [PATCH rdma-next 1/4] IB/core: Improve ODP to use hmm_range_fault()
In reply to Jason Gunthorpedrivers/infiniband/core/umem_odp.c void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt, · line 474
Quoted 9 lines · Jason Gunthorpe
Quoted 2 lines · Christoph Hellwig
Note that 0 is a valid DMA address. I think due the access bit this works, but it is a little subtle..
It should be checked again carefully.. This looks a bit questionable if the flags are what makes it work:
drivers/infiniband/core/umem_odp.c +2
-0+ dma_addr = dma &ODP_DMA_ADDR_MASK ; +if (dma_addr) {
I have changed the check to be on dma, the masking to get the dma_address will be just later on so this should be fine even for potential valid NULL dma address. Will be part of V1.
Quoted 7 lines
Quoted 3 lines
But more importantly except for (dma_addr_t)-1 (
DMA_MAPPING_ERROR) alldma_addr_tvalues are valid, so taking more than a single bit from adma_addr_tis not strictly speaking correct.This is the result of
dma_map_page(). The HW requires thatdma_map_page(page_size)returns a DMA address that ispage_sizealigned, even for huge pages.
Quoted 8 lines
So at least the last 12 bits of the DMA address are always 0, they can be used for flags. This is also the HW representation in the page table.
Last time you pointed at this code you agreed this alignment was met, is it still OK?
Jason
Yes, this was confirmed by Christoph in next mail.
Thanks,
Yishai
Re: [PATCH rdma-next 1/4] IB/core: Improve ODP to use hmm_range_fault()
In reply to Christoph HellwigQuoted 9 lines · Christoph Hellwig
Quoted 7 lines
In addition,
Moving to use the HMM enables to reduce page faults in the system as it exposes the snapshot mode, this will be introduced in next patches from this series.As part of this cleanup some flows and use the required data structures to work with HMM.
Just saying HMM here seems weird. The function is
hmm_range_fault. And it really needs to grow a better name eventually..
Sure, will rephrase.
Quoted 26 lines
Quoted 26 lines
unsigned long start;
unsigned long end;drivers/infiniband/core/umem_odp.c +7
-6-size_t pages; +size_t ndmas, npfns; start =ALIGN_DOWN (umem_odp->umem.address, page_size); if (check_add_overflow(umem_odp->umem.address,@@ -71,20 +72,21 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,if (unlikely (end < page_size))return -EOVERFLOW ; - pages = (end - start) >> umem_odp->page_shift; -if (!pages) + ndmas = (end - start) >> umem_odp->page_shift; +if (!ndmas)return -EINVAL ; - umem_odp->page_list =kvcalloc ( - pages,sizeof (*umem_odp->page_list),GFP_KERNEL ); -if (!umem_odp->page_list) + npfns = (end - start) >>PAGE_SHIFT ; + umem_odp->pfn_list =kvcalloc ( + npfns,sizeof (*umem_odp->pfn_list),GFP_KERNEL ); +if (!umem_odp->pfn_list)return -ENOMEM ;
Quoted 8 lines
drivers/infiniband/core/umem_odp.c static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp, · line 89
Quoted 7 lines
drivers/infiniband/core/umem_odp.c +2
-2umem_odp->dma_list =kvcalloc ( - pages,sizeof (*umem_odp->dma_list),GFP_KERNEL ); + ndmas,sizeof (*umem_odp->dma_list),GFP_KERNEL );if (!umem_odp->dma_list) { ret = -ENOMEM ; - goto out_page_list; + goto out_pfn_list;Why do you rename these variables? We're still mapping pages.
The pfn_list is allocated in a size to match the system page size as used by hmm_range_fault() by contrast to the dma_list that its size depends on umem page size. As of that I have separated to 2 variables and renamed to clarify their usage (i.e no struct page ** is allocated but unsigned long *)
Quoted 9 lines
Quoted 9 lines
drivers/infiniband/core/umem_odp.c +2
-0static int ib_umem_odp_map_dma_single_page (struct ib_umem_odp *umem_odp, +unsigned int dma_index,struct page *page, +u64 access_mask) {struct ib_device *dev = umem_odp->umem.ibdev;dma_addr_t dma_addr;
Quoted 3 lines
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 311
Quoted 1 line
drivers/infiniband/core/umem_odp.c +1
-0+if (umem_odp->dma_list[dma_index]) {Note that 0 is a valid DMA address. I think due the access bit this works, but it is a little subtle..
I have added a note to clarify the usage as you have suggested in the next email will be part of V1, the flags will do the work.
By the way see below [1], existing code assumes that NULL is not valid but new code won't rely on any more.
[1] https://elixir.bootlin.com/linux/v5.3-rc7/source/drivers/infiniband/core/umem_odp.c#L727
Quoted 26 lines
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 313
Quoted 2 lines
drivers/infiniband/core/umem_odp.c +2
-0+ umem_odp->dma_list[dma_index] &=ODP_DMA_ADDR_MASK ; + umem_odp->dma_list[dma_index] |= access_mask;This looks a little weird. Instead of &=
ODP_DMA_ADDR_MASKI'd do a &= ~ACCESS_MASK as that makes the code a lot more logical.But more importantly except for (dma_addr_t)-1 (
DMA_MAPPING_ERROR) alldma_addr_tvalues are valid, so taking more than a single bit from adma_addr_tis not strictly speaking correct.drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 319
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +4
-0+return 0 ; } + dma_addr = +ib_dma_map_page (dev, page,0 ,BIT (umem_odp->page_shift), +DMA_BIDIRECTIONAL );The use of the BIT macro which is already obsfucating in its intended place is really out of place here.
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 325
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +6
-0+if (ib_dma_mapping_error (dev, dma_addr)) +return -EFAULT ; + + umem_odp->dma_list[dma_index] = dma_addr | access_mask; + umem_odp->npages++; +return 0 ;I'd change the calling conventions and write the whole thing as:
Quoted 7 lines
static int ib_umem_odp_map_dma_single_page(struct
ib_umem_odp*umem_odp
unsigned intdma_index, struct page *page, u64access_mask)
{struct ib_device *dev = umem_odp->umem.ibdev; dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index]; if (!dma_addr) {
You referred to if (!*dma_addr), right ? no problem will use your suggestion below.
Quoted 8 lines
*dma_addr = ib_dma_map_page(dev, page, 0, 1 << umem_odp->page_shift, DMA_BIDIRECTIONAL); if (ib_dma_mapping_error(dev, *dma_addr)) return -EFAULT; umem_odp->npages++; } *dma_addr &= ~ODP_ACCESS_MASK;
There isn't such a mask, I found it clear to use ODP_DMA_ADDR_MASK.
Quoted 11 lines
*dma_addr |= access_mask; return 0; }drivers/infiniband/hw/mlx5/odp.c static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, · line 692
Quoted 5 lines
drivers/infiniband/hw/mlx5/odp.c +5
-0+/* +* No need to check whether the MTTs really belong to +* this MR, since ib_umem_odp_map_dma_and_lock already +* checks this. +*/ You could easily squeeze the three lines into two while still staying under 80 characters for each line.
No problem, will change.
Yishai
PATCH 2/4 IB/core: Enable ODP sync without faulting
In reply to Leon RomanovskyFrom: Yishai Hadas <[email protected]>
Enable ODP sync without faulting, this improves performance by reducing the number of page faults in the system.
The gain from this option is that the device page table can be aligned with the presented pages in the CPU page table without causing page faults.
As of that, the overhead on data path from hardware point of view to trigger a fault which end-up by calling the driver to bring the pages will be dropped.
--- drivers/infiniband/core/umem_odp.c | 35 +++++++++++++++++++++--------- drivers/infiniband/hw/mlx5/odp.c | 2 +- include/rdma/ib_umem_odp.h | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-)
drivers/infiniband/core/umem_odp.c +25 -10
@@ -343,9 +343,10 @@ static int ib_umem_odp_map_dma_single_page(* the return value. * @access_mask: bit mask of the requested access permissions for the given * range. +* @fault: is faulting required for the given range */ int ib_umem_odp_map_dma_and_lock (struct ib_umem_odp *umem_odp,u64 user_virt, -u64 bcnt,u64 access_mask) +u64 bcnt,u64 access_mask,bool fault) __acquires(&umem_odp->umem_mutex) {struct task_struct *owning_process =NULL ;
@@ -381,10 +382,12 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, range.end =ALIGN (user_virt + bcnt,1UL << page_shift); pfn_start_idx = (range.start -ib_umem_start (umem_odp)) >>PAGE_SHIFT ; num_pfns = (range.end - range.start) >>PAGE_SHIFT ; - range.default_flags =HMM_PFN_REQ_FAULT ; +if (fault) { + range.default_flags =HMM_PFN_REQ_FAULT ; -if (access_mask &ODP_WRITE_ALLOWED_BIT ) - range.default_flags |=HMM_PFN_REQ_WRITE ; +if (access_mask &ODP_WRITE_ALLOWED_BIT ) + range.default_flags |=HMM_PFN_REQ_WRITE ; + } range.hmm_pfns = &(umem_odp->pfn_list[pfn_start_idx]); timeout = jiffies +msecs_to_jiffies (HMM_RANGE_DEFAULT_TIMEOUT );
@@ -413,12 +416,24 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,for (pfn_index =0 ; pfn_index < num_pfns; pfn_index +=1 << (page_shift -PAGE_SHIFT ), dma_index++) { -/* -* Since we asked for hmm_range_fault() to populate pages, -* it shouldn't return an error entry on success. -*/ -WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); -WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + +if (fault) { +/* +* Since we asked for hmm_range_fault() to populate pages, +* it shouldn't return an error entry on success. +*/ +WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); +WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + }else { +if (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )) { +WARN_ON (umem_odp->dma_list[dma_index]); +continue ; + } + access_mask = (range.hmm_pfns[pfn_index] &HMM_PFN_WRITE ) ? + (ODP_READ_ALLOWED_BIT |ODP_WRITE_ALLOWED_BIT ) : +ODP_READ_ALLOWED_BIT ; + } + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]);/* If a hugepage was detected and ODP wasn't set for, the umem * page_shift will be used, the opposite case is an error.
drivers/infiniband/hw/mlx5/odp.c +1 -1
@@ -681,7 +681,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,if (odp->umem.writable && !downgrade) access_mask |=ODP_WRITE_ALLOWED_BIT ; - np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask); + np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask, true);if (np <0 )return np;
include/rdma/ib_umem_odp.h +1 -1
@@ -94,7 +94,7 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr,void ib_umem_odp_release (struct ib_umem_odp *umem_odp);int ib_umem_odp_map_dma_and_lock (struct ib_umem_odp *umem_odp,u64 start_offset, -u64 bcnt,u64 access_mask); +u64 bcnt,u64 access_mask,bool fault);void ib_umem_odp_unmap_dma_pages (struct ib_umem_odp *umem_odp,u64 start_offset,u64 bound);
Signature · 1 line
2.26.2
Re: [PATCH rdma-next 2/4] IB/core: Enable ODP sync without faulting
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 422
Quoted 3 lines
drivers/infiniband/core/umem_odp.c +3
-0+if (fault) { +/* +* Since we asked for hmm_range_fault() to populate pages,
Totally pointless line over 80 characters.
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 435
Quoted 4 lines
drivers/infiniband/core/umem_odp.c +4
-0+ access_mask = (range.hmm_pfns[pfn_index] &HMM_PFN_WRITE ) ? + (ODP_READ_ALLOWED_BIT |ODP_WRITE_ALLOWED_BIT ) : +ODP_READ_ALLOWED_BIT ; + }
Another weird overly long line, caused by rather osfucated code. This really should be something like:
access_mask = ODP_READ_ALLOWED_BIT; if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) access_mask |= ODP_WRITE_ALLOWED_BIT;
Re: [PATCH rdma-next 2/4] IB/core: Enable ODP sync without faulting
Quoted 5 lines · Christoph Hellwig
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 422
Quoted 3 lines
drivers/infiniband/core/umem_odp.c +3
-0+if (fault) { +/* +* Since we asked for hmm_range_fault() to populate pages, Totally pointless line over 80 characters.
checkpatch.pl was updated to allow 100 symbols.
Regarding "pointless", at least for me it wasn't clear why we don't have HMM_PFN_ERROR check in non-fault path so I asked form Yishai to add this comment.
Quoted 9 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 435
Quoted 5 lines
drivers/infiniband/core/umem_odp.c +4
-0+ access_mask = (range.hmm_pfns[pfn_index] &HMM_PFN_WRITE ) ? + (ODP_READ_ALLOWED_BIT |ODP_WRITE_ALLOWED_BIT ) : +ODP_READ_ALLOWED_BIT ; + }Another weird overly long line, caused by rather osfucated code. This really should be something like:
Quoted 3 lines
access_mask = ODP_READ_ALLOWED_BIT; if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) access_mask |= ODP_WRITE_ALLOWED_BIT;
Sure
Quoted 1 line
Re: [PATCH rdma-next 2/4] IB/core: Enable ODP sync without faulting
Quoted 8 lines · Leon Romanovsky
Quoted 5 lines · Christoph Hellwig
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 422
Quoted 3 lines
drivers/infiniband/core/umem_odp.c +3
-0+if (fault) { +/* +* Since we asked for hmm_range_fault() to populate pages, Totally pointless line over 80 characters.
checkpatch.pl was updated to allow 100 symbols.
checkpatch.pl doesn't matter, it is in fact pretty silly. The coding style document says:
"The preferred limit on the length of a single line is 80 columns.
Statements longer than 80 columns should be broken into sensible chunks, unless exceeding 80 columns significantly increases readability and does not hide information."
and that absolutely is not the case here, you could trivially move the last word into the next line and would not even increase the line count.
Re: [PATCH rdma-next 2/4] IB/core: Enable ODP sync without faulting
Quoted 13 lines · Christoph Hellwig
Quoted 8 lines · Leon Romanovsky
Quoted 5 lines · Christoph Hellwig
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 422
Quoted 3 lines
drivers/infiniband/core/umem_odp.c +3
-0+if (fault) { +/* +* Since we asked for hmm_range_fault() to populate pages, Totally pointless line over 80 characters.
checkpatch.pl was updated to allow 100 symbols.
checkpatch.pl doesn't matter, it is in fact pretty silly. The coding style document says:
Quoted 8 lines
"The preferred limit on the length of a single line is 80 columns.
Statements longer than 80 columns should be broken into sensible chunks, unless exceeding 80 columns significantly increases readability and does not hide information."
and that absolutely is not the case here, you could trivially move the last word into the next line and would not even increase the line count.
I'm not arguing, just explained why it was missed.
Thanks
Re: [PATCH rdma-next 2/4] IB/core: Enable ODP sync without faulting
In reply to Christoph HellwigQuoted 4 lines · Christoph Hellwig
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 422
Quoted 3 lines
drivers/infiniband/core/umem_odp.c +3
-0+if (fault) { +/* +* Since we asked for hmm_range_fault() to populate pages, Totally pointless line over 80 characters.
I'll leave the comment as was asked by Leon, will fix to match 80 characters.
Quoted 11 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, · line 435
Quoted 4 lines
drivers/infiniband/core/umem_odp.c +4
-0+ access_mask = (range.hmm_pfns[pfn_index] &HMM_PFN_WRITE ) ? + (ODP_READ_ALLOWED_BIT |ODP_WRITE_ALLOWED_BIT ) : +ODP_READ_ALLOWED_BIT ; + }Another weird overly long line, caused by rather osfucated code. This really should be something like:
access_mask = ODP_READ_ALLOWED_BIT; if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) access_mask |= ODP_WRITE_ALLOWED_BIT;
No problem, will be part of V1.
PATCH 3/4 RDMA/mlx5: Extend advice MR to support non faulting mode
In reply to Leon RomanovskyFrom: Yishai Hadas <[email protected]>
Extend advice MR to support non faulting mode, this improves performance by eliminating page faults and bring only the existing CPU pages.
--- drivers/infiniband/hw/mlx5/mr.c | 3 ++- drivers/infiniband/hw/mlx5/odp.c | 7 ++++++- include/uapi/rdma/ib_user_ioctl_verbs.h | 1 + 3 files changed, 9 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +2 -1
@@ -1321,7 +1321,8 @@ int mlx5_ib_advise_mr(struct ib_pd *pd, struct uverbs_attr_bundle *attrs) {if (advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && - advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE ) + advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && + advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT )return -EOPNOTSUPP ; return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
drivers/infiniband/hw/mlx5/odp.c +6 -1
@@ -665,6 +665,7 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) }#define MLX5_PF_FLAGS_DOWNGRADE BIT(1) +#define MLX5_PF_FLAGS_SNAPSHOT BIT(2)static int pagefault_real_mr (struct mlx5_ib_mr *mr,struct ib_umem_odp *odp,u64 user_va,size_t bcnt,u32 *bytes_mapped,u32 flags)
@@ -673,6 +674,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,bool downgrade = flags &MLX5_PF_FLAGS_DOWNGRADE ;u64 access_mask;u64 start_idx; +bool fault = !(flags &MLX5_PF_FLAGS_SNAPSHOT ); page_shift = odp->page_shift; start_idx = (user_va -ib_umem_start (odp)) >> page_shift;
@@ -681,7 +683,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,if (odp->umem.writable && !downgrade) access_mask |=ODP_WRITE_ALLOWED_BIT ; - np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask, true); + np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask, fault);if (np <0 )return np;
@@ -1852,6 +1854,9 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,if (advice ==IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH ) pf_flags |=MLX5_PF_FLAGS_DOWNGRADE ; +if (advice ==IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT ) + pf_flags |=MLX5_PF_FLAGS_SNAPSHOT ; +if (flags &IB_UVERBS_ADVISE_MR_FLAG_FLUSH )return mlx5_ib_prefetch_sg_list (pd, advice, pf_flags, sg_list, num_sge);
include/uapi/rdma/ib_user_ioctl_verbs.h +1 -0
@@ -208,6 +208,7 @@ enum ib_uverbs_read_counters_flags {enum ib_uverbs_advise_mr_advice {IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH ,IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE , +IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT , };enum ib_uverbs_advise_mr_flag {
Signature · 1 line
2.26.2
PATCH 4/4 RDMA/mlx5: Sync device with CPU pages upon ODP MR registration
In reply to Leon RomanovskyFrom: Yishai Hadas <[email protected]>
Sync device with CPU pages upon ODP MR registration.
This reduce potential page faults down the road and improve performance.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 6 ++++++ drivers/infiniband/hw/mlx5/mr.c | 11 +++++++---- drivers/infiniband/hw/mlx5/odp.c | 22 +++++++++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +6 -0
@@ -1283,6 +1283,7 @@ void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,int mlx5_ib_advise_mr_prefetch (struct ib_pd *pd,enum ib_uverbs_advise_mr_advice advice,u32 flags,struct ib_sge *sg_list,u32 num_sge); +int mlx5_ib_init_odp_mr (struct mlx5_ib_mr *mr,u64 user_va,size_t bcnt,bool enable);#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline void mlx5_ib_internal_fill_odp_caps (struct mlx5_ib_dev *dev) {
@@ -1304,6 +1305,11 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, {return -EOPNOTSUPP ; } +static inline int mlx5_ib_init_odp_mr (struct mlx5_ib_mr *mr,u64 user_va, +size_t bcnt,bool enable) +{ +return -EOPNOTSUPP ; +}#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ extern const struct mmu_interval_notifier_ops mlx5_mn_ops;
drivers/infiniband/hw/mlx5/mr.c +7 -4
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->umem = umem;set_mr_fields (dev, mr, npages, length, access_flags); -if (xlt_with_umr) { +if (xlt_with_umr && !(access_flags &IB_ACCESS_ON_DEMAND )) {/* * If the MR was created with reg_create then it will be * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,*/ int update_xlt_flags =MLX5_IB_UPD_XLT_ENABLE ; -if (access_flags &IB_ACCESS_ON_DEMAND ) - update_xlt_flags |=MLX5_IB_UPD_XLT_ZAP ; - err =mlx5_ib_update_xlt (mr,0 , ncont, page_shift, update_xlt_flags);if (err) {
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,dereg_mr (dev, mr);return ERR_PTR (err); } + + err =mlx5_ib_init_odp_mr (mr, start, length, xlt_with_umr); +if (err) { +dereg_mr (dev, mr); +return ERR_PTR (err); + } }return &mr->ibmr;
drivers/infiniband/hw/mlx5/odp.c +21 -1
@@ -666,6 +666,7 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)#define MLX5_PF_FLAGS_DOWNGRADE BIT(1)#define MLX5_PF_FLAGS_SNAPSHOT BIT(2) +#define MLX5_PF_FLAGS_ENABLE BIT(3)static int pagefault_real_mr (struct mlx5_ib_mr *mr,struct ib_umem_odp *odp,u64 user_va,size_t bcnt,u32 *bytes_mapped,u32 flags)
@@ -675,6 +676,10 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,u64 access_mask;u64 start_idx;bool fault = !(flags &MLX5_PF_FLAGS_SNAPSHOT ); +u32 xlt_flags =MLX5_IB_UPD_XLT_ATOMIC ; + +if (flags &MLX5_PF_FLAGS_ENABLE ) + xlt_flags |=MLX5_IB_UPD_XLT_ENABLE ; page_shift = odp->page_shift; start_idx = (user_va -ib_umem_start (odp)) >> page_shift;
@@ -693,7 +698,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,* checks this. */ ret =mlx5_ib_update_xlt (mr, start_idx, np, - page_shift,MLX5_IB_UPD_XLT_ATOMIC ); + page_shift, xlt_flags);mutex_unlock (&odp->umem_mutex);if (ret <0 ) {
@@ -828,6 +833,21 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, flags); } +int mlx5_ib_init_odp_mr (struct mlx5_ib_mr *mr,u64 user_va,size_t bcnt, +bool enable) +{ +u32 flags =MLX5_PF_FLAGS_SNAPSHOT ; +int ret; + +if (enable) + flags |=MLX5_PF_FLAGS_ENABLE ; + + ret =pagefault_real_mr (mr,to_ib_umem_odp (mr->umem), + user_va, bcnt,NULL , + flags); +return ret >=0 ?0 : ret; +} +struct pf_frame {struct pf_frame *next;u32 key;
Signature · 1 line
2.26.2