Improve ODP by using HMM API
Subsystems (2)
v2 cover Improve ODP by using HMM API
From: Leon Romanovsky <[email protected]>
Changelog:
v2:
* Patch #1 – Drop redundant mask.
* Patch #4 – Use address and length directly from umem_odp.
v1: https://lore.kernel.org/lkml/[email protected]
* Cleaned code.
* Support a potential valid dma_address of NULL by flags detection, note was added to clarify things as was asked.
* Fix 80 character lines in few places.
v0: https://lore.kernel.org/lkml/[email protected]
Based on:
https://lore.kernel.org/lkml/[email protected]/
---------------------------------------------------------------------------------------
From Yishai:
This series improves ODP performance by moving to use the HMM API as of below.
The get_user_pages_remote() functionality was replaced by HMM:
- No need anymore to allocate and free memory to hold its output per call.
- No need anymore to use the put_page() to unpin the pages.
- The logic to detect contiguous pages is done based on the returned order from HMM, no need to run per page, and evaluate.
Moving to use the HMM enables to reduce page faults in the system by using the snapshot mode. This mode allows existing pages in the CPU to become presented to the device without faulting.
This non-faulting mode may be used explicitly by an application with some new option of advice MR (i.e. PREFETCH_NO_FAULT) and is used upon ODP MR registration internally as part of initiating the device page table.
To achieve the above, internal changes in the ODP data structures were done and some flows were cleaned-up/adapted accordingly.
Thanks
Yishai Hadas (4):
IB/core: Improve ODP to use hmm_range_fault() IB/core: Enable ODP sync without faulting
RDMA/mlx5: Extend advice MR to support non faulting mode RDMA/mlx5: Sync device with CPU pages upon ODP MR registration
drivers/infiniband/Kconfig | 1 + drivers/infiniband/core/umem_odp.c | 285 ++++++++++-------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 + drivers/infiniband/hw/mlx5/mr.c | 14 +- drivers/infiniband/hw/mlx5/odp.c | 48 ++-- include/rdma/ib_umem_odp.h | 21 +- include/uapi/rdma/ib_user_ioctl_verbs.h | 1 + 7 files changed, 173 insertions(+), 202 deletions(-)
--
2.26.2
PATCH v2 1/4 IB/core: Improve ODP to use hmm_range_fault()
From: Yishai Hadas <[email protected]>
Move to use hmm_range_fault() which improve performance in few aspects comparing previous functionality of get_user_pages_remote().
This includes:
- Dropping the need to allocate and free memory to hold its output.
- No need any more to use the put_page() to unpin the pages.
- The logic to detect contiguous pages is done based on the returned order, no need to run per page and evaluate.
In addition,
Moving to use hmm_range_fault() enables to reduce page faults in the system as it exposes the snapshot mode, this will be introduced in next patches from this series.
As part of this, cleanup some flows and use the required data structures to work with hmm_range_fault().
--- drivers/infiniband/Kconfig | 1 + drivers/infiniband/core/umem_odp.c | 272 +++++++++++------------------ drivers/infiniband/hw/mlx5/odp.c | 24 +-- include/rdma/ib_umem_odp.h | 21 +-- 4 files changed, 120 insertions(+), 198 deletions(-)
drivers/infiniband/Kconfig +1 -0
@@ -48,6 +48,7 @@ config INFINIBAND_ON_DEMAND_PAGING depends on INFINIBAND_USER_MEM select MMU_NOTIFIER select INTERVAL_TREE + select HMM_MIRROR default y help On demand paging support for the InfiniBand subsystem.
drivers/infiniband/core/umem_odp.c +104 -168
401 lines not on this page · raw message
drivers/infiniband/hw/mlx5/odp.c +7 -17
@@ -671,7 +671,6 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, {int page_shift, ret, np;bool downgrade = flags &MLX5_PF_FLAGS_DOWNGRADE ; -unsigned long current_seq;u64 access_mask;u64 start_idx;
@@ -682,25 +681,16 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,if (odp->umem.writable && !downgrade) access_mask |=ODP_WRITE_ALLOWED_BIT ; - current_seq =mmu_interval_read_begin (&odp->notifier); - - np =ib_umem_odp_map_dma_pages (odp, user_va, bcnt, access_mask, - current_seq); + np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask);if (np <0 )return np; -mutex_lock (&odp->umem_mutex); -if (!mmu_interval_read_retry (&odp->notifier, current_seq)) { -/* -* No need to check whether the MTTs really belong to -* this MR, since ib_umem_odp_map_dma_pages already -* checks this. -*/ - ret =mlx5_ib_update_xlt (mr, start_idx, np, - page_shift,MLX5_IB_UPD_XLT_ATOMIC ); - }else { - ret = -EAGAIN ; - } +/* +* No need to check whether the MTTs really belong to this MR, since +* ib_umem_odp_map_dma_and_lock already checks this. +*/ + ret =mlx5_ib_update_xlt (mr, start_idx, np, + page_shift,MLX5_IB_UPD_XLT_ATOMIC );mutex_unlock (&odp->umem_mutex);if (ret <0 ) {
include/rdma/ib_umem_odp.h +8 -13
@@ -14,17 +14,13 @@ struct ib_umem_odp {struct mmu_interval_notifier notifier;struct pid *tgid; +/* An array of the pfns included in the on-demand paging umem. */ +unsigned long *pfn_list; +/* -* An array of the pages included in the on-demand paging umem. -* Indices of pages that are currently not mapped into the device will -* contain NULL. -*/ -struct page **page_list; -/* -* An array of the same size as page_list, with DMA addresses mapped -* for pages the pages in page_list. The lower two bits designate -* access permissions. See ODP_READ_ALLOWED_BIT and -* ODP_WRITE_ALLOWED_BIT. +* An array with DMA addresses mapped for pfns in pfn_list. +* The lower two bits designate access permissions. +* See ODP_READ_ALLOWED_BIT and ODP_WRITE_ALLOWED_BIT. */ dma_addr_t *dma_list;/*
@@ -97,9 +93,8 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr, const struct mmu_interval_notifier_ops *ops);void ib_umem_odp_release (struct ib_umem_odp *umem_odp); -int ib_umem_odp_map_dma_pages (struct ib_umem_odp *umem_odp,u64 start_offset, -u64 bcnt,u64 access_mask, -unsigned long current_seq); +int ib_umem_odp_map_dma_and_lock (struct ib_umem_odp *umem_odp,u64 start_offset, +u64 bcnt,u64 access_mask);void ib_umem_odp_unmap_dma_pages (struct ib_umem_odp *umem_odp,u64 start_offset,u64 bound);
--
2.26.2
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 44 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +4
-23@@ -287,87 +289,48 @@ EXPORT_SYMBOL(ib_umem_odp_release);* Map for DMA and insert a single page into the on-demand paging page tables. * * @umem: the umem to insert the page to. -* @page_index: index in the umem to add the page to. +* @dma_index: index in the umem to add the dma to. * @page: the page struct to map and add. * @access_mask: access permissions needed for this page. * @current_seq: sequence number for synchronization with invalidations. * the sequence number is taken from * umem_odp->notifiers_seq. * -* The function returns -EFAULT if the DMA mapping operation fails. It returns -* -EAGAIN if a concurrent invalidation prevents us from updating the page. +* The function returns -EFAULT if the DMA mapping operation fails. * -* The page is released via put_page even if the operation failed. For on-demand -* pinning, the page is released whenever it isn't stored in the umem. */ static int ib_umem_odp_map_dma_single_page (struct ib_umem_odp *umem_odp, -unsigned int page_index, +unsigned int dma_index,struct page *page, -u64 access_mask, -unsigned long current_seq) +u64 access_mask) {struct ib_device *dev = umem_odp->umem.ibdev; -dma_addr_t dma_addr; -int ret =0 ; - -if (mmu_interval_check_retry (&umem_odp->notifier, current_seq)) { - ret = -EAGAIN ; - goto out; - } -if (!(umem_odp->dma_list[page_index])) { - dma_addr = -ib_dma_map_page (dev, page,0 ,BIT (umem_odp->page_shift), -DMA_BIDIRECTIONAL ); -if (ib_dma_mapping_error (dev, dma_addr)) { - ret = -EFAULT ; - goto out; - }
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 316
Quoted 10 lines
drivers/infiniband/core/umem_odp.c +8
-2- umem_odp->dma_list[page_index] = dma_addr | access_mask; - umem_odp->page_list[page_index] = page; +dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index]; + +if (!*dma_addr) { + *dma_addr =ib_dma_map_page (dev, page,0 , +1 << umem_odp->page_shift, +DMA_BIDIRECTIONAL ); +if (ib_dma_mapping_error (dev, *dma_addr)) +return -EFAULT ;
This leaves *dma_addr set to ib_dma_mapping_error, which means the next try to map it will fail the if (!dma_addr) and produce a corrupted dma address.
*dma_addr should be set to 0 here
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 5 lines · Jason Gunthorpe
This leaves
*dma_addrset toib_dma_mapping_error, which means the next try to map it will fail the if (!dma_addr) and produce a corrupted dma address.
*dma_addrshould be set to 0 here
Maybe the code should use DMA_MAPPING_ERROR as the sentinel for a not mapped entry?
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 9 lines · Christoph Hellwig
Quoted 5 lines · Jason Gunthorpe
This leaves
*dma_addrset toib_dma_mapping_error, which means the next try to map it will fail the if (!dma_addr) and produce a corrupted dma address.
*dma_addrshould be set to 0 hereMaybe the code should use
DMA_MAPPING_ERRORas the sentinel for a not mapped entry?
My eventual hope is to be able to send this DMA page list to the HW without having to parse and copy it like is done today. We already have it in a linear array that can be DMA'd from.
However, the HW knows 0 means need-fault (the flag bits are zero), it doesn't know what to do with DMA_MAPPING_ERROR..
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 6 lines · Jason Gunthorpe
My eventual hope is to be able to send this DMA page list to the HW without having to parse and copy it like is done today. We already have it in a linear array that can be DMA'd from.
However, the HW knows 0 means need-fault (the flag bits are zero), it doesn't know what to do with
DMA_MAPPING_ERROR..
I think you are falling into the same traps as the original hmm code. The above might be true for mlx5 hardware, but this is generic infrastructure. Making any assumptions about being able to directly pass it on to hardware is just futile. Nevermind such pesky things as endianess conversions.
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 5 lines · Christoph Hellwig
Quoted 4 lines · Jason Gunthorpe
My eventual hope is to be able to send this DMA page list to the HW without having to parse and copy it like is done today. We already have it in a linear array that can be DMA'd from.
Quoted 8 lines
Quoted 2 lines
However, the HW knows 0 means need-fault (the flag bits are zero), it doesn't know what to do with
DMA_MAPPING_ERROR..I think you are falling into the same traps as the original hmm code. The above might be true for mlx5 hardware, but this is generic infrastructure. Making any assumptions about being able to directly pass it on to hardware is just futile. Nevermind such pesky things as endianess conversions.
I was thinking to avoid that - the DMA related code would be pulled into the driver, and the core code would just do the hmm_range_fault() for the umem.
The driver should be the owner of the DMA list and it should be kept in the driver specific format.
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
In reply to Jason GunthorpeQuoted 59 lines · Jason Gunthorpe
Quoted 44 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +4
-23@@ -287,87 +289,48 @@ EXPORT_SYMBOL(ib_umem_odp_release);* Map for DMA and insert a single page into the on-demand paging page tables. * * @umem: the umem to insert the page to. -* @page_index: index in the umem to add the page to. +* @dma_index: index in the umem to add the dma to. * @page: the page struct to map and add. * @access_mask: access permissions needed for this page. * @current_seq: sequence number for synchronization with invalidations. * the sequence number is taken from * umem_odp->notifiers_seq. * -* The function returns -EFAULT if the DMA mapping operation fails. It returns -* -EAGAIN if a concurrent invalidation prevents us from updating the page. +* The function returns -EFAULT if the DMA mapping operation fails. * -* The page is released via put_page even if the operation failed. For on-demand -* pinning, the page is released whenever it isn't stored in the umem. */ static int ib_umem_odp_map_dma_single_page (struct ib_umem_odp *umem_odp, -unsigned int page_index, +unsigned int dma_index,struct page *page, -u64 access_mask, -unsigned long current_seq) +u64 access_mask) {struct ib_device *dev = umem_odp->umem.ibdev; -dma_addr_t dma_addr; -int ret =0 ; - -if (mmu_interval_check_retry (&umem_odp->notifier, current_seq)) { - ret = -EAGAIN ; - goto out; - } -if (!(umem_odp->dma_list[page_index])) { - dma_addr = -ib_dma_map_page (dev, page,0 ,BIT (umem_odp->page_shift), -DMA_BIDIRECTIONAL ); -if (ib_dma_mapping_error (dev, dma_addr)) { - ret = -EFAULT ; - goto out; - }drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 316
Quoted 10 lines
drivers/infiniband/core/umem_odp.c +8
-2- umem_odp->dma_list[page_index] = dma_addr | access_mask; - umem_odp->page_list[page_index] = page; +dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index]; + +if (!*dma_addr) { + *dma_addr =ib_dma_map_page (dev, page,0 , +1 << umem_odp->page_shift, +DMA_BIDIRECTIONAL ); +if (ib_dma_mapping_error (dev, *dma_addr)) +return -EFAULT ;This leaves
*dma_addrset toib_dma_mapping_error, which means the next try to map it will fail the if (!dma_addr) and produce a corrupted dma address.
Quoted 3 lines
*dma_addrshould be set to 0 hereJason
This makes sense, do we need V3 for this or that you can add locally ? the other notes in the other mail are quite straight forward as well or can be some nice to have I believe.
Yishai
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
In reply to Leon Romanovskydrivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320
Quoted 12 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +12
-0+if (!*dma_addr) { + *dma_addr =ib_dma_map_page (dev, page,0 , +1 << umem_odp->page_shift, +DMA_BIDIRECTIONAL ); +if (ib_dma_mapping_error (dev, *dma_addr)) { + *dma_addr =0 ; +return -EFAULT ; + } + umem_odp->npages++; + } + + *dma_addr |= access_mask;
This does need some masking, the purpose of this is to update the access flags in the case we hit a fault on a dma mapped thing. Looks like this can happen on a read-only page becoming writable again (wp_page_reuse() doesn't trigger notifiers)
It should also have a comment to that effect.
something like:
if (*dma_addr) {
/*
* If the page is already dma mapped it means it went through a
* non-invalidating trasition, like read-only to writable. Resync the
* flags.
*/
*dma_addr = (*dma_addr & (~ODP_DMA_ADDR_MASK)) | access_mask;
return;
}new_dma_addr = ib_dma_map_page()
[..]*dma_addr = new_dma_addr | access_mask
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, · line 427
Quoted 12 lines
drivers/infiniband/core/umem_odp.c +10
-0+WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); +WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]); +/* If a hugepage was detected and ODP wasn't set for, the umem +* page_shift will be used, the opposite case is an error. +*/ +if (hmm_order +PAGE_SHIFT < page_shift) { + ret = -EINVAL ; +pr_debug ("%s: un-expected hmm_order %d, page_shift %d\n" , + __func__, hmm_order, page_shift);break ; }
I think this break should be a continue here. There is no reason not to go to the next aligned PFN and try to sync as much as possible.
This should also
WARN_ON(umem_odp->dma_list[dma_index]);
And all the pr_debugs around this code being touched should become mlx5_ib_dbg
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 23 lines · Jason Gunthorpe
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320
Quoted 12 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +12
-0+if (!*dma_addr) { + *dma_addr =ib_dma_map_page (dev, page,0 , +1 << umem_odp->page_shift, +DMA_BIDIRECTIONAL ); +if (ib_dma_mapping_error (dev, *dma_addr)) { + *dma_addr =0 ; +return -EFAULT ; + } + umem_odp->npages++; + } + + *dma_addr |= access_mask;This does need some masking, the purpose of this is to update the access flags in the case we hit a fault on a dma mapped thing. Looks like this can happen on a read-only page becoming writable again (
wp_page_reuse()doesn't trigger notifiers)It should also have a comment to that effect.
something like:
Quoted 7 lines
if (
*dma_addr) {
/*
* If the page is already dma mapped it means it went through a
* non-invalidating trasition, like read-only to writable. Resync the
* flags.
*/
*dma_addr= (*dma_addr& (~ODP_DMA_ADDR_MASK)) |access_mask;
Did you mean
*dma_addr = (*dma_addr & (ODP_DMA_ADDR_MASK)) | access_mask;
Otherwise we may lose the dma_addr itself and just have the access flags. (see ODP_DMA_ADDR_MASK).
Also, if we went through a read->write access without invalidation why do we need to mask at all ? the new access_mask should have the write access.
Quoted 7 lines
return; }
new_dma_addr=ib_dma_map_page()
[..]*dma_addr=new_dma_addr|access_mask
Quoted 14 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, · line 427
Quoted 12 lines
drivers/infiniband/core/umem_odp.c +10
-0+WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); +WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]); +/* If a hugepage was detected and ODP wasn't set for, the umem +* page_shift will be used, the opposite case is an error. +*/ +if (hmm_order +PAGE_SHIFT < page_shift) { + ret = -EINVAL ; +pr_debug ("%s: un-expected hmm_order %d, page_shift %d\n" , + __func__, hmm_order, page_shift);break ; }I think this break should be a continue here. There is no reason not to go to the next aligned PFN and try to sync as much as possible.
This might happen if the application didn't honor the contract to use hugepages for the full range despite that it sets IB_ACCESS_HUGETLB, right ?
Do we still need to sync as much as possible in that case ? I believe that we may consider return an error in this case to let application be aware of as was before this series.
Quoted 6 lines
This should also
WARN_ON(umem_odp->dma_list[dma_index]);And all the
pr_debugsaround this code being touched should becomemlx5_ib_dbg
We are in IB core, why mlx5_ib_debug ?
Quoted 2 lines
Jason
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 31 lines · Yishai Hadas
Quoted 23 lines · Jason Gunthorpe
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320
Quoted 12 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +12
-0+if (!*dma_addr) { + *dma_addr =ib_dma_map_page (dev, page,0 , +1 << umem_odp->page_shift, +DMA_BIDIRECTIONAL ); +if (ib_dma_mapping_error (dev, *dma_addr)) { + *dma_addr =0 ; +return -EFAULT ; + } + umem_odp->npages++; + } + + *dma_addr |= access_mask;This does need some masking, the purpose of this is to update the access flags in the case we hit a fault on a dma mapped thing. Looks like this can happen on a read-only page becoming writable again (
wp_page_reuse()doesn't trigger notifiers)It should also have a comment to that effect.
something like:
Quoted 7 lines
if (
*dma_addr) {
/*
* If the page is already dma mapped it means it went through a
* non-invalidating trasition, like read-only to writable. Resync the
* flags.
*/
*dma_addr= (*dma_addr& (~ODP_DMA_ADDR_MASK)) |access_mask;
Quoted 3 lines
Did you mean
*dma_addr= (*dma_addr& (ODP_DMA_ADDR_MASK)) |access_mask;
Probably
Quoted 3 lines
flags. (see
ODP_DMA_ADDR_MASK). Also, if we went through aread->writeaccess without invalidation why do we need to mask at all ? the newaccess_maskshould have the write access.
Feels like a good idea to be safe here
Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +6
-0+WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); +WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]); +/* If a hugepage was detected and ODP wasn't set for, the umem +* page_shift will be used, the opposite case is an error. +*/ Quoted 8 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, · line 427
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +4
-0+if (hmm_order +PAGE_SHIFT < page_shift) { + ret = -EINVAL ; +pr_debug ("%s: un-expected hmm_order %d, page_shift %d\n" , + __func__, hmm_order, page_shift);break ; }I think this break should be a continue here. There is no reason not to go to the next aligned PFN and try to sync as much as possible.
Quoted 2 lines
This might happen if the application didn't honor the contract to use hugepages for the full range despite that it sets
IB_ACCESS_HUGETLB, right ?
Yes
Quoted 3 lines
Do we still need to sync as much as possible in that case ? I believe that we may consider return an error in this case to let application be aware of as was before this series.
We might be prefetching or something weird where it could make sense.
Quoted 7 lines
Quoted 6 lines
This should also
WARN_ON(umem_odp->dma_list[dma_index]);And all the
pr_debugsaround this code being touched should becomemlx5_ib_dbgWe are in IB core, why
mlx5_ib_debug?
oops, dev_dbg
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 34 lines · Jason Gunthorpe
Quoted 33 lines · Yishai Hadas
Quoted 23 lines · Jason Gunthorpe
drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320
Quoted 12 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +12
-0+if (!*dma_addr) { + *dma_addr =ib_dma_map_page (dev, page,0 , +1 << umem_odp->page_shift, +DMA_BIDIRECTIONAL ); +if (ib_dma_mapping_error (dev, *dma_addr)) { + *dma_addr =0 ; +return -EFAULT ; + } + umem_odp->npages++; + } + + *dma_addr |= access_mask;This does need some masking, the purpose of this is to update the access flags in the case we hit a fault on a dma mapped thing. Looks like this can happen on a read-only page becoming writable again (
wp_page_reuse()doesn't trigger notifiers)It should also have a comment to that effect.
something like:
Quoted 7 lines
if (
*dma_addr) {
/*
* If the page is already dma mapped it means it went through a
* non-invalidating trasition, like read-only to writable. Resync the
* flags.
*/
*dma_addr= (*dma_addr& (~ODP_DMA_ADDR_MASK)) |access_mask;Did you mean
Quoted 7 lines
Quoted 1 line
*dma_addr= (*dma_addr& (ODP_DMA_ADDR_MASK)) |access_mask;Probably
Quoted 3 lines
flags. (see
ODP_DMA_ADDR_MASK). Also, if we went through aread->writeaccess without invalidation why do we need to mask at all ? the newaccess_maskshould have the write access.Feels like a good idea to be safe here
It followed your note from V1 that the extra mask was really redundant, the original code also didn't have it, but up-to-you.
Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, · line 427
Quoted 13 lines
drivers/infiniband/core/umem_odp.c +10
-0+WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); +WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]); +/* If a hugepage was detected and ODP wasn't set for, the umem +* page_shift will be used, the opposite case is an error. +*/ +if (hmm_order +PAGE_SHIFT < page_shift) { + ret = -EINVAL ; +pr_debug ("%s: un-expected hmm_order %d, page_shift %d\n" , + __func__, hmm_order, page_shift);break ; }I think this break should be a continue here. There is no reason not to go to the next aligned PFN and try to sync as much as possible.
Quoted 8 lines
Quoted 2 lines
This might happen if the application didn't honor the contract to use hugepages for the full range despite that it sets
IB_ACCESS_HUGETLB, right ?Yes
Quoted 3 lines
Do we still need to sync as much as possible in that case ? I believe that we may consider return an error in this case to let application be aware of as was before this series.
We might be prefetching or something weird where it could make sense.
Not sure about the exact scenario rather than an application issue, this follows the original code in this area, maybe better sets an error in the clear application error.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
This should also
WARN_ON(umem_odp->dma_list[dma_index]);
Can you add it locally if you prefer the above approach ?
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines
And all the
pr_debugsaround this code being touched should becomemlx5_ib_dbgWe are in IB core, why
mlx5_ib_debug?oops,
dev_dbg
Can it can done locally ?
Quoted 1 line
Jason
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 6 lines · Yishai Hadas
Quoted 4 lines
Quoted 3 lines
flags. (see
ODP_DMA_ADDR_MASK). Also, if we went through aread->writeaccess without invalidation why do we need to mask at all ? the newaccess_maskshould have the write access.Feels like a good idea to be safe here
It followed your note from V1 that the extra mask was really redundant, the original code also didn't have it, but up-to-you.
It wasn't hard to read from the diff that this was being done in all cases, not just as the result of ib_dma_map_page(). This is why I think it would be clearer with the control flow I suggested
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines
Quoted 2 lines
And all the
pr_debugsaround this code being touched should becomemlx5_ib_dbgWe are in IB core, why
mlx5_ib_debug?oops,
dev_dbgCan it can done locally ?
As long as the ib_device is avaiable the print function should be used
Actually it is ibdev_dbg() these days
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
In reply to Jason GunthorpeQuoted 13 lines · Jason Gunthorpe
Quoted 13 lines
Quoted 13 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, · line 427
Quoted 11 lines
drivers/infiniband/core/umem_odp.c +9
-0+WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]); +/* If a hugepage was detected and ODP wasn't set for, the umem +* page_shift will be used, the opposite case is an error. +*/ +if (hmm_order +PAGE_SHIFT < page_shift) { + ret = -EINVAL ; +pr_debug ("%s: un-expected hmm_order %d, page_shift %d\n" , + __func__, hmm_order, page_shift);break ; }I think this break should be a continue here. There is no reason not to go to the next aligned PFN and try to sync as much as possible.
Quoted 10 lines
Quoted 2 lines
This might happen if the application didn't honor the contract to use hugepages for the full range despite that it sets
IB_ACCESS_HUGETLB, right ?Yes
Quoted 3 lines
Do we still need to sync as much as possible in that case ? I believe that we may consider return an error in this case to let application be aware of as was before this series.
We might be prefetching or something weird where it could make sense.
In addition to my previous note here as of below [1], ignoring the clear error case might break some testing that expects to get an error in this case when the contract was not honored.
Also not sure how the HW will behave, won't that cause an extra / infinite call to the driver to page fault for the missing data as the result will be success but no dma will be provided ? As of that I believe that better leave the code as is, what do you think ?
[1] "Not sure about the exact scenario rather than an application issue, this follows the original code in this area, maybe better sets an error in the clear application error."
Yishai
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 21 lines · Yishai Hadas
Quoted 20 lines · Jason Gunthorpe
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
drivers/infiniband/core/umem_odp.c +5
-0+WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]); +/* If a hugepage was detected and ODP wasn't set for, the umem +* page_shift will be used, the opposite case is an error. +*/ Quoted 8 lines
drivers/infiniband/core/umem_odp.c int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, · line 427
Quoted 6 lines
drivers/infiniband/core/umem_odp.c +4
-0+if (hmm_order +PAGE_SHIFT < page_shift) { + ret = -EINVAL ; +pr_debug ("%s: un-expected hmm_order %d, page_shift %d\n" , + __func__, hmm_order, page_shift);break ; }I think this break should be a continue here. There is no reason not to go to the next aligned PFN and try to sync as much as possible.
Quoted 2 lines
This might happen if the application didn't honor the contract to use hugepages for the full range despite that it sets
IB_ACCESS_HUGETLB, right ?Yes
Quoted 3 lines
Do we still need to sync as much as possible in that case ? I believe that we may consider return an error in this case to let application be aware of as was before this series.
Quoted 6 lines
Quoted 2 lines
We might be prefetching or something weird where it could make sense.
In addition to my previous note here as of below [1], ignoring the clear error case might break some testing that expects to get an error in this case when the contract was not honored.
The error code should be preserved, but not all callers care, like prefetch for instance
Quoted 4 lines
Also not sure how the HW will behave, won't that cause an extra / infinite call to the driver to page fault for the missing data as the result will be success but no dma will be provided ?
As of that I believe that better leave the code as is, what do you think ?
The HW must trigger fail if it reaches a pfn that isn't valid. The return code is an indirect indication this happened.
Please send an update tomorre with these small changes since it is late for me now
Jason
Re: [PATCH rdma-next v2 1/4] IB/core: Improve ODP to use hmm_range_fault()
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines
In addition to my previous note here as of below [1], ignoring the clear error case might break some testing that expects to get an error in this case when the contract was not honored.
The error code should be preserved, but not all callers care, like prefetch for instance
Right, but for now we have a single caller (i.e. pagefault_real_mr) which doesn't know in the general case whether it was a prefetch. This can be considered as of some future improvement that should be handled carefully, I prefer in this stage to stay with previous behavior in this area.
Quoted 4 lines
Please send an update tomorre with these small changes since it is late for me now
Jason
Sure, V3 may be sent soon, thanks for your feedback.
Yishai
PATCH v2 2/4 IB/core: Enable ODP sync without faulting
In reply to Leon RomanovskyFrom: Yishai Hadas <[email protected]>
Enable ODP sync without faulting, this improves performance by reducing the number of page faults in the system.
The gain from this option is that the device page table can be aligned with the presented pages in the CPU page table without causing page faults.
As of that, the overhead on data path from hardware point of view to trigger a fault which end-up by calling the driver to bring the pages will be dropped.
--- drivers/infiniband/core/umem_odp.c | 35 +++++++++++++++++++++--------- drivers/infiniband/hw/mlx5/odp.c | 2 +- include/rdma/ib_umem_odp.h | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-)
drivers/infiniband/core/umem_odp.c +25 -10
@@ -339,9 +339,10 @@ static int ib_umem_odp_map_dma_single_page(* the return value. * @access_mask: bit mask of the requested access permissions for the given * range. +* @fault: is faulting required for the given range */ int ib_umem_odp_map_dma_and_lock (struct ib_umem_odp *umem_odp,u64 user_virt, -u64 bcnt,u64 access_mask) +u64 bcnt,u64 access_mask,bool fault) __acquires(&umem_odp->umem_mutex) {struct task_struct *owning_process =NULL ;
@@ -377,10 +378,12 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, range.end =ALIGN (user_virt + bcnt,1UL << page_shift); pfn_start_idx = (range.start -ib_umem_start (umem_odp)) >>PAGE_SHIFT ; num_pfns = (range.end - range.start) >>PAGE_SHIFT ; - range.default_flags =HMM_PFN_REQ_FAULT ; +if (fault) { + range.default_flags =HMM_PFN_REQ_FAULT ; -if (access_mask &ODP_WRITE_ALLOWED_BIT ) - range.default_flags |=HMM_PFN_REQ_WRITE ; +if (access_mask &ODP_WRITE_ALLOWED_BIT ) + range.default_flags |=HMM_PFN_REQ_WRITE ; + } range.hmm_pfns = &(umem_odp->pfn_list[pfn_start_idx]); timeout = jiffies +msecs_to_jiffies (HMM_RANGE_DEFAULT_TIMEOUT );
@@ -409,12 +412,24 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,for (pfn_index =0 ; pfn_index < num_pfns; pfn_index +=1 << (page_shift -PAGE_SHIFT ), dma_index++) { -/* -* Since we asked for hmm_range_fault() to populate pages, -* it shouldn't return an error entry on success. -*/ -WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); -WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + +if (fault) { +/* +* Since we asked for hmm_range_fault() to populate +* pages it shouldn't return an error entry on success. +*/ +WARN_ON (range.hmm_pfns[pfn_index] &HMM_PFN_ERROR ); +WARN_ON (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )); + }else { +if (!(range.hmm_pfns[pfn_index] &HMM_PFN_VALID )) { +WARN_ON (umem_odp->dma_list[dma_index]); +continue ; + } + access_mask =ODP_READ_ALLOWED_BIT ; +if (range.hmm_pfns[pfn_index] &HMM_PFN_WRITE ) + access_mask |=ODP_WRITE_ALLOWED_BIT ; + } + hmm_order =hmm_pfn_to_map_order (range.hmm_pfns[pfn_index]);/* If a hugepage was detected and ODP wasn't set for, the umem * page_shift will be used, the opposite case is an error.
drivers/infiniband/hw/mlx5/odp.c +1 -1
@@ -681,7 +681,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,if (odp->umem.writable && !downgrade) access_mask |=ODP_WRITE_ALLOWED_BIT ; - np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask); + np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask, true);if (np <0 )return np;
include/rdma/ib_umem_odp.h +1 -1
@@ -94,7 +94,7 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr,void ib_umem_odp_release (struct ib_umem_odp *umem_odp);int ib_umem_odp_map_dma_and_lock (struct ib_umem_odp *umem_odp,u64 start_offset, -u64 bcnt,u64 access_mask); +u64 bcnt,u64 access_mask,bool fault);void ib_umem_odp_unmap_dma_pages (struct ib_umem_odp *umem_odp,u64 start_offset,u64 bound);
--
2.26.2
PATCH v2 3/4 RDMA/mlx5: Extend advice MR to support non faulting mode
In reply to Leon RomanovskyFrom: Yishai Hadas <[email protected]>
Extend advice MR to support non faulting mode, this improves performance by eliminating page faults and bring only the existing CPU pages.
--- drivers/infiniband/hw/mlx5/mr.c | 3 ++- drivers/infiniband/hw/mlx5/odp.c | 7 ++++++- include/uapi/rdma/ib_user_ioctl_verbs.h | 1 + 3 files changed, 9 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +2 -1
@@ -1321,7 +1321,8 @@ int mlx5_ib_advise_mr(struct ib_pd *pd, struct uverbs_attr_bundle *attrs) {if (advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && - advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE ) + advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && + advice !=IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT )return -EOPNOTSUPP ; return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
drivers/infiniband/hw/mlx5/odp.c +6 -1
@@ -665,6 +665,7 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) }#define MLX5_PF_FLAGS_DOWNGRADE BIT(1) +#define MLX5_PF_FLAGS_SNAPSHOT BIT(2)static int pagefault_real_mr (struct mlx5_ib_mr *mr,struct ib_umem_odp *odp,u64 user_va,size_t bcnt,u32 *bytes_mapped,u32 flags)
@@ -673,6 +674,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,bool downgrade = flags &MLX5_PF_FLAGS_DOWNGRADE ;u64 access_mask;u64 start_idx; +bool fault = !(flags &MLX5_PF_FLAGS_SNAPSHOT ); page_shift = odp->page_shift; start_idx = (user_va -ib_umem_start (odp)) >> page_shift;
@@ -681,7 +683,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,if (odp->umem.writable && !downgrade) access_mask |=ODP_WRITE_ALLOWED_BIT ; - np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask, true); + np =ib_umem_odp_map_dma_and_lock (odp, user_va, bcnt, access_mask, fault);if (np <0 )return np;
@@ -1851,6 +1853,9 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,if (advice ==IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH ) pf_flags |=MLX5_PF_FLAGS_DOWNGRADE ; +if (advice ==IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT ) + pf_flags |=MLX5_PF_FLAGS_SNAPSHOT ; +if (flags &IB_UVERBS_ADVISE_MR_FLAG_FLUSH )return mlx5_ib_prefetch_sg_list (pd, advice, pf_flags, sg_list, num_sge);
include/uapi/rdma/ib_user_ioctl_verbs.h +1 -0
@@ -208,6 +208,7 @@ enum ib_uverbs_read_counters_flags {enum ib_uverbs_advise_mr_advice {IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH ,IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE , +IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT , };enum ib_uverbs_advise_mr_flag {
--
2.26.2
PATCH v2 4/4 RDMA/mlx5: Sync device with CPU pages upon ODP MR registration
In reply to Leon RomanovskyFrom: Yishai Hadas <[email protected]>
Sync device with CPU pages upon ODP MR registration.
This reduce potential page faults down the road and improve performance.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +++++ drivers/infiniband/hw/mlx5/mr.c | 11 +++++++---- drivers/infiniband/hw/mlx5/odp.c | 21 ++++++++++++++++++++- 3 files changed, 32 insertions(+), 5 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +5 -0
@@ -1283,6 +1283,7 @@ void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,int mlx5_ib_advise_mr_prefetch (struct ib_pd *pd,enum ib_uverbs_advise_mr_advice advice,u32 flags,struct ib_sge *sg_list,u32 num_sge); +int mlx5_ib_init_odp_mr (struct mlx5_ib_mr *mr,bool enable);#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline void mlx5_ib_internal_fill_odp_caps (struct mlx5_ib_dev *dev) {
@@ -1304,6 +1305,10 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, {return -EOPNOTSUPP ; } +static inline int mlx5_ib_init_odp_mr (struct mlx5_ib_mr *mr,bool enable) +{ +return -EOPNOTSUPP ; +}#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ extern const struct mmu_interval_notifier_ops mlx5_mn_ops;
drivers/infiniband/hw/mlx5/mr.c +7 -4
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->umem = umem;set_mr_fields (dev, mr, npages, length, access_flags); -if (xlt_with_umr) { +if (xlt_with_umr && !(access_flags &IB_ACCESS_ON_DEMAND )) {/* * If the MR was created with reg_create then it will be * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,*/ int update_xlt_flags =MLX5_IB_UPD_XLT_ENABLE ; -if (access_flags &IB_ACCESS_ON_DEMAND ) - update_xlt_flags |=MLX5_IB_UPD_XLT_ZAP ; - err =mlx5_ib_update_xlt (mr,0 , ncont, page_shift, update_xlt_flags);if (err) {
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,dereg_mr (dev, mr);return ERR_PTR (err); } + + err =mlx5_ib_init_odp_mr (mr, xlt_with_umr); +if (err) { +dereg_mr (dev, mr); +return ERR_PTR (err); + } }return &mr->ibmr;
drivers/infiniband/hw/mlx5/odp.c +20 -1
@@ -666,6 +666,7 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)#define MLX5_PF_FLAGS_DOWNGRADE BIT(1)#define MLX5_PF_FLAGS_SNAPSHOT BIT(2) +#define MLX5_PF_FLAGS_ENABLE BIT(3)static int pagefault_real_mr (struct mlx5_ib_mr *mr,struct ib_umem_odp *odp,u64 user_va,size_t bcnt,u32 *bytes_mapped,u32 flags)
@@ -675,6 +676,10 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,u64 access_mask;u64 start_idx;bool fault = !(flags &MLX5_PF_FLAGS_SNAPSHOT ); +u32 xlt_flags =MLX5_IB_UPD_XLT_ATOMIC ; + +if (flags &MLX5_PF_FLAGS_ENABLE ) + xlt_flags |=MLX5_IB_UPD_XLT_ENABLE ; page_shift = odp->page_shift; start_idx = (user_va -ib_umem_start (odp)) >> page_shift;
@@ -692,7 +697,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,* ib_umem_odp_map_dma_and_lock already checks this. */ ret =mlx5_ib_update_xlt (mr, start_idx, np, - page_shift,MLX5_IB_UPD_XLT_ATOMIC ); + page_shift, xlt_flags);mutex_unlock (&odp->umem_mutex);if (ret <0 ) {
@@ -827,6 +832,20 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, flags); } +int mlx5_ib_init_odp_mr (struct mlx5_ib_mr *mr,bool enable) +{ +u32 flags =MLX5_PF_FLAGS_SNAPSHOT ; +int ret; + +if (enable) + flags |=MLX5_PF_FLAGS_ENABLE ; + + ret =pagefault_real_mr (mr,to_ib_umem_odp (mr->umem), + mr->umem->address, mr->umem->length,NULL , + flags); +return ret >=0 ?0 : ret; +} +struct pf_frame {struct pf_frame *next;u32 key;
--
2.26.2