Skip to contentSkip to conversation

Improve ODP by using HMM API

PATCH Superseded Leon Romanovsky 3 participants 13 messages Sep 17, 2020 – Sep 21, 2020

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#1 lore raw

PATCH 1/4 IB/core: Improve ODP to use hmm_range_fault()

In reply to Leon Romanovsky

From: Yishai Hadas <[email protected]>

Move to use hmm_range_fault() which improve performance in few aspects comparing previous functionality of get_user_pages_remote().

This includes:
- Dropping the need to allocate and free memory to hold its output.
- No need any more to use the put_page() to unpin the pages.
- The logic to detect contiguous pages is done based on the returned order, no need to run per page and evaluate.

In addition,
Moving to use hmm_range_fault() enables to reduce page faults in the system as it exposes the snapshot mode, this will be introduced in next patches from this series.

As part of this, cleanup some flows and use the required data structures to work with hmm_range_fault().

---
 drivers/infiniband/Kconfig         |   1 +
 drivers/infiniband/core/umem_odp.c | 273 +++++++++++------------------
 drivers/infiniband/hw/mlx5/odp.c   |  24 +--
 include/rdma/ib_umem_odp.h         |  21 +--
 4 files changed, 121 insertions(+), 198 deletions(-)
drivers/infiniband/Kconfig +1 -0
index 91b023341b77..32a51432ec4f 100644
@@ -48,6 +48,7 @@ config INFINIBAND_ON_DEMAND_PAGING
 	depends on INFINIBAND_USER_MEM
 	select MMU_NOTIFIER
 	select INTERVAL_TREE
+	select HMM_MIRROR
 	default y
 	help
 	  On demand paging support for the InfiniBand subsystem.
drivers/infiniband/core/umem_odp.c +105 -168

402 lines not on this page · raw message

drivers/infiniband/hw/mlx5/odp.c +7 -17
index e40a80c6636c..0f203141a6ad 100644
@@ -671,7 +671,6 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 {
 	int page_shift, ret, np;
 	bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE;
-	unsigned long current_seq;
 	u64 access_mask;
 	u64 start_idx;
 
@@ -682,25 +681,16 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 	if (odp->umem.writable && !downgrade)
 		access_mask |= ODP_WRITE_ALLOWED_BIT;
 
-	current_seq = mmu_interval_read_begin(&odp->notifier);
-
-	np = ib_umem_odp_map_dma_pages(odp, user_va, bcnt, access_mask,
-				       current_seq);
+	np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask);
 	if (np < 0)
 		return np;
 
-	mutex_lock(&odp->umem_mutex);
-	if (!mmu_interval_read_retry(&odp->notifier, current_seq)) {
-		/*
-		 * No need to check whether the MTTs really belong to
-		 * this MR, since ib_umem_odp_map_dma_pages already
-		 * checks this.
-		 */
-		ret = mlx5_ib_update_xlt(mr, start_idx, np,
-					 page_shift, MLX5_IB_UPD_XLT_ATOMIC);
-	} else {
-		ret = -EAGAIN;
-	}
+	/*
+	 * No need to check whether the MTTs really belong to this MR, since
+	 * ib_umem_odp_map_dma_and_lock already checks this.
+	 */
+	ret = mlx5_ib_update_xlt(mr, start_idx, np,
+				 page_shift, MLX5_IB_UPD_XLT_ATOMIC);
 	mutex_unlock(&odp->umem_mutex);
 
 	if (ret < 0) {
include/rdma/ib_umem_odp.h +8 -13
index d16d2c17e733..a53b62ac8a9d 100644
@@ -14,17 +14,13 @@ struct ib_umem_odp {
 	struct mmu_interval_notifier notifier;
 	struct pid *tgid;
 
+	/* An array of the pfns included in the on-demand paging umem. */
+	unsigned long *pfn_list;
+
 	/*
-	 * An array of the pages included in the on-demand paging umem.
-	 * Indices of pages that are currently not mapped into the device will
-	 * contain NULL.
-	 */
-	struct page		**page_list;
-	/*
-	 * An array of the same size as page_list, with DMA addresses mapped
-	 * for pages the pages in page_list. The lower two bits designate
-	 * access permissions. See ODP_READ_ALLOWED_BIT and
-	 * ODP_WRITE_ALLOWED_BIT.
+	 * An array with DMA addresses mapped for pfns in pfn_list.
+	 * The lower two bits designate access permissions.
+	 * See ODP_READ_ALLOWED_BIT and ODP_WRITE_ALLOWED_BIT.
 	 */
 	dma_addr_t		*dma_list;
 	/*
@@ -97,9 +93,8 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr,
 			const struct mmu_interval_notifier_ops *ops);
 void ib_umem_odp_release(struct ib_umem_odp *umem_odp);
 
-int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 start_offset,
-			      u64 bcnt, u64 access_mask,
-			      unsigned long current_seq);
+int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 start_offset,
+				 u64 bcnt, u64 access_mask);
 
 void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 start_offset,
 				 u64 bound);

--
2.26.2

#8 lore raw

PATCH 4/4 RDMA/mlx5: Sync device with CPU pages upon ODP MR registration

In reply to Leon Romanovsky

From: Yishai Hadas <[email protected]>

Sync device with CPU pages upon ODP MR registration.
This reduce potential page faults down the road and improve performance.

---
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  6 ++++++
 drivers/infiniband/hw/mlx5/mr.c      | 11 +++++++----
 drivers/infiniband/hw/mlx5/odp.c     | 22 +++++++++++++++++++++-
 3 files changed, 34 insertions(+), 5 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +6 -0
index 6ab3efb75b21..8e77a262e44c 100644
@@ -1283,6 +1283,7 @@ void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
 			       enum ib_uverbs_advise_mr_advice advice,
 			       u32 flags, struct ib_sge *sg_list, u32 num_sge);
+int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, u64 user_va, size_t bcnt, bool enable);
 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
 {
@@ -1304,6 +1305,11 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
 {
 	return -EOPNOTSUPP;
 }
+static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, u64 user_va,
+				      size_t bcnt, bool enable)
+{
+	return -EOPNOTSUPP;
+}
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
 extern const struct mmu_interval_notifier_ops mlx5_mn_ops;
drivers/infiniband/hw/mlx5/mr.c +7 -4
index dea65e511a3e..234a5d25a072 100644
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	mr->umem = umem;
 	set_mr_fields(dev, mr, npages, length, access_flags);
 
-	if (xlt_with_umr) {
+	if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
 		/*
 		 * If the MR was created with reg_create then it will be
 		 * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		 */
 		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
 
-		if (access_flags & IB_ACCESS_ON_DEMAND)
-			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
-
 		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
 					 update_xlt_flags);
 		if (err) {
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			dereg_mr(dev, mr);
 			return ERR_PTR(err);
 		}
+
+		err = mlx5_ib_init_odp_mr(mr, start, length, xlt_with_umr);
+		if (err) {
+			dereg_mr(dev, mr);
+			return ERR_PTR(err);
+		}
 	}
 
 	return &mr->ibmr;
drivers/infiniband/hw/mlx5/odp.c +21 -1
index 28b7227d31bf..61bff5ecb356 100644
@@ -666,6 +666,7 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)
 
 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
 #define MLX5_PF_FLAGS_SNAPSHOT BIT(2)
+#define MLX5_PF_FLAGS_ENABLE BIT(3)
 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 			     u64 user_va, size_t bcnt, u32 *bytes_mapped,
 			     u32 flags)
@@ -675,6 +676,10 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 	u64 access_mask;
 	u64 start_idx;
 	bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT);
+	u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC;
+
+	if (flags & MLX5_PF_FLAGS_ENABLE)
+		xlt_flags |= MLX5_IB_UPD_XLT_ENABLE;
 
 	page_shift = odp->page_shift;
 	start_idx = (user_va - ib_umem_start(odp)) >> page_shift;
@@ -692,7 +697,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 	 * ib_umem_odp_map_dma_and_lock already checks this.
 	 */
 	ret = mlx5_ib_update_xlt(mr, start_idx, np,
-				 page_shift, MLX5_IB_UPD_XLT_ATOMIC);
+				 page_shift, xlt_flags);
 	mutex_unlock(&odp->umem_mutex);
 
 	if (ret < 0) {
@@ -827,6 +832,21 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt,
 				     flags);
 }
 
+int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, u64 user_va, size_t bcnt,
+			bool enable)
+{
+	u32 flags = MLX5_PF_FLAGS_SNAPSHOT;
+	int ret;
+
+	if (enable)
+		flags |= MLX5_PF_FLAGS_ENABLE;
+
+	ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem),
+				user_va, bcnt, NULL,
+				flags);
+	return ret >= 0 ? 0 : ret;
+}
+
 struct pf_frame {
 	struct pf_frame *next;
 	u32 key;

--
2.26.2

#7 lore raw

PATCH 3/4 RDMA/mlx5: Extend advice MR to support non faulting mode

In reply to Leon Romanovsky

From: Yishai Hadas <[email protected]>

Extend advice MR to support non faulting mode, this improves performance by eliminating page faults and bring only the existing CPU pages.

---
 drivers/infiniband/hw/mlx5/mr.c         | 3 ++-
 drivers/infiniband/hw/mlx5/odp.c        | 7 ++++++-
 include/uapi/rdma/ib_user_ioctl_verbs.h | 1 +
 3 files changed, 9 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +2 -1
index 6b0d6109afc6..dea65e511a3e 100644
@@ -1321,7 +1321,8 @@ int mlx5_ib_advise_mr(struct ib_pd *pd,
 		      struct uverbs_attr_bundle *attrs)
 {
 	if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &&
-	    advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE)
+	    advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE &&
+	    advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT)
 		return -EOPNOTSUPP;
 
 	return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
drivers/infiniband/hw/mlx5/odp.c +6 -1
index 5bd5e19d76a2..28b7227d31bf 100644
@@ -665,6 +665,7 @@ void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)
 }
 
 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
+#define MLX5_PF_FLAGS_SNAPSHOT BIT(2)
 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 			     u64 user_va, size_t bcnt, u32 *bytes_mapped,
 			     u32 flags)
@@ -673,6 +674,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 	bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE;
 	u64 access_mask;
 	u64 start_idx;
+	bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT);
 
 	page_shift = odp->page_shift;
 	start_idx = (user_va - ib_umem_start(odp)) >> page_shift;
@@ -681,7 +683,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 	if (odp->umem.writable && !downgrade)
 		access_mask |= ODP_WRITE_ALLOWED_BIT;
 
-	np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, true);
+	np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault);
 	if (np < 0)
 		return np;
 
@@ -1851,6 +1853,9 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
 	if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH)
 		pf_flags |= MLX5_PF_FLAGS_DOWNGRADE;
 
+	if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT)
+		pf_flags |= MLX5_PF_FLAGS_SNAPSHOT;
+
 	if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH)
 		return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list,
 						num_sge);
include/uapi/rdma/ib_user_ioctl_verbs.h +1 -0
index cfea82acfe57..22483799cd07 100644
@@ -208,6 +208,7 @@ enum ib_uverbs_read_counters_flags {
 enum ib_uverbs_advise_mr_advice {
 	IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH,
 	IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE,
+	IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT,
 };
 
 enum ib_uverbs_advise_mr_flag {

--
2.26.2

#0 lore raw

cover Improve ODP by using HMM API

From: Leon Romanovsky <[email protected]>

Changelog:
v1:
* Cleaned code.
* Support a potential valid dma_address of NULL by flags detection, note was added to clarify things as was asked.
* Fix 80 character lines in few places.
v0: https://lore.kernel.org/lkml/[email protected]

Based on:
https://lore.kernel.org/lkml/[email protected]/

---------------------------------------------------------------------------------------
From Yishai:

This series improves ODP performance by moving to use the HMM API as of below.

The get_user_pages_remote() functionality was replaced by HMM:
- No need anymore to allocate and free memory to hold its output per call.
- No need anymore to use the put_page() to unpin the pages.
- The logic to detect contiguous pages is done based on the returned order from HMM, no need to run per page, and evaluate.

Moving to use the HMM enables to reduce page faults in the system by using the snapshot mode. This mode allows existing pages in the CPU to become presented to the device without faulting.

This non-faulting mode may be used explicitly by an application with some new option of advice MR (i.e. PREFETCH_NO_FAULT) and is used upon ODP MR registration internally as part of initiating the device page table.

To achieve the above, internal changes in the ODP data structures were done and some flows were cleaned-up/adapted accordingly.

Thanks

Yishai Hadas (4):
IB/core: Improve ODP to use hmm_range_fault() IB/core: Enable ODP sync without faulting
RDMA/mlx5: Extend advice MR to support non faulting mode RDMA/mlx5: Sync device with CPU pages upon ODP MR registration

 drivers/infiniband/Kconfig              |   1 +
 drivers/infiniband/core/umem_odp.c      | 286 ++++++++++--------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h    |   6 +
 drivers/infiniband/hw/mlx5/mr.c         |  14 +-
 drivers/infiniband/hw/mlx5/odp.c        |  49 ++--
 include/rdma/ib_umem_odp.h              |  21 +-
 include/uapi/rdma/ib_user_ioctl_verbs.h |   1 +
 7 files changed, 176 insertions(+), 202 deletions(-)

--
2.26.2

#6 lore raw

PATCH 2/4 IB/core: Enable ODP sync without faulting

From: Yishai Hadas <[email protected]>

Enable ODP sync without faulting, this improves performance by reducing the number of page faults in the system.

The gain from this option is that the device page table can be aligned with the presented pages in the CPU page table without causing page faults.

As of that, the overhead on data path from hardware point of view to trigger a fault which end-up by calling the driver to bring the pages will be dropped.

---
 drivers/infiniband/core/umem_odp.c | 35 +++++++++++++++++++++---------
 drivers/infiniband/hw/mlx5/odp.c   |  2 +-
 include/rdma/ib_umem_odp.h         |  2 +-
 3 files changed, 27 insertions(+), 12 deletions(-)
drivers/infiniband/core/umem_odp.c +25 -10
index 3d702aa66d6d..2ba1b46b4c7d 100644
@@ -340,9 +340,10 @@ static int ib_umem_odp_map_dma_single_page(
  *        the return value.
  * @access_mask: bit mask of the requested access permissions for the given
  *               range.
+ * @fault: is faulting required for the given range
  */
 int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,
-				 u64 bcnt, u64 access_mask)
+				 u64 bcnt, u64 access_mask, bool fault)
 			__acquires(&umem_odp->umem_mutex)
 {
 	struct task_struct *owning_process  = NULL;
@@ -378,10 +379,12 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,
 	range.end = ALIGN(user_virt + bcnt, 1UL << page_shift);
 	pfn_start_idx = (range.start - ib_umem_start(umem_odp)) >> PAGE_SHIFT;
 	num_pfns = (range.end - range.start) >> PAGE_SHIFT;
-	range.default_flags = HMM_PFN_REQ_FAULT;
+	if (fault) {
+		range.default_flags = HMM_PFN_REQ_FAULT;
 
-	if (access_mask & ODP_WRITE_ALLOWED_BIT)
-		range.default_flags |= HMM_PFN_REQ_WRITE;
+		if (access_mask & ODP_WRITE_ALLOWED_BIT)
+			range.default_flags |= HMM_PFN_REQ_WRITE;
+	}
 
 	range.hmm_pfns = &(umem_odp->pfn_list[pfn_start_idx]);
 	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
@@ -410,12 +413,24 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,
 
 	for (pfn_index = 0; pfn_index < num_pfns;
 		pfn_index += 1 << (page_shift - PAGE_SHIFT), dma_index++) {
-		/*
-		 * Since we asked for hmm_range_fault() to populate pages,
-		 * it shouldn't return an error entry on success.
-		 */
-		WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR);
-		WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID));
+
+		if (fault) {
+			/*
+			 * Since we asked for hmm_range_fault() to populate
+			 * pages it shouldn't return an error entry on success.
+			 */
+			WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR);
+			WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID));
+		} else {
+			if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) {
+				WARN_ON(umem_odp->dma_list[dma_index]);
+				continue;
+			}
+			access_mask = ODP_READ_ALLOWED_BIT;
+			if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE)
+				access_mask |= ODP_WRITE_ALLOWED_BIT;
+		}
+
 		hmm_order = hmm_pfn_to_map_order(range.hmm_pfns[pfn_index]);
 		/* If a hugepage was detected and ODP wasn't set for, the umem
 		 * page_shift will be used, the opposite case is an error.
drivers/infiniband/hw/mlx5/odp.c +1 -1
index 0f203141a6ad..5bd5e19d76a2 100644
@@ -681,7 +681,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 	if (odp->umem.writable && !downgrade)
 		access_mask |= ODP_WRITE_ALLOWED_BIT;
 
-	np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask);
+	np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, true);
 	if (np < 0)
 		return np;
 
include/rdma/ib_umem_odp.h +1 -1
index a53b62ac8a9d..0844c1d05ac6 100644
@@ -94,7 +94,7 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr,
 void ib_umem_odp_release(struct ib_umem_odp *umem_odp);
 
 int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 start_offset,
-				 u64 bcnt, u64 access_mask);
+				 u64 bcnt, u64 access_mask, bool fault);
 
 void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 start_offset,
 				 u64 bound);

--
2.26.2

maintainer
#2 lore raw

Re: [PATCH rdma-next v1 1/4] IB/core: Improve ODP to use hmm_range_fault()

In reply to Leon Romanovsky

drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320

Quoted 4 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +1 -3
-out:
-	put_page(page);
-	return ret;
+	*dma_addr &= ODP_DMA_ADDR_MASK;

I thought we agreed to get rid of this because it must be zero?

Jason

maintainer
#9 lore raw

Re: [PATCH rdma-next v1 4/4] RDMA/mlx5: Sync device with CPU pages upon ODP MR registration

In reply to Leon Romanovsky
Quoted 23 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +2 -4
index dea65e511a3e..234a5d25a072 100644
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	mr->umem = umem;
 	set_mr_fields(dev, mr, npages, length, access_flags);
 
-	if (xlt_with_umr) {
+	if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
 		/*
 		 * If the MR was created with reg_create then it will be
 		 * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		 */
 		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
 
-		if (access_flags & IB_ACCESS_ON_DEMAND)
-			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
-
 		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
 					 update_xlt_flags);
 		if (err) {

drivers/infiniband/hw/mlx5/mr.c struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · line 1468

Quoted 5 lines
drivers/infiniband/hw/mlx5/mr.c +2 -0
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			dereg_mr(dev, mr);
 			return ERR_PTR(err);
 		}
+
+		err = mlx5_ib_init_odp_mr(mr, start, length, xlt_with_umr);

No reason to pass start/length, that is already in the mr

Jason

#3 lore raw

Re: [PATCH rdma-next v1 1/4] IB/core: Improve ODP to use hmm_range_fault()

In reply to Jason Gunthorpe
Quoted 9 lines · Jason Gunthorpe

drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320

Quoted 4 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +1 -3
-out:
-	put_page(page);
-	return ret;
+	*dma_addr &= ODP_DMA_ADDR_MASK;

I thought we agreed to get rid of this because it must be zero?

Jason

At the end we agreed to not rely on as potentially dma_addr may be zero, the access flags are used  to mark an entry as valid in this case (see also ib_umem_odp_unmap_dma_pages).

Yishai

maintainer
#4 lore raw

Re: [PATCH rdma-next v1 1/4] IB/core: Improve ODP to use hmm_range_fault()

Quoted 9 lines · Yishai Hadas
Quoted 8 lines · Jason Gunthorpe

drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320

Quoted 4 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +1 -3
-out:
-	put_page(page);
-	return ret;
+	*dma_addr &= ODP_DMA_ADDR_MASK;

I thought we agreed to get rid of this because it must be zero?

Quoted 5 lines
Quoted 1 line

Jason

At the end we agreed to not rely on as potentially dma_addr may be zero, the access flags are used  to mark an entry as valid in this case (see also ib_umem_odp_unmap_dma_pages).

I mean the mask, that masking is always a NOP or it is really broken

Jason

#10 lore raw

Re: [PATCH rdma-next v1 4/4] RDMA/mlx5: Sync device with CPU pages upon ODP MR registration

In reply to Jason Gunthorpe
Quoted 32 lines · Jason Gunthorpe
Quoted 23 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +2 -4
index dea65e511a3e..234a5d25a072 100644
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  	mr->umem = umem;
  	set_mr_fields(dev, mr, npages, length, access_flags);
 
-	if (xlt_with_umr) {
+	if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
  		/*
  		 * If the MR was created with reg_create then it will be
  		 * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  		 */
  		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
 
-		if (access_flags & IB_ACCESS_ON_DEMAND)
-			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
-
  		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
  					 update_xlt_flags);
  		if (err) {

drivers/infiniband/hw/mlx5/mr.c struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · line 1468

Quoted 5 lines
drivers/infiniband/hw/mlx5/mr.c +2 -0
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  			dereg_mr(dev, mr);
  			return ERR_PTR(err);
  		}
+
+		err = mlx5_ib_init_odp_mr(mr, start, length, xlt_with_umr);

No reason to pass start/length, that is already in the mr

Quoted 1 line

Jason

The start / iova is set on 'ib_mr' in the uverbs layer post returning from reg_user_mr for all drivers [1], so the function just got both. Makes sense ?

[1] https://elixir.bootlin.com/linux/v5.9-rc6/source/drivers/infiniband/core/uverbs_cmd.c#L746

Yishai

maintainer
#11 lore raw

Re: [PATCH rdma-next v1 4/4] RDMA/mlx5: Sync device with CPU pages upon ODP MR registration

Quoted 33 lines · Yishai Hadas
Quoted 32 lines · Jason Gunthorpe
Quoted 23 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +2 -4
index dea65e511a3e..234a5d25a072 100644
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  	mr->umem = umem;
  	set_mr_fields(dev, mr, npages, length, access_flags);
 
-	if (xlt_with_umr) {
+	if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
  		/*
  		 * If the MR was created with reg_create then it will be
  		 * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  		 */
  		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
 
-		if (access_flags & IB_ACCESS_ON_DEMAND)
-			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
-
  		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
  					 update_xlt_flags);
  		if (err) {

drivers/infiniband/hw/mlx5/mr.c struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · line 1468

Quoted 5 lines
drivers/infiniband/hw/mlx5/mr.c +2 -0
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
  			dereg_mr(dev, mr);
  			return ERR_PTR(err);
  		}
+
+		err = mlx5_ib_init_odp_mr(mr, start, length, xlt_with_umr);

No reason to pass start/length, that is already in the mr

Quoted 5 lines
Quoted 1 line

Jason

The start / iova is set on 'ib_mr' in the uverbs layer post returning from reg_user_mr for all drivers [1], so the function just got both. Makes sense ?

The information is stored in the umem_odp

Jason

#5 lore raw

Re: [PATCH rdma-next v1 1/4] IB/core: Improve ODP to use hmm_range_fault()

In reply to Jason Gunthorpe
Quoted 10 lines · Jason Gunthorpe
Quoted 9 lines · Yishai Hadas
Quoted 8 lines · Jason Gunthorpe

drivers/infiniband/core/umem_odp.c EXPORT_SYMBOL(ib_umem_odp_release); · line 320

Quoted 4 lines · Leon Romanovsky
drivers/infiniband/core/umem_odp.c +1 -3
-out:
-	put_page(page);
-	return ret;
+	*dma_addr &= ODP_DMA_ADDR_MASK;

I thought we agreed to get rid of this because it must be zero?

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Jason

At the end we agreed to not rely on as potentially dma_addr may be zero, the access flags are used  to mark an entry as valid in this case (see also ib_umem_odp_unmap_dma_pages).

I mean the mask, that masking is always a NOP or it is really broken

Jason

I tend to agree that this is redundant as the invalidation flow should to the cleanup in case was some change in access from previous setting. However,
Don't we want to have it for clearness at least as of the non-faulting mode (i.e. snapshot) that we just add in this series which get current CPU page access and sets it ?

Yishai

#12 lore raw

Re: [PATCH rdma-next v1 4/4] RDMA/mlx5: Sync device with CPU pages upon ODP MR registration

In reply to Jason Gunthorpe
Quoted 34 lines · Jason Gunthorpe
Quoted 33 lines · Yishai Hadas
Quoted 32 lines · Jason Gunthorpe
Quoted 23 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +2 -4
index dea65e511a3e..234a5d25a072 100644
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1431,7 +1431,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
   	mr->umem = umem;
   	set_mr_fields(dev, mr, npages, length, access_flags);
 
-	if (xlt_with_umr) {
+	if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
   		/*
   		 * If the MR was created with reg_create then it will be
   		 * configured properly but left disabled. It is safe to go ahead
@@ -1439,9 +1439,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
   		 */
   		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
 
-		if (access_flags & IB_ACCESS_ON_DEMAND)
-			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
-
   		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
   					 update_xlt_flags);
   		if (err) {

drivers/infiniband/hw/mlx5/mr.c struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · line 1468

Quoted 5 lines
drivers/infiniband/hw/mlx5/mr.c +2 -0
@@ -1467,6 +1464,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
   			dereg_mr(dev, mr);
   			return ERR_PTR(err);
   		}
+
+		err = mlx5_ib_init_odp_mr(mr, start, length, xlt_with_umr);

No reason to pass start/length, that is already in the mr

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Jason

The start / iova is set on 'ib_mr' in the uverbs layer post returning from reg_user_mr for all drivers [1], so the function just got both. Makes sense ?

The information is stored in the umem_odp

Jason

OK, we may take it from there, will be part of V2.

Yishai