Skip to contentSkip to conversation

On-Demand Paging on SoftRoCE

PATCH Superseded Daisuke Matsuda 4 participants 13 messages Apr 19, 2023 – Apr 27, 2023

Subsystems (2)

v4 of 9 Superseded Latest is v9 →

lore
Share & tools
lore
#1 lore raw

PATCH v4 1/8 RDMA/rxe: Tentative workqueue implementation

In reply to Daisuke Matsuda

This is a mock patch and not intended to be merged. In order to implement ODP on rxe, it is necessary to convert triple tasklets to workqueue. I expect Bob Pearson will submit his patch to do this very soon.

---
 drivers/infiniband/sw/rxe/rxe.c      |  9 ++-
 drivers/infiniband/sw/rxe/rxe_task.c | 84 ++++++++++++++++++----------
 drivers/infiniband/sw/rxe/rxe_task.h |  6 +-
 3 files changed, 67 insertions(+), 32 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +8 -1
index 7a7e713de52d..54c723a6edda 100644
@@ -212,10 +212,16 @@ static int __init rxe_module_init(void)
 {
 	int err;
 
-	err = rxe_net_init();
+	err = rxe_alloc_wq();
 	if (err)
 		return err;
 
+	err = rxe_net_init();
+	if (err) {
+		rxe_destroy_wq();
+		return err;
+	}
+
 	rdma_link_register(&rxe_link_ops);
 	pr_info("loaded\n");
 	return 0;
@@ -226,6 +232,7 @@ static void __exit rxe_module_exit(void)
 	rdma_link_unregister(&rxe_link_ops);
 	ib_unregister_driver(RDMA_DRIVER_RXE);
 	rxe_net_exit();
+	rxe_destroy_wq();
 
 	pr_info("unloaded\n");
 }
drivers/infiniband/sw/rxe/rxe_task.c +54 -30

200 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_task.h +5 -1
index facb7c8e3729..a63e258b3d66 100644
@@ -22,7 +22,7 @@ enum {
  * called again.
  */
 struct rxe_task {
-	struct tasklet_struct	tasklet;
+	struct work_struct	work;
 	int			state;
 	spinlock_t		lock;
 	struct rxe_qp		*qp;
@@ -32,6 +32,10 @@ struct rxe_task {
 	long			num_done;
 };
 
+int rxe_alloc_wq(void);
+
+void rxe_destroy_wq(void);
+
 /*
  * init rxe_task structure
  *	qp  => parameter to pass to func
Signature · 1 line
2.39.1
#2 lore raw

PATCH v4 2/8 RDMA/rxe: Always schedule works before accessing user MRs

In reply to Daisuke Matsuda

Both responder and completer can sleep to execute page-fault when used with ODP. It happens when they are going to access user MRs, so works must be scheduled in such cases.

---
 drivers/infiniband/sw/rxe/rxe_comp.c | 12 ++++++++++--
 drivers/infiniband/sw/rxe/rxe_loc.h  |  4 ++--
 drivers/infiniband/sw/rxe/rxe_recv.c |  4 ++--
 drivers/infiniband/sw/rxe/rxe_resp.c | 14 +++++++++-----
 4 files changed, 23 insertions(+), 11 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +10 -2
index db18ace74d2b..b71bd9cc00d0 100644
@@ -126,13 +126,21 @@ void retransmit_timer(struct timer_list *t)
 	spin_unlock_bh(&qp->state_lock);
 }
 
-void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
+void rxe_comp_queue_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
+	struct rxe_qp *qp = pkt->qp;
 	int must_sched;
 
 	skb_queue_tail(&qp->resp_pkts, skb);
 
-	must_sched = skb_queue_len(&qp->resp_pkts) > 1;
+	/* Schedule the task if processing Read responses or Atomic acks.
+	 * In these cases, completer may sleep to access ODP-enabled MRs.
+	 */
+	if (pkt->mask | (RXE_PAYLOAD_MASK || RXE_ATMACK_MASK))
+		must_sched = 1;
+	else
+		must_sched = skb_queue_len(&qp->resp_pkts) > 1;
+
 	if (must_sched != 0)
 		rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
 
drivers/infiniband/sw/rxe/rxe_loc.h +2 -2
index 804b15e929dd..bf28ac13c3f5 100644
@@ -179,9 +179,9 @@ int rxe_icrc_init(struct rxe_dev *rxe);
 int rxe_icrc_check(struct sk_buff *skb, struct rxe_pkt_info *pkt);
 void rxe_icrc_generate(struct sk_buff *skb, struct rxe_pkt_info *pkt);
 
-void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb);
+void rxe_resp_queue_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb);
 
-void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb);
+void rxe_comp_queue_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb);
 
 static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)
 {
drivers/infiniband/sw/rxe/rxe_recv.c +2 -2
index 2f953cc74256..0d869615508a 100644
@@ -181,9 +181,9 @@ static int hdr_check(struct rxe_pkt_info *pkt)
 static inline void rxe_rcv_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
 	if (pkt->mask & RXE_REQ_MASK)
-		rxe_resp_queue_pkt(pkt->qp, skb);
+		rxe_resp_queue_pkt(pkt, skb);
 	else
-		rxe_comp_queue_pkt(pkt->qp, skb);
+		rxe_comp_queue_pkt(pkt, skb);
 }
 
 static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
drivers/infiniband/sw/rxe/rxe_resp.c +9 -5
index 68f6cd188d8e..f915128ed32a 100644
@@ -47,15 +47,19 @@ static char *resp_state_name[] = {
 };
 
 /* rxe_recv calls here to add a request packet to the input queue */
-void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
+void rxe_resp_queue_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
-	int must_sched;
-	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
+	int must_sched = 1;
+	struct rxe_qp *qp = pkt->qp;
 
 	skb_queue_tail(&qp->req_pkts, skb);
 
-	must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) ||
-			(skb_queue_len(&qp->req_pkts) > 1);
+	/* responder can sleep to access an ODP-enabled MR. Always schedule
+	 * tasks for non-zero-byte operations, RDMA Read, and Atomic.
+	 */
+	if ((skb_queue_len(&qp->req_pkts) == 1) && (payload_size(pkt) == 0)
+	    && !(pkt->mask & RXE_READ_OR_ATOMIC_MASK))
+		must_sched = 0;
 
 	if (must_sched)
 		rxe_sched_task(&qp->resp.task);
Signature · 1 line
2.39.1
#5 lore raw

PATCH v4 4/8 RDMA/rxe: Move resp_states definition to rxe_verbs.h

In reply to Daisuke Matsuda

To use the resp_states values in rxe_loc.h, it is necessary to move the definition to rxe_verbs.h, where other internal states of this driver are defined.

---
 drivers/infiniband/sw/rxe/rxe.h       | 37 ---------------------------
 drivers/infiniband/sw/rxe/rxe_verbs.h | 37 +++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 37 deletions(-)
drivers/infiniband/sw/rxe/rxe.h +0 -37
index d33dd6cf83d3..9b4d044a1264 100644
@@ -100,43 +100,6 @@
 #define rxe_info_mw(mw, fmt, ...) ibdev_info_ratelimited((mw)->ibmw.device, \
 		"mw#%d %s:  " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
 
-/* responder states */
-enum resp_states {
-	RESPST_NONE,
-	RESPST_GET_REQ,
-	RESPST_CHK_PSN,
-	RESPST_CHK_OP_SEQ,
-	RESPST_CHK_OP_VALID,
-	RESPST_CHK_RESOURCE,
-	RESPST_CHK_LENGTH,
-	RESPST_CHK_RKEY,
-	RESPST_EXECUTE,
-	RESPST_READ_REPLY,
-	RESPST_ATOMIC_REPLY,
-	RESPST_ATOMIC_WRITE_REPLY,
-	RESPST_PROCESS_FLUSH,
-	RESPST_COMPLETE,
-	RESPST_ACKNOWLEDGE,
-	RESPST_CLEANUP,
-	RESPST_DUPLICATE_REQUEST,
-	RESPST_ERR_MALFORMED_WQE,
-	RESPST_ERR_UNSUPPORTED_OPCODE,
-	RESPST_ERR_MISALIGNED_ATOMIC,
-	RESPST_ERR_PSN_OUT_OF_SEQ,
-	RESPST_ERR_MISSING_OPCODE_FIRST,
-	RESPST_ERR_MISSING_OPCODE_LAST_C,
-	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
-	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
-	RESPST_ERR_RNR,
-	RESPST_ERR_RKEY_VIOLATION,
-	RESPST_ERR_INVALIDATE_RKEY,
-	RESPST_ERR_LENGTH,
-	RESPST_ERR_CQ_OVERFLOW,
-	RESPST_ERROR,
-	RESPST_DONE,
-	RESPST_EXIT,
-};
-
 void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
 
 int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);
drivers/infiniband/sw/rxe/rxe_verbs.h +37 -0
index 26a20f088692..b6fbd9b3d086 100644
@@ -127,6 +127,43 @@ struct rxe_comp_info {
 	struct rxe_task		task;
 };
 
+/* responder states */
+enum resp_states {
+	RESPST_NONE,
+	RESPST_GET_REQ,
+	RESPST_CHK_PSN,
+	RESPST_CHK_OP_SEQ,
+	RESPST_CHK_OP_VALID,
+	RESPST_CHK_RESOURCE,
+	RESPST_CHK_LENGTH,
+	RESPST_CHK_RKEY,
+	RESPST_EXECUTE,
+	RESPST_READ_REPLY,
+	RESPST_ATOMIC_REPLY,
+	RESPST_ATOMIC_WRITE_REPLY,
+	RESPST_PROCESS_FLUSH,
+	RESPST_COMPLETE,
+	RESPST_ACKNOWLEDGE,
+	RESPST_CLEANUP,
+	RESPST_DUPLICATE_REQUEST,
+	RESPST_ERR_MALFORMED_WQE,
+	RESPST_ERR_UNSUPPORTED_OPCODE,
+	RESPST_ERR_MISALIGNED_ATOMIC,
+	RESPST_ERR_PSN_OUT_OF_SEQ,
+	RESPST_ERR_MISSING_OPCODE_FIRST,
+	RESPST_ERR_MISSING_OPCODE_LAST_C,
+	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
+	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
+	RESPST_ERR_RNR,
+	RESPST_ERR_RKEY_VIOLATION,
+	RESPST_ERR_INVALIDATE_RKEY,
+	RESPST_ERR_LENGTH,
+	RESPST_ERR_CQ_OVERFLOW,
+	RESPST_ERROR,
+	RESPST_DONE,
+	RESPST_EXIT,
+};
+
 enum rdatm_res_state {
 	rdatm_res_state_next,
 	rdatm_res_state_new,
Signature · 1 line
2.39.1
#8 lore raw

PATCH v4 7/8 RDMA/rxe: Add support for Send/Recv/Write/Read with ODP

In reply to Daisuke Matsuda

rxe_mr_copy() is used widely to copy data to/from a user MR. requester uses it to load payloads of requesting packets; responder uses it to process Send, Write, and Read operaetions; completer uses it to copy data from response packets of Read and Atomic operations to a user MR.

Allow these operations to be used with ODP by adding a subordinate function rxe_odp_mr_copy(). It is comprised of the following steps:
1. Check the driver page table(umem_odp->dma_list) to see if pages being accessed are present with appropriate permission.
2. If necessary, trigger page fault to map the pages.
3. Update the MR xarray using PFNs in umem_odp->pfn_list.
4. Execute data copy to/from the pages.

umem_mutex is used to ensure that dma_list (an array of addresses of an MR) is not changed while it is being checked and that mapped pages are not invalidated before data copy completes.

---
 drivers/infiniband/sw/rxe/rxe.c     |  10 +++
 drivers/infiniband/sw/rxe/rxe_loc.h |   8 ++
 drivers/infiniband/sw/rxe/rxe_mr.c  |   2 +-
 drivers/infiniband/sw/rxe/rxe_odp.c | 109 ++++++++++++++++++++++++++++
 4 files changed, 128 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe.c +10 -0
index f2284d27229b..207a022156f0 100644
@@ -79,6 +79,16 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 
 		/* IB_ODP_SUPPORT_IMPLICIT is not supported right now. */
 		rxe->attr.odp_caps.general_caps |= IB_ODP_SUPPORT;
+
+		rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND;
+		rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_RECV;
+		rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
+
+		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND;
+		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;
+		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;
+		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;
+		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 	}
 }
 
drivers/infiniband/sw/rxe/rxe_loc.h +8 -0
index 0f91e23c151e..35c2ccb2fdd9 100644
@@ -206,6 +206,8 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
 			 u64 iova, int access_flags, struct rxe_mr *mr);
+int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
+		    enum rxe_mr_copy_dir dir);
 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 static inline int
 rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -213,6 +215,12 @@ rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 {
 	return -EOPNOTSUPP;
 }
+static inline int
+rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
+		int length, enum rxe_mr_copy_dir dir)
+{
+	return -EOPNOTSUPP;
+}
 
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
drivers/infiniband/sw/rxe/rxe_mr.c +1 -1
index cd368cd096c8..0e3cda59d702 100644
@@ -319,7 +319,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
 	}
 
 	if (mr->odp_enabled)
-		return -EOPNOTSUPP;
+		return rxe_odp_mr_copy(mr, iova, addr, length, dir);
 	else
 		return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
 }
drivers/infiniband/sw/rxe/rxe_odp.c +109 -0

112 lines not on this page · raw message

Signature · 1 line
2.39.1
#6 lore raw

PATCH v4 5/8 RDMA/rxe: Add page invalidation support

In reply to Daisuke Matsuda

On page invalidation, an MMU notifier callback is invoked to unmap DMA addresses and update the driver page table(umem_odp->dma_list). It also set the corresponding entries in MR xarray to NULL to prevent any access. The callback is registered when an ODP-enabled MR is created.

---
 drivers/infiniband/sw/rxe/Makefile  |  2 ++
 drivers/infiniband/sw/rxe/rxe_odp.c | 56 +++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c
drivers/infiniband/sw/rxe/Makefile +2 -0
index 5395a581f4bb..93134f1d1d0c 100644
@@ -23,3 +23,5 @@ rdma_rxe-y := \
 	rxe_task.o \
 	rxe_net.o \
 	rxe_hw_counters.o
+
+rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_odp.c new file +56 -0

56 lines not on this page · raw message

Signature · 1 line
2.39.1
#7 lore raw

PATCH v4 6/8 RDMA/rxe: Allow registering MRs for On-Demand Paging

In reply to Daisuke Matsuda

Allow applications to register an ODP-enabled MR, in which case the flag IB_ACCESS_ON_DEMAND is passed to rxe_reg_user_mr(). However, there is no RDMA operation supported right now. They will be enabled later in the subsequent two patches.

rxe_odp_do_pagefault() is called to initialize an ODP-enabled MR. It syncs process address space from the CPU page table to the driver page table (dma_list/pfn_list in umem_odp) when called with RXE_PAGEFAULT_SNAPSHOT flag. Additionally, It can be used to trigger page fault when pages being accessed are not present or do not have proper read/write permissions, and possibly to prefetch pages in the future.

---
 drivers/infiniband/sw/rxe/rxe.c       |   7 ++
 drivers/infiniband/sw/rxe/rxe_loc.h   |  14 +++
 drivers/infiniband/sw/rxe/rxe_mr.c    |   9 +-
 drivers/infiniband/sw/rxe/rxe_odp.c   | 120 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c  |  15 +++-
 drivers/infiniband/sw/rxe/rxe_verbs.c |   5 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |   2 +
 7 files changed, 166 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +7 -0
index 54c723a6edda..f2284d27229b 100644
@@ -73,6 +73,13 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 			rxe->ndev->dev_addr);
 
 	rxe->max_ucontext			= RXE_MAX_UCONTEXT;
+
+	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
+		rxe->attr.kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
+
+		/* IB_ODP_SUPPORT_IMPLICIT is not supported right now. */
+		rxe->attr.odp_caps.general_caps |= IB_ODP_SUPPORT;
+	}
 }
 
 /* initialize port attributes */
drivers/infiniband/sw/rxe/rxe_loc.h +14 -0
index 3476726425d9..0f91e23c151e 100644
@@ -202,4 +202,18 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)
 	return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type];
 }
 
+/* rxe_odp.c */
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
+			 u64 iova, int access_flags, struct rxe_mr *mr);
+#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
+static inline int
+rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
+		     int access_flags, struct rxe_mr *mr)
+{
+	return -EOPNOTSUPP;
+}
+
+#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
+
 #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mr.c +8 -1
index ffbac6f5e828..cd368cd096c8 100644
@@ -318,7 +318,10 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
 		return err;
 	}
 
-	return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
+	if (mr->odp_enabled)
+		return -EOPNOTSUPP;
+	else
+		return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
 }
 
 /* copy data in or out of a wqe, i.e. sg list
@@ -527,6 +530,10 @@ int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
 	struct page *page;
 	u64 *va;
 
+	/* ODP is not supported right now. WIP. */
+	if (mr->odp_enabled)
+		return RESPST_ERR_UNSUPPORTED_OPCODE;
+
 	/* See IBA oA19-28 */
 	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
 		rxe_dbg_mr(mr, "mr not in valid state");
drivers/infiniband/sw/rxe/rxe_odp.c +120 -0

129 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +11 -4
index f915128ed32a..b40c47477be3 100644
@@ -641,6 +641,10 @@ static enum resp_states process_flush(struct rxe_qp *qp,
 	struct rxe_mr *mr = qp->resp.mr;
 	struct resp_res *res = qp->resp.res;
 
+	/* ODP is not supported right now. WIP. */
+	if (mr->odp_enabled)
+		return RESPST_ERR_UNSUPPORTED_OPCODE;
+
 	/* oA19-14, oA19-15 */
 	if (res && res->replay)
 		return RESPST_ACKNOWLEDGE;
@@ -694,10 +698,13 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 	if (!res->replay) {
 		u64 iova = qp->resp.va + qp->resp.offset;
 
-		err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
-					  atmeth_comp(pkt),
-					  atmeth_swap_add(pkt),
-					  &res->atomic.orig_val);
+		if (mr->odp_enabled)
+			err = RESPST_ERR_UNSUPPORTED_OPCODE;
+		else
+			err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
+						  atmeth_comp(pkt),
+						  atmeth_swap_add(pkt),
+						  &res->atomic.orig_val);
 		if (err)
 			return err;
 
drivers/infiniband/sw/rxe/rxe_verbs.c +4 -1
index dea605b7f683..9c23defdc7b5 100644
@@ -1274,7 +1274,10 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start,
 	mr->ibmr.pd = ibpd;
 	mr->ibmr.device = ibpd->device;
 
-	err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
+	if (access & IB_ACCESS_ON_DEMAND)
+		err = rxe_odp_mr_init_user(rxe, start, length, iova, access, mr);
+	else
+		err = rxe_mr_init_user(rxe, start, length, iova, access, mr);
 	if (err) {
 		rxe_dbg_mr(mr, "reg_user_mr failed, err = %d", err);
 		goto err_cleanup;
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -0
index b6fbd9b3d086..de5a982c7c7e 100644
@@ -333,6 +333,8 @@ struct rxe_mr {
 	u32			nbuf;
 
 	struct xarray		page_list;
+
+	bool			odp_enabled;
 };
 
 static inline unsigned int mr_page_size(struct rxe_mr *mr)
Signature · 1 line
2.39.1
#9 lore raw

PATCH v4 8/8 RDMA/rxe: Add support for the traditional Atomic operations with ODP

In reply to Daisuke Matsuda

Enable 'fetch and add' and 'compare and swap' operations to manipulate data in an ODP-enabled MR. This is comprised of the following steps:
1. Check the driver page table(umem_odp->dma_list) to see if the target page is both readable and writable.
2. If not, then trigger page fault to map the page.
3. Update the entry in the MR xarray.
4. Execute the operation.

umem_mutex is used to ensure that dma_list (an array of addresses of an MR) is not changed while it is being checked and that the target page is not invalidated before data access completes.

---
 drivers/infiniband/sw/rxe/rxe.c      |  1 +
 drivers/infiniband/sw/rxe/rxe_loc.h  |  9 +++++++++
 drivers/infiniband/sw/rxe/rxe_odp.c  | 26 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c |  5 ++++-
 4 files changed, 40 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe.c +1 -0
index 207a022156f0..abd3267c2873 100644
@@ -88,6 +88,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;
 		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;
 		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;
+		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
 		rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 	}
 }
drivers/infiniband/sw/rxe/rxe_loc.h +9 -0
index 35c2ccb2fdd9..6ff10b15fa32 100644
@@ -208,6 +208,9 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
 			 u64 iova, int access_flags, struct rxe_mr *mr);
 int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		    enum rxe_mr_copy_dir dir);
+int rxe_odp_mr_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			 u64 compare, u64 swap_add, u64 *orig_val);
+
 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 static inline int
 rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -221,6 +224,12 @@ rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
 {
 	return -EOPNOTSUPP;
 }
+static inline int
+rxe_odp_mr_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+		     u64 compare, u64 swap_add, u64 *orig_val)
+{
+	return RESPST_ERR_UNSUPPORTED_OPCODE;
+}
 
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
drivers/infiniband/sw/rxe/rxe_odp.c +26 -0
index cbe5d0c3fcc4..194b1fab98b7 100644
@@ -283,3 +283,29 @@ int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 
 	return err;
 }
+
+int rxe_odp_mr_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			 u64 compare, u64 swap_add, u64 *orig_val)
+{
+	int err;
+	struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem);
+
+	/* If pagefault is not required, umem mutex will be held until the
+	 * atomic operation completes. Otherwise, it is released and locked
+	 * again in rxe_odp_map_range() to let invalidation handler do its
+	 * work meanwhile.
+	 */
+	mutex_lock(&umem_odp->umem_mutex);
+
+	/* Atomic operations manipulate a single char. */
+	err = rxe_odp_map_range(mr, iova, sizeof(char), 0);
+	if (err)
+		return err;
+
+	err = rxe_mr_do_atomic_op(mr, iova, opcode, compare,
+				  swap_add, orig_val);
+
+	mutex_unlock(&umem_odp->umem_mutex);
+
+	return err;
+}
drivers/infiniband/sw/rxe/rxe_resp.c +4 -1
index b40c47477be3..99ad1dec10c7 100644
@@ -699,7 +699,10 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 		u64 iova = qp->resp.va + qp->resp.offset;
 
 		if (mr->odp_enabled)
-			err = RESPST_ERR_UNSUPPORTED_OPCODE;
+			err = rxe_odp_mr_atomic_op(mr, iova, pkt->opcode,
+						   atmeth_comp(pkt),
+						   atmeth_swap_add(pkt),
+						   &res->atomic.orig_val);
 		else
 			err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
 						  atmeth_comp(pkt),
Signature · 1 line
2.39.1
#4 lore raw

PATCH v4 3/8 RDMA/rxe: Make MR functions accessible from other rxe source code

In reply to Daisuke Matsuda

Some functions in rxe_mr.c are going to be used in rxe_odp.c, which is to be created in the subsequent patch. List the declarations of the functions in rxe_loc.h.

---
 drivers/infiniband/sw/rxe/rxe_loc.h | 14 ++++++++++++++
 drivers/infiniband/sw/rxe/rxe_mr.c  | 18 ++++--------------
 2 files changed, 18 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +14 -0
index bf28ac13c3f5..3476726425d9 100644
@@ -60,7 +60,9 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
 
 /* rxe_mr.c */
 u8 rxe_get_next_key(u32 last_key);
+void rxe_mr_init(int access, struct rxe_mr *mr);
 void rxe_mr_init_dma(int access, struct rxe_mr *mr);
+int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt);
 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr);
 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
@@ -71,6 +73,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
+int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
+		       unsigned int length, enum rxe_mr_copy_dir dir);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 			u64 compare, u64 swap_add, u64 *orig_val);
 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
@@ -82,6 +86,16 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 void rxe_mr_cleanup(struct rxe_pool_elem *elem);
 
+static inline unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
+{
+	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
+}
+
+static inline unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
+{
+	return iova & (mr_page_size(mr) - 1);
+}
+
 /* rxe_mw.c */
 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
 int rxe_dealloc_mw(struct ib_mw *ibmw);
drivers/infiniband/sw/rxe/rxe_mr.c +4 -14
index 0e538fafcc20..ffbac6f5e828 100644
@@ -49,7 +49,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 				| IB_ACCESS_REMOTE_WRITE	\
 				| IB_ACCESS_REMOTE_ATOMIC)
 
-static void rxe_mr_init(int access, struct rxe_mr *mr)
+void rxe_mr_init(int access, struct rxe_mr *mr)
 {
 	u32 lkey = mr->elem.index << 8 | rxe_get_next_key(-1);
 	u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
@@ -77,16 +77,6 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr)
 	mr->ibmr.type = IB_MR_TYPE_DMA;
 }
 
-static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
-{
-	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
-}
-
-static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
-{
-	return iova & (mr_page_size(mr) - 1);
-}
-
 static bool is_pmem_page(struct page *pg)
 {
 	unsigned long paddr = page_to_phys(pg);
@@ -96,7 +86,7 @@ static bool is_pmem_page(struct page *pg)
 				 IORES_DESC_PERSISTENT_MEMORY);
 }
 
-static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
+int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
 {
 	XA_STATE(xas, &mr->page_list, 0);
 	struct sg_page_iter sg_iter;
@@ -247,8 +237,8 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl,
 	return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
 }
 
-static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
-			      unsigned int length, enum rxe_mr_copy_dir dir)
+int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
+		       unsigned int length, enum rxe_mr_copy_dir dir)
 {
 	unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
 	unsigned long index = rxe_mr_iova_to_index(mr, iova);
Signature · 1 line
2.39.1
#0 lore raw

v4 cover On-Demand Paging on SoftRoCE

This patch series implements the On-Demand Paging feature on SoftRoCE(rxe) driver, which has been available only in mlx5 driver[1] so far.

The first patch of this series is provided for testing purpose, and it should be dropped in the end. It converts triple tasklets to use workqueue in order to let them sleep during page-fault. Bob Pearson says he will post the patch to do this, and I think we can adopt that. The other patches in this series are, I believe, completed works.

I omitted some contents like the motive behind this series for simplicity. Please see the cover letter of v3 for more details[2].

[Overview]
When applications register a memory region(MR), RDMA drivers normally pin pages in the MR so that physical addresses are never changed during RDMA communication. This requires the MR to fit in physical memory and inevitably leads to memory pressure. On the other hand, On-Demand Paging (ODP) allows applications to register MRs without pinning pages. They are paged-in when the driver requires and paged-out when the OS reclaims. As a result, it is possible to register a large MR that does not fit in physical memory without taking up so much physical memory.

[How does ODP work?]
"struct ib_umem_odp" is used to manage pages. It is created for each ODP-enabled MR on its registration. This struct holds a pair of arrays (dma_list/pfn_list) that serve as a driver page table. DMA addresses and PFNs are stored in the driver page table. They are updated on page-in and page-out, both of which use the common interfaces in the ib_uverbs layer.

Page-in can occur when requester, responder or completer access an MR in order to process RDMA operations. If they find that the pages being accessed are not present on physical memory or requisite permissions are not set on the pages, they provoke page fault to make the pages present with proper permissions and at the same time update the driver page table. After confirming the presence of the pages, they execute memory access such as read, write or atomic operations.

Page-out is triggered by page reclaim or filesystem events (e.g. metadata update of a file that is being used as an MR). When creating an ODP-enabled MR, the driver registers an MMU notifier callback. When the kernel issues a page invalidation notification, the callback is provoked to unmap DMA addresses and update the driver page table. After that, the kernel releases the pages.

[Supported operations]
All traditional operations are supported on RC connection. The new Atomic write[3] and RDMA Flush[4] operations are not included in this patchset. I will post them later after this patchset is merged. On UD connection, Send, Recv, and SRQ-Recv are supported.

[How to test ODP?]
There are only a few resources available for testing. pyverbs testcases in rdma-core and perftest[5] are recommendable ones. Other than them, the ibv_rc_pingpong command can also used for testing. Note that you may have to build perftest from upstream because older versions do not handle ODP capabilities correctly.

The tree is available from github:
https://github.com/daimatsuda/linux/tree/odp_v4
While this series is based on commit f605f26ea196, the tree includes an additional bugfix, which is yet to be merged as of today (Apr 19th, 2023).
https://lore.kernel.org/linux-rdma/[email protected]/

[Future work]
My next work is to enable the new Atomic write[3] and RDMA Flush[4] operations with ODP. After that, I am going to implement the prefetch feature. It allows applications to trigger page fault using ibv_advise_mr(3) to optimize performance. Some existing software like librpma[6] use this feature. Additionally, I think we can also add the implicit ODP feature in the future.

[1] [RFC 00/20] On demand paging
https://www.spinics.net/lists/linux-rdma/msg18906.html

[2] [PATCH for-next v3 0/7] On-Demand Paging on SoftRoCE
https://lore.kernel.org/lkml/[email protected]/

[3] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation
https://lore.kernel.org/linux-rdma/[email protected]/

[4] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation
https://lore.kernel.org/lkml/[email protected]/

[5] linux-rdma/perftest: Infiniband Verbs Performance Tests
https://github.com/linux-rdma/perftest

[6] librpma: Remote Persistent Memory Access Library
https://github.com/pmem/rpma

v3->v4:
1) Re-designed functions that access MRs to use the MR xarray.
2) Rebased onto the latest jgg-for-next tree.

v2->v3:
1) Removed a patch that changes the common ib_uverbs layer.
2) Re-implemented patches for conversion to workqueue.
3) Fixed compile errors (happened when CONFIG_INFINIBAND_ON_DEMAND_PAGING=n).
4) Fixed some functions that returned incorrect errors.
5) Temporarily disabled ODP for RDMA Flush and Atomic Write.

v1->v2:
1) Fixed a crash issue reported by Haris Iqbal.
2) Tried to make lock patters clearer as pointed out by Romanovsky.
3) Minor clean ups and fixes.

Daisuke Matsuda (8):
RDMA/rxe: Tentative workqueue implementation
RDMA/rxe: Always schedule works before accessing user MRs RDMA/rxe: Make MR functions accessible from other rxe source code RDMA/rxe: Move resp_states definition to rxe_verbs.h RDMA/rxe: Add page invalidation support
RDMA/rxe: Allow registering MRs for On-Demand Paging RDMA/rxe: Add support for Send/Recv/Write/Read with ODP RDMA/rxe: Add support for the traditional Atomic operations with ODP

 drivers/infiniband/sw/rxe/Makefile    |   2 +
 drivers/infiniband/sw/rxe/rxe.c       |  27 ++-
 drivers/infiniband/sw/rxe/rxe.h       |  37 ---
 drivers/infiniband/sw/rxe/rxe_comp.c  |  12 +-
 drivers/infiniband/sw/rxe/rxe_loc.h   |  49 +++-
 drivers/infiniband/sw/rxe/rxe_mr.c    |  27 +--
 drivers/infiniband/sw/rxe/rxe_odp.c   | 311 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_recv.c  |   4 +-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  32 ++-
 drivers/infiniband/sw/rxe/rxe_task.c  |  84 ++++---
 drivers/infiniband/sw/rxe/rxe_task.h  |   6 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c |   5 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  39 ++++
 13 files changed, 535 insertions(+), 100 deletions(-)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c

base-commit: f605f26ea196a3b49bea249330cbd18dba61a33e

Signature · 1 line
2.39.1
#10 lore raw

The work queue patch has been submitted and is waiting for some action. -- Bob

-----Original Message-----
From: Daisuke Matsuda <[email protected]>
Sent: Wednesday, April 19, 2023 12:52 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Daisuke Matsuda <[email protected]>
Subject: [PATCH for-next v4 0/8] On-Demand Paging on SoftRoCE

This patch series implements the On-Demand Paging feature on SoftRoCE(rxe) driver, which has been available only in mlx5 driver[1] so far.

The first patch of this series is provided for testing purpose, and it should be dropped in the end. It converts triple tasklets to use workqueue in order to let them sleep during page-fault. Bob Pearson says he will post the patch to do this, and I think we can adopt that. The other patches in this series are, I believe, completed works.

I omitted some contents like the motive behind this series for simplicity. Please see the cover letter of v3 for more details[2].

[Overview]
When applications register a memory region(MR), RDMA drivers normally pin pages in the MR so that physical addresses are never changed during RDMA communication. This requires the MR to fit in physical memory and inevitably leads to memory pressure. On the other hand, On-Demand Paging (ODP) allows applications to register MRs without pinning pages. They are paged-in when the driver requires and paged-out when the OS reclaims. As a result, it is possible to register a large MR that does not fit in physical memory without taking up so much physical memory.

[How does ODP work?]
"struct ib_umem_odp" is used to manage pages. It is created for each ODP-enabled MR on its registration. This struct holds a pair of arrays
(dma_list/pfn_list) that serve as a driver page table. DMA addresses and PFNs are stored in the driver page table. They are updated on page-in and page-out, both of which use the common interfaces in the ib_uverbs layer.

Page-in can occur when requester, responder or completer access an MR in order to process RDMA operations. If they find that the pages being accessed are not present on physical memory or requisite permissions are not set on the pages, they provoke page fault to make the pages present with proper permissions and at the same time update the driver page table. After confirming the presence of the pages, they execute memory access such as read, write or atomic operations.

Page-out is triggered by page reclaim or filesystem events (e.g. metadata update of a file that is being used as an MR). When creating an ODP-enabled MR, the driver registers an MMU notifier callback. When the kernel issues a page invalidation notification, the callback is provoked to unmap DMA addresses and update the driver page table. After that, the kernel releases the pages.

[Supported operations]
All traditional operations are supported on RC connection. The new Atomic write[3] and RDMA Flush[4] operations are not included in this patchset. I will post them later after this patchset is merged. On UD connection, Send, Recv, and SRQ-Recv are supported.

[How to test ODP?]
There are only a few resources available for testing. pyverbs testcases in rdma-core and perftest[5] are recommendable ones. Other than them, the ibv_rc_pingpong command can also used for testing. Note that you may have to build perftest from upstream because older versions do not handle ODP capabilities correctly.

The tree is available from github:
https://github.com/daimatsuda/linux/tree/odp_v4
While this series is based on commit f605f26ea196, the tree includes an additional bugfix, which is yet to be merged as of today (Apr 19th, 2023).
https://lore.kernel.org/linux-rdma/[email protected]/

[Future work]
My next work is to enable the new Atomic write[3] and RDMA Flush[4] operations with ODP. After that, I am going to implement the prefetch feature. It allows applications to trigger page fault using ibv_advise_mr(3) to optimize performance. Some existing software like librpma[6] use this feature. Additionally, I think we can also add the implicit ODP feature in the future.

[1] [RFC 00/20] On demand paging
https://www.spinics.net/lists/linux-rdma/msg18906.html

[2] [PATCH for-next v3 0/7] On-Demand Paging on SoftRoCE https://lore.kernel.org/lkml/[email protected]/

[3] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation https://lore.kernel.org/linux-rdma/[email protected]/

[4] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation https://lore.kernel.org/lkml/[email protected]/

[5] linux-rdma/perftest: Infiniband Verbs Performance Tests https://github.com/linux-rdma/perftest

[6] librpma: Remote Persistent Memory Access Library https://github.com/pmem/rpma

v3->v4:
1) Re-designed functions that access MRs to use the MR xarray.
2) Rebased onto the latest jgg-for-next tree.

v2->v3:
1) Removed a patch that changes the common ib_uverbs layer.
2) Re-implemented patches for conversion to workqueue.
3) Fixed compile errors (happened when CONFIG_INFINIBAND_ON_DEMAND_PAGING=n).
4) Fixed some functions that returned incorrect errors.
5) Temporarily disabled ODP for RDMA Flush and Atomic Write.

v1->v2:
1) Fixed a crash issue reported by Haris Iqbal.
2) Tried to make lock patters clearer as pointed out by Romanovsky.
3) Minor clean ups and fixes.

Daisuke Matsuda (8):
RDMA/rxe: Tentative workqueue implementation
RDMA/rxe: Always schedule works before accessing user MRs RDMA/rxe: Make MR functions accessible from other rxe source code RDMA/rxe: Move resp_states definition to rxe_verbs.h RDMA/rxe: Add page invalidation support
RDMA/rxe: Allow registering MRs for On-Demand Paging RDMA/rxe: Add support for Send/Recv/Write/Read with ODP RDMA/rxe: Add support for the traditional Atomic operations with ODP

 drivers/infiniband/sw/rxe/Makefile    |   2 +
 drivers/infiniband/sw/rxe/rxe.c       |  27 ++-
 drivers/infiniband/sw/rxe/rxe.h       |  37 ---
 drivers/infiniband/sw/rxe/rxe_comp.c  |  12 +-
 drivers/infiniband/sw/rxe/rxe_loc.h   |  49 +++-
 drivers/infiniband/sw/rxe/rxe_mr.c    |  27 +--
 drivers/infiniband/sw/rxe/rxe_odp.c   | 311 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_recv.c  |   4 +-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  32 ++-  drivers/infiniband/sw/rxe/rxe_task.c  |  84 ++++---
 drivers/infiniband/sw/rxe/rxe_task.h  |   6 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c |   5 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  39 ++++
 13 files changed, 535 insertions(+), 100 deletions(-)  create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c

base-commit: f605f26ea196a3b49bea249330cbd18dba61a33e

--
2.39.1

#3 lore raw

Re: [PATCH for-next v4 2/8] RDMA/rxe: Always schedule works before accessing user MRs

In reply to Daisuke Matsuda
kernel test robot noticed the following build warnings:

Hi Daisuke,

kernel test robot noticed the following build warnings:

[auto build test WARNING on f605f26ea196a3b49bea249330cbd18dba61a33e]

url:    https://github.com/intel-lab-lkp/linux/commits/Daisuke-Matsuda/RDMA-rxe-Tentative-workqueue-implementation/20230419-135731
base:   f605f26ea196a3b49bea249330cbd18dba61a33e
patch link:    https://lore.kernel.org/r/7441c59fcea601c03c70ec03b5d17a69032e51f8.1681882651.git.matsuda-daisuke%40fujitsu.com
patch subject: [PATCH for-next v4 2/8] RDMA/rxe: Always schedule works before accessing user MRs
config: x86_64-randconfig-a011-20230417 (https://download.01.org/0day-ci/archive/20230420/[email protected]/config)
compiler: clang version 14.0.6 (https://github.com/llvm/llvm-project f28c006a5895fc0e329fe15fead81e37457cb1d1)

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/493fb0777100e2e1b6358176e84b4b29372105ae
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Daisuke-Matsuda/RDMA-rxe-Tentative-workqueue-implementation/20230419-135731 git checkout 493fb0777100e2e1b6358176e84b4b29372105ae
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 olddefconfig COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash drivers/infiniband/sw/rxe/

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <[email protected]>
| Link: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> drivers/infiniband/sw/rxe/rxe_comp.c:139:36: warning: converting the enum constant to a boolean [-Wint-in-bool-context]
           if (pkt->mask | (RXE_PAYLOAD_MASK || RXE_ATMACK_MASK))
                                             ^

1 warning generated.

vim +139 drivers/infiniband/sw/rxe/rxe_comp.c

128
129 void rxe_comp_queue_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb)
130 {
131 struct rxe_qp *qp = pkt->qp;
132 int must_sched;
133
134 skb_queue_tail(&qp->resp_pkts, skb);
135
136 /* Schedule the task if processing Read responses or Atomic acks.
137 * In these cases, completer may sleep to access ODP-enabled MRs.
138 */

 > 139		if (pkt->mask | (RXE_PAYLOAD_MASK || RXE_ATMACK_MASK))

140 must_sched = 1;
141 else
142 must_sched = skb_queue_len(&qp->resp_pkts) > 1;
143
144 if (must_sched != 0)
145 rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
146
147 if (must_sched)
148 rxe_sched_task(&qp->comp.task);
149 else
150 rxe_run_task(&qp->comp.task);
151 }
152

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests
#11 lore raw
In reply to Pearson, Robert B
Quoted 2 lines · Robert B

The work queue patch has been submitted and is waiting for some action. -- Bob

Hi,
Could you tell me which is it? I am willing to review it.

This seems to be your latest work queue patch:
https://lore.kernel.org/all/TYCPR01MB8455A2D0B3303FD90B3BB6F1E58B9@TYCPR01MB8455.jpnprd01.prod.outlook.com/
I cannot find any one newer on the mailing list nor on the Patchwork.

Daisuke

Quoted 126 lines

-----Original Message-----
From: Daisuke Matsuda <[email protected]>
Sent: Wednesday, April 19, 2023 12:52 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Daisuke Matsuda <[email protected]>
Subject: [PATCH for-next v4 0/8] On-Demand Paging on SoftRoCE

This patch series implements the On-Demand Paging feature on SoftRoCE(rxe) driver, which has been available only in mlx5 driver[1] so far.

The first patch of this series is provided for testing purpose, and it should be dropped in the end. It converts triple tasklets to use workqueue in order to let them sleep during page-fault. Bob Pearson says he will post the patch to do this, and I think we can adopt that. The other patches in this series are, I believe, completed works.

I omitted some contents like the motive behind this series for simplicity. Please see the cover letter of v3 for more details[2].

[Overview]
When applications register a memory region(MR), RDMA drivers normally pin pages in the MR so that physical addresses are never changed during RDMA communication. This requires the MR to fit in physical memory and inevitably leads to memory pressure. On the other hand, On-Demand Paging
(ODP) allows applications to register MRs without pinning pages. They are paged-in when the driver requires and paged-out when the OS reclaims. As a result, it is possible to register a large MR that does not fit in physical memory without taking up so much physical memory.

[How does ODP work?]
"struct ib_umem_odp" is used to manage pages. It is created for each ODP-enabled MR on its registration. This struct holds a pair of arrays
(dma_list/pfn_list) that serve as a driver page table. DMA addresses and PFNs are stored in the driver page table. They are updated on page-in and page-out, both of which use the common interfaces in the ib_uverbs layer.

Page-in can occur when requester, responder or completer access an MR in order to process RDMA operations. If they find that the pages being accessed are not present on physical memory or requisite permissions are not set on the pages, they provoke page fault to make the pages present with proper permissions and at the same time update the driver page table.
After confirming the presence of the pages, they execute memory access such as read, write or atomic operations.

Page-out is triggered by page reclaim or filesystem events (e.g. metadata update of a file that is being used as an MR). When creating an ODP-enabled MR, the driver registers an MMU notifier callback. When the kernel issues a page invalidation notification, the callback is provoked to unmap DMA addresses and update the driver page table. After that, the kernel releases the pages.

[Supported operations]
All traditional operations are supported on RC connection. The new Atomic write[3] and RDMA Flush[4] operations are not included in this patchset. I will post them later after this patchset is merged. On UD connection, Send, Recv, and SRQ-Recv are supported.

[How to test ODP?]
There are only a few resources available for testing. pyverbs testcases in rdma-core and perftest[5] are recommendable ones. Other than them, the ibv_rc_pingpong command can also used for testing. Note that you may have to build perftest from upstream because older versions do not handle ODP capabilities correctly.

The tree is available from github:
https://github.com/daimatsuda/linux/tree/odp_v4
While this series is based on commit f605f26ea196, the tree includes an additional bugfix, which is yet to be merged as of today (Apr 19th, 2023).
https://lore.kernel.org/linux-rdma/[email protected]/

[Future work]
My next work is to enable the new Atomic write[3] and RDMA Flush[4] operations with ODP. After that, I am going to implement the prefetch feature. It allows applications to trigger page fault using
ibv_advise_mr(3) to optimize performance. Some existing software like librpma[6] use this feature. Additionally, I think we can also add the implicit ODP feature in the future.

[1] [RFC 00/20] On demand paging
https://www.spinics.net/lists/linux-rdma/msg18906.html

[2] [PATCH for-next v3 0/7] On-Demand Paging on SoftRoCE
https://lore.kernel.org/lkml/[email protected]/

[3] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation
https://lore.kernel.org/linux-rdma/[email protected]/

[4] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation
https://lore.kernel.org/lkml/[email protected]/

[5] linux-rdma/perftest: Infiniband Verbs Performance Tests https://github.com/linux-rdma/perftest

[6] librpma: Remote Persistent Memory Access Library https://github.com/pmem/rpma

v3->v4:
1) Re-designed functions that access MRs to use the MR xarray.
2) Rebased onto the latest jgg-for-next tree.

v2->v3:
1) Removed a patch that changes the common ib_uverbs layer.
2) Re-implemented patches for conversion to workqueue.
3) Fixed compile errors (happened when CONFIG_INFINIBAND_ON_DEMAND_PAGING=n).
4) Fixed some functions that returned incorrect errors.
5) Temporarily disabled ODP for RDMA Flush and Atomic Write.

v1->v2:
1) Fixed a crash issue reported by Haris Iqbal.
2) Tried to make lock patters clearer as pointed out by Romanovsky.
3) Minor clean ups and fixes.

Daisuke Matsuda (8):
RDMA/rxe: Tentative workqueue implementation
RDMA/rxe: Always schedule works before accessing user MRs RDMA/rxe: Make MR functions accessible from other rxe source code RDMA/rxe: Move resp_states definition to rxe_verbs.h RDMA/rxe: Add page invalidation support
RDMA/rxe: Allow registering MRs for On-Demand Paging RDMA/rxe: Add support for Send/Recv/Write/Read with ODP RDMA/rxe: Add support for the traditional Atomic operations with ODP

 drivers/infiniband/sw/rxe/Makefile    |   2 +
 drivers/infiniband/sw/rxe/rxe.c       |  27 ++-
 drivers/infiniband/sw/rxe/rxe.h       |  37 ---
 drivers/infiniband/sw/rxe/rxe_comp.c  |  12 +-
 drivers/infiniband/sw/rxe/rxe_loc.h   |  49 +++-
 drivers/infiniband/sw/rxe/rxe_mr.c    |  27 +--
 drivers/infiniband/sw/rxe/rxe_odp.c   | 311 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_recv.c  |   4 +-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  32 ++-  drivers/infiniband/sw/rxe/rxe_task.c  |  84 ++++---
 drivers/infiniband/sw/rxe/rxe_task.h  |   6 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c |   5 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  39 ++++
 13 files changed, 535 insertions(+), 100 deletions(-)  create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c

base-commit: f605f26ea196a3b49bea249330cbd18dba61a33e

--
2.39.1

#12 lore raw
Quoted 4 lines · Daisuke Matsuda (Fujitsu)
Quoted 2 lines · Robert B

The work queue patch has been submitted and is waiting for some action. -- Bob

Quoted 8 lines

Hi,
Could you tell me which is it? I am willing to review it.

This seems to be your latest work queue patch:
https://lore.kernel.org/all/TYCPR01MB8455A2D0B3303FD90B3BB6F1E58B9@TYCPR01MB8455.jpnprd01.prod.outlook.com/
I cannot find any one newer on the mailing list nor on the Patchwork.

Daisuke

Daisuke,

Sorry for the delay. I've been on another project for a few days.
I can't either. After the fix qp counting in task.c the work queue patch is almost trivial. I'll send it again.

Bob