Skip to contentSkip to conversation

Providers/rxe: Implement XRC transport for rxe

PATCH Superseded Bob Pearson 2 participants 8 messages Jul 30, 2021 – Aug 13, 2021 InfiniBand

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Providers/rxe: Implement XRC transport for rxe

This set of patches implements xrc transport and extended srq required for xrc transport in the rxe provider. It matches a kernel rxe driver patch set with a similar name.

This patch set should be applied to the current rdma-core with the "Replace AV by AH for UD sends (v2)" patch set which is a prerequisite.

With this patch set 4 of the 5 xrc tests in the python tests pass for both regular and extended QP create verbs. The XRC/ODP test does not pass since rxe does not currently support ODP.

Bob Pearson (5):
Update kernel headers
Providers/rxe: Support alloc/dealloc xrcd
Providers/rxe: Support extended create srq
Providers/rxe: Support get srq number
Providers/rxe: Support XRC traffic

 kernel-headers/rdma/rdma_user_rxe.h |   6 +-
 providers/rxe/rxe-abi.h             |   2 +
 providers/rxe/rxe.c                 | 235 ++++++++++++++++++++++------
 providers/rxe/rxe.h                 |   5 +-
 4 files changed, 200 insertions(+), 48 deletions(-)
Signature · 1 line
2.30.2
#1 lore raw

PATCH 1/5 Update kernel headers

To commit ?? ("RDMA/rxe: Enable receiving XRC packets").

---
 kernel-headers/rdma/rdma_user_rxe.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
kernel-headers/rdma/rdma_user_rxe.h +5 -1
index ad7da77d..1d80586c 100644
@@ -82,6 +82,10 @@ struct rxe_send_wr {
 		__u32		invalidate_rkey;
 	} ex;
 	union {
+		struct {
+			__aligned_u64 pad[4];
+			__u32	srq_num;
+		} xrc;
 		struct {
 			__aligned_u64 remote_addr;
 			__u32	rkey;
@@ -101,7 +105,7 @@ struct rxe_send_wr {
 			__u16	reserved;
 			__u32	ah_num;
 			__u32	pad[4];
-			struct rxe_av av;	/* deprecated */
+			struct rxe_av av;
 		} ud;
 		struct {
 			__aligned_u64	addr;
Signature · 1 line
2.30.2
#2 lore raw

PATCH 2/5 Providers/rxe: Support alloc/dealloc xrcd

In reply to Bob Pearson

Add support for ibv_alloc_xrcd and ibv_dealloc_xrcd verbs.

---
 providers/rxe/rxe.c | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
providers/rxe/rxe.c +40 -0
index 0776279c..3bb9f01c 100644
@@ -128,6 +128,44 @@ static int rxe_dealloc_pd(struct ibv_pd *pd)
 	return ret;
 }
 
+static struct ibv_xrcd *rxe_open_xrcd(struct ibv_context *context,
+					struct ibv_xrcd_init_attr *attr)
+{
+	struct verbs_xrcd *xrcd;
+	int xrcd_size = sizeof(*xrcd);
+	struct ibv_open_xrcd cmd = {};
+	size_t cmd_size = sizeof(cmd);
+	struct ib_uverbs_open_xrcd_resp resp = {};
+	size_t resp_size = sizeof(resp);
+	int ret;
+
+	xrcd = calloc(1, sizeof(*xrcd));
+	if (!xrcd)
+		return NULL;
+
+	ret = ibv_cmd_open_xrcd(context, xrcd, xrcd_size, attr,
+				&cmd, cmd_size, &resp, resp_size);
+	if (ret) {
+		free(xrcd);
+		return NULL;
+	}
+
+	return &xrcd->xrcd;
+}
+
+static int rxe_close_xrcd(struct ibv_xrcd *ibxrcd)
+{
+	struct verbs_xrcd *xrcd = container_of(ibxrcd,
+					struct verbs_xrcd, xrcd);
+	int ret;
+
+	ret = ibv_cmd_close_xrcd(xrcd);
+	if (!ret)
+		free(xrcd);
+
+	return ret;
+}
+
 static struct ibv_mw *rxe_alloc_mw(struct ibv_pd *ibpd, enum ibv_mw_type type)
 {
 	int ret;
@@ -1742,6 +1780,8 @@ static const struct verbs_context_ops rxe_ctx_ops = {
 	.query_port = rxe_query_port,
 	.alloc_pd = rxe_alloc_pd,
 	.dealloc_pd = rxe_dealloc_pd,
+	.open_xrcd = rxe_open_xrcd,
+	.close_xrcd = rxe_close_xrcd,
 	.reg_mr = rxe_reg_mr,
 	.dereg_mr = rxe_dereg_mr,
 	.alloc_mw = rxe_alloc_mw,
Signature · 1 line
2.30.2
#3 lore raw

PATCH 3/5 Providers/rxe: Support extended create srq

In reply to Bob Pearson

Add support for the ibv_create_srq_ex verb.

---
 providers/rxe/rxe-abi.h |  2 ++
 providers/rxe/rxe.c     | 54 +++++++++++++++++++++++++++++++++++------
 providers/rxe/rxe.h     |  5 ++--
 3 files changed, 50 insertions(+), 11 deletions(-)
providers/rxe/rxe-abi.h +2 -0
index 020201a9..07e90d81 100644
@@ -51,6 +51,8 @@ DECLARE_DRV_CMD(urxe_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
 		empty, rxe_create_qp_resp);
 DECLARE_DRV_CMD(urxe_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
 		empty, rxe_create_srq_resp);
+DECLARE_DRV_CMD(urxe_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
+		empty, rxe_create_srq_resp);
 DECLARE_DRV_CMD(urxe_modify_srq, IB_USER_VERBS_CMD_MODIFY_SRQ,
 		rxe_modify_srq_cmd, empty);
 DECLARE_DRV_CMD(urxe_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ,
providers/rxe/rxe.c +46 -8

97 lines not on this page · raw message

providers/rxe/rxe.h +2 -3
index 6882d9c7..f3215f2e 100644
@@ -91,10 +91,9 @@ struct rxe_qp {
 };
 
 struct rxe_srq {
-	struct ibv_srq		ibv_srq;
+	struct verbs_srq	vsrq;
 	struct mminfo		mmap_info;
 	struct rxe_wq		rq;
-	uint32_t		srq_num;
 };
 
 #define to_rxxx(xxx, type) container_of(ib##xxx, struct rxe_##type, ibv_##xxx)
@@ -121,7 +120,7 @@ static inline struct rxe_qp *to_rqp(struct ibv_qp *ibqp)
 
 static inline struct rxe_srq *to_rsrq(struct ibv_srq *ibsrq)
 {
-	return to_rxxx(srq, srq);
+	return container_of(ibsrq, struct rxe_srq, vsrq.srq);
 }
 
 static inline struct rxe_ah *to_rah(struct ibv_ah *ibah)
Signature · 1 line
2.30.2
#4 lore raw

PATCH 4/5 Providers/rxe: Support get srq number

In reply to Bob Pearson

Add support for ibv_get_srq_num verb.

---
 providers/rxe/rxe.c | 9 +++++++++
 1 file changed, 9 insertions(+)
providers/rxe/rxe.c +9 -0
index 9cdddb8c..d4538713 100644
@@ -696,6 +696,14 @@ static struct ibv_srq *rxe_create_srq_ex(struct ibv_context *context,
 	return &srq->vsrq.srq;
 }
 
+static int rxe_get_srq_num(struct ibv_srq *ibsrq, uint32_t *srq_num)
+{
+	struct rxe_srq *srq = to_rsrq(ibsrq);
+
+	*srq_num = srq->vsrq.srq_num;
+	return 0;
+}
+
 static int rxe_modify_srq(struct ibv_srq *ibsrq,
 		   struct ibv_srq_attr *attr, int attr_mask)
 {
@@ -1836,6 +1844,7 @@ static const struct verbs_context_ops rxe_ctx_ops = {
 	.query_srq = rxe_query_srq,
 	.destroy_srq = rxe_destroy_srq,
 	.post_srq_recv = rxe_post_srq_recv,
+	.get_srq_num = rxe_get_srq_num,
 	.create_qp = rxe_create_qp,
 	.create_qp_ex = rxe_create_qp_ex,
 	.query_qp = rxe_query_qp,
Signature · 1 line
2.30.2
#5 lore raw

PATCH 5/5 Providers/rxe: Support XRC traffic

In reply to Bob Pearson

Extended create_qp and create_qp_ex verbs to support XRC QP types. Extended WRs to support XRC operations.

---
 providers/rxe/rxe.c | 132 ++++++++++++++++++++++++++++++++------------
 1 file changed, 96 insertions(+), 36 deletions(-)
providers/rxe/rxe.c +96 -36
index d4538713..4fbdb689 100644
@@ -875,9 +875,10 @@ static void wr_atomic_fetch_add(struct ibv_qp_ex *ibqp, uint32_t rkey,
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_ATOMIC_FETCH_AND_ADD;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.wr.atomic.remote_addr = remote_addr;
 	wqe->wr.wr.atomic.compare_add = add;
 	wqe->wr.wr.atomic.rkey = rkey;
@@ -899,8 +900,9 @@ static void wr_bind_mw(struct ibv_qp_ex *ibqp, struct ibv_mw *ibmw,
 	memset(wqe, 0, sizeof(*wqe));
 
 	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_BIND_MW;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.wr.mw.addr = info->addr;
 	wqe->wr.wr.mw.length = info->length;
 	wqe->wr.wr.mw.mr_lkey = info->mr->lkey;
@@ -922,9 +924,10 @@ static void wr_local_inv(struct ibv_qp_ex *ibqp, uint32_t invalidate_rkey)
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_LOCAL_INV;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.ex.invalidate_rkey = invalidate_rkey;
 	wqe->ssn = qp->ssn++;
 
@@ -942,9 +945,10 @@ static void wr_rdma_read(struct ibv_qp_ex *ibqp, uint32_t rkey,
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_RDMA_READ;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.wr.rdma.remote_addr = remote_addr;
 	wqe->wr.wr.rdma.rkey = rkey;
 	wqe->iova = remote_addr;
@@ -964,9 +968,10 @@ static void wr_rdma_write(struct ibv_qp_ex *ibqp, uint32_t rkey,
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_RDMA_WRITE;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.wr.rdma.remote_addr = remote_addr;
 	wqe->wr.wr.rdma.rkey = rkey;
 	wqe->iova = remote_addr;
@@ -986,9 +991,10 @@ static void wr_rdma_write_imm(struct ibv_qp_ex *ibqp, uint32_t rkey,
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.wr.rdma.remote_addr = remote_addr;
 	wqe->wr.wr.rdma.rkey = rkey;
 	wqe->wr.ex.imm_data = imm_data;
@@ -1008,9 +1014,10 @@ static void wr_send(struct ibv_qp_ex *ibqp)
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_SEND;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->ssn = qp->ssn++;
 
 	advance_qp_cur_index(qp);
@@ -1026,9 +1033,10 @@ static void wr_send_imm(struct ibv_qp_ex *ibqp, __be32 imm_data)
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_SEND_WITH_IMM;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.ex.imm_data = imm_data;
 	wqe->ssn = qp->ssn++;
 
@@ -1045,9 +1053,10 @@ static void wr_send_inv(struct ibv_qp_ex *ibqp, uint32_t invalidate_rkey)
 
 	memset(wqe, 0, sizeof(*wqe));
 
-	wqe->wr.wr_id = qp->vqp.qp_ex.wr_id;
+	wqe->wr.wr_id = ibqp->wr_id;
+	wqe->wr.send_flags = ibqp->wr_flags;
 	wqe->wr.opcode = IBV_WR_SEND_WITH_INV;
-	wqe->wr.send_flags = qp->vqp.qp_ex.wr_flags;
+
 	wqe->wr.ex.invalidate_rkey = invalidate_rkey;
 	wqe->ssn = qp->ssn++;
 
@@ -1074,6 +1083,18 @@ static void wr_set_ud_addr(struct ibv_qp_ex *ibqp, struct ibv_ah *ibah,
 		memcpy(&wqe->wr.wr.ud.av, &ah->av, sizeof(ah->av));
 }
 
+static void wr_set_xrc_srqn(struct ibv_qp_ex *ibqp, uint32_t remote_srqn)
+{
+	struct rxe_qp *qp = container_of(ibqp, struct rxe_qp, vqp.qp_ex);
+	struct rxe_send_wqe *wqe = addr_from_index(qp->sq.queue,
+						   qp->cur_index - 1);
+
+	if (qp->err)
+		return;
+
+	wqe->wr.wr.xrc.srq_num = remote_srqn;
+}
+
 static void wr_set_inline_data(struct ibv_qp_ex *ibqp, void *addr,
 			       size_t length)
 {
@@ -1212,7 +1233,8 @@ static int map_queue_pair(int cmd_fd, struct rxe_qp *qp,
 			  struct ibv_qp_init_attr *attr,
 			  struct rxe_create_qp_resp *resp)
 {
-	if (attr->srq) {
+	if (attr->srq || qp_type(qp) == IBV_QPT_XRC_RECV ||
+	    qp_type(qp) == IBV_QPT_XRC_SEND) {
 		qp->rq.max_sge = 0;
 		qp->rq.queue = NULL;
 		qp->rq_mmap_info.size = 0;
@@ -1228,23 +1250,44 @@ static int map_queue_pair(int cmd_fd, struct rxe_qp *qp,
 		pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE);
 	}
 
-	qp->sq.max_sge = attr->cap.max_send_sge;
-	qp->sq.max_inline = attr->cap.max_inline_data;
-	qp->sq.queue = mmap(NULL, resp->sq_mi.size, PROT_READ | PROT_WRITE,
-			    MAP_SHARED,
-			    cmd_fd, resp->sq_mi.offset);
-	if ((void *)qp->sq.queue == MAP_FAILED) {
-		if (qp->rq_mmap_info.size)
-			munmap(qp->rq.queue, qp->rq_mmap_info.size);
-		return errno;
-	}
+	if (qp_type(qp) != IBV_QPT_XRC_RECV) {
+		qp->sq.max_sge = attr->cap.max_send_sge;
+		qp->sq.max_inline = attr->cap.max_inline_data;
+		qp->sq.queue = mmap(NULL, resp->sq_mi.size, PROT_READ | PROT_WRITE,
+				    MAP_SHARED,
+				    cmd_fd, resp->sq_mi.offset);
+		if ((void *)qp->sq.queue == MAP_FAILED) {
+			if (qp->rq_mmap_info.size)
+				munmap(qp->rq.queue, qp->rq_mmap_info.size);
+			return errno;
+		}
 
-	qp->sq_mmap_info = resp->sq_mi;
-	pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE);
+		qp->sq_mmap_info = resp->sq_mi;
+		pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE);
+	}
 
 	return 0;
 }
 
+static int map_queue_pair_ex(int cmd_fd, struct rxe_qp *qp,
+			     struct ibv_qp_init_attr_ex *attr,
+			     struct rxe_create_qp_resp *resp)
+{
+	switch (attr->qp_type) {
+	case IBV_QPT_RC:
+	case IBV_QPT_UC:
+	case IBV_QPT_UD:
+	case IBV_QPT_XRC_SEND:
+		return map_queue_pair(cmd_fd, qp,
+				(struct ibv_qp_init_attr *)attr, resp);
+	case IBV_QPT_XRC_RECV:
+		return 0;
+	default:
+		errno = EINVAL;
+		return errno;
+	}
+}
+
 static struct ibv_qp *rxe_create_qp(struct ibv_pd *ibpd,
 				    struct ibv_qp_init_attr *attr)
 {
@@ -1283,7 +1326,7 @@ err:
 enum {
 	RXE_QP_CREATE_FLAGS_SUP = 0,
 
-	RXE_QP_COMP_MASK_SUP = IBV_QP_INIT_ATTR_PD |
+	RXE_QP_COMP_MASK_SUP = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD |
 		IBV_QP_INIT_ATTR_CREATE_FLAGS | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS,
 
 	RXE_SUP_RC_QP_SEND_OPS_FLAGS =
@@ -1300,6 +1343,13 @@ enum {
 
 	RXE_SUP_UD_QP_SEND_OPS_FLAGS =
 		IBV_QP_EX_WITH_SEND | IBV_QP_EX_WITH_SEND_WITH_IMM,
+
+	RXE_SUP_XRC_QP_SEND_OPS_FLAGS =
+		IBV_QP_EX_WITH_RDMA_WRITE | IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM |
+		IBV_QP_EX_WITH_SEND | IBV_QP_EX_WITH_SEND_WITH_IMM |
+		IBV_QP_EX_WITH_RDMA_READ | IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP |
+		IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD | IBV_QP_EX_WITH_LOCAL_INV |
+		IBV_QP_EX_WITH_BIND_MW | IBV_QP_EX_WITH_SEND_WITH_INV,
 };
 
 static int check_qp_init_attr(struct ibv_qp_init_attr_ex *attr)
@@ -1325,6 +1375,10 @@ static int check_qp_init_attr(struct ibv_qp_init_attr_ex *attr)
 			if (attr->send_ops_flags & ~RXE_SUP_UD_QP_SEND_OPS_FLAGS)
 				goto err;
 			break;
+		case IBV_QPT_XRC_SEND:
+			if (attr->send_ops_flags & ~RXE_SUP_XRC_QP_SEND_OPS_FLAGS)
+				goto err;
+			break;
 		default:
 			goto err;
 		}
@@ -1369,6 +1423,7 @@ static void set_qp_send_ops(struct rxe_qp *qp, uint64_t flags)
 		qp->vqp.qp_ex.wr_send_inv = wr_send_inv;
 
 	qp->vqp.qp_ex.wr_set_ud_addr = wr_set_ud_addr;
+	qp->vqp.qp_ex.wr_set_xrc_srqn = wr_set_xrc_srqn;
 	qp->vqp.qp_ex.wr_set_inline_data = wr_set_inline_data;
 	qp->vqp.qp_ex.wr_set_inline_data_list = wr_set_inline_data_list;
 	qp->vqp.qp_ex.wr_set_sge = wr_set_sge;
@@ -1390,8 +1445,9 @@ static struct ibv_qp *rxe_create_qp_ex(struct ibv_context *context,
 	size_t resp_size = sizeof(resp);
 
 	ret = check_qp_init_attr(attr);
-	if (ret)
+	if (ret) {
 		goto err;
+	}
 
 	qp = calloc(1, sizeof(*qp));
 	if (!qp)
@@ -1408,9 +1464,8 @@ static struct ibv_qp *rxe_create_qp_ex(struct ibv_context *context,
 
 	qp->vqp.comp_mask |= VERBS_QP_EX;
 
-	ret = map_queue_pair(context->cmd_fd, qp,
-			     (struct ibv_qp_init_attr *)attr,
-			     &resp.drv_payload);
+	ret = map_queue_pair_ex(context->cmd_fd, qp, attr,
+				&resp.drv_payload);
 	if (ret)
 		goto err_destroy;
 
@@ -1484,7 +1539,9 @@ static int validate_send_wr(struct rxe_qp *qp, struct ibv_send_wr *ibwr,
 			return -EINVAL;
 		if (ibwr->imm_data)
 			return -EINVAL;
-		if ((qp_type(qp) != IBV_QPT_RC) && (qp_type(qp) != IBV_QPT_UC))
+		if ((qp_type(qp) != IBV_QPT_RC) &&
+		    (qp_type(qp) != IBV_QPT_UC) &&
+		    (qp_type(qp) != IBV_QPT_XRC_SEND))
 			return -EINVAL;
 	}
 
@@ -1547,6 +1604,9 @@ static void convert_send_wr(struct rxe_qp *qp, struct rxe_send_wr *kwr,
 	default:
 		break;
 	}
+
+	if (qp_type(qp) == IBV_QPT_XRC_SEND)
+		kwr->wr.xrc.srq_num = uwr->qp_type.xrc.remote_srqn;
 }
 
 static int init_send_wqe(struct rxe_qp *qp, struct rxe_wq *sq,
Signature · 1 line
2.30.2
#6 lore raw

Re: [PATCH 5/5] Providers/rxe: Support XRC traffic

Quoted 7 lines · Bob Pearson

Extended create_qp and create_qp_ex verbs to support XRC QP types. Extended WRs to support XRC operations.

---
 providers/rxe/rxe.c | 132 ++++++++++++++++++++++++++++++++------------
 1 file changed, 96 insertions(+), 36 deletions(-)

<...>

providers/rxe/rxe.c static void wr_set_ud_addr(struct ibv_qp_ex *ibqp, struct ibv_ah *ibah, · line 1093

Quoted 8 lines
providers/rxe/rxe.c +8 -0
+static void wr_set_xrc_srqn(struct ibv_qp_ex *ibqp, uint32_t remote_srqn)
+{
+	struct rxe_qp *qp = container_of(ibqp, struct rxe_qp, vqp.qp_ex);
+	struct rxe_send_wqe *wqe = addr_from_index(qp->sq.queue,
+						   qp->cur_index - 1);
+
+	if (qp->err)
+		return;

Why is that?

Quoted 4 lines
providers/rxe/rxe.c +4 -0
+
+	wqe->wr.wr.xrc.srq_num = remote_srqn;
+}
+
#7 lore raw

Re: [PATCH 5/5] Providers/rxe: Support XRC traffic

Quoted 22 lines · Leon Romanovsky
Quoted 7 lines · Bob Pearson

Extended create_qp and create_qp_ex verbs to support XRC QP types. Extended WRs to support XRC operations.

---
 providers/rxe/rxe.c | 132 ++++++++++++++++++++++++++++++++------------
 1 file changed, 96 insertions(+), 36 deletions(-)

<...>

providers/rxe/rxe.c static void wr_set_ud_addr(struct ibv_qp_ex *ibqp, struct ibv_ah *ibah, · line 1093

Quoted 8 lines
providers/rxe/rxe.c +8 -0
+static void wr_set_xrc_srqn(struct ibv_qp_ex *ibqp, uint32_t remote_srqn)
+{
+	struct rxe_qp *qp = container_of(ibqp, struct rxe_qp, vqp.qp_ex);
+	struct rxe_send_wqe *wqe = addr_from_index(qp->sq.queue,
+						   qp->cur_index - 1);
+
+	if (qp->err)
+		return;

Why is that?

Quoted 4 lines

providers/rxe/rxe.c static void wr_set_ud_addr(struct ibv_qp_ex *ibqp, struct ibv_ah *ibah, · line 1097

Quoted 4 lines
providers/rxe/rxe.c +4 -0
+
+	wqe->wr.wr.xrc.srq_num = remote_srqn;
+}
+

qp->err is used to detect overrun in the send WQ. Each of the 'builders' calls check_qp_queue_full() and sets qp->err if there isn't any more room in the send queue. Since the routines are of type void there is no way to tell the caller that something bad has happened. Once the builder fails the 'setters' have to fail as well because there is no place to write the WQE parameters. If this happens the user space caller will will see the operations succeed up to the point that it ran out of room and then stop. This is sort of a half solution to the problem. The alternative is to say that the user can never make this mistake and just keep building WQEs wrapping around the queue.

Bob