Skip to contentSkip to conversation

providers/rxe: Implement the xrc transport protocol

PATCH In review Bob Pearson 2 participants 4 messages Sep 17, 2022 – Sep 26, 2022 InfiniBand

lore
Share & tools
lore
#0 lore raw

cover providers/rxe: Implement the xrc transport protocol

Make changes to implement the xrc transport protocol including
- Implement ibv_create_srq_ex
- Implement ibv_open/close_xrcd
- Implement ibv_get_srq_num
- Implement xrc support for qp_ex

The patch "providers/rxe: Remove redundant num_sge fields" is a pre-requisite for this patch.

Bob Pearson (2):
Update kernel headers
providers/rxe: Implement the xrc transport protocol

 kernel-headers/rdma/ib_user_ioctl_verbs.h |   2 +
 kernel-headers/rdma/ib_user_verbs.h       |  16 +
 kernel-headers/rdma/rdma_user_rxe.h       |  14 +-
 providers/rxe/rxe-abi.h                   |   2 +
 providers/rxe/rxe.c                       | 387 +++++++++++++++-------
 providers/rxe/rxe.h                       |  21 +-
 6 files changed, 318 insertions(+), 124 deletions(-)

base-commit: 41c28b03d2b7cfc982eedd2e7491b01df984f5d7

Signature · 1 line
2.34.1
#1 lore raw

PATCH 1/2 Update kernel headers

To commit ?? ("RDMA/rxe: Extend rxe_resp.c to support xrc qps").

---
 kernel-headers/rdma/rdma_user_rxe.h       | 14 ++++++++++----
 1 files changed, 10 insertions(+), 4 deletions(-)
kernel-headers/rdma/rdma_user_rxe.h +3 -4
index 73f679df..c44db5fa 100644
@@ -74,7 +74,7 @@ struct rxe_av {
 
 struct rxe_send_wr {
 	__aligned_u64		wr_id;
-	__u32			reserved;
+	__u32			srq_num;	/* xrc only */
 	__u32			opcode;
 	__u32			send_flags;
 	union {
@@ -166,7 +173,7 @@ struct rxe_send_wqe {
 
 struct rxe_recv_wqe {
 	__aligned_u64		wr_id;
-	__u32			reserved;
+	__u32			num_sge;
 	__u32			padding;
 	struct rxe_dma_info	dma;
 };
@@ -191,8 +198,7 @@ struct rxe_create_qp_resp {
 
 struct rxe_create_srq_resp {
 	struct mminfo mi;
-	__u32 srq_num;
-	__u32 reserved;
+	__u32 reserved[2];
 };
 
 struct rxe_modify_srq_cmd {
Signature · 1 line
2.34.1
#2 lore raw

PATCH 2/2 providers/rxe: Implement the xrc transport protocol

In reply to Bob Pearson

Make changes to implement the xrc transport protocol.

---
 providers/rxe/rxe-abi.h |   2 +
 providers/rxe/rxe.c     | 387 ++++++++++++++++++++++++++++------------
 providers/rxe/rxe.h     |  21 ++-
 3 files changed, 290 insertions(+), 120 deletions(-)
providers/rxe/rxe-abi.h +2 -0
index 020201a9..07e90d81 100644
@@ -51,6 +51,8 @@ DECLARE_DRV_CMD(urxe_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
 		empty, rxe_create_qp_resp);
 DECLARE_DRV_CMD(urxe_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
 		empty, rxe_create_srq_resp);
+DECLARE_DRV_CMD(urxe_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
+		empty, rxe_create_srq_resp);
 DECLARE_DRV_CMD(urxe_modify_srq, IB_USER_VERBS_CMD_MODIFY_SRQ,
 		rxe_modify_srq_cmd, empty);
 DECLARE_DRV_CMD(urxe_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ,
providers/rxe/rxe.c +270 -117

599 lines not on this page · raw message

providers/rxe/rxe.h +18 -3
index 6882d9c7..2023ecae 100644
@@ -40,6 +40,8 @@
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <rdma/rdma_user_rxe.h>
+#include <unistd.h>
+
 #include "rxe-abi.h"
 
 struct rxe_device {
@@ -90,13 +92,21 @@ struct rxe_qp {
 	int			err;
 };
 
+struct rxe_xrcd {
+	struct verbs_xrcd	vxrcd;
+};
+
 struct rxe_srq {
-	struct ibv_srq		ibv_srq;
+	struct verbs_srq	vsrq;
 	struct mminfo		mmap_info;
 	struct rxe_wq		rq;
-	uint32_t		srq_num;
 };
 
+static inline unsigned int srq_num(struct rxe_srq *srq)
+{
+	return srq->vsrq.srq_num;
+}
+
 #define to_rxxx(xxx, type) container_of(ib##xxx, struct rxe_##type, ibv_##xxx)
 
 static inline struct rxe_context *to_rctx(struct ibv_context *ibctx)
@@ -119,9 +129,14 @@ static inline struct rxe_qp *to_rqp(struct ibv_qp *ibqp)
 	return container_of(ibqp, struct rxe_qp, vqp.qp);
 }
 
+static inline struct rxe_xrcd *to_rxrcd(struct ibv_xrcd *ibxrcd)
+{
+	return container_of(ibxrcd, struct rxe_xrcd, vxrcd.xrcd);
+}
+
 static inline struct rxe_srq *to_rsrq(struct ibv_srq *ibsrq)
 {
-	return to_rxxx(srq, srq);
+	return container_of(ibsrq, struct rxe_srq, vsrq.srq);
 }
 
 static inline struct rxe_ah *to_rah(struct ibv_ah *ibah)
Signature · 1 line
2.34.1
maintainer
#3 lore raw
Quoted 8 lines · Bob Pearson

Make changes to implement the xrc transport protocol.

---
 providers/rxe/rxe-abi.h |   2 +
 providers/rxe/rxe.c     | 387 ++++++++++++++++++++++++++++------------
 providers/rxe/rxe.h     |  21 ++-
 3 files changed, 290 insertions(+), 120 deletions(-)

Please send as a PR to rdma-core

Jason