Skip to contentSkip to conversation

Providers/rxe: Replace AV by AH for UD sends

PATCH Superseded Bob Pearson 1 participant 3 messages Jun 29, 2021

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover Providers/rxe: Replace AV by AH for UD sends

These patches contains the changes to the rxe provider to match the kernel patch series with a similar name.

Bob Pearson (2):
Update kernel headers
Providers/rxe: Replace AV by AH for UD sends

 kernel-headers/rdma/rdma_user_rxe.h |  16 +++-
 providers/rxe/rxe-abi.h             |   2 +
 providers/rxe/rxe.c                 | 117 ++++++++++++++++++----------
 providers/rxe/rxe.h                 |   8 +-
 4 files changed, 100 insertions(+), 43 deletions(-)
Signature · 1 line
2.30.2
#1 lore raw

PATCH 1/2 Update kernel headers

To commit ?? ("RDMA/rxe: Convert kernel UD post send to use ah_num").

---
 kernel-headers/rdma/rdma_user_rxe.h | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)
kernel-headers/rdma/rdma_user_rxe.h +13 -1
index e283c222..e544832e 100644
@@ -98,6 +98,8 @@ struct rxe_send_wr {
 			__u32	remote_qpn;
 			__u32	remote_qkey;
 			__u16	pkey_index;
+			__u16	reserved;
+			__u32	ah_num;
 		} ud;
 		struct {
 			__aligned_u64	addr;
@@ -148,7 +150,12 @@ struct rxe_dma_info {
 
 struct rxe_send_wqe {
 	struct rxe_send_wr	wr;
-	struct rxe_av		av;
+	union {
+		struct rxe_av av;
+		struct {
+			__u32		reserved[0];
+		} ex;
+	};
 	__u32			status;
 	__u32			state;
 	__aligned_u64		iova;
@@ -168,6 +175,11 @@ struct rxe_recv_wqe {
 	struct rxe_dma_info	dma;
 };
 
+struct rxe_create_ah_resp {
+	__u32 ah_num;
+	__u32 reserved;
+};
+
 struct rxe_create_cq_resp {
 	struct mminfo mi;
 };
Signature · 1 line
2.30.2
#2 lore raw

PATCH 2/2 Providers/rxe: Replace AV by AH for UD sends

In reply to Bob Pearson

Currently the rxe provider computes an AV for each AH created. These are then passed to the kernel in each UD send WQE. This rxe provider patch matches a set of kernel patches that creates an index for the pool of AHs which are passed back to the rxe provider which in turn returns them in the UD send WQEs instead of AVs. This falls back to the older AV based WQE if either the rxe provider or the kernel does not have these patches applied.

---
 providers/rxe/rxe-abi.h |   2 +
 providers/rxe/rxe.c     | 117 ++++++++++++++++++++++++++--------------
 providers/rxe/rxe.h     |   8 ++-
 3 files changed, 85 insertions(+), 42 deletions(-)
providers/rxe/rxe-abi.h +2 -0
index 0689c3f4..020201a9 100644
@@ -39,6 +39,8 @@
 #include <rdma/rdma_user_rxe.h>
 #include <kernel-abi/rdma_user_rxe.h>
 
+DECLARE_DRV_CMD(urxe_create_ah, IB_USER_VERBS_CMD_CREATE_AH,
+		empty, rxe_create_ah_resp);
 DECLARE_DRV_CMD(urxe_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
 		empty, rxe_create_cq_resp);
 DECLARE_DRV_CMD(urxe_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
providers/rxe/rxe.c +77 -40

178 lines not on this page · raw message

providers/rxe/rxe.h +6 -2
index ddb8fe89..6882d9c7 100644
@@ -67,6 +67,7 @@ struct rxe_cq {
 struct rxe_ah {
 	struct ibv_ah		ibv_ah;
 	struct rxe_av		av;
+	int			ah_num;
 };
 
 struct rxe_wq {
@@ -89,8 +90,6 @@ struct rxe_qp {
 	int			err;
 };
 
-#define qp_type(qp)		((qp)->vqp.qp.qp_type)
-
 struct rxe_srq {
 	struct ibv_srq		ibv_srq;
 	struct mminfo		mmap_info;
@@ -130,4 +129,9 @@ static inline struct rxe_ah *to_rah(struct ibv_ah *ibah)
 	return to_rxxx(ah, ah);
 }
 
+static inline enum ibv_qp_type qp_type(struct rxe_qp *qp)
+{
+	return qp->vqp.qp.qp_type;
+}
+
 #endif /* RXE_H */
Signature · 1 line
2.30.2