Providers/rxe: Replace AV by AH for UD sends
v2 cover Providers/rxe: Replace AV by AH for UD sends
These patches contains the changes to the rxe provider to match the kernel patch series with a similar name.
This patch series is identical to the one sent with the v3 RDMA/rxe:
kernel patch set but still cleanly applied to the current rdma-core tree.
commit 0217e47de29c5429679c7ebf73d38a95623d2785 (master)
v2:
As suggested by Jason Gunthorp moved the AV struct out of rxe_send_wqe and into rxe_wr in the ud struct in the wr union at the original offset. This maintains ABI compatibility with old kernels or rdma-core but simplifies the code.
Bob Pearson (2):
Update kernel headers
Providers/rxe: Replace AV by AH for UD sends
kernel-headers/rdma/rdma_user_rxe.h | 16 +++- providers/rxe/rxe-abi.h | 2 + providers/rxe/rxe.c | 117 ++++++++++++++++++---------- providers/rxe/rxe.h | 8 +- 4 files changed, 100 insertions(+), 43 deletions(-)
Signature · 1 line
2.30.2
PATCH v2 1/2 Update kernel headers
To commit ?? ("RDMA/rxe: Convert kernel UD post send to use ah_num").
--- kernel-headers/rdma/rdma_user_rxe.h | 10 +++++++++- providers/rxe/rxe.c | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-)
kernel-headers/rdma/rdma_user_rxe.h +9 -1
@@ -98,6 +98,10 @@ struct rxe_send_wr {__u32 remote_qpn;__u32 remote_qkey;__u16 pkey_index; +__u16 reserved; +__u32 ah_num; +__u32 pad[4 ]; +struct rxe_av av;/* deprecated */ } ud;struct {__aligned_u64 addr;
@@ -148,7 +152,6 @@ struct rxe_dma_info {struct rxe_send_wqe {struct rxe_send_wr wr; -struct rxe_av av;__u32 status;__u32 state;__aligned_u64 iova;
@@ -168,6 +171,11 @@ struct rxe_recv_wqe {struct rxe_dma_info dma; }; +struct rxe_create_ah_resp { +__u32 ah_num; +__u32 reserved; +}; +struct rxe_create_cq_resp {struct mminfo mi; };
providers/rxe/rxe.c +2 -2
@@ -982,7 +982,7 @@ static void wr_set_ud_addr(struct ibv_qp_ex *ibqp, struct ibv_ah *ibah,if (qp->err)return ; -memcpy (&wqe->av, &ah->av,sizeof (ah->av)); +memcpy (&wqe->wr.wr.ud.av, &ah->av,sizeof (ah->av)); wqe->wr.wr.ud.remote_qpn = remote_qpn; wqe->wr.wr.ud.remote_qkey = remote_qkey; }
@@ -1467,7 +1467,7 @@ static int init_send_wqe(struct rxe_qp *qp, struct rxe_wq *sq,convert_send_wr (&wqe->wr, ibwr);if (qp_type (qp) ==IBV_QPT_UD ) -memcpy (&wqe->av, &to_rah (ibwr->wr.ud.ah)->av, +memcpy (&wqe->wr.wr.ud.av, &to_rah (ibwr->wr.ud.ah)->av,sizeof (struct rxe_av ));if (ibwr->send_flags &IBV_SEND_INLINE ) {
Signature · 1 line
2.30.2
PATCH v2 2/2 Providers/rxe: Replace AV by AH for UD sends
In reply to Bob PearsonCurrently the rxe provider computes an AV for each AH created. These are then passed to the kernel in each UD send WQE. This rxe provider patch matches a set of kernel patches that creates an index for the pool of AHs which are passed back to the rxe provider which in turn returns them in the UD send WQEs instead of AVs. This falls back to the older AV based WQE if either the rxe provider or the kernel does not have these patches applied.
--- providers/rxe/rxe-abi.h | 2 + providers/rxe/rxe.c | 117 ++++++++++++++++++++++++++-------------- providers/rxe/rxe.h | 8 ++- 3 files changed, 85 insertions(+), 42 deletions(-)
providers/rxe/rxe-abi.h +2 -0
@@ -39,6 +39,8 @@#include <rdma/rdma_user_rxe.h> #include <kernel-abi/rdma_user_rxe.h> +DECLARE_DRV_CMD (urxe_create_ah,IB_USER_VERBS_CMD_CREATE_AH , + empty, rxe_create_ah_resp);DECLARE_DRV_CMD (urxe_create_cq,IB_USER_VERBS_CMD_CREATE_CQ , empty, rxe_create_cq_resp); DECLARE_DRV_CMD(urxe_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
providers/rxe/rxe.c +77 -40
178 lines not on this page · raw message
providers/rxe/rxe.h +6 -2
@@ -67,6 +67,7 @@ struct rxe_cq {struct rxe_ah {struct ibv_ah ibv_ah;struct rxe_av av; +int ah_num; };struct rxe_wq {
@@ -89,8 +90,6 @@ struct rxe_qp {int err; }; -#define qp_type (qp) ((qp)->vqp.qp.qp_type) -struct rxe_srq {struct ibv_srq ibv_srq;struct mminfo mmap_info;
@@ -130,4 +129,9 @@ static inline struct rxe_ah *to_rah(struct ibv_ah *ibah) return to_rxxx(ah, ah); } +static inline enum ibv_qp_type qp_type (struct rxe_qp *qp) +{ +return qp->vqp.qp.qp_type; +} +#endif /* RXE_H */
Signature · 1 line
2.30.2