RDMA/rtrs: Use new shared CQ mechanism
Subsystems (2)
PATCH v2 1/2 RDMA/rtrs: Use new shared CQ mechanism
Has the driver use shared CQs providing ~10%-20% improvement during test.
Instead of opening a CQ for each QP per connection, a CQ for each QP will be provided by the RDMA core driver that will be shared between the QPs on that core reducing interrupt overhead.
--- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 10 +++++----- drivers/infiniband/ulp/rtrs/rtrs-pri.h | 1 + drivers/infiniband/ulp/rtrs/rtrs-srv.c | 10 +++++----- drivers/infiniband/ulp/rtrs/rtrs.c | 9 +++++---- 4 files changed, 16 insertions(+), 14 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-clt.c +5 -5
@@ -325,7 +325,7 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)static void rtrs_clt_fast_reg_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_clt_con *con = cq->cq_context; +struct rtrs_clt_con *con =to_clt_con (wc->qp->qp_context);if (unlikely (wc->status !=IB_WC_SUCCESS )) { rtrs_err(con->c.sess, "Failed IB_WR_REG_MR: %s\n",
@@ -345,7 +345,7 @@ static void rtrs_clt_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) {struct rtrs_clt_io_req *req =container_of (wc->wr_cqe,typeof (*req), inv_cqe); -struct rtrs_clt_con *con = cq->cq_context; +struct rtrs_clt_con *con =to_clt_con (wc->qp->qp_context);if (unlikely (wc->status !=IB_WC_SUCCESS )) { rtrs_err(con->c.sess, "Failed IB_WR_LOCAL_INV: %s\n",
@@ -586,7 +586,7 @@ static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe)static void rtrs_clt_rdma_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_clt_con *con = cq->cq_context; +struct rtrs_clt_con *con =to_clt_con (wc->qp->qp_context);struct rtrs_clt_sess *sess =to_clt_sess (con->c.sess);u32 imm_type, imm_payload;bool w_inval = false;
@@ -2241,7 +2241,7 @@ static int init_conns(struct rtrs_clt_sess *sess)static void rtrs_clt_info_req_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_clt_con *con = cq->cq_context; +struct rtrs_clt_con *con =to_clt_con (wc->qp->qp_context);struct rtrs_clt_sess *sess =to_clt_sess (con->c.sess);struct rtrs_iu *iu;
@@ -2323,7 +2323,7 @@ static int process_info_rsp(struct rtrs_clt_sess *sess,static void rtrs_clt_info_rsp_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_clt_con *con = cq->cq_context; +struct rtrs_clt_con *con =to_clt_con (wc->qp->qp_context);struct rtrs_clt_sess *sess =to_clt_sess (con->c.sess);struct rtrs_msg_info_rsp *msg;enum rtrs_clt_state state;
drivers/infiniband/ulp/rtrs/rtrs-pri.h +1 -0
@@ -91,6 +91,7 @@ struct rtrs_con {struct ib_cq *cq;struct rdma_cm_id *cm_id;unsigned int cid; +u16 cq_size; };struct rtrs_sess {
drivers/infiniband/ulp/rtrs/rtrs-srv.c +5 -5
@@ -199,7 +199,7 @@ static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess)static void rtrs_srv_reg_mr_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_srv_con *con = cq->cq_context; +struct rtrs_srv_con *con =to_srv_con (wc->qp->qp_context);struct rtrs_sess *s = con->c.sess;struct rtrs_srv_sess *sess =to_srv_sess (s);
@@ -720,7 +720,7 @@ static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess)static void rtrs_srv_info_rsp_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_srv_con *con = cq->cq_context; +struct rtrs_srv_con *con =to_srv_con (wc->qp->qp_context);struct rtrs_sess *s = con->c.sess;struct rtrs_srv_sess *sess =to_srv_sess (s);struct rtrs_iu *iu;
@@ -862,7 +862,7 @@ static int process_info_req(struct rtrs_srv_con *con,static void rtrs_srv_info_req_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_srv_con *con = cq->cq_context; +struct rtrs_srv_con *con =to_srv_con (wc->qp->qp_context);struct rtrs_sess *s = con->c.sess;struct rtrs_srv_sess *sess =to_srv_sess (s);struct rtrs_msg_info_req *msg;
@@ -1110,7 +1110,7 @@ static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) {struct rtrs_srv_mr *mr =container_of (wc->wr_cqe,typeof (*mr), inv_cqe); -struct rtrs_srv_con *con = cq->cq_context; +struct rtrs_srv_con *con =to_srv_con (wc->qp->qp_context);struct rtrs_sess *s = con->c.sess;struct rtrs_srv_sess *sess =to_srv_sess (s);struct rtrs_srv *srv = sess->srv;
@@ -1167,7 +1167,7 @@ static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con)static void rtrs_srv_rdma_done (struct ib_cq *cq,struct ib_wc *wc) { -struct rtrs_srv_con *con = cq->cq_context; +struct rtrs_srv_con *con =to_srv_con (wc->qp->qp_context);struct rtrs_sess *s = con->c.sess;struct rtrs_srv_sess *sess =to_srv_sess (s);struct rtrs_srv *srv = sess->srv;
drivers/infiniband/ulp/rtrs/rtrs.c +5 -4
@@ -218,14 +218,15 @@ static int create_cq(struct rtrs_con *con, int cq_vector, u16 cq_size,struct rdma_cm_id *cm_id = con->cm_id;struct ib_cq *cq; - cq =ib_alloc_cq (cm_id->device, con, cq_size, - cq_vector, poll_ctx); + cq =ib_cq_pool_get (cm_id->device, cq_size, + cq_vector, poll_ctx);if (IS_ERR (cq)) {rtrs_err (con->sess,"Creating completion queue failed, errno: %ld\n" ,PTR_ERR (cq));return PTR_ERR (cq); } con->cq = cq; + con->cq_size = cq_size;return 0 ; }
@@ -273,7 +274,7 @@ int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con, err =create_qp (con, sess->dev->ib_pd, max_send_wr, max_recv_wr, max_send_sge);if (err) { -ib_free_cq (con->cq); +ib_cq_pool_put (con->cq, con->cq_size); con->cq =NULL ;return err; }
@@ -290,7 +291,7 @@ void rtrs_cq_qp_destroy(struct rtrs_con *con) con->qp =NULL ; }if (con->cq) { -ib_free_cq (con->cq); +ib_cq_pool_put (con->cq, con->cq_size); con->cq =NULL ; } }
Signature · 1 line
2.25.1
PATCH v2 2/2 RDMA/rtrs-clt: Use rdma_event_msg in log
It's easier to understand string instead of enum.
--- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-clt.c +6 -3
@@ -1853,12 +1853,14 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id,case RDMA_CM_EVENT_UNREACHABLE :case RDMA_CM_EVENT_ADDR_CHANGE :case RDMA_CM_EVENT_TIMEWAIT_EXIT : -rtrs_wrn (s,"CM error event %d\n" , ev->event); +rtrs_wrn (s,"CM error (CM event: %s, err: %d)\n" , +rdma_event_msg (ev->event), ev->status); cm_err = -ECONNRESET ;break ;case RDMA_CM_EVENT_ADDR_ERROR :case RDMA_CM_EVENT_ROUTE_ERROR : -rtrs_wrn (s,"CM error event %d\n" , ev->event); +rtrs_wrn (s,"CM error (CM event: %s, err: %d)\n" , +rdma_event_msg (ev->event), ev->status); cm_err = -EHOSTUNREACH ;break ;case RDMA_CM_EVENT_DEVICE_REMOVAL :
@@ -1868,7 +1870,8 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id,rtrs_clt_close_conns (sess, false);return 0 ;default : -rtrs_err (s,"Unexpected RDMA CM event (%d)\n" , ev->event); +rtrs_err (s,"Unexpected RDMA CM error (CM event: %s, err: %d)\n" , +rdma_event_msg (ev->event), ev->status); cm_err = -ECONNRESET ;break ; }
Signature · 1 line
2.25.1
Re: [PATCHv2 for-next 2/2] RDMA/rtrs-clt: Use rdma_event_msg in log
Quoted 7 lines · Jack Wang
It's easier to understand string instead of enum.
Signed-off-by Jack Wang <[email protected]>Reviewed-by Leon Romanovsky <[email protected]>--- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
Applied to for-next, thanks
Jason
Quoted 6 lines · Jack Wang
Has the driver use shared CQs providing ~10%-20% improvement during test.
Instead of opening a CQ for each QP per connection, a CQ for each QP will be provided by the RDMA core driver that will be shared between the QPs on that core reducing interrupt overhead.
Quoted 8 lines
Signed-off-by Jack Wang <[email protected]>Reviewed-by Leon Romanovsky <[email protected]>--- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 10 +++++----- drivers/infiniband/ulp/rtrs/rtrs-pri.h | 1 + drivers/infiniband/ulp/rtrs/rtrs-srv.c | 10 +++++----- drivers/infiniband/ulp/rtrs/rtrs.c | 9 +++++---- 4 files changed, 16 insertions(+), 14 deletions(-)
Applied to for-next, thanks
Jason