RDMA/rxe: Do some cleanup
Subsystems (2)
v3 cover RDMA/rxe: Do some cleanup
V2->V3:
1) Rebase on the latest kernel
2) Drop the ternary expression
Xiao Yang (5):
RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user RDMA/rxe: Remove the common is_user member of struct rxe_qp RDMA/rxe: Change the is_user member of struct rxe_cq to bool RDMA/rxe: Set partial attributes when completion status !=
IBV_WC_SUCCESS
RDMA/rxe: Remove duplicate settings
drivers/infiniband/sw/rxe/rxe_comp.c | 51 +++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_cq.c | 3 +- drivers/infiniband/sw/rxe/rxe_qp.c | 5 +-- drivers/infiniband/sw/rxe/rxe_req.c | 4 +-- drivers/infiniband/sw/rxe/rxe_resp.c | 14 +++----- drivers/infiniband/sw/rxe/rxe_verbs.c | 41 +++++---------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 3 +- 7 files changed, 49 insertions(+), 72 deletions(-)
Signature · 1 line
2.25.1
PATCH v3 1/5 RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
1) post_one_send() always processes kernel's send queue.
2) rxe_poll_cq() always processes kernel's completion queue.
--- drivers/infiniband/sw/rxe/rxe_verbs.c | 29 ++++++--------------------- 1 file changed, 6 insertions(+), 23 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.c +6 -23
@@ -625,7 +625,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,struct rxe_sq *sq = &qp->sq;struct rxe_send_wqe *send_wqe;unsigned long flags; -int full; err =validate_send_wr (qp, ibwr, mask, length);if (err)
@@ -633,27 +632,16 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,spin_lock_irqsave (&qp->sq.sq_lock, flags); -if (qp->is_user) - full =queue_full (sq->queue,QUEUE_TYPE_FROM_USER ); -else - full =queue_full (sq->queue,QUEUE_TYPE_KERNEL ); - -if (unlikely (full)) { +if (unlikely (queue_full (sq->queue,QUEUE_TYPE_KERNEL ))) {spin_unlock_irqrestore (&qp->sq.sq_lock, flags);return -ENOMEM ; } -if (qp->is_user) - send_wqe =producer_addr (sq->queue,QUEUE_TYPE_FROM_USER ); -else - send_wqe =producer_addr (sq->queue,QUEUE_TYPE_KERNEL ); + send_wqe =producer_addr (sq->queue,QUEUE_TYPE_KERNEL );init_send_wqe (qp, ibwr, mask, length, send_wqe); -if (qp->is_user) -advance_producer (sq->queue,QUEUE_TYPE_FROM_USER ); -else -advance_producer (sq->queue,QUEUE_TYPE_KERNEL ); +advance_producer (sq->queue,QUEUE_TYPE_KERNEL );spin_unlock_irqrestore (&qp->sq.sq_lock, flags);
@@ -845,18 +833,13 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)spin_lock_irqsave (&cq->cq_lock, flags);for (i =0 ; i < num_entries; i++) { -if (cq->is_user) - cqe =queue_head (cq->queue,QUEUE_TYPE_TO_USER ); -else - cqe =queue_head (cq->queue,QUEUE_TYPE_KERNEL ); + cqe =queue_head (cq->queue,QUEUE_TYPE_KERNEL );if (!cqe)break ;memcpy (wc++, &cqe->ibwc,sizeof (*wc)); -if (cq->is_user) -advance_consumer (cq->queue,QUEUE_TYPE_TO_USER ); -else -advance_consumer (cq->queue,QUEUE_TYPE_KERNEL ); + +advance_consumer (cq->queue,QUEUE_TYPE_KERNEL ); }spin_unlock_irqrestore (&cq->cq_lock, flags);
Signature · 1 line
2.25.1
PATCH v3 2/5 RDMA/rxe: Remove the common is_user member of struct rxe_qp
In reply to Xiao YangThe following commit introduced separate is_user members for struct rxe_sq/rxe_rq/rxe_srq but no code uses the separate is_user member of struct rxe_sq and lots of code still use the common is_user member of struct rxe_qp. So it is clear to make all code use separate is_user members uniformly and remove the common is_user member.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_qp.c | 5 +++-- drivers/infiniband/sw/rxe/rxe_req.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_resp.c | 10 +++++----- drivers/infiniband/sw/rxe/rxe_verbs.c | 12 +++--------- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 - 6 files changed, 16 insertions(+), 22 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +3 -3
@@ -142,7 +142,7 @@ static inline enum comp_state get_wqe(struct rxe_qp *qp,/* we come here whether or not we found a response packet to see if * there are any posted WQEs */ -if (qp->is_user) +if (qp->sq.is_user) wqe =queue_head (qp->sq.queue,QUEUE_TYPE_FROM_USER );else wqe =queue_head (qp->sq.queue,QUEUE_TYPE_KERNEL );
@@ -385,7 +385,7 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe, {memset (cqe,0 ,sizeof (*cqe)); -if (!qp->is_user) { +if (!qp->sq.is_user) {struct ib_wc *wc = &cqe->ibwc; wc->wr_id = wqe->wr.wr_id;
@@ -432,7 +432,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)if (post)make_send_cqe (qp, wqe, &cqe); -if (qp->is_user) +if (qp->sq.is_user)advance_consumer (qp->sq.queue,QUEUE_TYPE_FROM_USER );else advance_consumer (qp->sq.queue,QUEUE_TYPE_KERNEL );
drivers/infiniband/sw/rxe/rxe_qp.c +3 -2
@@ -248,7 +248,8 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,return err; } -if (qp->is_user) + qp->sq.is_user = uresp; +if (qp->sq.is_user) qp->req.wqe_index =producer_index (qp->sq.queue,QUEUE_TYPE_FROM_USER );else
@@ -313,7 +314,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,spin_lock_init (&qp->rq.producer_lock);spin_lock_init (&qp->rq.consumer_lock); - qp->rq.is_user = qp->is_user; + qp->rq.is_user = uresp;skb_queue_head_init (&qp->resp_pkts);
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
@@ -49,7 +49,7 @@ static void req_retry(struct rxe_qp *qp)unsigned int cons;unsigned int prod; -if (qp->is_user) { +if (qp->sq.is_user) { cons =consumer_index (q,QUEUE_TYPE_FROM_USER ); prod =producer_index (q,QUEUE_TYPE_FROM_USER ); }else {
@@ -121,7 +121,7 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)unsigned int cons;unsigned int prod; -if (qp->is_user) { +if (qp->sq.is_user) { wqe =queue_head (q,QUEUE_TYPE_FROM_USER ); cons =consumer_index (q,QUEUE_TYPE_FROM_USER ); prod =producer_index (q,QUEUE_TYPE_FROM_USER );
drivers/infiniband/sw/rxe/rxe_resp.c +5 -5
@@ -303,7 +303,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)spin_lock_bh (&srq->rq.consumer_lock); -if (qp->is_user) +if (srq->is_user) wqe =queue_head (q,QUEUE_TYPE_FROM_USER );else wqe =queue_head (q,QUEUE_TYPE_KERNEL );
@@ -322,7 +322,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)memcpy (&qp->resp.srq_wqe, wqe, size); qp->resp.wqe = &qp->resp.srq_wqe.wqe; -if (qp->is_user) { +if (srq->is_user) {advance_consumer (q,QUEUE_TYPE_FROM_USER ); count =queue_count (q,QUEUE_TYPE_FROM_USER ); }else {
@@ -357,7 +357,7 @@ static enum resp_states check_resource(struct rxe_qp *qp, qp->resp.status =IB_WC_WR_FLUSH_ERR ;return RESPST_COMPLETE ; }else if (!srq) { -if (qp->is_user) +if (qp->rq.is_user) qp->resp.wqe =queue_head (qp->rq.queue,QUEUE_TYPE_FROM_USER );else
@@ -389,7 +389,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,if (srq)return get_srq_wqe (qp); -if (qp->is_user) +if (qp->rq.is_user) qp->resp.wqe =queue_head (qp->rq.queue,QUEUE_TYPE_FROM_USER );else
@@ -937,7 +937,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,/* have copy for srq and reference for !srq */ if (!qp->srq) { -if (qp->is_user) +if (qp->rq.is_user)advance_consumer (qp->rq.queue,QUEUE_TYPE_FROM_USER );else advance_consumer (qp->rq.queue,QUEUE_TYPE_KERNEL );
drivers/infiniband/sw/rxe/rxe_verbs.c +3 -9
@@ -413,14 +413,8 @@ static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init,if (err)return err; -if (udata) { -if (udata->inlen) -return -EINVAL ; - - qp->is_user = true; - }else { - qp->is_user = false; - } +if (udata && udata->inlen) +return -EINVAL ; err =rxe_add_to_pool (&rxe->qp_pool, qp);if (err)
@@ -709,7 +703,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,return -EINVAL ; } -if (qp->is_user) { +if (qp->sq.is_user) {/* Utilize process context to do protocol processing */ rxe_run_task (&qp->req.task,0 );return 0 ;
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -1
@@ -215,7 +215,6 @@ struct rxe_qp {struct ib_qp_attr attr;unsigned int valid;unsigned int mtu; -bool is_user;struct rxe_pd *pd;struct rxe_srq *srq;
Signature · 1 line
2.25.1
PATCH v3 3/5 RDMA/rxe: Change the is_user member of struct rxe_cq to bool
In reply to Xiao YangMake all is_user members of struct rxe_sq/rxe_cq/rxe_srq/rxe_cq has the same type.
--- drivers/infiniband/sw/rxe/rxe_cq.c | 3 +-- drivers/infiniband/sw/rxe/rxe_verbs.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_cq.c +1 -2
@@ -81,8 +81,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,return err; } -if (uresp) - cq->is_user =1 ; + cq->is_user = uresp; cq->is_dying = false;
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -1
@@ -64,7 +64,7 @@ struct rxe_cq {spinlock_t cq_lock;u8 notify;bool is_dying; -int is_user; +bool is_user;struct tasklet_struct comp_task; };
Signature · 1 line
2.25.1
PATCH v3 4/5 RDMA/rxe: Set partial attributes when completion status != IBV_WC_SUCCESS
In reply to Xiao YangAs ibv_poll_cq()'s manual said, only partial attributes are valid when completion status != IBV_WC_SUCCESS.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 45 +++++++++++++++------------- 1 file changed, 25 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +25 -20
@@ -383,30 +383,35 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp,static void make_send_cqe (struct rxe_qp *qp,struct rxe_send_wqe *wqe,struct rxe_cqe *cqe) { +struct ib_wc *wc = &cqe->ibwc; +struct ib_uverbs_wc *uwc = &cqe->uibwc; +memset (cqe,0 ,sizeof (*cqe));if (!qp->sq.is_user) { -struct ib_wc *wc = &cqe->ibwc; - - wc->wr_id = wqe->wr.wr_id; - wc->status = wqe->status; - wc->opcode =wr_to_wc_opcode (wqe->wr.opcode); -if (wqe->wr.opcode ==IB_WR_RDMA_WRITE_WITH_IMM || - wqe->wr.opcode ==IB_WR_SEND_WITH_IMM ) - wc->wc_flags =IB_WC_WITH_IMM ; - wc->byte_len = wqe->dma.length; - wc->qp = &qp->ibqp; + wc->wr_id = wqe->wr.wr_id; + wc->status = wqe->status; + wc->qp = &qp->ibqp; }else { -struct ib_uverbs_wc *uwc = &cqe->uibwc; - - uwc->wr_id = wqe->wr.wr_id; - uwc->status = wqe->status; - uwc->opcode =wr_to_wc_opcode (wqe->wr.opcode); -if (wqe->wr.opcode ==IB_WR_RDMA_WRITE_WITH_IMM || - wqe->wr.opcode ==IB_WR_SEND_WITH_IMM ) - uwc->wc_flags =IB_WC_WITH_IMM ; - uwc->byte_len = wqe->dma.length; - uwc->qp_num = qp->ibqp.qp_num; + uwc->wr_id = wqe->wr.wr_id; + uwc->status = wqe->status; + uwc->qp_num = qp->ibqp.qp_num; + } + +if (wqe->status ==IB_WC_SUCCESS ) { +if (!qp->sq.is_user) { + wc->opcode =wr_to_wc_opcode (wqe->wr.opcode); +if (wqe->wr.opcode ==IB_WR_RDMA_WRITE_WITH_IMM || + wqe->wr.opcode ==IB_WR_SEND_WITH_IMM ) + wc->wc_flags =IB_WC_WITH_IMM ; + wc->byte_len = wqe->dma.length; + }else { + uwc->opcode =wr_to_wc_opcode (wqe->wr.opcode); +if (wqe->wr.opcode ==IB_WR_RDMA_WRITE_WITH_IMM || + wqe->wr.opcode ==IB_WR_SEND_WITH_IMM ) + uwc->wc_flags =IB_WC_WITH_IMM ; + uwc->byte_len = wqe->dma.length; + } } }
Signature · 1 line
2.25.1
PATCH v3 5/5 RDMA/rxe: Remove duplicate settings
In reply to Xiao YangRemove duplicate settings for vendor_err and qp_num.
--- drivers/infiniband/sw/rxe/rxe_resp.c | 4 ---- 1 file changed, 4 deletions(-)
drivers/infiniband/sw/rxe/rxe_resp.c +0 -4
@@ -876,7 +876,6 @@ static enum resp_states do_complete(struct rxe_qp *qp, wc->opcode = (pkt->mask &RXE_IMMDT_MASK && pkt->mask &RXE_WRITE_MASK ) ?IB_WC_RECV_RDMA_WITH_IMM :IB_WC_RECV ; - wc->vendor_err =0 ; wc->byte_len = (pkt->mask &RXE_IMMDT_MASK && pkt->mask &RXE_WRITE_MASK ) ? qp->resp.length : wqe->dma.length - wqe->dma.resid;
@@ -897,8 +896,6 @@ static enum resp_states do_complete(struct rxe_qp *qp, uwc->ex.invalidate_rkey =ieth_rkey (pkt); } - uwc->qp_num = qp->ibqp.qp_num; -if (pkt->mask &RXE_DETH_MASK ) uwc->src_qp =deth_sqp (pkt);
@@ -930,7 +927,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,if (pkt->mask &RXE_DETH_MASK ) wc->src_qp =deth_sqp (pkt); - wc->qp = &qp->ibqp; wc->port_num = qp->attr.port_num; } }
Signature · 1 line
2.25.1
Jason Gunthorpe
maintainerQuoted 12 lines · Xiao Yang
V2->V3:
1) Rebase on the latest kernel
2) Drop the ternary expressionXiao Yang (5):
RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user RDMA/rxe: Remove the commonis_usermember of structrxe_qpRDMA/rxe: Change theis_usermember of structrxe_cqto bool RDMA/rxe: Set partial attributes when completion status !=
IBV_WC_SUCCESS
RDMA/rxe: Remove duplicate settings
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_comp.c | 51 +++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_cq.c | 3 +- drivers/infiniband/sw/rxe/rxe_qp.c | 5 +-- drivers/infiniband/sw/rxe/rxe_req.c | 4 +-- drivers/infiniband/sw/rxe/rxe_resp.c | 14 +++----- drivers/infiniband/sw/rxe/rxe_verbs.c | 41 +++++---------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 3 +- 7 files changed, 49 insertions(+), 72 deletions(-)
This doesn't apply, can you rebase and resend please
Jason
Quoted 21 lines · Jason Gunthorpe
Quoted 12 lines · Xiao Yang
V2->V3:
1) Rebase on the latest kernel
2) Drop the ternary expressionXiao Yang (5):
RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user RDMA/rxe: Remove the commonis_usermember of structrxe_qpRDMA/rxe: Change theis_usermember of structrxe_cqto bool RDMA/rxe: Set partial attributes when completion status !=
IBV_WC_SUCCESS
RDMA/rxe: Remove duplicate settingsQuoted 8 lines
drivers/infiniband/sw/rxe/rxe_comp.c | 51 +++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_cq.c | 3 +- drivers/infiniband/sw/rxe/rxe_qp.c | 5 +-- drivers/infiniband/sw/rxe/rxe_req.c | 4 +-- drivers/infiniband/sw/rxe/rxe_resp.c | 14 +++----- drivers/infiniband/sw/rxe/rxe_verbs.c | 41 +++++---------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 3 +- 7 files changed, 49 insertions(+), 72 deletions(-)
Quoted 1 line
This doesn't apply, can you rebase and resend please
Hi Jason,
Sure, I will rebase and resend soon.
Best Regards,
Xiao Yang
Quoted 1 line
Jason