RDMA/rxe: Remove qp->resp.state
Subsystems (2)
PATCH 1/5 RDMA/rxe: Remove qp->resp.state
The rxe driver has four different QP state variables,
qp->attr.qp_state,
qp->req.state,
qp->comp.state, and
qp->resp.state.All of these basically carry the same information.
This patch replaces uses of qp->resp.state by qp->attr.qp_state. This is the first of three patches which will remove all but the qp->attr.qp_state variable. This will bring the driver closer to the IBA description.
--- drivers/infiniband/sw/rxe/rxe_net.c | 2 +- drivers/infiniband/sw/rxe/rxe_qp.c | 5 ----- drivers/infiniband/sw/rxe/rxe_recv.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 10 +++++----- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 - 6 files changed, 8 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_net.c +1 -1
@@ -414,7 +414,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,struct rxe_dev *rxe =to_rdev (qp->ibqp.device);if ((is_request && (qp->req.state !=QP_STATE_READY )) || - (!is_request && (qp->resp.state !=QP_STATE_READY ))) { + (!is_request && (qp_state (qp) <=IB_QPS_RTR ))) {rxe_dbg_qp (qp,"Packet dropped. QP is not in ready state\n" ); goto drop; }
drivers/infiniband/sw/rxe/rxe_qp.c +0 -5
@@ -287,7 +287,6 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, qp->resp.opcode =OPCODE_NONE ; qp->resp.msn =0 ; - qp->resp.state =QP_STATE_RESET ;return 0 ; }
@@ -479,7 +478,6 @@ static void rxe_qp_reset(struct rxe_qp *qp)/* move qp to the reset state */ qp->req.state =QP_STATE_RESET ; qp->comp.state =QP_STATE_RESET ; - qp->resp.state =QP_STATE_RESET ;/* drain work and packet queuesc */ rxe_requester (qp);
@@ -532,7 +530,6 @@ static void rxe_qp_drain(struct rxe_qp *qp)void rxe_qp_error (struct rxe_qp *qp) { qp->req.state =QP_STATE_ERROR ; - qp->resp.state =QP_STATE_ERROR ; qp->comp.state =QP_STATE_ERROR ; qp->attr.qp_state =IB_QPS_ERR ;
@@ -663,13 +660,11 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,case IB_QPS_INIT :rxe_dbg_qp (qp,"state -> INIT\n" ); qp->req.state =QP_STATE_INIT ; - qp->resp.state =QP_STATE_INIT ; qp->comp.state =QP_STATE_INIT ;break ;case IB_QPS_RTR :rxe_dbg_qp (qp,"state -> RTR\n" ); - qp->resp.state =QP_STATE_READY ;break ;case IB_QPS_RTS :
drivers/infiniband/sw/rxe/rxe_recv.c +1 -1
@@ -39,7 +39,7 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, }if (pkt->mask &RXE_REQ_MASK ) { -if (unlikely (qp->resp.state !=QP_STATE_READY )) +if (unlikely (qp_state (qp) <=IB_QPS_RTR ))return -EINVAL ; }else if (unlikely (qp->req.state <QP_STATE_READY || qp->req.state >QP_STATE_DRAINED ))
drivers/infiniband/sw/rxe/rxe_resp.c +5 -5
@@ -1137,7 +1137,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,return RESPST_ERR_CQ_OVERFLOW ; finish: -if (unlikely (qp->resp.state ==QP_STATE_ERROR )) +if (unlikely (qp_state (qp) ==IB_QPS_ERR ))return RESPST_CHK_RESOURCE ;if (unlikely (!pkt))return RESPST_DONE ;
@@ -1464,10 +1464,10 @@ int rxe_responder(struct rxe_qp *qp)struct rxe_pkt_info *pkt =NULL ;int ret; -if (!qp->valid || qp->resp.state ==QP_STATE_ERROR || - qp->resp.state ==QP_STATE_RESET ) { -bool notify = qp->valid && - (qp->resp.state ==QP_STATE_ERROR ); +if (!qp->valid ||qp_state (qp) ==IB_QPS_ERR || +qp_state (qp) ==IB_QPS_RESET ) { +bool notify = qp->valid && (qp_state (qp) ==IB_QPS_ERR ); +drain_req_pkts (qp);flush_recv_queue (qp, notify); goto exit;
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -1
@@ -1012,7 +1012,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,spin_unlock_irqrestore (&rq->producer_lock, flags); -if (qp->resp.state ==QP_STATE_ERROR ) +if (qp_state (qp) ==IB_QPS_ERR )rxe_sched_task (&qp->resp.task); err_out:
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -1
@@ -173,7 +173,6 @@ struct resp_res { };struct rxe_resp_info { -enum rxe_qp_state state;u32 msn;u32 psn;u32 ack_psn;
Signature · 1 line
2.37.2
PATCH 2/5 RDMA/rxe: Remove qp->comp.state
The rxe driver has four different QP state variables,
qp->attr.qp_state,
qp->req.state,
qp->comp.state, and
qp->resp.state.All of these basically carry the same information.
This patch replaces uses of qp->comp.state by qp->attr.qp_state. This is the second of three patches which will remove all but the qp->attr.qp_state variable. This will bring the driver closer to the IBA description.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_qp.c | 5 ----- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 - 3 files changed, 4 insertions(+), 10 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +4 -4
@@ -619,10 +619,10 @@ int rxe_completer(struct rxe_qp *qp)enum comp_state state;int ret; -if (!qp->valid || qp->comp.state ==QP_STATE_ERROR || - qp->comp.state ==QP_STATE_RESET ) { -bool notify = qp->valid && - (qp->comp.state ==QP_STATE_ERROR ); +if (!qp->valid ||qp_state (qp) ==IB_QPS_ERR || +qp_state (qp) ==IB_QPS_RESET ) { +bool notify = qp->valid && (qp_state (qp) ==IB_QPS_ERR ); +drain_resp_pkts (qp);flush_send_queue (qp, notify); goto exit;
drivers/infiniband/sw/rxe/rxe_qp.c +0 -5
@@ -232,7 +232,6 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, QUEUE_TYPE_FROM_CLIENT); qp->req.state =QP_STATE_RESET ; - qp->comp.state =QP_STATE_RESET ; qp->req.opcode =-1 ; qp->comp.opcode =-1 ;
@@ -477,7 +476,6 @@ static void rxe_qp_reset(struct rxe_qp *qp)/* move qp to the reset state */ qp->req.state =QP_STATE_RESET ; - qp->comp.state =QP_STATE_RESET ;/* drain work and packet queuesc */ rxe_requester (qp);
@@ -530,7 +528,6 @@ static void rxe_qp_drain(struct rxe_qp *qp)void rxe_qp_error (struct rxe_qp *qp) { qp->req.state =QP_STATE_ERROR ; - qp->comp.state =QP_STATE_ERROR ; qp->attr.qp_state =IB_QPS_ERR ;/* drain work and packet queues */
@@ -660,7 +657,6 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,case IB_QPS_INIT :rxe_dbg_qp (qp,"state -> INIT\n" ); qp->req.state =QP_STATE_INIT ; - qp->comp.state =QP_STATE_INIT ;break ;case IB_QPS_RTR :
@@ -670,7 +666,6 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,case IB_QPS_RTS :rxe_dbg_qp (qp,"state -> RTS\n" ); qp->req.state =QP_STATE_READY ; - qp->comp.state =QP_STATE_READY ;break ;case IB_QPS_SQD :
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -1
@@ -127,7 +127,6 @@ struct rxe_req_info { };struct rxe_comp_info { -enum rxe_qp_state state;u32 psn;int opcode;int timeout;
Signature · 1 line
2.37.2
PATCH 3/5 RDMA/rxe: Remove qp->req.state
In reply to Bob PearsonThe rxe driver has four different QP state variables,
qp->attr.qp_state,
qp->req.state,
qp->comp.state, and
qp->resp.state.All of these basically carry the same information.
This patch replaces uses of qp->req.state by qp->attr.qp_state and enum rxe_qp_state. This is the third of three patches which will remove all but the qp->attr.qp_state variable. This will bring the driver closer to the IBA description.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 9 +++-- drivers/infiniband/sw/rxe/rxe_net.c | 4 +-- drivers/infiniband/sw/rxe/rxe_qp.c | 49 ++++++++------------------- drivers/infiniband/sw/rxe/rxe_recv.c | 9 ++--- drivers/infiniband/sw/rxe/rxe_req.c | 15 ++++---- drivers/infiniband/sw/rxe/rxe_verbs.c | 4 +-- drivers/infiniband/sw/rxe/rxe_verbs.h | 10 ------ 7 files changed, 34 insertions(+), 66 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +4 -5
@@ -491,12 +491,11 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, } } -if (unlikely (qp->req.state ==QP_STATE_DRAIN )) { +if (unlikely (qp_state (qp) ==IB_QPS_SQD )) {/* state_lock used by requester & completer */ spin_lock_bh (&qp->state_lock); -if ((qp->req.state ==QP_STATE_DRAIN ) && - (qp->comp.psn == qp->req.psn)) { - qp->req.state =QP_STATE_DRAINED ; +if (qp->attr.sq_draining && qp->comp.psn == qp->req.psn) { + qp->attr.sq_draining =0 ;spin_unlock_bh (&qp->state_lock);if (qp->ibqp.event_handler) {
@@ -723,7 +722,7 @@ int rxe_completer(struct rxe_qp *qp)* (4) the timeout parameter is set */ if ((qp_type (qp) ==IB_QPT_RC ) && - (qp->req.state ==QP_STATE_READY ) && + (qp_state (qp) >=IB_QPS_RTS ) && (psn_compare (qp->req.psn, qp->comp.psn) >0 ) && qp->qp_timeout_jiffies) mod_timer(&qp->retrans_timer,
drivers/infiniband/sw/rxe/rxe_net.c +2 -2
@@ -413,8 +413,8 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,int is_request = pkt->mask &RXE_REQ_MASK ;struct rxe_dev *rxe =to_rdev (qp->ibqp.device); -if ((is_request && (qp->req.state !=QP_STATE_READY )) || - (!is_request && (qp_state (qp) <=IB_QPS_RTR ))) { +if ((is_request && (qp_state (qp) <IB_QPS_RTS )) || + (!is_request && (qp_state (qp) <IB_QPS_RTR ))) {rxe_dbg_qp (qp,"Packet dropped. QP is not in ready state\n" ); goto drop; }
drivers/infiniband/sw/rxe/rxe_qp.c +14 -35
105 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_recv.c +5 -4
@@ -39,11 +39,12 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, }if (pkt->mask &RXE_REQ_MASK ) { -if (unlikely (qp_state (qp) <=IB_QPS_RTR )) +if (unlikely (qp_state (qp) <IB_QPS_RTR ))return -EINVAL ; - }else if (unlikely (qp->req.state <QP_STATE_READY || - qp->req.state >QP_STATE_DRAINED )) -return -EINVAL ; + }else { +if (unlikely (qp_state (qp) <IB_QPS_RTS )) +return -EINVAL ; + }return 0 ; }
drivers/infiniband/sw/rxe/rxe_req.c +7 -8
@@ -120,13 +120,13 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) cons =queue_get_consumer (q,QUEUE_TYPE_FROM_CLIENT ); prod =queue_get_producer (q,QUEUE_TYPE_FROM_CLIENT ); -if (unlikely (qp->req.state ==QP_STATE_DRAIN )) { +if (unlikely (qp_state (qp) ==IB_QPS_SQD )) {/* check to see if we are drained; * state_lock used by requester and completer */ spin_lock_bh (&qp->state_lock); do { - if (qp->req.state != QP_STATE_DRAIN) { + if (!qp->attr.sq_draining) { /* comp just finished */ spin_unlock_bh(&qp->state_lock); break;
@@ -139,7 +139,7 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)break ; } - qp->req.state =QP_STATE_DRAINED ; + qp->attr.sq_draining =0 ;spin_unlock_bh (&qp->state_lock);if (qp->ibqp.event_handler) {
@@ -159,8 +159,7 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) wqe =queue_addr_from_index (q, index); -if (unlikely ((qp->req.state ==QP_STATE_DRAIN || - qp->req.state ==QP_STATE_DRAINED ) && +if (unlikely ((qp_state (qp) ==IB_QPS_SQD ) && (wqe->state != wqe_state_processing)))return NULL ;
@@ -656,7 +655,7 @@ int rxe_requester(struct rxe_qp *qp)if (unlikely (!qp->valid)) goto exit; -if (unlikely (qp->req.state ==QP_STATE_ERROR )) { +if (unlikely (qp_state (qp) ==IB_QPS_ERR )) { wqe =req_next_wqe (qp);if (wqe)/*
@@ -667,7 +666,7 @@ int rxe_requester(struct rxe_qp *qp) goto exit; } -if (unlikely (qp->req.state ==QP_STATE_RESET )) { +if (unlikely (qp_state (qp) ==IB_QPS_RESET )) { qp->req.wqe_index =queue_get_consumer (q,QUEUE_TYPE_FROM_CLIENT ); qp->req.opcode =-1 ;
@@ -836,7 +835,7 @@ int rxe_requester(struct rxe_qp *qp)/* update wqe_index for each wqe completion */ qp->req.wqe_index =queue_next_index (qp->sq.queue, qp->req.wqe_index); wqe->state = wqe_state_error; - qp->req.state =QP_STATE_ERROR ; + qp->attr.qp_state =IB_QPS_ERR ;rxe_sched_task (&qp->comp.task); exit: ret = -EAGAIN ;
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -2
@@ -881,7 +881,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr,if (!err)rxe_sched_task (&qp->req.task); -if (unlikely (qp->req.state ==QP_STATE_ERROR )) +if (unlikely (qp_state (qp) ==IB_QPS_ERR ))rxe_sched_task (&qp->comp.task);return err;
@@ -900,7 +900,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, goto err_out; } -if (unlikely (qp->req.state <QP_STATE_READY )) { +if (unlikely (qp_state (qp) <IB_QPS_RTS )) { *bad_wr = wr; err = -EINVAL ;rxe_dbg_qp (qp,"qp not ready to send" );
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -10
@@ -102,17 +102,7 @@ struct rxe_srq {int error; }; -enum rxe_qp_state { -QP_STATE_RESET , -QP_STATE_INIT , -QP_STATE_READY , -QP_STATE_DRAIN ,/* req only */ -QP_STATE_DRAINED ,/* req only */ -QP_STATE_ERROR -}; -struct rxe_req_info { -enum rxe_qp_state state;int wqe_index;u32 psn;int opcode;
Signature · 1 line
2.37.2
PATCH 4/5 RDMA/rxe: Move code to check if drained to subroutine
In reply to Bob PearsonMove two blocks of code in rxe_comp.c and rxe_req.c to subroutines that check if draining is complete in the SQD state and, if so, generate a SQ_DRAINED event.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 35 ++++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_req.c | 32 ++++++++++++++----------- 2 files changed, 38 insertions(+), 29 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +20 -15
@@ -477,20 +477,8 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) } } -static inline enum comp_state complete_ack (struct rxe_qp *qp, -struct rxe_pkt_info *pkt, -struct rxe_send_wqe *wqe) +static void comp_check_sq_drain_done (struct rxe_qp *qp) { -if (wqe->has_rd_atomic) { - wqe->has_rd_atomic =0 ; -atomic_inc (&qp->req.rd_atomic); -if (qp->req.need_rd_atomic) { - qp->comp.timeout_retry =0 ; - qp->req.need_rd_atomic =0 ; -rxe_sched_task (&qp->req.task); - } - } -if (unlikely (qp_state (qp) ==IB_QPS_SQD )) {/* state_lock used by requester & completer */ spin_lock_bh (&qp->state_lock);
@@ -507,10 +495,27 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, qp->ibqp.event_handler (&ev, qp->ibqp.qp_context); } - }else { -spin_unlock_bh (&qp->state_lock); +return ; } +spin_unlock_bh (&qp->state_lock); } +} + +static inline enum comp_state complete_ack (struct rxe_qp *qp, +struct rxe_pkt_info *pkt, +struct rxe_send_wqe *wqe) +{ +if (wqe->has_rd_atomic) { + wqe->has_rd_atomic =0 ; +atomic_inc (&qp->req.rd_atomic); +if (qp->req.need_rd_atomic) { + qp->comp.timeout_retry =0 ; + qp->req.need_rd_atomic =0 ; +rxe_sched_task (&qp->req.task); + } + } + +comp_check_sq_drain_done (qp);do_complete (qp, wqe);
drivers/infiniband/sw/rxe/rxe_req.c +18 -14
@@ -108,17 +108,12 @@ void rnr_nak_timer(struct timer_list *t)rxe_sched_task (&qp->req.task); } -static struct rxe_send_wqe *req_next_wqe (struct rxe_qp *qp) +static void req_check_sq_drain_done (struct rxe_qp *qp) { -struct rxe_send_wqe *wqe;struct rxe_queue *q = qp->sq.queue;unsigned int index = qp->req.wqe_index; -unsigned int cons; -unsigned int prod; - - wqe =queue_head (q,QUEUE_TYPE_FROM_CLIENT ); - cons =queue_get_consumer (q,QUEUE_TYPE_FROM_CLIENT ); - prod =queue_get_producer (q,QUEUE_TYPE_FROM_CLIENT ); +unsigned int cons =queue_get_consumer (q,QUEUE_TYPE_FROM_CLIENT ); +struct rxe_send_wqe *wqe =queue_addr_from_index (q, cons);if (unlikely (qp_state (qp) ==IB_QPS_SQD )) {/* check to see if we are drained;
@@ -126,18 +121,14 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)*/ spin_lock_bh (&qp->state_lock); do { - if (!qp->attr.sq_draining) { + if (!qp->attr.sq_draining) /* comp just finished */ - spin_unlock_bh(&qp->state_lock); break; - } if (wqe && ((index != cons) || - (wqe->state != wqe_state_posted))) { + (wqe->state != wqe_state_posted))) /* comp not done yet */ - spin_unlock_bh(&qp->state_lock); break; - } qp->attr.sq_draining = 0; spin_unlock_bh(&qp->state_lock);
@@ -151,9 +142,22 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) qp->ibqp.event_handler (&ev, qp->ibqp.qp_context); } +return ; }while (0 ); +spin_unlock_bh (&qp->state_lock); } +} +static struct rxe_send_wqe *req_next_wqe (struct rxe_qp *qp) +{ +struct rxe_send_wqe *wqe; +struct rxe_queue *q = qp->sq.queue; +unsigned int index = qp->req.wqe_index; +unsigned int prod; + +req_check_sq_drain_done (qp); + + prod =queue_get_producer (q,QUEUE_TYPE_FROM_CLIENT );if (index == prod)return NULL ;
Signature · 1 line
2.37.2
PATCH 5/5 RDMA/rxe: Protect QP state with qp->state_lock
In reply to Bob PearsonCurrently the rxe driver makes little effort to make the changes to qp state (which includes qp->attr.qp_state, qp->attr.sq_draining and qp->valid) atomic between different client threads and IO threads. In particular a common template is for an RDMA application to call ib_modify_qp() to move a qp to ERR state and then wait until all the packet and work queues have drained before calling ib_destroy_qp(). None of these state changes are protected by locks to assure that the changes are executed atomically and that memory barriers are included. This has been observed to lead to incorrect behavior around qp cleanup.
This patch continues the work of the previous patches in this series and adds locking code around qp state changes and lookups.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 48 ++++-- drivers/infiniband/sw/rxe/rxe_net.c | 3 + drivers/infiniband/sw/rxe/rxe_qp.c | 153 +++++++++-------- drivers/infiniband/sw/rxe/rxe_recv.c | 10 +- drivers/infiniband/sw/rxe/rxe_req.c | 71 +++++--- drivers/infiniband/sw/rxe/rxe_resp.c | 12 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 238 +++++++++++++++----------- 7 files changed, 317 insertions(+), 218 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +30 -18
@@ -118,10 +118,12 @@ void retransmit_timer(struct timer_list *t)rxe_dbg_qp (qp,"retransmit timer fired\n" ); +spin_lock_bh (&qp->state_lock);if (qp->valid) { qp->comp.timeout =1 ;rxe_sched_task (&qp->comp.task); } +spin_unlock_bh (&qp->state_lock); }void rxe_comp_queue_pkt (struct rxe_qp *qp,struct sk_buff *skb)
@@ -479,9 +481,8 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)static void comp_check_sq_drain_done (struct rxe_qp *qp) { +spin_lock_bh (&qp->state_lock);if (unlikely (qp_state (qp) ==IB_QPS_SQD )) { -/* state_lock used by requester & completer */ -spin_lock_bh (&qp->state_lock);if (qp->attr.sq_draining && qp->comp.psn == qp->req.psn) { qp->attr.sq_draining =0 ;spin_unlock_bh (&qp->state_lock);
@@ -497,8 +498,8 @@ static void comp_check_sq_drain_done(struct rxe_qp *qp) }return ; } -spin_unlock_bh (&qp->state_lock); } +spin_unlock_bh (&qp->state_lock); } static inline enum comp_state complete_ack(struct rxe_qp *qp,
@@ -614,6 +615,26 @@ static void free_pkt(struct rxe_pkt_info *pkt)ib_device_put (dev); } +/* reset the retry timer if +* - QP is type RC +* - there is a packet sent by the requester that +* might be acked (we still might get spurious +* timeouts but try to keep them as few as possible) +* - the timeout parameter is set +* - the QP is alive +*/ +static void reset_retry_timer (struct rxe_qp *qp) +{ +if (qp_type (qp) ==IB_QPT_RC && qp->qp_timeout_jiffies) { +spin_lock_bh (&qp->state_lock); +if (qp_state (qp) >=IB_QPS_RTS && +psn_compare (qp->req.psn, qp->comp.psn) >0 ) +mod_timer (&qp->retrans_timer, + jiffies + qp->qp_timeout_jiffies); +spin_unlock_bh (&qp->state_lock); + } +} +int rxe_completer (struct rxe_qp *qp) {struct rxe_dev *rxe =to_rdev (qp->ibqp.device);
@@ -623,14 +644,17 @@ int rxe_completer(struct rxe_qp *qp)enum comp_state state;int ret; +spin_lock_bh (&qp->state_lock);if (!qp->valid ||qp_state (qp) ==IB_QPS_ERR || -qp_state (qp) ==IB_QPS_RESET ) { +qp_state (qp) ==IB_QPS_RESET ) {bool notify = qp->valid && (qp_state (qp) ==IB_QPS_ERR );drain_resp_pkts (qp);flush_send_queue (qp, notify); +spin_unlock_bh (&qp->state_lock); goto exit; } +spin_unlock_bh (&qp->state_lock);if (qp->comp.timeout) { qp->comp.timeout_retry =1 ;
@@ -718,20 +742,7 @@ int rxe_completer(struct rxe_qp *qp)break ; } -/* re reset the timeout counter if -* (1) QP is type RC -* (2) the QP is alive -* (3) there is a packet sent by the requester that -* might be acked (we still might get spurious -* timeouts but try to keep them as few as possible) -* (4) the timeout parameter is set -*/ -if ((qp_type (qp) ==IB_QPT_RC ) && - (qp_state (qp) >=IB_QPS_RTS ) && - (psn_compare (qp->req.psn, qp->comp.psn) >0 ) && - qp->qp_timeout_jiffies) -mod_timer (&qp->retrans_timer, - jiffies + qp->qp_timeout_jiffies); +reset_retry_timer (qp); goto exit;case COMPST_ERROR_RETRY :
@@ -793,6 +804,7 @@ int rxe_completer(struct rxe_qp *qp)*/ qp->req.wait_for_rnr_timer =1 ;rxe_dbg_qp (qp,"set rnr nak timer\n" ); +// TODO who protects from destroy_qp?? mod_timer (&qp->rnr_nak_timer, jiffies +rnrnak_jiffies (aeth_syn (pkt) & ~AETH_TYPE_MASK ));
drivers/infiniband/sw/rxe/rxe_net.c +3 -0
@@ -413,11 +413,14 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,int is_request = pkt->mask &RXE_REQ_MASK ;struct rxe_dev *rxe =to_rdev (qp->ibqp.device); +spin_lock_bh (&qp->state_lock);if ((is_request && (qp_state (qp) <IB_QPS_RTS )) || (!is_request && (qp_state (qp) <IB_QPS_RTR ))) { +spin_unlock_bh (&qp->state_lock);rxe_dbg_qp (qp,"Packet dropped. QP is not in ready state\n" ); goto drop; } +spin_unlock_bh (&qp->state_lock);rxe_icrc_generate (skb, pkt);
drivers/infiniband/sw/rxe/rxe_qp.c +85 -68
216 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_recv.c +8 -2
@@ -38,13 +38,19 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,return -EINVAL ; } +spin_lock_bh (&qp->state_lock);if (pkt->mask &RXE_REQ_MASK ) { -if (unlikely (qp_state (qp) <IB_QPS_RTR )) +if (unlikely (qp_state (qp) <IB_QPS_RTR )) { +spin_unlock_bh (&qp->state_lock);return -EINVAL ; + } }else { -if (unlikely (qp_state (qp) <IB_QPS_RTS )) +if (unlikely (qp_state (qp) <IB_QPS_RTS )) { +spin_unlock_bh (&qp->state_lock);return -EINVAL ; + } } +spin_unlock_bh (&qp->state_lock);return 0 ; }
drivers/infiniband/sw/rxe/rxe_req.c +47 -24
130 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +10 -2
@@ -1137,8 +1137,13 @@ static enum resp_states do_complete(struct rxe_qp *qp,return RESPST_ERR_CQ_OVERFLOW ; finish: -if (unlikely (qp_state (qp) ==IB_QPS_ERR )) +spin_lock_bh (&qp->state_lock); +if (unlikely (qp_state (qp) ==IB_QPS_ERR )) { +spin_unlock_bh (&qp->state_lock);return RESPST_CHK_RESOURCE ; + } +spin_unlock_bh (&qp->state_lock); +if (unlikely (!pkt))return RESPST_DONE ;if (qp_type (qp) ==IB_QPT_RC )
@@ -1464,14 +1469,17 @@ int rxe_responder(struct rxe_qp *qp)struct rxe_pkt_info *pkt =NULL ;int ret; +spin_lock_bh (&qp->state_lock);if (!qp->valid ||qp_state (qp) ==IB_QPS_ERR || -qp_state (qp) ==IB_QPS_RESET ) { +qp_state (qp) ==IB_QPS_RESET ) {bool notify = qp->valid && (qp_state (qp) ==IB_QPS_ERR );drain_req_pkts (qp);flush_recv_queue (qp, notify); +spin_unlock_bh (&qp->state_lock); goto exit; } +spin_unlock_bh (&qp->state_lock); qp->resp.aeth_syndrome =AETH_ACK_UNLIMITED ;
drivers/infiniband/sw/rxe/rxe_verbs.c +134 -104
375 lines not on this page · raw message
Signature · 1 line
2.37.2
Quoted 13 lines · Bob Pearson
The rxe driver has four different QP state variables,
qp->attr.qp_state, qp->req.state, qp->comp.state, and qp->resp.state.All of these basically carry the same information.
This patch replaces uses of
qp->resp.statebyqp->attr.qp_state. This is the first of three patches which will remove all but theqp->attr.qp_statevariable. This will bring the driver closer to the IBA description.Signed-off-by Bob Pearson <[email protected]>
Quoted 8 lines
--- drivers/infiniband/sw/rxe/rxe_net.c | 2 +- drivers/infiniband/sw/rxe/rxe_qp.c | 5 ----- drivers/infiniband/sw/rxe/rxe_recv.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 10 +++++----- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 - 6 files changed, 8 insertions(+), 14 deletions(-)
Applied this series to for-next
Thanks,
Jason