RDMA/rxe: Various bug fixes.
Subsystems (2)
cover RDMA/rxe: Various bug fixes.
Currently some blktest test cases fail to execute correctly. This series of patches allows these test cases to pass and implements some other related changes.
The first patch is a repeat of an earlier patch after rebasing to for-next version 5.14.0-rc6+. It adds memory barriers to kernel to kernel queues. The logic for this is the same as an earlier patch that only treated user to kernel queues. Without this patch kernel to kernel queues are expected to intermittently fail at low frequency.
The second patch is also a repeat after rebasing to for-next version 5.14.0-rc6+. It fixes a multicast bug.
The third patch separates the keys in rxe_mr and ib_mr. This allows the sequence
do {
ib_post_send( IB_WR_LOCAL_INV )
ib_update_fast_reg_key()
ib_map_mr_sg()
ib_post_send( IB_WR_REG_MR )
} while ( !done )to operate correctly since now the changes to the local copy of the keys only happens at the final post_send. The same fix is applied to MWs since they need to synchronize with IB_WR_BIND_MW work requests.
The fourth patch creates duplicate mapping tables for fast MRs. This prevents rkeys referencing fast MRs from accessing data from an updated map after the call to ib_map_mr_sg() call by keeping the new and old mappings separate.
The final patch cleans up the state and type enums used by MRs.
Bob Pearson (5):
RDMA/rxe: Add memory barriers to kernel queues RDMA/rxe: Fix memory allocation while locked RDMA/rxe: Separate HW and SW l/rkeys RDMA/rxe: Create duplicate mapping tables for FMRs RDMA/rxe: Cleanup MR status and type enums
drivers/infiniband/sw/rxe/rxe_comp.c | 10 +- drivers/infiniband/sw/rxe/rxe_cq.c | 25 +-- drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_mcast.c | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 269 +++++++++++++++++++------- drivers/infiniband/sw/rxe/rxe_mw.c | 36 ++-- drivers/infiniband/sw/rxe/rxe_qp.c | 10 +- drivers/infiniband/sw/rxe/rxe_queue.h | 73 ++----- drivers/infiniband/sw/rxe/rxe_req.c | 35 +--- drivers/infiniband/sw/rxe/rxe_resp.c | 38 +--- drivers/infiniband/sw/rxe/rxe_srq.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 92 +++------ drivers/infiniband/sw/rxe/rxe_verbs.h | 48 ++--- 13 files changed, 307 insertions(+), 335 deletions(-)
Signature · 1 line
2.30.2
PATCH v2 1/5 RDMA/rxe: Add memory barriers to kernel queues
Earlier patches added memory barriers to protect user space to kernel space communications. This patch extends that to the case where queues are used between kernel space threads.
---
v2:
Rebase on version 5.14.
drivers/infiniband/sw/rxe/rxe_comp.c | 10 +--- drivers/infiniband/sw/rxe/rxe_cq.c | 25 ++------- drivers/infiniband/sw/rxe/rxe_qp.c | 10 ++-- drivers/infiniband/sw/rxe/rxe_queue.h | 73 ++++++++------------------- drivers/infiniband/sw/rxe/rxe_req.c | 21 ++------ drivers/infiniband/sw/rxe/rxe_resp.c | 38 ++++---------- drivers/infiniband/sw/rxe/rxe_srq.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 53 ++++--------------- 8 files changed, 55 insertions(+), 177 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +2 -8
@@ -142,10 +142,7 @@ static inline enum comp_state get_wqe(struct rxe_qp *qp,/* we come here whether or not we found a response packet to see if * there are any posted WQEs */ -if (qp->is_user) - wqe =queue_head (qp->sq.queue,QUEUE_TYPE_FROM_USER ); -else - wqe =queue_head (qp->sq.queue,QUEUE_TYPE_KERNEL ); + wqe =queue_head (qp->sq.queue,QUEUE_TYPE_FROM_CLIENT ); *wqe_p = wqe;/* no WQE or requester has not started it yet */
@@ -432,10 +429,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)if (post)make_send_cqe (qp, wqe, &cqe); -if (qp->is_user) -advance_consumer (qp->sq.queue,QUEUE_TYPE_FROM_USER ); -else -advance_consumer (qp->sq.queue,QUEUE_TYPE_KERNEL ); +advance_consumer (qp->sq.queue,QUEUE_TYPE_FROM_CLIENT );if (post)rxe_cq_post (qp->scq, &cqe,0 );
drivers/infiniband/sw/rxe/rxe_cq.c +5 -20
@@ -25,11 +25,7 @@ int rxe_cq_chk_attr(struct rxe_dev *rxe, struct rxe_cq *cq, }if (cq) { -if (cq->is_user) - count =queue_count (cq->queue,QUEUE_TYPE_TO_USER ); -else - count =queue_count (cq->queue,QUEUE_TYPE_KERNEL ); - + count =queue_count (cq->queue,QUEUE_TYPE_TO_CLIENT );if (cqe < count) {pr_warn ("cqe(%d) < current # elements in queue (%d)" , cqe, count);
@@ -65,7 +61,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,int err;enum queue_type type; - type = uresp ?QUEUE_TYPE_TO_USER :QUEUE_TYPE_KERNEL ; + type =QUEUE_TYPE_TO_CLIENT ; cq->queue =rxe_queue_init (rxe, &cqe,sizeof (struct rxe_cqe ), type);if (!cq->queue) {
@@ -117,11 +113,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)spin_lock_irqsave (&cq->cq_lock, flags); -if (cq->is_user) - full =queue_full (cq->queue,QUEUE_TYPE_TO_USER ); -else - full =queue_full (cq->queue,QUEUE_TYPE_KERNEL ); - + full =queue_full (cq->queue,QUEUE_TYPE_TO_CLIENT );if (unlikely (full)) {spin_unlock_irqrestore (&cq->cq_lock, flags);if (cq->ibcq.event_handler) {
@@ -134,17 +126,10 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)return -EBUSY ; } -if (cq->is_user) - addr =producer_addr (cq->queue,QUEUE_TYPE_TO_USER ); -else - addr =producer_addr (cq->queue,QUEUE_TYPE_KERNEL ); - + addr =producer_addr (cq->queue,QUEUE_TYPE_TO_CLIENT );memcpy (addr, cqe,sizeof (*cqe)); -if (cq->is_user) -advance_producer (cq->queue,QUEUE_TYPE_TO_USER ); -else -advance_producer (cq->queue,QUEUE_TYPE_KERNEL ); +advance_producer (cq->queue,QUEUE_TYPE_TO_CLIENT );spin_unlock_irqrestore (&cq->cq_lock, flags);
drivers/infiniband/sw/rxe/rxe_qp.c +3 -7
@@ -231,7 +231,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp->sq.max_inline = init->cap.max_inline_data = wqe_size; wqe_size +=sizeof (struct rxe_send_wqe ); - type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + type =QUEUE_TYPE_FROM_CLIENT ; qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, wqe_size, type);if (!qp->sq.queue)
@@ -248,12 +248,8 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,return err; } -if (qp->is_user) qp->req.wqe_index =producer_index (qp->sq.queue, -QUEUE_TYPE_FROM_USER ); -else - qp->req.wqe_index =producer_index (qp->sq.queue, -QUEUE_TYPE_KERNEL ); +QUEUE_TYPE_FROM_CLIENT ); qp->req.state =QP_STATE_RESET ; qp->req.opcode =-1 ;
@@ -293,7 +289,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,pr_debug ("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n" ,qp_num (qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size); - type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + type =QUEUE_TYPE_FROM_CLIENT ; qp->rq.queue =rxe_queue_init (rxe, &qp->rq.max_wr, wqe_size, type);if (!qp->rq.queue)
drivers/infiniband/sw/rxe/rxe_queue.h +20 -53
201 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_req.c +5 -16
@@ -49,13 +49,8 @@ static void req_retry(struct rxe_qp *qp)unsigned int cons;unsigned int prod; -if (qp->is_user) { - cons =consumer_index (q,QUEUE_TYPE_FROM_USER ); - prod =producer_index (q,QUEUE_TYPE_FROM_USER ); - }else { - cons =consumer_index (q,QUEUE_TYPE_KERNEL ); - prod =producer_index (q,QUEUE_TYPE_KERNEL ); - } + cons =consumer_index (q,QUEUE_TYPE_FROM_CLIENT ); + prod =producer_index (q,QUEUE_TYPE_FROM_CLIENT ); qp->req.wqe_index = cons; qp->req.psn = qp->comp.psn;
@@ -121,15 +116,9 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)unsigned int cons;unsigned int prod; -if (qp->is_user) { - wqe =queue_head (q,QUEUE_TYPE_FROM_USER ); - cons =consumer_index (q,QUEUE_TYPE_FROM_USER ); - prod =producer_index (q,QUEUE_TYPE_FROM_USER ); - }else { - wqe =queue_head (q,QUEUE_TYPE_KERNEL ); - cons =consumer_index (q,QUEUE_TYPE_KERNEL ); - prod =producer_index (q,QUEUE_TYPE_KERNEL ); - } + wqe =queue_head (q,QUEUE_TYPE_FROM_CLIENT ); + cons =consumer_index (q,QUEUE_TYPE_FROM_CLIENT ); + prod =producer_index (q,QUEUE_TYPE_FROM_CLIENT );if (unlikely (qp->req.state ==QP_STATE_DRAIN )) {/* check to see if we are drained;
drivers/infiniband/sw/rxe/rxe_resp.c +9 -29
@@ -303,10 +303,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)spin_lock_bh (&srq->rq.consumer_lock); -if (qp->is_user) - wqe =queue_head (q,QUEUE_TYPE_FROM_USER ); -else - wqe =queue_head (q,QUEUE_TYPE_KERNEL ); + wqe =queue_head (q,QUEUE_TYPE_FROM_CLIENT );if (!wqe) {spin_unlock_bh (&srq->rq.consumer_lock);return RESPST_ERR_RNR ;
@@ -322,13 +319,8 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)memcpy (&qp->resp.srq_wqe, wqe, size); qp->resp.wqe = &qp->resp.srq_wqe.wqe; -if (qp->is_user) { -advance_consumer (q,QUEUE_TYPE_FROM_USER ); - count =queue_count (q,QUEUE_TYPE_FROM_USER ); - }else { -advance_consumer (q,QUEUE_TYPE_KERNEL ); - count =queue_count (q,QUEUE_TYPE_KERNEL ); - } +advance_consumer (q,QUEUE_TYPE_FROM_CLIENT ); + count =queue_count (q,QUEUE_TYPE_FROM_CLIENT );if (srq->limit && srq->ibsrq.event_handler && (count < srq->limit)) { srq->limit =0 ;
@@ -357,12 +349,8 @@ static enum resp_states check_resource(struct rxe_qp *qp, qp->resp.status =IB_WC_WR_FLUSH_ERR ;return RESPST_COMPLETE ; }else if (!srq) { -if (qp->is_user) - qp->resp.wqe =queue_head (qp->rq.queue, -QUEUE_TYPE_FROM_USER ); -else - qp->resp.wqe =queue_head (qp->rq.queue, -QUEUE_TYPE_KERNEL ); + qp->resp.wqe =queue_head (qp->rq.queue, +QUEUE_TYPE_FROM_CLIENT );if (qp->resp.wqe) { qp->resp.status =IB_WC_WR_FLUSH_ERR ;return RESPST_COMPLETE ;
@@ -389,12 +377,8 @@ static enum resp_states check_resource(struct rxe_qp *qp,if (srq)return get_srq_wqe (qp); -if (qp->is_user) - qp->resp.wqe =queue_head (qp->rq.queue, -QUEUE_TYPE_FROM_USER ); -else - qp->resp.wqe =queue_head (qp->rq.queue, -QUEUE_TYPE_KERNEL ); + qp->resp.wqe =queue_head (qp->rq.queue, +QUEUE_TYPE_FROM_CLIENT );return (qp->resp.wqe) ?RESPST_CHK_LENGTH :RESPST_ERR_RNR ; }
@@ -936,12 +920,8 @@ static enum resp_states do_complete(struct rxe_qp *qp, }/* have copy for srq and reference for !srq */ -if (!qp->srq) { -if (qp->is_user) -advance_consumer (qp->rq.queue,QUEUE_TYPE_FROM_USER ); -else -advance_consumer (qp->rq.queue,QUEUE_TYPE_KERNEL ); - } +if (!qp->srq) +advance_consumer (qp->rq.queue,QUEUE_TYPE_FROM_CLIENT ); qp->resp.wqe =NULL ;
drivers/infiniband/sw/rxe/rxe_srq.c +1 -1
@@ -93,7 +93,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,spin_lock_init (&srq->rq.producer_lock);spin_lock_init (&srq->rq.consumer_lock); - type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + type =QUEUE_TYPE_FROM_CLIENT ; q =rxe_queue_init (rxe, &srq->rq.max_wr, srq_wqe_size, type);if (!q) {
drivers/infiniband/sw/rxe/rxe_verbs.c +10 -43
107 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v2 2/5 RDMA/rxe: Fix memory allocation while locked
In reply to Bob Pearsonrxe_mcast_add_grp_elem() in rxe_mcast.c calls rxe_alloc() while holding spinlocks which in turn calls kzalloc(size, GFP_KERNEL) which is incorrect. This patch replaces rxe_alloc() by rxe_alloc_locked() which uses GFP_ATOMIC. This bug was caused by the below mentioned commit and failing to handle the need for the atomic allocate.
---
v2:
rebase on version 5.14
drivers/infiniband/sw/rxe/rxe_mcast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +1 -1
@@ -85,7 +85,7 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, goto out; } - elem =rxe_alloc (&rxe->mc_elem_pool); + elem =rxe_alloc_locked (&rxe->mc_elem_pool);if (!elem) { err = -ENOMEM ; goto out;
Signature · 1 line
2.30.2
PATCH 3/5 RDMA/rxe: Separate HW and SW l/rkeys
In reply to Bob PearsonSeparate software and simulated hardware lkeys and rkeys for MRs and MWs. This makes struct ib_mr and struct ib_mw isolated from hardware changes triggered by executing work requests.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 1 + drivers/infiniband/sw/rxe/rxe_mr.c | 69 ++++++++++++++++++++++----- drivers/infiniband/sw/rxe/rxe_mw.c | 30 ++++++------ drivers/infiniband/sw/rxe/rxe_req.c | 14 ++---- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 ++----- 5 files changed, 81 insertions(+), 51 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
@@ -86,6 +86,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);int advance_dma_data (struct rxe_dma_info *dma,unsigned int length);int rxe_invalidate_mr (struct rxe_qp *qp,u32 rkey); +int rxe_reg_fast_mr (struct rxe_qp *qp,struct rxe_send_wqe *wqe);int rxe_dereg_mr (struct ib_mr *ibmr,struct ib_udata *udata);void rxe_mr_cleanup (struct rxe_pool_entry *arg);
drivers/infiniband/sw/rxe/rxe_mr.c +58 -11
99 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_mw.c +14 -16
@@ -21,7 +21,7 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) }rxe_add_index (mw); - ibmw->rkey = (mw->pelem.index <<8 ) |rxe_get_next_key (-1 ); + mw->rkey = ibmw->rkey = (mw->pelem.index <<8 ) |rxe_get_next_key (-1 ); mw->state = (mw->ibmw.type ==IB_MW_TYPE_2 ) ?RXE_MW_STATE_FREE :RXE_MW_STATE_VALID ;spin_lock_init (&mw->lock);
@@ -71,6 +71,8 @@ int rxe_dealloc_mw(struct ib_mw *ibmw)static int rxe_check_bind_mw (struct rxe_qp *qp,struct rxe_send_wqe *wqe,struct rxe_mw *mw,struct rxe_mr *mr) { +u32 key = wqe->wr.wr.mw.rkey &0xff ; +if (mw->ibmw.type ==IB_MW_TYPE_1 ) {if (unlikely (mw->state !=RXE_MW_STATE_VALID )) { pr_err_once(
@@ -108,7 +110,7 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, } } -if (unlikely ((wqe->wr.wr.mw.rkey &0xff ) == (mw->ibmw.rkey &0xff ))) { +if (unlikely (key == (mw->rkey &0xff ))) {pr_err_once ("attempt to bind MW with same key\n" );return -EINVAL ; }
@@ -161,13 +163,9 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,static void rxe_do_bind_mw (struct rxe_qp *qp,struct rxe_send_wqe *wqe,struct rxe_mw *mw,struct rxe_mr *mr) { -u32 rkey; -u32 new_rkey; - - rkey = mw->ibmw.rkey; - new_rkey = (rkey &0xffffff00 ) | (wqe->wr.wr.mw.rkey &0x000000ff ); +u32 key = wqe->wr.wr.mw.rkey &0xff ; - mw->ibmw.rkey = new_rkey; + mw->rkey = (mw->rkey & ~0xff ) | key; mw->access = wqe->wr.wr.mw.access; mw->state =RXE_MW_STATE_VALID ; mw->addr = wqe->wr.wr.mw.addr;
@@ -197,29 +195,29 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)struct rxe_mw *mw;struct rxe_mr *mr;struct rxe_dev *rxe =to_rdev (qp->ibqp.device); +u32 mw_rkey = wqe->wr.wr.mw.mw_rkey; +u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;unsigned long flags; - mw =rxe_pool_get_index (&rxe->mw_pool, - wqe->wr.wr.mw.mw_rkey >>8 ); + mw =rxe_pool_get_index (&rxe->mw_pool, mw_rkey >>8 );if (unlikely (!mw)) { ret = -EINVAL ; goto err; } -if (unlikely (mw->ibmw.rkey != wqe->wr.wr.mw.mw_rkey)) { +if (unlikely (mw->rkey != mw_rkey)) { ret = -EINVAL ; goto err_drop_mw; }if (likely (wqe->wr.wr.mw.length)) { - mr =rxe_pool_get_index (&rxe->mr_pool, - wqe->wr.wr.mw.mr_lkey >>8 ); + mr =rxe_pool_get_index (&rxe->mr_pool, mr_lkey >>8 );if (unlikely (!mr)) { ret = -EINVAL ; goto err_drop_mw; } -if (unlikely (mr->ibmr.lkey != wqe->wr.wr.mw.mr_lkey)) { +if (unlikely (mr->lkey != mr_lkey)) { ret = -EINVAL ; goto err_drop_mr; }
@@ -292,7 +290,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) goto err; } -if (rkey != mw->ibmw.rkey) { +if (rkey != mw->rkey) { ret = -EINVAL ; goto err_drop_ref; }
@@ -323,7 +321,7 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)if (!mw)return NULL ; - if (unlikely((rxe_mw_rkey(mw) != rkey) || rxe_mw_pd(mw) != pd || + if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd || (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) || (mw->length == 0) || (access && !(access & mw->access)) ||
drivers/infiniband/sw/rxe/rxe_req.c +5 -9
@@ -561,7 +561,6 @@ static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,static int rxe_do_local_ops (struct rxe_qp *qp,struct rxe_send_wqe *wqe) {u8 opcode = wqe->wr.opcode; -struct rxe_mr *mr;u32 rkey;int ret;
@@ -579,14 +578,11 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) }break ;case IB_WR_REG_MR : - mr =to_rmr (wqe->wr.wr.reg.mr); -rxe_add_ref (mr); - mr->state =RXE_MR_STATE_VALID ; - mr->access = wqe->wr.wr.reg.access; - mr->ibmr.lkey = wqe->wr.wr.reg.key; - mr->ibmr.rkey = wqe->wr.wr.reg.key; - mr->iova = wqe->wr.wr.reg.mr->iova; -rxe_drop_ref (mr); + ret =rxe_reg_fast_mr (qp, wqe); +if (unlikely (ret)) { + wqe->status =IB_WC_LOC_QP_OP_ERR ; +return ret; + }break ;case IB_WR_BIND_MW : ret =rxe_bind_mw (qp, wqe);
drivers/infiniband/sw/rxe/rxe_verbs.h +3 -15
@@ -313,6 +313,8 @@ struct rxe_mr {struct ib_umem *umem; +u32 lkey; +u32 rkey;enum rxe_mr_state state;enum rxe_mr_type type;u64 va;
@@ -350,6 +352,7 @@ struct rxe_mw {enum rxe_mw_state state;struct rxe_qp *qp;/* Type 2 only */ struct rxe_mr *mr; +u32 rkey;int access;u64 addr;u64 length;
@@ -474,26 +477,11 @@ static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)return to_rpd (mr->ibmr.pd); } -static inline u32 mr_lkey (struct rxe_mr *mr) -{ -return mr->ibmr.lkey; -} - -static inline u32 mr_rkey (struct rxe_mr *mr) -{ -return mr->ibmr.rkey; -} -static inline struct rxe_pd *rxe_mw_pd (struct rxe_mw *mw) {return to_rpd (mw->ibmw.pd); } -static inline u32 rxe_mw_rkey (struct rxe_mw *mw) -{ -return mw->ibmw.rkey; -} -int rxe_register_device (struct rxe_dev *rxe,const char *ibdev_name);void rxe_mc_cleanup (struct rxe_pool_entry *arg);
Signature · 1 line
2.30.2
PATCH 4/5 RDMA/rxe: Create duplicate mapping tables for FMRs
In reply to Bob PearsonFor fast memory regions create duplicate mapping tables so ib_map_mr_sg() can build a new mapping table which is then swapped into place synchronously with the execution of an IB_WR_REG_MR work request.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 1 + drivers/infiniband/sw/rxe/rxe_mr.c | 196 +++++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_mw.c | 6 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 39 ++--- drivers/infiniband/sw/rxe/rxe_verbs.h | 21 +-- 5 files changed, 162 insertions(+), 101 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
@@ -87,6 +87,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);int advance_dma_data (struct rxe_dma_info *dma,unsigned int length);int rxe_invalidate_mr (struct rxe_qp *qp,u32 rkey);int rxe_reg_fast_mr (struct rxe_qp *qp,struct rxe_send_wqe *wqe); +int rxe_mr_set_page (struct ib_mr *ibmr,u64 addr);int rxe_dereg_mr (struct ib_mr *ibmr,struct ib_udata *udata);void rxe_mr_cleanup (struct rxe_pool_entry *arg);
drivers/infiniband/sw/rxe/rxe_mr.c +133 -63
354 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_mw.c +3 -3
@@ -142,15 +142,15 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,/* C10-75 */ if (mw->access &IB_ZERO_BASED ) { -if (unlikely (wqe->wr.wr.mw.length > mr->length)) { +if (unlikely (wqe->wr.wr.mw.length > mr->cur_map_set->length)) {pr_err_once ("attempt to bind a ZB MW outside of the MR\n" );return -EINVAL ; } }else { -if (unlikely ((wqe->wr.wr.mw.addr < mr->iova) || +if (unlikely ((wqe->wr.wr.mw.addr < mr->cur_map_set->iova) || ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) > - (mr->iova + mr->length)))) { + (mr->cur_map_set->iova + mr->cur_map_set->length)))) {pr_err_once ("attempt to bind a VA MW outside of the MR\n" );return -EINVAL ;
drivers/infiniband/sw/rxe/rxe_verbs.c +12 -27
@@ -954,41 +954,26 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,return ERR_PTR (err); } -static int rxe_set_page (struct ib_mr *ibmr,u64 addr) -{ -struct rxe_mr *mr =to_rmr (ibmr); -struct rxe_map *map; -struct rxe_phys_buf *buf; - -if (unlikely (mr->nbuf == mr->num_buf)) -return -ENOMEM ; - - map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; - mr->nbuf++; - -return 0 ; -} - +/* build next_map_set from scatterlist +* The IB_WR_REG_MR WR will swap map_sets +*/ static int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset) {struct rxe_mr *mr =to_rmr (ibmr); +struct rxe_map_set *set = mr->next_map_set;int n; - mr->nbuf =0 ; + set->nbuf =0 ; - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_mr_set_page); - mr->va = ibmr->iova; - mr->iova = ibmr->iova; - mr->length = ibmr->length; - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = mr->iova & mr->page_mask; + set->va = ibmr->iova; + set->iova = ibmr->iova; + set->length = ibmr->length; + set->page_shift =ilog2 (ibmr->page_size); + set->page_mask = ibmr->page_size -1 ; + set->offset = set->iova & set->page_mask;return n; }
drivers/infiniband/sw/rxe/rxe_verbs.h +13 -8
@@ -300,6 +300,17 @@ struct rxe_map {struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; }; +struct rxe_map_set { +struct rxe_map **map; +u64 va; +u64 iova; +size_t length; +u32 offset; +u32 nbuf; +int page_shift; +int page_mask; +}; +static inline int rkey_is_mw (u32 rkey) {u32 index = rkey >>8 ;
@@ -317,26 +328,20 @@ struct rxe_mr {u32 rkey;enum rxe_mr_state state;enum rxe_mr_type type; -u64 va; -u64 iova; -size_t length; -u32 offset;int access; -int page_shift; -int page_mask;int map_shift;int map_mask;u32 num_buf; -u32 nbuf;u32 max_buf;u32 num_map;atomic_t num_mw; -struct rxe_map **map; +struct rxe_map_set *cur_map_set; +struct rxe_map_set *next_map_set; };enum rxe_mw_state {
Signature · 1 line
2.30.2
PATCH 5/5 RDMA/rxe: Cleanup MR status and type enums
In reply to Bob PearsonEliminate RXE_MR_STATE_ZOMBIE which is not compatible with IBA. RXE_MR_STATE_INVALID is better.
Replace RXE_MR_TYPE_XXX by IB_MR_TYPE_XXX which covers all the needed types.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 30 ++++++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_verbs.h | 9 +------- 2 files changed, 21 insertions(+), 18 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +20 -10
@@ -27,16 +27,19 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)struct rxe_map_set *set = mr->cur_map_set;;switch (mr->type) { -case RXE_MR_TYPE_DMA : +case IB_MR_TYPE_DMA :return 0 ; -case RXE_MR_TYPE_MR : +case IB_MR_TYPE_USER : +case IB_MR_TYPE_MEM_REG :if (iova < set->iova || length > set->length || iova > set->iova + set->length - length)return -EFAULT ;return 0 ;default : +pr_warn ("%s: mr type (%d) not supported\n" , + __func__, mr->type);return -EFAULT ; } }
@@ -59,7 +62,7 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) mr->rkey = mr->ibmr.rkey = rkey; mr->state =RXE_MR_STATE_INVALID ; - mr->type =RXE_MR_TYPE_NONE ; + mr->type =-1 ; mr->map_shift =ilog2 (RXE_BUF_PER_MAP ); }
@@ -156,7 +159,7 @@ void rxe_mr_init_dma(struct rxe_pd *pd, int access, struct rxe_mr *mr) mr->ibmr.pd = &pd->ibpd; mr->access = access; mr->state =RXE_MR_STATE_VALID ; - mr->type =RXE_MR_TYPE_DMA ; + mr->type =IB_MR_TYPE_DMA ; } int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova,
@@ -226,7 +229,7 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, mr->umem = umem; mr->access = access; mr->state =RXE_MR_STATE_VALID ; - mr->type =RXE_MR_TYPE_MR ; + mr->type =IB_MR_TYPE_USER ; set->length = length; set->iova = iova;
@@ -257,7 +260,7 @@ int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr) mr->ibmr.pd = &pd->ibpd; mr->max_buf = max_pages; mr->state =RXE_MR_STATE_FREE ; - mr->type =RXE_MR_TYPE_MR ; + mr->type =IB_MR_TYPE_MEM_REG ;return 0 ;
@@ -360,7 +363,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,if (length ==0 )return 0 ; -if (mr->type ==RXE_MR_TYPE_DMA ) { +if (mr->type ==IB_MR_TYPE_DMA ) {u8 *src, *dest; src = (dir ==RXE_TO_MR_OBJ ) ? addr : ((void *)(uintptr_t )iova);
@@ -628,8 +631,15 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)/* user can only register MR in free state */ if (unlikely (mr->state !=RXE_MR_STATE_FREE )) { -pr_warn ("%s: mr->lkey = 0x%x not free\n" , - __func__, mr->lkey); +pr_warn ("%s: mr->state = %d not free\n" , + __func__, mr->state); +return -EINVAL ; + } + +/* user can only register MR of type IB_MR_TYPE_MEM_REG */ +if (unlikely (mr->type !=IB_MR_TYPE_MEM_REG )) { +pr_warn ("%s: mr->type = %d wrong type\n" , + __func__, mr->type);return -EINVAL ; }
@@ -683,7 +693,7 @@ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)return -EINVAL ; } - mr->state =RXE_MR_STATE_ZOMBIE ; + mr->state =RXE_MR_STATE_INVALID ;rxe_drop_ref (mr_pd (mr));rxe_drop_index (mr);rxe_drop_ref (mr);
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -8
@@ -267,18 +267,11 @@ struct rxe_qp { };enum rxe_mr_state { -RXE_MR_STATE_ZOMBIE ,RXE_MR_STATE_INVALID ,RXE_MR_STATE_FREE ,RXE_MR_STATE_VALID , }; -enum rxe_mr_type { -RXE_MR_TYPE_NONE , -RXE_MR_TYPE_DMA , -RXE_MR_TYPE_MR , -}; -enum rxe_mr_copy_dir {RXE_TO_MR_OBJ ,RXE_FROM_MR_OBJ ,
@@ -327,7 +320,7 @@ struct rxe_mr {u32 lkey;u32 rkey;enum rxe_mr_state state; -enum rxe_mr_type type; +enum ib_mr_type type;int access;int map_shift;
Signature · 1 line
2.30.2
Quoted 2 lines · Bob Pearson
The first patch is a repeat of an earlier patch after rebasing to for-next version 5.14.0-rc6+.
Hi Bob,
What is for-next version 5.14.0-rc6+? I tried to apply this series on v5.14, v5.14-rc6 and Jason's for-rc branch but 'git am' refused to apply this series on all these kernel versions ...
Thanks,
Bart.
Quoted 11 lines · Bart Van Assche
Quoted 2 lines · Bob Pearson
The first patch is a repeat of an earlier patch after rebasing to for-next version 5.14.0-rc6+.
Hi Bob,
What is for-next version 5.14.0-rc6+? I tried to apply this series on v5.14, v5.14-rc6 and Jason's for-rc branch but 'git am' refused to apply this series on all these kernel versions ...
Thanks,
Bart.
Bart,
I think I fixed the git am issue and resubmitted to the correct queue. You were on copy. Please try to see if this gets you to joy for blktest. The one case you mentioned srp/002 is working OK for me now.
Bob