RDMA/rxe: Replace mr page map with an xarray
Subsystems (2)
cover RDMA/rxe: Replace mr page map with an xarray
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.
Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
drivers/infiniband/sw/rxe/rxe_loc.h | 6 +- drivers/infiniband/sw/rxe/rxe_mr.c | 518 ++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_resp.c | 73 ++-- drivers/infiniband/sw/rxe/rxe_verbs.c | 36 -- drivers/infiniband/sw/rxe/rxe_verbs.h | 32 +- 5 files changed, 327 insertions(+), 338 deletions(-)
base-commit: 6978837ce42f8bea85041fc08c854f4e28852b3e
Signature · 1 line
2.37.2
PATCH 1/6 RDMA/rxe: Cleanup mr_check_range
Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
@@ -24,8 +24,6 @@ u8 rxe_get_next_key(u32 last_key)int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length) { - -switch (mr->ibmr.type) {case IB_MR_TYPE_DMA :return 0 ;
@@ -39,7 +37,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)default :rxe_dbg_mr (mr,"type (%d) not supported\n" , mr->ibmr.type); -return -EFAULT ; +return -EINVAL ; } }
Signature · 1 line
2.37.2
PATCH 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
In reply to Bob PearsonMove rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 36 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 36 --------------------------- 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -68,6 +68,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir); +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
@@ -202,6 +202,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)return err; } +static int rxe_set_page (struct ib_mr *ibmr,u64 addr) +{ +struct rxe_mr *mr =to_rmr (ibmr); +struct rxe_map *map; +struct rxe_phys_buf *buf; + +if (unlikely (mr->nbuf == mr->num_buf)) +return -ENOMEM ; + + map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; + buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; + + buf->addr = addr; + buf->size = ibmr->page_size; + mr->nbuf++; + +return 0 ; +} + +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset) +{ +struct rxe_mr *mr =to_rmr (ibmr); +int n; + + mr->nbuf =0 ; + + n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + + mr->page_shift =ilog2 (ibmr->page_size); + mr->page_mask = ibmr->page_size -1 ; + mr->offset = ibmr->iova & mr->page_mask; + +return n; +} +static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,return ERR_PTR (err); } -static int rxe_set_page (struct ib_mr *ibmr,u64 addr) -{ -struct rxe_mr *mr =to_rmr (ibmr); -struct rxe_map *map; -struct rxe_phys_buf *buf; - -if (unlikely (mr->nbuf == mr->num_buf)) -return -ENOMEM ; - - map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; - mr->nbuf++; - -return 0 ; -} - -static int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, -int sg_nents,unsigned int *sg_offset) -{ -struct rxe_mr *mr =to_rmr (ibmr); -int n; - - mr->nbuf =0 ; - - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); - - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; - -return n; -} -static ssize_t parent_show (struct device *device,struct device_attribute *attr,char *buf) {
Signature · 1 line
2.37.2
PATCH 3/6 RDMA-rxe: Isolate mr code from atomic_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 30 ++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_resp.c | 47 +++++++--------------------- 3 files changed, 44 insertions(+), 35 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -71,6 +71,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length); +int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +30 -0
@@ -484,6 +484,36 @@ int copy_data(return err; } +static DEFINE_SPINLOCK (atomic_ops_lock); + +int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val) +{ +u64 *vaddr =iova_to_vaddr (mr, iova,sizeof (u64)); +u64 value; + +/* needs to match rxe_resp.c */ +if (mr->state !=RXE_MR_STATE_VALID || !vaddr) +return -EFAULT ; +if (vaddr &7 ) +return -EINVAL ; + +spin_lock_bh (&atomic_ops_lock); + value = *orig_val = *vaddr; + +if (opcode ==IB_OPCODE_RC_COMPARE_SWAP ) { +if (value == compare) + value = swap_add; + }else { + value += swap_add; + } + + *vaddr = value; +spin_unlock_bh (&atomic_ops_lock); + +return 0 ; +} +int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) {struct rxe_sge *sge = &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +12 -35
70 lines not on this page · raw message
Signature · 1 line
2.37.2
PATCH 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 1 + drivers/infiniband/sw/rxe/rxe_mr.c | 41 +++++++++++++++++++++++++++- drivers/infiniband/sw/rxe/rxe_resp.c | 25 ++++++----------- 3 files changed, 50 insertions(+), 17 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
@@ -73,6 +73,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val); +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,void *addr);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +40 -1
53 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +9 -16
@@ -654,12 +654,12 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, }static enum resp_states atomic_write_reply (struct rxe_qp *qp, -struct rxe_pkt_info *pkt) +struct rxe_pkt_info *pkt) { -u64 src, *dst;struct resp_res *res = qp->resp.res;struct rxe_mr *mr = qp->resp.mr; -int payload =payload_size (pkt); +void *addr =payload_addr (pkt); +int err;if (!res) { res =rxe_prepare_res (qp, pkt,RXE_ATOMIC_WRITE_MASK );
@@ -668,22 +668,15 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp,if (!res->replay) {#ifdef CONFIG_64BIT -if (mr->state !=RXE_MR_STATE_VALID ) -return RESPST_ERR_RKEY_VIOLATION ; - -memcpy (&src,payload_addr (pkt), payload); +u64 iova = qp->resp.va + qp->resp.offset; - dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) + err =rxe_mr_do_atomic_write (mr, iova, addr); +if (err ==-1 )return RESPST_ERR_MISALIGNED_ATOMIC ; +else +return RESPST_ERR_RKEY_VIOLATION ; -/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); - -/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); - + qp->resp.resid -=8 ; qp->resp.msn++;/* next expected psn, read handles this separately */
Signature · 1 line
2.37.2
Jason Gunthorpe
maintainerRe: [PATCH for-next 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 499
Quoted 11 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_mr.c +1
-1@@ -495,7 +495,7 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,/* needs to match rxe_resp.c */ if (mr->state !=RXE_MR_STATE_VALID || !vaddr)return -EFAULT ; -if (vaddr &7 ) +if ((uintptr_t )vaddr &7 )return -EINVAL ;
??
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_mr.c +0
-4@@ -668,22 +668,15 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp,if (!res->replay) {#ifdef CONFIG_64BIT -if (mr->state !=RXE_MR_STATE_VALID ) -return RESPST_ERR_RKEY_VIOLATION ; - -memcpy (&src,payload_addr (pkt), payload);
drivers/infiniband/sw/rxe/rxe_mr.c static enum resp_states atomic_write_reply(struct rxe_qp *qp, · line 677
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +5
-3+u64 iova = qp->resp.va + qp->resp.offset; - dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) + err =rxe_mr_do_atomic_write (mr, iova, addr); +if (err ==-1 )return RESPST_ERR_MISALIGNED_ATOMIC ; +else +return RESPST_ERR_RKEY_VIOLATION ;
This doesn't look right?? Return on both else branches?
Jason
PATCH 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c
In reply to Bob PearsonCleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 28 +++++++++++++-------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 11 ++++++++--- 2 files changed, 21 insertions(+), 18 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +13 -15
@@ -58,6 +58,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) mr->lkey = mr->ibmr.lkey = lkey; mr->rkey = mr->ibmr.rkey = rkey; + mr->ibmr.page_size =PAGE_SIZE ; + mr->page_mask =PAGE_MASK ; + mr->page_shift =PAGE_SHIFT ; mr->state =RXE_MR_STATE_INVALID ; }
@@ -138,9 +141,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } - mr->page_shift =PAGE_SHIFT ; - mr->page_mask =PAGE_SIZE -1 ; - num_buf =0 ; map = mr->map;if (length >0 ) {
@@ -160,7 +160,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } buf->addr = (uintptr_t )vaddr; - buf->size =PAGE_SIZE ; + buf->size =mr_page_size (mr); num_buf++; buf++;
@@ -169,7 +169,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, mr->umem = umem; mr->access = access; - mr->offset =ib_umem_offset (umem); + mr->page_offset =ib_umem_offset (umem); mr->state =RXE_MR_STATE_VALID ; mr->ibmr.type =IB_MR_TYPE_USER ;
@@ -225,29 +225,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { struct rxe_mr *mr = to_rmr(ibmr); -int n; +unsigned int page_size =mr_page_size (mr); - mr->nbuf =0 ; - - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + mr->page_shift =ilog2 (page_size); + mr->page_mask = ~((u64 )page_size -1 ); + mr->page_offset = ibmr->iova & (page_size -1 ); - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; + mr->nbuf =0 ; -return n; +return ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); }static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) { -size_t offset = iova - mr->ibmr.iova + mr->offset; +size_t offset = iova - mr->ibmr.iova + mr->page_offset;int map_index;int buf_index;u64 length;if (likely (mr->page_shift)) { - *offset_out = offset & mr->page_mask; + *offset_out = offset & (mr_page_size (mr) -1 ); offset >>= mr->page_shift; *n_out = offset & mr->map_mask; *m_out = offset >> mr->map_shift;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
@@ -304,11 +304,11 @@ struct rxe_mr {u32 lkey;u32 rkey;enum rxe_mr_state state; -u32 offset;int access; -int page_shift; -int page_mask; +unsigned int page_offset; +unsigned int page_shift; +u64 page_mask;int map_shift;int map_mask;
@@ -323,6 +323,11 @@ struct rxe_mr {struct rxe_map **map; }; +static inline unsigned int mr_page_size (struct rxe_mr *mr) +{ +return mr ? mr->ibmr.page_size :PAGE_SIZE ; +} +enum rxe_mw_state {RXE_MW_STATE_INVALID =RXE_MR_STATE_INVALID ,RXE_MW_STATE_FREE =RXE_MR_STATE_FREE ,
Signature · 1 line
2.37.2
PATCH 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
In reply to Bob PearsonReplace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.
This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 1 - drivers/infiniband/sw/rxe/rxe_mr.c | 481 ++++++++++++-------------- drivers/infiniband/sw/rxe/rxe_resp.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 21 +- 4 files changed, 226 insertions(+), 282 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +0 -1
@@ -70,7 +70,6 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, void *addr, int length, enum rxe_mr_copy_dir dir);int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset); -void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val);int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,void *addr);
drivers/infiniband/sw/rxe/rxe_mr.c +221 -260
643 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +3 -2
@@ -635,7 +635,7 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,atmeth_comp (pkt),atmeth_swap_add (pkt), &res->atomic.orig_val); -if (err == -EINVAL ) +if (err ==-1 )return RESPST_ERR_MISALIGNED_ATOMIC ;else if (err)return RESPST_ERR_RKEY_VIOLATION ;
@@ -659,6 +659,7 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp,struct resp_res *res = qp->resp.res;struct rxe_mr *mr = qp->resp.mr;void *addr =payload_addr (pkt); +unsigned int length =8 ;int err;if (!res) {
@@ -676,7 +677,7 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp,else return RESPST_ERR_RKEY_VIOLATION ; - qp->resp.resid -=8 ; + qp->resp.resid -= length; qp->resp.msn++;/* next expected psn, read handles this separately */
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
@@ -277,17 +277,6 @@ enum rxe_mr_lookup_type { RXE_LOOKUP_REMOTE, }; -#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) - -struct rxe_phys_buf { -u64 addr; -u64 size; -}; - -struct rxe_map { -struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; -}; -static inline int rkey_is_mw (u32 rkey) {u32 index = rkey >>8 ;
@@ -305,22 +294,16 @@ struct rxe_mr {u32 rkey;enum rxe_mr_state state;int access; +atomic_t num_mw;unsigned int page_offset;unsigned int page_shift;u64 page_mask; -int map_shift; -int map_mask;u32 num_buf;u32 nbuf; -u32 max_buf; -u32 num_map; - -atomic_t num_mw; - -struct rxe_map **map; +struct xarray pages; };static inline unsigned int mr_page_size (struct rxe_mr *mr)
Signature · 1 line
2.37.2