RDMA/rxe: Replace mr page map with an xarray
Subsystems (2)
v4 cover RDMA/rxe: Replace mr page map with an xarray
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.
v4:
Responded to a comment by Zhu and cleaned up error passing between rxe_mr.c and rxe_resp.c.
Other various cleanups including more careful use of unsigned ints. Rebased to current for-next.
v3:
Fixed an error reported by kernel test robot
v2:
Rebased to 6.2.0-rc1+
Minor cleanups
Fixed error reported by Jason in 4/6 missing if after else.
Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
drivers/infiniband/sw/rxe/rxe_loc.h | 13 +- drivers/infiniband/sw/rxe/rxe_mr.c | 610 ++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_resp.c | 118 ++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 36 -- drivers/infiniband/sw/rxe/rxe_verbs.h | 37 +- 5 files changed, 405 insertions(+), 409 deletions(-)
base-commit: 1ec82317a1daac78c04b0c15af89018ccf9fa2b7
Signature · 1 line
2.37.2
PATCH v4 1/6 RDMA/rxe: Cleanup mr_check_range
Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
@@ -26,8 +26,6 @@ u8 rxe_get_next_key(u32 last_key)int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length) { - -switch (mr->ibmr.type) {case IB_MR_TYPE_DMA :return 0 ;
@@ -41,7 +39,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)default :rxe_dbg_mr (mr,"type (%d) not supported\n" , mr->ibmr.type); -return -EFAULT ; +return -EINVAL ; } }
Signature · 1 line
2.37.2
PATCH v4 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
In reply to Bob PearsonMove rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 36 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 36 --------------------------- 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -69,6 +69,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir); +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
@@ -223,6 +223,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)return err; } +static int rxe_set_page (struct ib_mr *ibmr,u64 addr) +{ +struct rxe_mr *mr =to_rmr (ibmr); +struct rxe_map *map; +struct rxe_phys_buf *buf; + +if (unlikely (mr->nbuf == mr->num_buf)) +return -ENOMEM ; + + map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; + buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; + + buf->addr = addr; + buf->size = ibmr->page_size; + mr->nbuf++; + +return 0 ; +} + +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset) +{ +struct rxe_mr *mr =to_rmr (ibmr); +int n; + + mr->nbuf =0 ; + + n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + + mr->page_shift =ilog2 (ibmr->page_size); + mr->page_mask = ibmr->page_size -1 ; + mr->offset = ibmr->iova & mr->page_mask; + +return n; +} +static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,return ERR_PTR (err); } -static int rxe_set_page (struct ib_mr *ibmr,u64 addr) -{ -struct rxe_mr *mr =to_rmr (ibmr); -struct rxe_map *map; -struct rxe_phys_buf *buf; - -if (unlikely (mr->nbuf == mr->num_buf)) -return -ENOMEM ; - - map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; - mr->nbuf++; - -return 0 ; -} - -static int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, -int sg_nents,unsigned int *sg_offset) -{ -struct rxe_mr *mr =to_rmr (ibmr); -int n; - - mr->nbuf =0 ; - - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); - - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; - -return n; -} -static ssize_t parent_show (struct device *device,struct device_attribute *attr,char *buf) {
Signature · 1 line
2.37.2
PATCH v4 3/6 RDMA-rxe: Isolate mr code from atomic_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_mr.c | 83 ++++++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_resp.c | 49 +++++----------- drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 4 files changed, 76 insertions(+), 63 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -72,6 +72,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length); +int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +55 -28
111 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +14 -35
73 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
@@ -273,6 +273,11 @@ enum rxe_mr_state { RXE_MR_STATE_VALID, }; +/* some rxe specific error conditions */ +enum rxe_mr_error { +RXE_ERR_NOT_ALIGNED =0x1001 ,/* misaligned address */ +}; +enum rxe_mr_copy_dir {RXE_TO_MR_OBJ ,RXE_FROM_MR_OBJ ,
Signature · 1 line
2.37.2
PATCH v4 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 1 + drivers/infiniband/sw/rxe/rxe_mr.c | 34 ++++++++++++++ drivers/infiniband/sw/rxe/rxe_resp.c | 69 ++++++++++++---------------- 3 files changed, 64 insertions(+), 40 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val); +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +34 -0
@@ -565,6 +565,40 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,return 0 ; } +/* only implemented for 64 bit architectures */ +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value) +{ +#if definedCONFIG_64BIT +u64 *va; + +/* See IBA oA19-28 */ +if (unlikely (mr->state !=RXE_MR_STATE_VALID )) { +rxe_dbg_mr (mr,"mr not in valid state" ); +return -EINVAL ; + } + + va =iova_to_vaddr (mr, iova,sizeof (value)); +if (unlikely (!va)) { +rxe_dbg_mr (mr,"iova out of range" ); +return -ERANGE ; + } + +/* See IBA A19.4.2 */ +if (unlikely ((uintptr_t )va &0x7 || iova &0x7 )) { +rxe_dbg_mr (mr,"misaligned address" ); +return -RXE_ERR_NOT_ALIGNED ; + } + +/* Do atomic write after all prior operations have completed */ +smp_store_release (va, value); + +return 0 ; +#else +WARN_ON (1 ); +return -EINVAL ; +#endif +} +int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) {struct rxe_sge *sge = &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +29 -40
88 lines not on this page · raw message
Signature · 1 line
2.37.2
PATCH v4 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c
In reply to Bob PearsonCleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 31 ++++++++++++--------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++++++--- 2 files changed, 22 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +14 -17
@@ -62,6 +62,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) mr->lkey = mr->ibmr.lkey = lkey; mr->rkey = mr->ibmr.rkey = rkey; + mr->ibmr.page_size =PAGE_SIZE ; + mr->page_mask =PAGE_MASK ; + mr->page_shift =PAGE_SHIFT ; mr->state =RXE_MR_STATE_INVALID ; }
@@ -151,9 +154,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } - mr->page_shift =PAGE_SHIFT ; - mr->page_mask =PAGE_SIZE -1 ; - num_buf =0 ; map = mr->map;if (length >0 ) {
@@ -182,7 +182,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } buf->addr = (uintptr_t )vaddr; - buf->size =PAGE_SIZE ; + buf->size =mr_page_size (mr); num_buf++; buf++;
@@ -191,10 +191,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, mr->umem = umem; mr->access = access; - mr->offset =ib_umem_offset (umem); + mr->page_offset =ib_umem_offset (umem); mr->state =RXE_MR_STATE_VALID ; mr->ibmr.type =IB_MR_TYPE_USER ; - mr->ibmr.page_size =PAGE_SIZE ;return 0 ;
@@ -248,29 +247,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { struct rxe_mr *mr = to_rmr(ibmr); -int n; - - mr->nbuf =0 ; +unsigned int page_size =mr_page_size (mr); - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + mr->page_shift =ilog2 (page_size); + mr->page_mask = ~((u64 )page_size -1 ); + mr->page_offset = ibmr->iova & (page_size -1 ); - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; + mr->nbuf =0 ; -return n; +return ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); }static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) { -size_t offset = iova - mr->ibmr.iova + mr->offset; +size_t offset = iova - mr->ibmr.iova + mr->page_offset;int map_index;int buf_index;u64 length;if (likely (mr->page_shift)) { - *offset_out = offset & mr->page_mask; + *offset_out = offset & (mr_page_size (mr) -1 ); offset >>= mr->page_shift; *n_out = offset & mr->map_mask; *m_out = offset >> mr->map_shift;
@@ -329,7 +326,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)if (mr->ibmr.type ==IB_MR_TYPE_DMA )return -EFAULT ; - offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask; + offset = (iova - mr->ibmr.iova + mr->page_offset) & mr->page_mask;while (length >0 ) {u8 *va;int bytes;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
@@ -315,11 +315,11 @@ struct rxe_mr {u32 lkey;u32 rkey;enum rxe_mr_state state; -u32 offset;int access; -int page_shift; -int page_mask; +unsigned int page_offset; +unsigned int page_shift; +u64 page_mask;int map_shift;int map_mask;
@@ -334,6 +334,11 @@ struct rxe_mr {struct rxe_map **map; }; +static inline unsigned int mr_page_size (struct rxe_mr *mr) +{ +return mr ? mr->ibmr.page_size :PAGE_SIZE ; +} +enum rxe_mw_state {RXE_MW_STATE_INVALID =RXE_MR_STATE_INVALID ,RXE_MW_STATE_FREE =RXE_MR_STATE_FREE ,
Signature · 1 line
2.37.2
PATCH v4 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
In reply to Bob PearsonReplace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.
This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 8 +- drivers/infiniband/sw/rxe/rxe_mr.c | 544 ++++++++++++-------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 21 +- 3 files changed, 265 insertions(+), 308 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +4 -4
@@ -64,14 +64,14 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);int rxe_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova,int access,struct rxe_mr *mr);int rxe_mr_init_fast (int max_pages,struct rxe_mr *mr); -int rxe_flush_pmem_iova (struct rxe_mr *mr,u64 iova,int length); -int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length, -enum rxe_mr_copy_dir dir); +int rxe_flush_pmem_iova (struct rxe_mr *mr,u64 iova, +unsigned int length); +int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr, +unsigned int length,enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir);int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset); -void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val);int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value);
drivers/infiniband/sw/rxe/rxe_mr.c +259 -285
720 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
@@ -288,17 +288,6 @@ enum rxe_mr_lookup_type { RXE_LOOKUP_REMOTE, }; -#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) - -struct rxe_phys_buf { -u64 addr; -u64 size; -}; - -struct rxe_map { -struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; -}; -static inline int rkey_is_mw (u32 rkey) {u32 index = rkey >>8 ;
@@ -316,22 +305,16 @@ struct rxe_mr {u32 rkey;enum rxe_mr_state state;int access; +atomic_t num_mw;unsigned int page_offset;unsigned int page_shift;u64 page_mask; -int map_shift; -int map_mask;u32 num_buf;u32 nbuf; -u32 max_buf; -u32 num_map; - -atomic_t num_mw; - -struct rxe_map **map; +struct xarray page_list; };static inline unsigned int mr_page_size (struct rxe_mr *mr)
Signature · 1 line
2.37.2