RDMA/rxe: Replace mr page map with an xarray
Subsystems (2)
v7 cover RDMA/rxe: Replace mr page map with an xarray
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.
v7:
Made changes requested by Jason to return RESPST_ERR_XXX from rxe_mr.c to rxe_resp.c.
v6:
Backed out.
v5:
Responded to a note from [email protected] and restored calls to is_pmem_page() which were accidentally dropped in earlier versions.
v4:
Responded to a comment by Zhu and cleaned up error passing between rxe_mr.c and rxe_resp.c.
Other various cleanups including more careful use of unsigned ints. Rebased to current for-next.
v3:
Fixed an error reported by kernel test robot
v2:
Rebased to 6.2.0-rc1+
Minor cleanups
Fixed error reported by Jason in 4/6 missing if after else.
Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
drivers/infiniband/sw/rxe/rxe.h | 38 ++ drivers/infiniband/sw/rxe/rxe_loc.h | 12 +- drivers/infiniband/sw/rxe/rxe_mr.c | 605 ++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_resp.c | 143 ++---- drivers/infiniband/sw/rxe/rxe_verbs.c | 36 -- drivers/infiniband/sw/rxe/rxe_verbs.h | 32 +- 6 files changed, 425 insertions(+), 441 deletions(-)
base-commit: 1ec82317a1daac78c04b0c15af89018ccf9fa2b7
Signature · 1 line
2.37.2
PATCH v7 1/6 RDMA/rxe: Cleanup mr_check_range
Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
@@ -26,8 +26,6 @@ u8 rxe_get_next_key(u32 last_key)int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length) { - -switch (mr->ibmr.type) {case IB_MR_TYPE_DMA :return 0 ;
@@ -41,7 +39,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)default :rxe_dbg_mr (mr,"type (%d) not supported\n" , mr->ibmr.type); -return -EFAULT ; +return -EINVAL ; } }
Signature · 1 line
2.37.2
PATCH v7 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
In reply to Bob PearsonMove rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 36 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 36 --------------------------- 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -69,6 +69,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir); +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
@@ -223,6 +223,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)return err; } +static int rxe_set_page (struct ib_mr *ibmr,u64 addr) +{ +struct rxe_mr *mr =to_rmr (ibmr); +struct rxe_map *map; +struct rxe_phys_buf *buf; + +if (unlikely (mr->nbuf == mr->num_buf)) +return -ENOMEM ; + + map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; + buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; + + buf->addr = addr; + buf->size = ibmr->page_size; + mr->nbuf++; + +return 0 ; +} + +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset) +{ +struct rxe_mr *mr =to_rmr (ibmr); +int n; + + mr->nbuf =0 ; + + n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + + mr->page_shift =ilog2 (ibmr->page_size); + mr->page_mask = ibmr->page_size -1 ; + mr->offset = ibmr->iova & mr->page_mask; + +return n; +} +static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,return ERR_PTR (err); } -static int rxe_set_page (struct ib_mr *ibmr,u64 addr) -{ -struct rxe_mr *mr =to_rmr (ibmr); -struct rxe_map *map; -struct rxe_phys_buf *buf; - -if (unlikely (mr->nbuf == mr->num_buf)) -return -ENOMEM ; - - map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; - mr->nbuf++; - -return 0 ; -} - -static int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, -int sg_nents,unsigned int *sg_offset) -{ -struct rxe_mr *mr =to_rmr (ibmr); -int n; - - mr->nbuf =0 ; - - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); - - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; - -return n; -} -static ssize_t parent_show (struct device *device,struct device_attribute *attr,char *buf) {
Signature · 1 line
2.37.2
PATCH v7 3/6 RDMA-rxe: Isolate mr code from atomic_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.
Minor cleanups to rxe_check_range() and iova_to_vaddr(). Move enum resp_state to rxe.h
--- drivers/infiniband/sw/rxe/rxe.h | 38 +++++++++++++ drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_mr.c | 83 ++++++++++++++++++---------- drivers/infiniband/sw/rxe/rxe_resp.c | 82 ++++----------------------- 4 files changed, 105 insertions(+), 100 deletions(-)
drivers/infiniband/sw/rxe/rxe.h +38 -0
@@ -57,6 +57,44 @@#define rxe_dbg_mw (mw, fmt, ...) ibdev_dbg((mw)->ibmw.device, \ "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__) +/* responder states */ +enum resp_states { +RESPST_NONE , +RESPST_GET_REQ , +RESPST_CHK_PSN , +RESPST_CHK_OP_SEQ , +RESPST_CHK_OP_VALID , +RESPST_CHK_RESOURCE , +RESPST_CHK_LENGTH , +RESPST_CHK_RKEY , +RESPST_EXECUTE , +RESPST_READ_REPLY , +RESPST_ATOMIC_REPLY , +RESPST_ATOMIC_WRITE_REPLY , +RESPST_PROCESS_FLUSH , +RESPST_COMPLETE , +RESPST_ACKNOWLEDGE , +RESPST_CLEANUP , +RESPST_DUPLICATE_REQUEST , +RESPST_ERR_MALFORMED_WQE , +RESPST_ERR_UNSUPPORTED_OPCODE , +RESPST_ERR_MISALIGNED_ATOMIC , +RESPST_ERR_PSN_OUT_OF_SEQ , +RESPST_ERR_MISSING_OPCODE_FIRST , +RESPST_ERR_MISSING_OPCODE_LAST_C , +RESPST_ERR_MISSING_OPCODE_LAST_D1E , +RESPST_ERR_TOO_MANY_RDMA_ATM_REQ , +RESPST_ERR_RNR , +RESPST_ERR_RKEY_VIOLATION , +RESPST_ERR_INVALIDATE_RKEY , +RESPST_ERR_LENGTH , +RESPST_ERR_CQ_OVERFLOW , +RESPST_ERROR , +RESPST_RESET , +RESPST_DONE , +RESPST_EXIT , +}; +void rxe_set_mtu (struct rxe_dev *rxe,unsigned int dev_mtu);int rxe_add (struct rxe_dev *rxe,unsigned int mtu,const char *ibdev_name);
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -72,6 +72,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length); +int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +55 -28
111 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +10 -72
111 lines not on this page · raw message
Signature · 1 line
2.37.2
PATCH v7 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 38 ++++++++++++++++- drivers/infiniband/sw/rxe/rxe_resp.c | 61 ++++++++++------------------ 3 files changed, 59 insertions(+), 42 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
@@ -71,9 +71,9 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, void *addr, int length, enum rxe_mr_copy_dir dir);int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset); -void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val); +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +37 -1
@@ -297,7 +297,7 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, } } -void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length) +static void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length) {size_t offset;int m, n;
@@ -565,6 +565,42 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,return 0 ; } +/* only implemented for 64 bit architectures */ +#if definedCONFIG_64BIT +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value) +{ +u64 *va; + +/* See IBA oA19-28 */ +if (unlikely (mr->state !=RXE_MR_STATE_VALID )) { +rxe_dbg_mr (mr,"mr not in valid state" ); +return RESPST_ERR_RKEY_VIOLATION ; + } + + va =iova_to_vaddr (mr, iova,sizeof (value)); +if (unlikely (!va)) { +rxe_dbg_mr (mr,"iova out of range" ); +return RESPST_ERR_RKEY_VIOLATION ; + } + +/* See IBA A19.4.2 */ +if (unlikely ((uintptr_t )va &0x7 || iova &0x7 )) { +rxe_dbg_mr (mr,"misaligned address" ); +return RESPST_ERR_MISALIGNED_ATOMIC ; + } + +/* Do atomic write after all prior operations have completed */ +smp_store_release (va, value); + +return 0 ; +} +#else +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value) +{ +return RESPST_ERR_UNSUPPORTED_OPCODE ; +} +#endif +int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) {struct rxe_sge *sge = &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +21 -40
80 lines not on this page · raw message
Signature · 1 line
2.37.2
PATCH v7 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c
In reply to Bob PearsonCleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 31 ++++++++++++--------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++++++--- 2 files changed, 22 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +14 -17
@@ -62,6 +62,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) mr->lkey = mr->ibmr.lkey = lkey; mr->rkey = mr->ibmr.rkey = rkey; + mr->ibmr.page_size =PAGE_SIZE ; + mr->page_mask =PAGE_MASK ; + mr->page_shift =PAGE_SHIFT ; mr->state =RXE_MR_STATE_INVALID ; }
@@ -151,9 +154,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } - mr->page_shift =PAGE_SHIFT ; - mr->page_mask =PAGE_SIZE -1 ; - num_buf =0 ; map = mr->map;if (length >0 ) {
@@ -182,7 +182,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } buf->addr = (uintptr_t )vaddr; - buf->size =PAGE_SIZE ; + buf->size =mr_page_size (mr); num_buf++; buf++;
@@ -191,10 +191,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, mr->umem = umem; mr->access = access; - mr->offset =ib_umem_offset (umem); + mr->page_offset =ib_umem_offset (umem); mr->state =RXE_MR_STATE_VALID ; mr->ibmr.type =IB_MR_TYPE_USER ; - mr->ibmr.page_size =PAGE_SIZE ;return 0 ;
@@ -248,29 +247,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { struct rxe_mr *mr = to_rmr(ibmr); -int n; - - mr->nbuf =0 ; +unsigned int page_size =mr_page_size (mr); - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + mr->page_shift =ilog2 (page_size); + mr->page_mask = ~((u64 )page_size -1 ); + mr->page_offset = ibmr->iova & (page_size -1 ); - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; + mr->nbuf =0 ; -return n; +return ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); }static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) { -size_t offset = iova - mr->ibmr.iova + mr->offset; +size_t offset = iova - mr->ibmr.iova + mr->page_offset;int map_index;int buf_index;u64 length;if (likely (mr->page_shift)) { - *offset_out = offset & mr->page_mask; + *offset_out = offset & (mr_page_size (mr) -1 ); offset >>= mr->page_shift; *n_out = offset & mr->map_mask; *m_out = offset >> mr->map_shift;
@@ -329,7 +326,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)if (mr->ibmr.type ==IB_MR_TYPE_DMA )return -EFAULT ; - offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask; + offset = (iova - mr->ibmr.iova + mr->page_offset) & mr->page_mask;while (length >0 ) {u8 *va;int bytes;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
@@ -310,11 +310,11 @@ struct rxe_mr {u32 lkey;u32 rkey;enum rxe_mr_state state; -u32 offset;int access; -int page_shift; -int page_mask; +unsigned int page_offset; +unsigned int page_shift; +u64 page_mask;int map_shift;int map_mask;
@@ -329,6 +329,11 @@ struct rxe_mr {struct rxe_map **map; }; +static inline unsigned int mr_page_size (struct rxe_mr *mr) +{ +return mr ? mr->ibmr.page_size :PAGE_SIZE ; +} +enum rxe_mw_state {RXE_MW_STATE_INVALID =RXE_MR_STATE_INVALID ,RXE_MW_STATE_FREE =RXE_MR_STATE_FREE ,
Signature · 1 line
2.37.2
PATCH v7 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
In reply to Bob PearsonReplace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.
This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 6 +- drivers/infiniband/sw/rxe/rxe_mr.c | 537 ++++++++++++-------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 21 +- 3 files changed, 262 insertions(+), 302 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +3 -3
@@ -64,9 +64,9 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);int rxe_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova,int access,struct rxe_mr *mr);int rxe_mr_init_fast (int max_pages,struct rxe_mr *mr); -int rxe_flush_pmem_iova (struct rxe_mr *mr,u64 iova,int length); -int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length, -enum rxe_mr_copy_dir dir); +int rxe_flush_pmem_iova (struct rxe_mr *mr,u64 iova,unsigned int length); +int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr, +unsigned int length,enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir); int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
drivers/infiniband/sw/rxe/rxe_mr.c +257 -280
716 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
@@ -283,17 +283,6 @@ enum rxe_mr_lookup_type { RXE_LOOKUP_REMOTE, }; -#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) - -struct rxe_phys_buf { -u64 addr; -u64 size; -}; - -struct rxe_map { -struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; -}; -static inline int rkey_is_mw (u32 rkey) {u32 index = rkey >>8 ;
@@ -311,22 +300,16 @@ struct rxe_mr {u32 rkey;enum rxe_mr_state state;int access; +atomic_t num_mw;unsigned int page_offset;unsigned int page_shift;u64 page_mask; -int map_shift; -int map_mask;u32 num_buf;u32 nbuf; -u32 max_buf; -u32 num_map; - -atomic_t num_mw; - -struct rxe_map **map; +struct xarray page_list; };static inline unsigned int mr_page_size (struct rxe_mr *mr)
Signature · 1 line
2.37.2
Jason Gunthorpe
maintainerQuoted 10 lines · Bob Pearson
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into
rxe_mr.c. The sixth patch is the actual change.v7:
Made changes requested by Jason to return
RESPST_ERR_XXXfromrxe_mr.ctorxe_resp.c.
I took it to for-next, but I made these changes, please check:
drivers/infiniband/sw/rxe/rxe_mr.c +4 -12
@@ -114,7 +114,8 @@ static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)if (persistent && !is_pmem_page (page)) {rxe_dbg_mr (mr,"Page can't be persistent\n" ); -return -EINVAL ; +xas_set_err (&xas, -EINVAL ); +break ; }xas_store (&xas, page);
@@ -213,7 +214,6 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova) {struct rxe_mr *mr =to_rmr (ibmr);struct page *page =virt_to_page (iova & mr->page_mask); -XA_STATE (xas, &mr->page_list, mr->nbuf);bool persistent = !!(mr->access &IB_ACCESS_FLUSH_PERSISTENT );int err;
@@ -225,13 +225,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)if (unlikely (mr->nbuf == mr->num_buf))return -ENOMEM ; -do { -xas_lock (&xas); -xas_store (&xas, page); -xas_unlock (&xas); - }while (xas_nomem (&xas,GFP_KERNEL )); - - err =xas_error (&xas); + err =xa_err (xa_store (&mr->page_list, mr->nbuf, page,GFP_KERNEL ));if (err)return err;
@@ -458,10 +452,8 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) mr_page_size(mr) - page_offset); va =kmap_local_page (page); -if (!va) -return -EFAULT ; -arch_wb_cache_pmem (va + page_offset, bytes); +kunmap_local (va); length -= bytes; iova += bytes;
Quoted 53 lines · Jason Gunthorpe
Quoted 10 lines · Bob Pearson
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into
rxe_mr.c. The sixth patch is the actual change.v7:
Made changes requested by Jason to return
RESPST_ERR_XXXfromrxe_mr.ctorxe_resp.c.I took it to for-next, but I made these changes, please check:
drivers/infiniband/sw/rxe/rxe_mr.c +3
-9@@ -114,7 +114,8 @@ static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)if (persistent && !is_pmem_page (page)) {rxe_dbg_mr (mr,"Page can't be persistent\n" ); -return -EINVAL ; +xas_set_err (&xas, -EINVAL ); +break ; }xas_store (&xas, page);@@ -213,7 +214,6 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova) {struct rxe_mr *mr =to_rmr (ibmr);struct page *page =virt_to_page (iova & mr->page_mask); -XA_STATE (xas, &mr->page_list, mr->nbuf);bool persistent = !!(mr->access &IB_ACCESS_FLUSH_PERSISTENT );int err;@@ -225,13 +225,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)if (unlikely (mr->nbuf == mr->num_buf))return -ENOMEM ; -do { -xas_lock (&xas); -xas_store (&xas, page); -xas_unlock (&xas); - }while (xas_nomem (&xas,GFP_KERNEL )); - - err =xas_error (&xas); + err =xa_err (xa_store (&mr->page_list, mr->nbuf, page,GFP_KERNEL ));if (err)return err;@@ -458,10 +452,8 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) mr_page_size(mr) - page_offset);
drivers/infiniband/sw/rxe/rxe_mr.c int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) · line 459
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +1
-3va =kmap_local_page (page); -if (!va) -return -EFAULT ; -arch_wb_cache_pmem (va + page_offset, bytes); +kunmap_local (va); length -= bytes; iova += bytes;
Thanks. I'll check these.
Bob
Quoted 19 lines · Jason Gunthorpe
Quoted 10 lines · Bob Pearson
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into
rxe_mr.c. The sixth patch is the actual change.v7:
Made changes requested by Jason to return
RESPST_ERR_XXXfromrxe_mr.ctorxe_resp.c.I took it to for-next, but I made these changes, please check:
drivers/infiniband/sw/rxe/rxe_mr.c +0
-0
drivers/infiniband/sw/rxe/rxe_mr.c static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) · line 121
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_mr.c +2
-1@@ -114,7 +114,8 @@ static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)if (persistent && !is_pmem_page (page)) {rxe_dbg_mr (mr,"Page can't be persistent\n" ); -return -EINVAL ; +xas_set_err (&xas, -EINVAL ); +break ; }xas_store (&xas, page);
Looks good. Good catch.
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +0
-1@@ -213,7 +214,6 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova) {struct rxe_mr *mr =to_rmr (ibmr);struct page *page =virt_to_page (iova & mr->page_mask); -XA_STATE (xas, &mr->page_list, mr->nbuf);bool persistent = !!(mr->access &IB_ACCESS_FLUSH_PERSISTENT );int err;@@ -225,13 +225,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)if (unlikely (mr->nbuf == mr->num_buf))
drivers/infiniband/sw/rxe/rxe_mr.c static int rxe_set_page(struct ib_mr *ibmr, u64 iova) · line 228
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +1
-7return -ENOMEM ; -do { -xas_lock (&xas); -xas_store (&xas, page); -xas_unlock (&xas); - }while (xas_nomem (&xas,GFP_KERNEL )); - - err =xas_error (&xas); + err =xa_err (xa_store (&mr->page_list, mr->nbuf, page,GFP_KERNEL ));
Looks good.
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_mr.c +0
-0if (err)return err;@@ -458,10 +452,8 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) mr_page_size(mr) - page_offset);
drivers/infiniband/sw/rxe/rxe_mr.c int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) · line 459
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +1
-3va =kmap_local_page (page); -if (!va) -return -EFAULT ; -arch_wb_cache_pmem (va + page_offset, bytes); +kunmap_local (va); length -= bytes; iova += bytes;
Looks good. Good catch. I take it kmap_local_page shouldn't fail.
Thanks,
Bob