RDMA/rxe: Replace mr page map with an xarray
Subsystems (2)
v5 cover RDMA/rxe: Replace mr page map with an xarray
This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.
v5:
Responded to a note from [email protected] and restored calls to is_pmem_page() which were accidentally dropped in earlier versions.
v4:
Responded to a comment by Zhu and cleaned up error passing between rxe_mr.c and rxe_resp.c.
Other various cleanups including more careful use of unsigned ints. Rebased to current for-next.
v3:
Fixed an error reported by kernel test robot
v2:
Rebased to 6.2.0-rc1+
Minor cleanups
Fixed error reported by Jason in 4/6 missing if after else.
Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
drivers/infiniband/sw/rxe/rxe_loc.h | 13 +- drivers/infiniband/sw/rxe/rxe_mr.c | 625 +++++++++++++++----------- drivers/infiniband/sw/rxe/rxe_resp.c | 118 ++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 36 -- drivers/infiniband/sw/rxe/rxe_verbs.h | 37 +- 5 files changed, 425 insertions(+), 404 deletions(-)
base-commit: 1ec82317a1daac78c04b0c15af89018ccf9fa2b7
Signature · 1 line
2.37.2
PATCH v5 1/6 RDMA/rxe: Cleanup mr_check_range
Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
@@ -26,8 +26,6 @@ u8 rxe_get_next_key(u32 last_key)int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length) { - -switch (mr->ibmr.type) {case IB_MR_TYPE_DMA :return 0 ;
@@ -41,7 +39,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)default :rxe_dbg_mr (mr,"type (%d) not supported\n" , mr->ibmr.type); -return -EFAULT ; +return -EINVAL ; } }
Signature · 1 line
2.37.2
PATCH v5 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
In reply to Bob PearsonMove rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 36 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 36 --------------------------- 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -69,6 +69,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir); +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
@@ -223,6 +223,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)return err; } +static int rxe_set_page (struct ib_mr *ibmr,u64 addr) +{ +struct rxe_mr *mr =to_rmr (ibmr); +struct rxe_map *map; +struct rxe_phys_buf *buf; + +if (unlikely (mr->nbuf == mr->num_buf)) +return -ENOMEM ; + + map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; + buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; + + buf->addr = addr; + buf->size = ibmr->page_size; + mr->nbuf++; + +return 0 ; +} + +int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, +int sg_nents,unsigned int *sg_offset) +{ +struct rxe_mr *mr =to_rmr (ibmr); +int n; + + mr->nbuf =0 ; + + n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + + mr->page_shift =ilog2 (ibmr->page_size); + mr->page_mask = ibmr->page_size -1 ; + mr->offset = ibmr->iova & mr->page_mask; + +return n; +} +static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,return ERR_PTR (err); } -static int rxe_set_page (struct ib_mr *ibmr,u64 addr) -{ -struct rxe_mr *mr =to_rmr (ibmr); -struct rxe_map *map; -struct rxe_phys_buf *buf; - -if (unlikely (mr->nbuf == mr->num_buf)) -return -ENOMEM ; - - map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; - mr->nbuf++; - -return 0 ; -} - -static int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg, -int sg_nents,unsigned int *sg_offset) -{ -struct rxe_mr *mr =to_rmr (ibmr); -int n; - - mr->nbuf =0 ; - - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); - - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; - -return n; -} -static ssize_t parent_show (struct device *device,struct device_attribute *attr,char *buf) {
Signature · 1 line
2.37.2
PATCH v5 3/6 RDMA-rxe: Isolate mr code from atomic_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 + drivers/infiniband/sw/rxe/rxe_mr.c | 83 ++++++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_resp.c | 49 +++++----------- drivers/infiniband/sw/rxe/rxe_verbs.h | 5 ++ 4 files changed, 76 insertions(+), 63 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -72,6 +72,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset);void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length); +int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +55 -28
111 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +14 -35
73 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
@@ -273,6 +273,11 @@ enum rxe_mr_state { RXE_MR_STATE_VALID, }; +/* some rxe specific error conditions */ +enum rxe_mr_error { +RXE_ERR_NOT_ALIGNED =0x1001 ,/* misaligned address */ +}; +enum rxe_mr_copy_dir {RXE_TO_MR_OBJ ,RXE_FROM_MR_OBJ ,
Signature · 1 line
2.37.2
Jason Gunthorpe
maintainerRe: [PATCH for-next v5 3/6] RDMA-rxe: Isolate mr code from atomic_reply()
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 751
Quoted 9 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_resp.c +9
-0+ err =rxe_mr_do_atomic_op (mr, iova, pkt->opcode, +atmeth_comp (pkt), +atmeth_swap_add (pkt), + &res->atomic.orig_val); +if (unlikely (err)) { +if (err == -RXE_ERR_NOT_ALIGNED ) +return RESPST_ERR_MISALIGNED_ATOMIC ; +else +return RESPST_ERR_RKEY_VIOLATION ;
Why not just return these RESPST_ constants directly from rxe_mr_do_atomic_op ?
Jason
Re: [PATCH for-next v5 3/6] RDMA-rxe: Isolate mr code from atomic_reply()
Quoted 15 lines · Jason Gunthorpe
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 751
Quoted 9 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_resp.c +9
-0+ err =rxe_mr_do_atomic_op (mr, iova, pkt->opcode, +atmeth_comp (pkt), +atmeth_swap_add (pkt), + &res->atomic.orig_val); +if (unlikely (err)) { +if (err == -RXE_ERR_NOT_ALIGNED ) +return RESPST_ERR_MISALIGNED_ATOMIC ; +else +return RESPST_ERR_RKEY_VIOLATION ;Why not just return these
RESPST_constants directly fromrxe_mr_do_atomic_op?
Quoted 1 line
Jason
The main reason is that the responder state machine is fairly complicated and hiding bits of it in other files just makes it more difficult to follow. Originally all the code was inline here but I felt it would be better to keep all the mr specific detail in one file.
Bob
PATCH v5 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()
In reply to Bob PearsonIsolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 1 + drivers/infiniband/sw/rxe/rxe_mr.c | 34 ++++++++++++++ drivers/infiniband/sw/rxe/rxe_resp.c | 69 ++++++++++++---------------- 3 files changed, 64 insertions(+), 40 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val); +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +34 -0
@@ -565,6 +565,40 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,return 0 ; } +/* only implemented for 64 bit architectures */ +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value) +{ +#if definedCONFIG_64BIT +u64 *va; + +/* See IBA oA19-28 */ +if (unlikely (mr->state !=RXE_MR_STATE_VALID )) { +rxe_dbg_mr (mr,"mr not in valid state" ); +return -EINVAL ; + } + + va =iova_to_vaddr (mr, iova,sizeof (value)); +if (unlikely (!va)) { +rxe_dbg_mr (mr,"iova out of range" ); +return -ERANGE ; + } + +/* See IBA A19.4.2 */ +if (unlikely ((uintptr_t )va &0x7 || iova &0x7 )) { +rxe_dbg_mr (mr,"misaligned address" ); +return -RXE_ERR_NOT_ALIGNED ; + } + +/* Do atomic write after all prior operations have completed */ +smp_store_release (va, value); + +return 0 ; +#else +WARN_ON (1 ); +return -EINVAL ; +#endif +} +int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) {struct rxe_sge *sge = &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +29 -40
88 lines not on this page · raw message
Signature · 1 line
2.37.2
Jason Gunthorpe
maintainerRe: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 571
Quoted 4 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_mr.c +4
-0+/* only implemented for 64 bit architectures */ +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value) +{ +#if definedCONFIG_64BIT
#ifdef
It is a little more typical style to provide an alternate version of the function when #ifdefing
Quoted 71 lines
drivers/infiniband/sw/rxe/rxe_mr.c +30
-0+u64 *va; + +/* See IBA oA19-28 */ +if (unlikely (mr->state !=RXE_MR_STATE_VALID )) { +rxe_dbg_mr (mr,"mr not in valid state" ); +return -EINVAL ; + } + + va =iova_to_vaddr (mr, iova,sizeof (value)); +if (unlikely (!va)) { +rxe_dbg_mr (mr,"iova out of range" ); +return -ERANGE ; + } + +/* See IBA A19.4.2 */ +if (unlikely ((uintptr_t )va &0x7 || iova &0x7 )) { +rxe_dbg_mr (mr,"misaligned address" ); +return -RXE_ERR_NOT_ALIGNED ; + } + +/* Do atomic write after all prior operations have completed */ +smp_store_release (va, value); + +return 0 ; +#else +WARN_ON (1 ); +return -EINVAL ; +#endif +} +int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) {struct rxe_sge *sge = &dma->sge[dma->cur_sge];drivers/infiniband/sw/rxe/rxe_resp.c +13
-14@@ -764,30 +764,40 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,return RESPST_ACKNOWLEDGE ; } -#ifdef CONFIG_64BIT -static enum resp_states do_atomic_write (struct rxe_qp *qp, -struct rxe_pkt_info *pkt) +static enum resp_states atomic_write_reply (struct rxe_qp *qp, +struct rxe_pkt_info *pkt) { -struct rxe_mr *mr = qp->resp.mr; -int payload =payload_size (pkt); -u64 src, *dst; - -if (mr->state !=RXE_MR_STATE_VALID ) -return RESPST_ERR_RKEY_VIOLATION ; +struct resp_res *res = qp->resp.res; +struct rxe_mr *mr; +u64 value; +u64 iova; +int err; -memcpy (&src,payload_addr (pkt), payload); +if (!res) { + res =rxe_prepare_res (qp, pkt,RXE_ATOMIC_WRITE_MASK ); + qp->resp.res = res; + } - dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) -return RESPST_ERR_MISALIGNED_ATOMIC ; +if (res->replay) +return RESPST_ACKNOWLEDGE ;
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_resp.c +4
-4-/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr (pkt); + iova = qp->resp.va + qp->resp.offset; -/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); +#if definedCONFIG_64BIT
Shouldn't need a #ifdef here
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 794
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c +6
-0+ err =rxe_mr_do_atomic_write (mr, iova, value); +if (unlikely (err)) { +if (err == -RXE_ERR_NOT_ALIGNED ) +return RESPST_ERR_MISALIGNED_ATOMIC ; +else +return RESPST_ERR_RKEY_VIOLATION ;
Again why not return the RESPST directly then the stub can return RESPST_ERR_UNSUPPORTED_OPCODE?
Jason
Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
Quoted 11 lines · Jason Gunthorpe
drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 571
Quoted 4 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_mr.c +4
-0+/* only implemented for 64 bit architectures */ +int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value) +{ +#if definedCONFIG_64BIT #ifdef
It is a little more typical style to provide an alternate version of the function when #ifdefing
I will do that.
Quoted 79 lines
Quoted 79 lines
drivers/infiniband/sw/rxe/rxe_mr.c +30
-0+u64 *va; + +/* See IBA oA19-28 */ +if (unlikely (mr->state !=RXE_MR_STATE_VALID )) { +rxe_dbg_mr (mr,"mr not in valid state" ); +return -EINVAL ; + } + + va =iova_to_vaddr (mr, iova,sizeof (value)); +if (unlikely (!va)) { +rxe_dbg_mr (mr,"iova out of range" ); +return -ERANGE ; + } + +/* See IBA A19.4.2 */ +if (unlikely ((uintptr_t )va &0x7 || iova &0x7 )) { +rxe_dbg_mr (mr,"misaligned address" ); +return -RXE_ERR_NOT_ALIGNED ; + } + +/* Do atomic write after all prior operations have completed */ +smp_store_release (va, value); + +return 0 ; +#else +WARN_ON (1 ); +return -EINVAL ; +#endif +} +int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) {struct rxe_sge *sge = &dma->sge[dma->cur_sge];drivers/infiniband/sw/rxe/rxe_resp.c +16
-16@@ -764,30 +764,40 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,return RESPST_ACKNOWLEDGE ; } -#ifdef CONFIG_64BIT -static enum resp_states do_atomic_write (struct rxe_qp *qp, -struct rxe_pkt_info *pkt) +static enum resp_states atomic_write_reply (struct rxe_qp *qp, +struct rxe_pkt_info *pkt) { -struct rxe_mr *mr = qp->resp.mr; -int payload =payload_size (pkt); -u64 src, *dst; - -if (mr->state !=RXE_MR_STATE_VALID ) -return RESPST_ERR_RKEY_VIOLATION ; +struct resp_res *res = qp->resp.res; +struct rxe_mr *mr; +u64 value; +u64 iova; +int err; -memcpy (&src,payload_addr (pkt), payload); +if (!res) { + res =rxe_prepare_res (qp, pkt,RXE_ATOMIC_WRITE_MASK ); + qp->resp.res = res; + } - dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) -return RESPST_ERR_MISALIGNED_ATOMIC ; +if (res->replay) +return RESPST_ACKNOWLEDGE ; -/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr (pkt); + iova = qp->resp.va + qp->resp.offset;
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788
Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-2-/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); +#if definedCONFIG_64BIT Shouldn't need a #ifdef here
This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.
I really don't understand why Fujitsu did it all this way instead of just using a spinlock for 32 bit architectures as a fallback. But if I want to keep to the spirit of their implementation this is fairly clear I think.
Bob
Quoted 12 lines
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 794
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_resp.c +6
-0+ err =rxe_mr_do_atomic_write (mr, iova, value); +if (unlikely (err)) { +if (err == -RXE_ERR_NOT_ALIGNED ) +return RESPST_ERR_MISALIGNED_ATOMIC ; +else +return RESPST_ERR_RKEY_VIOLATION ;Again why not return the RESPST directly then the stub can return
RESPST_ERR_UNSUPPORTED_OPCODE?Jason
Jason Gunthorpe
maintainerRe: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
Quoted 13 lines · Bob Pearson
Quoted 13 lines
Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_resp.c +5
-6- dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) -return RESPST_ERR_MISALIGNED_ATOMIC ; +if (res->replay) +return RESPST_ACKNOWLEDGE ; -/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr (pkt); + iova = qp->resp.va + qp->resp.offset;
Quoted 8 lines
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788
Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-2-/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); +#if definedCONFIG_64BIT Shouldn't need a #ifdef here
This avoids a new special error (i.e.
NOT_64_bit) and makes it clear we won't call the code in mr.
? That doesn't seem right
Quoted 3 lines
I really don't understand why Fujitsu did it all this way instead of just using a spinlock for 32 bit architectures as a fallback. But if I want to keep to the spirit of their implementation this is fairly clear I think.
IIRC the IBA definition is that this is supposed to be coherent with the host CPU, the spinlock version is for the non-atomic atomics which only has to be coherent with the "hca"
So a spinlock will not provide coherency with userspace that may be touching this same atomic memory.
Jason
Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
Quoted 21 lines · Jason Gunthorpe
Quoted 19 lines · Bob Pearson
Quoted 18 lines
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c +2
-4- dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) -return RESPST_ERR_MISALIGNED_ATOMIC ; +if (res->replay) +return RESPST_ACKNOWLEDGE ;drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_resp.c +4
-4-/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr (pkt); + iova = qp->resp.va + qp->resp.offset; -/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); +#if definedCONFIG_64BIT Shouldn't need a #ifdef here
Quoted 4 lines
Quoted 2 lines
This avoids a new special error (i.e.
NOT_64_bit) and makes it clear we won't call the code in mr.? That doesn't seem right
that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.
Quoted 11 lines
Quoted 4 lines
I really don't understand why Fujitsu did it all this way instead of just using a spinlock for 32 bit architectures as a fallback. But if I want to keep to the spirit of their implementation this is fairly clear I think.
IIRC the IBA definition is that this is supposed to be coherent with the host CPU, the spinlock version is for the non-atomic atomics which only has to be coherent with the "hca"
So a spinlock will not provide coherency with userspace that may be touching this same atomic memory.
Thanks. That makes sense.
Bob
Quoted 2 lines
Jason
Jason Gunthorpe
maintainerRe: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
Quoted 22 lines · Bob Pearson
Quoted 21 lines · Jason Gunthorpe
Quoted 19 lines · Bob Pearson
Quoted 18 lines
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c +2
-4- dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) -return RESPST_ERR_MISALIGNED_ATOMIC ; +if (res->replay) +return RESPST_ACKNOWLEDGE ;drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_resp.c +4
-4-/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr (pkt); + iova = qp->resp.va + qp->resp.offset; -/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); +#if definedCONFIG_64BIT Shouldn't need a #ifdef here
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
This avoids a new special error (i.e.
NOT_64_bit) and makes it clear we won't call the code in mr.? That doesn't seem right
that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.
I feel like it should be solvable without this ifdef though
Jason
Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
Quoted 28 lines · Jason Gunthorpe
Quoted 27 lines · Bob Pearson
Quoted 25 lines · Jason Gunthorpe
Quoted 13 lines · Bob Pearson
Quoted 13 lines
Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_resp.c +5
-6- dst =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, payload); -/* check vaddr is 8 bytes aligned. */ -if (!dst || (uintptr_t )dst &7 ) -return RESPST_ERR_MISALIGNED_ATOMIC ; +if (res->replay) +return RESPST_ACKNOWLEDGE ; -/* Do atomic write after all prior operations have completed */ -smp_store_release (dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr (pkt); + iova = qp->resp.va + qp->resp.offset;Quoted 8 lines
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788
Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-2-/* decrease resp.resid to zero */ - qp->resp.resid -=sizeof (payload); +#if definedCONFIG_64BIT Shouldn't need a #ifdef here
This avoids a new special error (i.e.
NOT_64_bit) and makes it clear we won't call the code in mr.? That doesn't seem right
Quoted 7 lines
Quoted 3 lines
that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.
I feel like it should be solvable without this ifdef though
Jason
You could get rid of the ifdef in the atomic_write_reply() routine but then the rxe_mr_do_atomic_write() routine would have to have a second version in the #else case that would have to return something different so that the third exit could be taken i.e.
whatever replaces the original -3. I really think this is simpler.
Bob
Jason Gunthorpe
maintainerRe: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()
Quoted 33 lines · Bob Pearson
Quoted 32 lines · Jason Gunthorpe
Quoted 30 lines · Bob Pearson
Quoted 21 lines · Jason Gunthorpe
Quoted 19 lines · Bob Pearson
Quoted 18 lines
Quoted 6 lines
(quoted diff) +2
-4- dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload); - /* check vaddr is 8 bytes aligned. */ - if (!dst || (uintptr_t)dst & 7) - return RESPST_ERR_MISALIGNED_ATOMIC; + if (res->replay) + return RESPST_ACKNOWLEDGE;Review
Quoted 10 lines
(quoted diff) +4
-4- /* Do atomic write after all prior operations have completed */ - smp_store_release(dst, src); + mr = qp->resp.mr; + value = *(u64 *)payload_addr(pkt); + iova = qp->resp.va + qp->resp.offset; - /* decrease resp.resid to zero */ - qp->resp.resid -= sizeof(payload); +#if defined CONFIG_64BITShouldn't need a #ifdef here
Quoted 4 lines
Quoted 2 lines
This avoids a new special error (i.e.
NOT_64_bit) and makes it clear we won't call the code in mr.? That doesn't seem right
that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.
Quoted 7 lines
Quoted 3 lines
I feel like it should be solvable without this ifdef though
Jason
You could get rid of the ifdef in the
atomic_write_reply()routine but then therxe_mr_do_atomic_write()routine would have to have a second version in the #else case that would have to return something different so that the third exit could be taken i.e.
whatever replaces the original -3. I really think this is simpler.
You really should just return the RESPST_* from these functions.
Jason
PATCH v5 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c
In reply to Bob PearsonCleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 31 ++++++++++++--------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++++++--- 2 files changed, 22 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +14 -17
@@ -62,6 +62,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) mr->lkey = mr->ibmr.lkey = lkey; mr->rkey = mr->ibmr.rkey = rkey; + mr->ibmr.page_size =PAGE_SIZE ; + mr->page_mask =PAGE_MASK ; + mr->page_shift =PAGE_SHIFT ; mr->state =RXE_MR_STATE_INVALID ; }
@@ -151,9 +154,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } - mr->page_shift =PAGE_SHIFT ; - mr->page_mask =PAGE_SIZE -1 ; - num_buf =0 ; map = mr->map;if (length >0 ) {
@@ -182,7 +182,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_release_umem; } buf->addr = (uintptr_t )vaddr; - buf->size =PAGE_SIZE ; + buf->size =mr_page_size (mr); num_buf++; buf++;
@@ -191,10 +191,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, mr->umem = umem; mr->access = access; - mr->offset =ib_umem_offset (umem); + mr->page_offset =ib_umem_offset (umem); mr->state =RXE_MR_STATE_VALID ; mr->ibmr.type =IB_MR_TYPE_USER ; - mr->ibmr.page_size =PAGE_SIZE ;return 0 ;
@@ -248,29 +247,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { struct rxe_mr *mr = to_rmr(ibmr); -int n; - - mr->nbuf =0 ; +unsigned int page_size =mr_page_size (mr); - n =ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); + mr->page_shift =ilog2 (page_size); + mr->page_mask = ~((u64 )page_size -1 ); + mr->page_offset = ibmr->iova & (page_size -1 ); - mr->page_shift =ilog2 (ibmr->page_size); - mr->page_mask = ibmr->page_size -1 ; - mr->offset = ibmr->iova & mr->page_mask; + mr->nbuf =0 ; -return n; +return ib_sg_to_pages (ibmr, sg, sg_nents, sg_offset, rxe_set_page); }static void lookup_iova (struct rxe_mr *mr,u64 iova,int *m_out,int *n_out,size_t *offset_out) { -size_t offset = iova - mr->ibmr.iova + mr->offset; +size_t offset = iova - mr->ibmr.iova + mr->page_offset;int map_index;int buf_index;u64 length;if (likely (mr->page_shift)) { - *offset_out = offset & mr->page_mask; + *offset_out = offset & (mr_page_size (mr) -1 ); offset >>= mr->page_shift; *n_out = offset & mr->map_mask; *m_out = offset >> mr->map_shift;
@@ -329,7 +326,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)if (mr->ibmr.type ==IB_MR_TYPE_DMA )return -EFAULT ; - offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask; + offset = (iova - mr->ibmr.iova + mr->page_offset) & mr->page_mask;while (length >0 ) {u8 *va;int bytes;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
@@ -315,11 +315,11 @@ struct rxe_mr {u32 lkey;u32 rkey;enum rxe_mr_state state; -u32 offset;int access; -int page_shift; -int page_mask; +unsigned int page_offset; +unsigned int page_shift; +u64 page_mask;int map_shift;int map_mask;
@@ -334,6 +334,11 @@ struct rxe_mr {struct rxe_map **map; }; +static inline unsigned int mr_page_size (struct rxe_mr *mr) +{ +return mr ? mr->ibmr.page_size :PAGE_SIZE ; +} +enum rxe_mw_state {RXE_MW_STATE_INVALID =RXE_MR_STATE_INVALID ,RXE_MW_STATE_FREE =RXE_MR_STATE_FREE ,
Signature · 1 line
2.37.2
PATCH v5 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray
In reply to Bob PearsonReplace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.
This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.
---
v5:
Responded to a note from [email protected] and restored calls to is_pmem_page() which were accidentally dropped in earlier versions.
drivers/infiniband/sw/rxe/rxe_loc.h | 8 +- drivers/infiniband/sw/rxe/rxe_mr.c | 559 +++++++++++++------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 21 +- 3 files changed, 285 insertions(+), 303 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +4 -4
@@ -64,14 +64,14 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);int rxe_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova,int access,struct rxe_mr *mr);int rxe_mr_init_fast (int max_pages,struct rxe_mr *mr); -int rxe_flush_pmem_iova (struct rxe_mr *mr,u64 iova,int length); -int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length, -enum rxe_mr_copy_dir dir); +int rxe_flush_pmem_iova (struct rxe_mr *mr,u64 iova, +unsigned int length); +int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr, +unsigned int length,enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir);int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset); -void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val);int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value);
drivers/infiniband/sw/rxe/rxe_mr.c +279 -280
737 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
@@ -288,17 +288,6 @@ enum rxe_mr_lookup_type { RXE_LOOKUP_REMOTE, }; -#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) - -struct rxe_phys_buf { -u64 addr; -u64 size; -}; - -struct rxe_map { -struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; -}; -static inline int rkey_is_mw (u32 rkey) {u32 index = rkey >>8 ;
@@ -316,22 +305,16 @@ struct rxe_mr {u32 rkey;enum rxe_mr_state state;int access; +atomic_t num_mw;unsigned int page_offset;unsigned int page_shift;u64 page_mask; -int map_shift; -int map_mask;u32 num_buf;u32 nbuf; -u32 max_buf; -u32 num_map; - -atomic_t num_mw; - -struct rxe_map **map; +struct xarray page_list; };static inline unsigned int mr_page_size (struct rxe_mr *mr)
Signature · 1 line
2.37.2