RDMA/rxe: Remove struct rxe_phys_buf
Subsystems (2)
PATCH v2 1/2 RDMA/rxe: Remove struct rxe_phys_buf
1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
2) Replace rxe_phys_buf[n].buf with addrs[n].
--- drivers/infiniband/sw/rxe/rxe_mr.c | 45 +++++++++------------------ drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +--- drivers/infiniband/sw/rxe/rxe_verbs.h | 9 ++---- 3 files changed, 18 insertions(+), 42 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +15 -30
@@ -115,7 +115,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, int access, struct rxe_mr *mr) { struct rxe_map **map; -struct rxe_phys_buf *buf =NULL ;struct ib_umem *umem;struct sg_page_iter sg_iter;int num_buf;
@@ -144,16 +143,14 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, mr->page_shift =PAGE_SHIFT ; mr->page_mask =PAGE_SIZE -1 ; + mr->ibmr.page_size =PAGE_SIZE ; - num_buf =0 ; + num_buf =0 ; map = mr->map;if (length >0 ) { - buf = map[0 ]->buf; - for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {if (num_buf >=RXE_BUF_PER_MAP ) { map++; - buf = map[0 ]->buf; num_buf =0 ; }
@@ -165,10 +162,8 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_cleanup_map; } - buf->addr = (uintptr_t )vaddr; - buf->size =PAGE_SIZE ; + map[0 ]->addrs[num_buf] = (uintptr_t )vaddr; num_buf++; - buf++; } }
@@ -216,9 +211,9 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, size_t *offset_out) {size_t offset = iova - mr->ibmr.iova + mr->offset; +u64 length = mr->ibmr.page_size;int map_index; -int buf_index; -u64 length; +int addr_index;if (likely (mr->page_shift)) { *offset_out = offset & mr->page_mask;
@@ -227,23 +222,20 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, *m_out = offset >> mr->map_shift; }else { map_index =0 ; - buf_index =0 ; - - length = mr->map[map_index]->buf[buf_index].size; + addr_index =0 ;while (offset >= length) { offset -= length; - buf_index++; + addr_index++; -if (buf_index ==RXE_BUF_PER_MAP ) { +if (addr_index ==RXE_BUF_PER_MAP ) { map_index++; - buf_index =0 ; + addr_index =0 ; } - length = mr->map[map_index]->buf[buf_index].size; } *m_out = map_index; - *n_out = buf_index; + *n_out = addr_index; *offset_out = offset; } }
@@ -273,13 +265,13 @@ void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)lookup_iova (mr, iova, &m, &n, &offset); -if (offset + length > mr->map[m]->buf[n].size) { +if (offset + length > mr->ibmr.page_size) {pr_warn ("crosses page boundary\n" ); addr =NULL ; goto out; } - addr = (void *)(uintptr_t )mr->map[m]->buf[n].addr + offset; + addr = (void *)(uintptr_t )mr->map[m]->addrs[n] + offset; out:return addr;
@@ -294,8 +286,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,int err;int bytes;u8 *va; -struct rxe_map **map; -struct rxe_phys_buf *buf;int m;int i;size_t offset;
@@ -325,17 +315,14 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,lookup_iova (mr, iova, &m, &i, &offset); - map = mr->map + m; - buf = map[0 ]->buf + i; -while (length >0 ) {u8 *src, *dest; - va = (u8 *)(uintptr_t )buf->addr + offset; + va = (u8 *)(uintptr_t )mr->map[m]->addrs[i] + offset; src = (dir ==RXE_TO_MR_OBJ ) ? addr : va; dest = (dir ==RXE_TO_MR_OBJ ) ? va : addr; - bytes = buf->size - offset; + bytes = mr->ibmr.page_size - offset;if (bytes > length) bytes = length;
@@ -346,13 +333,11 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, addr += bytes; offset =0 ; - buf++; i++;if (i ==RXE_BUF_PER_MAP ) { i =0 ; - map++; - buf = map[0 ]->buf; + m++; } }
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -5
@@ -948,16 +948,12 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr) {struct rxe_mr *mr =to_rmr (ibmr);struct rxe_map *map; -struct rxe_phys_buf *buf;if (unlikely (mr->nbuf == mr->num_buf))return -ENOMEM ; map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; + map->addrs[mr->nbuf %RXE_BUF_PER_MAP ] = addr; mr->nbuf++;return 0 ;
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -7
@@ -277,15 +277,10 @@ enum rxe_mr_lookup_type { RXE_LOOKUP_REMOTE, }; -#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) - -struct rxe_phys_buf { -u64 addr; -u64 size; -}; +#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(u64))struct rxe_map { -struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; +u64 addrs[RXE_BUF_PER_MAP ]; };static inline int rkey_is_mw (u32 rkey)
Signature · 1 line
2.25.1
PATCH v2 2/2 RDMA/rxe: Replace page_address() with kmap_local_page()
page_address() will be broken when new in-kernel memory protection scheme[1] is applied in the future so use kmap_local_page() instead.
[1]:
https://lore.kernel.org/lkml/[email protected]/
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 44 +++++++++++++++++----------- drivers/infiniband/sw/rxe/rxe_resp.c | 1 + 3 files changed, 30 insertions(+), 17 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
@@ -68,6 +68,8 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);int rxe_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova,int access,struct rxe_mr *mr);int rxe_mr_init_fast (int max_pages,struct rxe_mr *mr); +void *rxe_map_to_vaddr (struct rxe_mr *mr,int map_index,int addr_index,size_t offset); +void rxe_unmap_vaddr (struct rxe_mr *mr,void *vaddr);int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length,enum rxe_mr_copy_dir dir); int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
drivers/infiniband/sw/rxe/rxe_mr.c +27 -17
87 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +1 -0
@@ -652,6 +652,7 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, ret =RESPST_ACKNOWLEDGE ; out: +rxe_unmap_vaddr (mr, vaddr);return ret; }
Signature · 1 line
2.25.1
Hi Jason, Ira and others
Kindly ping. I hope you can review them.
Thank you in advance.
Best Regards,
Xiao Yang
Quoted 198 lines · Xiao Yang
1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
2) Replace rxe_phys_buf[n].buf with addrs[n].Signed-off-by Xiao Yang <[email protected]>--- drivers/infiniband/sw/rxe/rxe_mr.c | 45 +++++++++------------------ drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +--- drivers/infiniband/sw/rxe/rxe_verbs.h | 9 ++---- 3 files changed, 18 insertions(+), 42 deletions(-)drivers/infiniband/sw/rxe/rxe_mr.c +15
-30@@ -115,7 +115,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, int access, struct rxe_mr *mr) { struct rxe_map **map; -struct rxe_phys_buf *buf =NULL ;struct ib_umem *umem;struct sg_page_iter sg_iter;int num_buf;@@ -144,16 +143,14 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, mr->page_shift =PAGE_SHIFT ; mr->page_mask =PAGE_SIZE -1 ; + mr->ibmr.page_size =PAGE_SIZE ; - num_buf =0 ; + num_buf =0 ; map = mr->map;if (length >0 ) { - buf = map[0 ]->buf; - for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {if (num_buf >=RXE_BUF_PER_MAP ) { map++; - buf = map[0 ]->buf; num_buf =0 ; }@@ -165,10 +162,8 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, goto err_cleanup_map; } - buf->addr = (uintptr_t )vaddr; - buf->size =PAGE_SIZE ; + map[0 ]->addrs[num_buf] = (uintptr_t )vaddr; num_buf++; - buf++; } }@@ -216,9 +211,9 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, size_t *offset_out) {size_t offset = iova - mr->ibmr.iova + mr->offset; +u64 length = mr->ibmr.page_size;int map_index; -int buf_index; -u64 length; +int addr_index;if (likely (mr->page_shift)) { *offset_out = offset & mr->page_mask;@@ -227,23 +222,20 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, *m_out = offset >> mr->map_shift; }else { map_index =0 ; - buf_index =0 ; - - length = mr->map[map_index]->buf[buf_index].size; + addr_index =0 ;while (offset >= length) { offset -= length; - buf_index++; + addr_index++; -if (buf_index ==RXE_BUF_PER_MAP ) { +if (addr_index ==RXE_BUF_PER_MAP ) { map_index++; - buf_index =0 ; + addr_index =0 ; } - length = mr->map[map_index]->buf[buf_index].size; } *m_out = map_index; - *n_out = buf_index; + *n_out = addr_index; *offset_out = offset; } }@@ -273,13 +265,13 @@ void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)lookup_iova (mr, iova, &m, &n, &offset); -if (offset + length > mr->map[m]->buf[n].size) { +if (offset + length > mr->ibmr.page_size) {pr_warn ("crosses page boundary\n" ); addr =NULL ; goto out; } - addr = (void *)(uintptr_t )mr->map[m]->buf[n].addr + offset; + addr = (void *)(uintptr_t )mr->map[m]->addrs[n] + offset; out:return addr;@@ -294,8 +286,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,int err;int bytes;u8 *va; -struct rxe_map **map; -struct rxe_phys_buf *buf;int m;int i;size_t offset;@@ -325,17 +315,14 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,lookup_iova (mr, iova, &m, &i, &offset); - map = mr->map + m; - buf = map[0 ]->buf + i; -while (length >0 ) {u8 *src, *dest; - va = (u8 *)(uintptr_t )buf->addr + offset; + va = (u8 *)(uintptr_t )mr->map[m]->addrs[i] + offset; src = (dir ==RXE_TO_MR_OBJ ) ? addr : va; dest = (dir ==RXE_TO_MR_OBJ ) ? va : addr; - bytes = buf->size - offset; + bytes = mr->ibmr.page_size - offset;if (bytes > length) bytes = length;@@ -346,13 +333,11 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, addr += bytes; offset =0 ; - buf++; i++;if (i ==RXE_BUF_PER_MAP ) { i =0 ; - map++; - buf = map[0 ]->buf; + m++; } }drivers/infiniband/sw/rxe/rxe_verbs.c +1
-5@@ -948,16 +948,12 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr) {struct rxe_mr *mr =to_rmr (ibmr);struct rxe_map *map; -struct rxe_phys_buf *buf;if (unlikely (mr->nbuf == mr->num_buf))return -ENOMEM ; map = mr->map[mr->nbuf /RXE_BUF_PER_MAP ]; - buf = &map->buf[mr->nbuf %RXE_BUF_PER_MAP ]; - - buf->addr = addr; - buf->size = ibmr->page_size; + map->addrs[mr->nbuf %RXE_BUF_PER_MAP ] = addr; mr->nbuf++;return 0 ;drivers/infiniband/sw/rxe/rxe_verbs.h +2
-7@@ -277,15 +277,10 @@ enum rxe_mr_lookup_type { RXE_LOOKUP_REMOTE, }; -#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) - -struct rxe_phys_buf { -u64 addr; -u64 size; -}; +#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(u64))struct rxe_map { -struct rxe_phys_buf buf[RXE_BUF_PER_MAP ]; +u64 addrs[RXE_BUF_PER_MAP ]; };static inline int rkey_is_mw (u32 rkey)
Jason Gunthorpe
maintainerQuoted 2 lines · Xiao Yang
1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
2) Replace rxe_phys_buf[n].buf with addrs[n].
This almost certainly doesn't work, but here is a general sketch how all of this really should look:
drivers/infiniband/sw/rxe/rxe_loc.h +0 -1
@@ -68,7 +68,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, enum rxe_mr_copy_dir dir);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum rxe_mr_copy_dir dir); -void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum rxe_mr_lookup_type type);int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +86 -218
369 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +15 -6
@@ -631,22 +631,30 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, }if (!res->replay) { +u64 iova = qp->resp.va + qp->resp.offset; +unsigned int page_offset = iova %PAGE_SIZE ; +struct page *page; +if (mr->state !=RXE_MR_STATE_VALID ) { ret =RESPST_ERR_RKEY_VIOLATION ; goto out; } - vaddr =iova_to_vaddr (mr, qp->resp.va + qp->resp.offset, -sizeof (u64)); -/* check vaddr is 8 bytes aligned. */ -if (!vaddr || (uintptr_t )vaddr &7 ) { +if (iova &7 ) { ret =RESPST_ERR_MISALIGNED_ATOMIC ; goto out; } +/* +* FIXME: Need to ensure the xarray isn't changing while +* this is happening +*/ + page =xa_load (&mr->pages,rxe_mr_iova_to_index (iova)); + + vaddr =kmap_local_page (page);spin_lock_bh (&atomic_ops_lock); - res->atomic.orig_val = value = *vaddr; + res->atomic.orig_val = value = *(vaddr + page_offset);if (pkt->opcode ==IB_OPCODE_RC_COMPARE_SWAP ) {if (value ==atmeth_comp (pkt))
@@ -655,8 +663,9 @@ static enum resp_states atomic_reply(struct rxe_qp *qp, value +=atmeth_swap_add (pkt); } - *vaddr = value; + *(vaddr + page_offset) = value;spin_unlock_bh (&atomic_ops_lock); +kunmap_local (vaddr); qp->resp.msn++;
drivers/infiniband/sw/rxe/rxe_verbs.c +36 -16
67 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -1
@@ -320,7 +320,7 @@ struct rxe_mr {atomic_t num_mw; -struct rxe_map **map; +struct xarray pages; };enum rxe_mw_state {
Quoted 2 lines · Jason Gunthorpe
This almost certainly doesn't work, but here is a general sketch how all of this really should look:
Hi Jason,
Thank you very much for the sketch. I will try to understand it.
Best Regards,
Xiao Yang