Skip to contentSkip to conversation

RDMA/rxe: Remove struct rxe_phys_buf

PATCH In review Xiao Yang 2 participants 5 messages Nov 11, 2022 – Nov 20, 2022

Subsystems (2)
lore
Share & tools
lore

Xiao Yang

author
#0 lore raw

PATCH v2 1/2 RDMA/rxe: Remove struct rxe_phys_buf

1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
2) Replace rxe_phys_buf[n].buf with addrs[n].

---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 45 +++++++++------------------
 drivers/infiniband/sw/rxe/rxe_verbs.c |  6 +---
 drivers/infiniband/sw/rxe/rxe_verbs.h |  9 ++----
 3 files changed, 18 insertions(+), 42 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +15 -30
index bc081002bddc..4438eb8a3727 100644
@@ -115,7 +115,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr)
 {
 	struct rxe_map		**map;
-	struct rxe_phys_buf	*buf = NULL;
 	struct ib_umem		*umem;
 	struct sg_page_iter	sg_iter;
 	int			num_buf;
@@ -144,16 +143,14 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 
 	mr->page_shift = PAGE_SHIFT;
 	mr->page_mask = PAGE_SIZE - 1;
+	mr->ibmr.page_size = PAGE_SIZE;
 
-	num_buf			= 0;
+	num_buf	= 0;
 	map = mr->map;
 	if (length > 0) {
-		buf = map[0]->buf;
-
 		for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
 			if (num_buf >= RXE_BUF_PER_MAP) {
 				map++;
-				buf = map[0]->buf;
 				num_buf = 0;
 			}
 
@@ -165,10 +162,8 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 				goto err_cleanup_map;
 			}
 
-			buf->addr = (uintptr_t)vaddr;
-			buf->size = PAGE_SIZE;
+			map[0]->addrs[num_buf] = (uintptr_t)vaddr;
 			num_buf++;
-			buf++;
 
 		}
 	}
@@ -216,9 +211,9 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
 	size_t offset = iova - mr->ibmr.iova + mr->offset;
+	u64 length = mr->ibmr.page_size;
 	int			map_index;
-	int			buf_index;
-	u64			length;
+	int			addr_index;
 
 	if (likely(mr->page_shift)) {
 		*offset_out = offset & mr->page_mask;
@@ -227,23 +222,20 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 		*m_out = offset >> mr->map_shift;
 	} else {
 		map_index = 0;
-		buf_index = 0;
-
-		length = mr->map[map_index]->buf[buf_index].size;
+		addr_index = 0;
 
 		while (offset >= length) {
 			offset -= length;
-			buf_index++;
+			addr_index++;
 
-			if (buf_index == RXE_BUF_PER_MAP) {
+			if (addr_index == RXE_BUF_PER_MAP) {
 				map_index++;
-				buf_index = 0;
+				addr_index = 0;
 			}
-			length = mr->map[map_index]->buf[buf_index].size;
 		}
 
 		*m_out = map_index;
-		*n_out = buf_index;
+		*n_out = addr_index;
 		*offset_out = offset;
 	}
 }
@@ -273,13 +265,13 @@ void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)
 
 	lookup_iova(mr, iova, &m, &n, &offset);
 
-	if (offset + length > mr->map[m]->buf[n].size) {
+	if (offset + length > mr->ibmr.page_size) {
 		pr_warn("crosses page boundary\n");
 		addr = NULL;
 		goto out;
 	}
 
-	addr = (void *)(uintptr_t)mr->map[m]->buf[n].addr + offset;
+	addr = (void *)(uintptr_t)mr->map[m]->addrs[n] + offset;
 
 out:
 	return addr;
@@ -294,8 +286,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 	int			err;
 	int			bytes;
 	u8			*va;
-	struct rxe_map		**map;
-	struct rxe_phys_buf	*buf;
 	int			m;
 	int			i;
 	size_t			offset;
@@ -325,17 +315,14 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 
 	lookup_iova(mr, iova, &m, &i, &offset);
 
-	map = mr->map + m;
-	buf	= map[0]->buf + i;
-
 	while (length > 0) {
 		u8 *src, *dest;
 
-		va	= (u8 *)(uintptr_t)buf->addr + offset;
+		va	= (u8 *)(uintptr_t)mr->map[m]->addrs[i] + offset;
 		src = (dir == RXE_TO_MR_OBJ) ? addr : va;
 		dest = (dir == RXE_TO_MR_OBJ) ? va : addr;
 
-		bytes	= buf->size - offset;
+		bytes	= mr->ibmr.page_size - offset;
 
 		if (bytes > length)
 			bytes = length;
@@ -346,13 +333,11 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		addr	+= bytes;
 
 		offset	= 0;
-		buf++;
 		i++;
 
 		if (i == RXE_BUF_PER_MAP) {
 			i = 0;
-			map++;
-			buf = map[0]->buf;
+			m++;
 		}
 	}
 
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -5
index bcdfdadaebbc..13e4d660cb02 100644
@@ -948,16 +948,12 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
 	struct rxe_map *map;
-	struct rxe_phys_buf *buf;
 
 	if (unlikely(mr->nbuf == mr->num_buf))
 		return -ENOMEM;
 
 	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
-	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
-
-	buf->addr = addr;
-	buf->size = ibmr->page_size;
+	map->addrs[mr->nbuf % RXE_BUF_PER_MAP] = addr;
 	mr->nbuf++;
 
 	return 0;
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -7
index 22a299b0a9f0..d136f02d5b56 100644
@@ -277,15 +277,10 @@ enum rxe_mr_lookup_type {
 	RXE_LOOKUP_REMOTE,
 };
 
-#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
-
-struct rxe_phys_buf {
-	u64      addr;
-	u64      size;
-};
+#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(u64))
 
 struct rxe_map {
-	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
+	u64 addrs[RXE_BUF_PER_MAP];
 };
 
 static inline int rkey_is_mw(u32 rkey)
Signature · 1 line
2.25.1
author
#1 lore raw

PATCH v2 2/2 RDMA/rxe: Replace page_address() with kmap_local_page()

page_address() will be broken when new in-kernel memory protection scheme[1] is applied in the future so use kmap_local_page() instead.

[1]:
https://lore.kernel.org/lkml/[email protected]/

---
 drivers/infiniband/sw/rxe/rxe_loc.h  |  2 ++
 drivers/infiniband/sw/rxe/rxe_mr.c   | 44 +++++++++++++++++-----------
 drivers/infiniband/sw/rxe/rxe_resp.c |  1 +
 3 files changed, 30 insertions(+), 17 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index c2a5c8814a48..a63d29156a66 100644
@@ -68,6 +68,8 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);
 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr);
 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
+void *rxe_map_to_vaddr(struct rxe_mr *mr, int map_index, int addr_index, size_t offset);
+void rxe_unmap_vaddr(struct rxe_mr *mr, void *vaddr);
 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
drivers/infiniband/sw/rxe/rxe_mr.c +27 -17

87 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +1 -0
index c32bc12cc82f..31f9ba11a921 100644
@@ -652,6 +652,7 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 
 	ret = RESPST_ACKNOWLEDGE;
 out:
+	rxe_unmap_vaddr(mr, vaddr);
 	return ret;
 }
 
Signature · 1 line
2.25.1
#2 lore raw
In reply to Xiao Yang

Hi Jason, Ira and others

Kindly ping. I hope you can review them.
Thank you in advance.

Best Regards,
Xiao Yang

Quoted 198 lines · Xiao Yang

1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
2) Replace rxe_phys_buf[n].buf with addrs[n].

---
  drivers/infiniband/sw/rxe/rxe_mr.c    | 45 +++++++++------------------
  drivers/infiniband/sw/rxe/rxe_verbs.c |  6 +---
  drivers/infiniband/sw/rxe/rxe_verbs.h |  9 ++----
  3 files changed, 18 insertions(+), 42 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +15 -30
index bc081002bddc..4438eb8a3727 100644
@@ -115,7 +115,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
  		     int access, struct rxe_mr *mr)
  {
  	struct rxe_map		**map;
-	struct rxe_phys_buf	*buf = NULL;
  	struct ib_umem		*umem;
  	struct sg_page_iter	sg_iter;
  	int			num_buf;
@@ -144,16 +143,14 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 
  	mr->page_shift = PAGE_SHIFT;
  	mr->page_mask = PAGE_SIZE - 1;
+	mr->ibmr.page_size = PAGE_SIZE;
 
-	num_buf			= 0;
+	num_buf	= 0;
  	map = mr->map;
  	if (length > 0) {
-		buf = map[0]->buf;
-
  		for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
  			if (num_buf >= RXE_BUF_PER_MAP) {
  				map++;
-				buf = map[0]->buf;
  				num_buf = 0;
  			}
 
@@ -165,10 +162,8 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
  				goto err_cleanup_map;
  			}
 
-			buf->addr = (uintptr_t)vaddr;
-			buf->size = PAGE_SIZE;
+			map[0]->addrs[num_buf] = (uintptr_t)vaddr;
  			num_buf++;
-			buf++;
 
  		}
  	}
@@ -216,9 +211,9 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
  			size_t *offset_out)
  {
  	size_t offset = iova - mr->ibmr.iova + mr->offset;
+	u64 length = mr->ibmr.page_size;
  	int			map_index;
-	int			buf_index;
-	u64			length;
+	int			addr_index;
 
  	if (likely(mr->page_shift)) {
  		*offset_out = offset & mr->page_mask;
@@ -227,23 +222,20 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
  		*m_out = offset >> mr->map_shift;
  	} else {
  		map_index = 0;
-		buf_index = 0;
-
-		length = mr->map[map_index]->buf[buf_index].size;
+		addr_index = 0;
 
  		while (offset >= length) {
  			offset -= length;
-			buf_index++;
+			addr_index++;
 
-			if (buf_index == RXE_BUF_PER_MAP) {
+			if (addr_index == RXE_BUF_PER_MAP) {
  				map_index++;
-				buf_index = 0;
+				addr_index = 0;
  			}
-			length = mr->map[map_index]->buf[buf_index].size;
  		}
 
  		*m_out = map_index;
-		*n_out = buf_index;
+		*n_out = addr_index;
  		*offset_out = offset;
  	}
  }
@@ -273,13 +265,13 @@ void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)
 
  	lookup_iova(mr, iova, &m, &n, &offset);
 
-	if (offset + length > mr->map[m]->buf[n].size) {
+	if (offset + length > mr->ibmr.page_size) {
  		pr_warn("crosses page boundary\n");
  		addr = NULL;
  		goto out;
  	}
 
-	addr = (void *)(uintptr_t)mr->map[m]->buf[n].addr + offset;
+	addr = (void *)(uintptr_t)mr->map[m]->addrs[n] + offset;
 
  out:
  	return addr;
@@ -294,8 +286,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
  	int			err;
  	int			bytes;
  	u8			*va;
-	struct rxe_map		**map;
-	struct rxe_phys_buf	*buf;
  	int			m;
  	int			i;
  	size_t			offset;
@@ -325,17 +315,14 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 
  	lookup_iova(mr, iova, &m, &i, &offset);
 
-	map = mr->map + m;
-	buf	= map[0]->buf + i;
-
  	while (length > 0) {
  		u8 *src, *dest;
 
-		va	= (u8 *)(uintptr_t)buf->addr + offset;
+		va	= (u8 *)(uintptr_t)mr->map[m]->addrs[i] + offset;
  		src = (dir == RXE_TO_MR_OBJ) ? addr : va;
  		dest = (dir == RXE_TO_MR_OBJ) ? va : addr;
 
-		bytes	= buf->size - offset;
+		bytes	= mr->ibmr.page_size - offset;
 
  		if (bytes > length)
  			bytes = length;
@@ -346,13 +333,11 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
  		addr	+= bytes;
 
  		offset	= 0;
-		buf++;
  		i++;
 
  		if (i == RXE_BUF_PER_MAP) {
  			i = 0;
-			map++;
-			buf = map[0]->buf;
+			m++;
  		}
  	}
 
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -5
index bcdfdadaebbc..13e4d660cb02 100644
@@ -948,16 +948,12 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
  {
  	struct rxe_mr *mr = to_rmr(ibmr);
  	struct rxe_map *map;
-	struct rxe_phys_buf *buf;
 
  	if (unlikely(mr->nbuf == mr->num_buf))
  		return -ENOMEM;
 
  	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
-	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
-
-	buf->addr = addr;
-	buf->size = ibmr->page_size;
+	map->addrs[mr->nbuf % RXE_BUF_PER_MAP] = addr;
  	mr->nbuf++;
 
  	return 0;
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -7
index 22a299b0a9f0..d136f02d5b56 100644
@@ -277,15 +277,10 @@ enum rxe_mr_lookup_type {
  	RXE_LOOKUP_REMOTE,
  };
 
-#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
-
-struct rxe_phys_buf {
-	u64      addr;
-	u64      size;
-};
+#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(u64))
 
  struct rxe_map {
-	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
+	u64 addrs[RXE_BUF_PER_MAP];
  };
 
  static inline int rkey_is_mw(u32 rkey)
maintainer
#3 lore raw
In reply to Xiao Yang
Quoted 2 lines · Xiao Yang

1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
2) Replace rxe_phys_buf[n].buf with addrs[n].

This almost certainly doesn't work, but here is a general sketch how all of this really should look:

drivers/infiniband/sw/rxe/rxe_loc.h +0 -1
index a22476d27b3843..7539cf3e00db55 100644
@@ -68,7 +68,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +86 -218

369 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +15 -6
index 6761bcd1d4d8f7..c1ed200e797779 100644
@@ -631,22 +631,30 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 	}
 
 	if (!res->replay) {
+		u64 iova = qp->resp.va + qp->resp.offset;
+		unsigned int page_offset = iova % PAGE_SIZE;
+		struct page *page;
+
 		if (mr->state != RXE_MR_STATE_VALID) {
 			ret = RESPST_ERR_RKEY_VIOLATION;
 			goto out;
 		}
 
-		vaddr = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset,
-					sizeof(u64));
-
 		/* check vaddr is 8 bytes aligned. */
-		if (!vaddr || (uintptr_t)vaddr & 7) {
+		if (iova & 7) {
 			ret = RESPST_ERR_MISALIGNED_ATOMIC;
 			goto out;
 		}
 
+		/*
+		 * FIXME: Need to ensure the xarray isn't changing while
+		 * this is happening
+		 */
+		page = xa_load(&mr->pages, rxe_mr_iova_to_index(iova));
+
+		vaddr = kmap_local_page(page);
 		spin_lock_bh(&atomic_ops_lock);
-		res->atomic.orig_val = value = *vaddr;
+		res->atomic.orig_val = value = *(vaddr + page_offset);
 
 		if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP) {
 			if (value == atmeth_comp(pkt))
@@ -655,8 +663,9 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 			value += atmeth_swap_add(pkt);
 		}
 
-		*vaddr = value;
+		*(vaddr + page_offset) = value;
 		spin_unlock_bh(&atomic_ops_lock);
+		kunmap_local(vaddr);
 
 		qp->resp.msn++;
 
drivers/infiniband/sw/rxe/rxe_verbs.c +36 -16

67 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.h +1 -1
index 22a299b0a9f0a8..6eebbd7b91a687 100644
@@ -320,7 +320,7 @@ struct rxe_mr {
 
 	atomic_t		num_mw;
 
-	struct rxe_map		**map;
+	struct xarray		pages;
 };
 
 enum rxe_mw_state {
#4 lore raw
Quoted 2 lines · Jason Gunthorpe

This almost certainly doesn't work, but here is a general sketch how all of this really should look:

Hi Jason,

Thank you very much for the sketch. I will try to understand it.

Best Regards,
Xiao Yang