Skip to contentSkip to conversation

RDMA/rxe: Replace mr page map with an xarray

PATCH In review Bob Pearson 2 participants 10 messages Jan 20, 2023 – Jan 27, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v7 cover RDMA/rxe: Replace mr page map with an xarray

This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.

v7:

Made changes requested by Jason to return RESPST_ERR_XXX from rxe_mr.c to rxe_resp.c.
v6:
Backed out.
v5:
Responded to a note from [email protected] and restored calls to is_pmem_page() which were accidentally dropped in earlier versions.
v4:
Responded to a comment by Zhu and cleaned up error passing between rxe_mr.c and rxe_resp.c.
Other various cleanups including more careful use of unsigned ints. Rebased to current for-next.
v3:
Fixed an error reported by kernel test robot
v2:
Rebased to 6.2.0-rc1+
Minor cleanups
Fixed error reported by Jason in 4/6 missing if after else.

Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

 drivers/infiniband/sw/rxe/rxe.h       |  38 ++
 drivers/infiniband/sw/rxe/rxe_loc.h   |  12 +-
 drivers/infiniband/sw/rxe/rxe_mr.c    | 605 ++++++++++++++------------
 drivers/infiniband/sw/rxe/rxe_resp.c  | 143 ++----
 drivers/infiniband/sw/rxe/rxe_verbs.c |  36 --
 drivers/infiniband/sw/rxe/rxe_verbs.h |  32 +-
 6 files changed, 425 insertions(+), 441 deletions(-)

base-commit: 1ec82317a1daac78c04b0c15af89018ccf9fa2b7

Signature · 1 line
2.37.2
#1 lore raw

PATCH v7 1/6 RDMA/rxe: Cleanup mr_check_range

Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.

---
 drivers/infiniband/sw/rxe/rxe_mr.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
index 072eac4b65d2..632ee1e516a1 100644
@@ -26,8 +26,6 @@ u8 rxe_get_next_key(u32 last_key)
 
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 {
-
-
 	switch (mr->ibmr.type) {
 	case IB_MR_TYPE_DMA:
 		return 0;
@@ -41,7 +39,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 
 	default:
 		rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);
-		return -EFAULT;
+		return -EINVAL;
 	}
 }
 
Signature · 1 line
2.37.2
#2 lore raw

PATCH v7 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c

In reply to Bob Pearson

Move rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   |  2 ++
 drivers/infiniband/sw/rxe/rxe_mr.c    | 36 +++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.c | 36 ---------------------------
 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 948ce4902b10..29b6c2143045 100644
@@ -69,6 +69,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset);
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
index 632ee1e516a1..229c7259644c 100644
@@ -223,6 +223,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
 	return err;
 }
 
+static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
+{
+	struct rxe_mr *mr = to_rmr(ibmr);
+	struct rxe_map *map;
+	struct rxe_phys_buf *buf;
+
+	if (unlikely(mr->nbuf == mr->num_buf))
+		return -ENOMEM;
+
+	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
+	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
+
+	buf->addr = addr;
+	buf->size = ibmr->page_size;
+	mr->nbuf++;
+
+	return 0;
+}
+
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset)
+{
+	struct rxe_mr *mr = to_rmr(ibmr);
+	int n;
+
+	mr->nbuf = 0;
+
+	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
+
+	mr->page_shift = ilog2(ibmr->page_size);
+	mr->page_mask = ibmr->page_size - 1;
+	mr->offset = ibmr->iova & mr->page_mask;
+
+	return n;
+}
+
 static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
index 025b35bf014e..7a902e0a0607 100644
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	return ERR_PTR(err);
 }
 
-static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	struct rxe_map *map;
-	struct rxe_phys_buf *buf;
-
-	if (unlikely(mr->nbuf == mr->num_buf))
-		return -ENOMEM;
-
-	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
-	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
-
-	buf->addr = addr;
-	buf->size = ibmr->page_size;
-	mr->nbuf++;
-
-	return 0;
-}
-
-static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-			 int sg_nents, unsigned int *sg_offset)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-
-	mr->nbuf = 0;
-
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
-
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
-
-	return n;
-}
-
 static ssize_t parent_show(struct device *device,
 			   struct device_attribute *attr, char *buf)
 {
Signature · 1 line
2.37.2
#3 lore raw

PATCH v7 3/6 RDMA-rxe: Isolate mr code from atomic_reply()

In reply to Bob Pearson

Isolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.
Minor cleanups to rxe_check_range() and iova_to_vaddr(). Move enum resp_state to rxe.h

---
 drivers/infiniband/sw/rxe/rxe.h      | 38 +++++++++++++
 drivers/infiniband/sw/rxe/rxe_loc.h  |  2 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 83 ++++++++++++++++++----------
 drivers/infiniband/sw/rxe/rxe_resp.c | 82 ++++-----------------------
 4 files changed, 105 insertions(+), 100 deletions(-)
drivers/infiniband/sw/rxe/rxe.h +38 -0
index ab334900fcc3..2415f3704f57 100644
@@ -57,6 +57,44 @@
 #define rxe_dbg_mw(mw, fmt, ...) ibdev_dbg((mw)->ibmw.device,		\
 		"mw#%d %s:  " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__)
 
+/* responder states */
+enum resp_states {
+	RESPST_NONE,
+	RESPST_GET_REQ,
+	RESPST_CHK_PSN,
+	RESPST_CHK_OP_SEQ,
+	RESPST_CHK_OP_VALID,
+	RESPST_CHK_RESOURCE,
+	RESPST_CHK_LENGTH,
+	RESPST_CHK_RKEY,
+	RESPST_EXECUTE,
+	RESPST_READ_REPLY,
+	RESPST_ATOMIC_REPLY,
+	RESPST_ATOMIC_WRITE_REPLY,
+	RESPST_PROCESS_FLUSH,
+	RESPST_COMPLETE,
+	RESPST_ACKNOWLEDGE,
+	RESPST_CLEANUP,
+	RESPST_DUPLICATE_REQUEST,
+	RESPST_ERR_MALFORMED_WQE,
+	RESPST_ERR_UNSUPPORTED_OPCODE,
+	RESPST_ERR_MISALIGNED_ATOMIC,
+	RESPST_ERR_PSN_OUT_OF_SEQ,
+	RESPST_ERR_MISSING_OPCODE_FIRST,
+	RESPST_ERR_MISSING_OPCODE_LAST_C,
+	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
+	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
+	RESPST_ERR_RNR,
+	RESPST_ERR_RKEY_VIOLATION,
+	RESPST_ERR_INVALIDATE_RKEY,
+	RESPST_ERR_LENGTH,
+	RESPST_ERR_CQ_OVERFLOW,
+	RESPST_ERROR,
+	RESPST_RESET,
+	RESPST_DONE,
+	RESPST_EXIT,
+};
+
 void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
 
 int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 29b6c2143045..bcb1bbcf50df 100644
@@ -72,6 +72,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			u64 compare, u64 swap_add, u64 *orig_val);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +55 -28

111 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +10 -72

111 lines not on this page · raw message

Signature · 1 line
2.37.2
#4 lore raw

PATCH v7 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()

In reply to Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
 drivers/infiniband/sw/rxe/rxe_loc.h  |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c   | 38 ++++++++++++++++-
 drivers/infiniband/sw/rxe/rxe_resp.c | 61 ++++++++++------------------
 3 files changed, 59 insertions(+), 42 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index bcb1bbcf50df..ad8bd6bd728a 100644
@@ -71,9 +71,9 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 			u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +37 -1
index df9741474f1f..5c4ce43914fa 100644
@@ -297,7 +297,7 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 	}
 }
 
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)
+static void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)
 {
 	size_t offset;
 	int m, n;
@@ -565,6 +565,42 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 	return 0;
 }
 
+/* only implemented for 64 bit architectures */
+#if defined CONFIG_64BIT
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
+{
+	u64 *va;
+
+	/* See IBA oA19-28 */
+	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+		rxe_dbg_mr(mr, "mr not in valid state");
+		return RESPST_ERR_RKEY_VIOLATION;
+	}
+
+	va = iova_to_vaddr(mr, iova, sizeof(value));
+	if (unlikely(!va)) {
+		rxe_dbg_mr(mr, "iova out of range");
+		return RESPST_ERR_RKEY_VIOLATION;
+	}
+
+	/* See IBA A19.4.2 */
+	if (unlikely((uintptr_t)va & 0x7 || iova & 0x7)) {
+		rxe_dbg_mr(mr, "misaligned address");
+		return RESPST_ERR_MISALIGNED_ATOMIC;
+	}
+
+	/* Do atomic write after all prior operations have completed */
+	smp_store_release(va, value);
+
+	return 0;
+}
+#else
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
+{
+	return RESPST_ERR_UNSUPPORTED_OPCODE;
+}
+#endif
+
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
 	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +21 -40

80 lines not on this page · raw message

Signature · 1 line
2.37.2
#5 lore raw

PATCH v7 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c

In reply to Bob Pearson

Cleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.

---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 31 ++++++++++++---------------
 drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++++++---
 2 files changed, 22 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +14 -17
index 5c4ce43914fa..2181165ea40d 100644
@@ -62,6 +62,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
 	mr->lkey = mr->ibmr.lkey = lkey;
 	mr->rkey = mr->ibmr.rkey = rkey;
 
+	mr->ibmr.page_size = PAGE_SIZE;
+	mr->page_mask = PAGE_MASK;
+	mr->page_shift = PAGE_SHIFT;
 	mr->state = RXE_MR_STATE_INVALID;
 }
 
@@ -151,9 +154,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		goto err_release_umem;
 	}
 
-	mr->page_shift = PAGE_SHIFT;
-	mr->page_mask = PAGE_SIZE - 1;
-
 	num_buf			= 0;
 	map = mr->map;
 	if (length > 0) {
@@ -182,7 +182,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 				goto err_release_umem;
 			}
 			buf->addr = (uintptr_t)vaddr;
-			buf->size = PAGE_SIZE;
+			buf->size = mr_page_size(mr);
 			num_buf++;
 			buf++;
 
@@ -191,10 +191,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 
 	mr->umem = umem;
 	mr->access = access;
-	mr->offset = ib_umem_offset(umem);
+	mr->page_offset = ib_umem_offset(umem);
 	mr->state = RXE_MR_STATE_VALID;
 	mr->ibmr.type = IB_MR_TYPE_USER;
-	mr->ibmr.page_size = PAGE_SIZE;
 
 	return 0;
 
@@ -248,29 +247,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-
-	mr->nbuf = 0;
+	unsigned int page_size = mr_page_size(mr);
 
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
+	mr->page_shift = ilog2(page_size);
+	mr->page_mask = ~((u64)page_size - 1);
+	mr->page_offset = ibmr->iova & (page_size - 1);
 
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
+	mr->nbuf = 0;
 
-	return n;
+	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
 }
 
 static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
-	size_t offset = iova - mr->ibmr.iova + mr->offset;
+	size_t offset = iova - mr->ibmr.iova + mr->page_offset;
 	int			map_index;
 	int			buf_index;
 	u64			length;
 
 	if (likely(mr->page_shift)) {
-		*offset_out = offset & mr->page_mask;
+		*offset_out = offset & (mr_page_size(mr) - 1);
 		offset >>= mr->page_shift;
 		*n_out = offset & mr->map_mask;
 		*m_out = offset >> mr->map_shift;
@@ -329,7 +326,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)
 	if (mr->ibmr.type == IB_MR_TYPE_DMA)
 		return -EFAULT;
 
-	offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;
+	offset = (iova - mr->ibmr.iova + mr->page_offset) & mr->page_mask;
 	while (length > 0) {
 		u8 *va;
 		int bytes;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
index 19ddfa890480..bfc94caaeec5 100644
@@ -310,11 +310,11 @@ struct rxe_mr {
 	u32			lkey;
 	u32			rkey;
 	enum rxe_mr_state	state;
-	u32			offset;
 	int			access;
 
-	int			page_shift;
-	int			page_mask;
+	unsigned int		page_offset;
+	unsigned int		page_shift;
+	u64			page_mask;
 	int			map_shift;
 	int			map_mask;
 
@@ -329,6 +329,11 @@ struct rxe_mr {
 	struct rxe_map		**map;
 };
 
+static inline unsigned int mr_page_size(struct rxe_mr *mr)
+{
+	return mr ? mr->ibmr.page_size : PAGE_SIZE;
+}
+
 enum rxe_mw_state {
 	RXE_MW_STATE_INVALID	= RXE_MR_STATE_INVALID,
 	RXE_MW_STATE_FREE	= RXE_MR_STATE_FREE,
Signature · 1 line
2.37.2
#6 lore raw

PATCH v7 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

In reply to Bob Pearson

Replace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.

This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   |   6 +-
 drivers/infiniband/sw/rxe/rxe_mr.c    | 537 ++++++++++++--------------
 drivers/infiniband/sw/rxe/rxe_verbs.h |  21 +-
 3 files changed, 262 insertions(+), 302 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +3 -3
index ad8bd6bd728a..1bb0cb479eb1 100644
@@ -64,9 +64,9 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);
 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr);
 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
-int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length);
-int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
-		enum rxe_mr_copy_dir dir);
+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length);
+int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
+		unsigned int length, enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
drivers/infiniband/sw/rxe/rxe_mr.c +257 -280

716 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
index bfc94caaeec5..c269ae2a3224 100644
@@ -283,17 +283,6 @@ enum rxe_mr_lookup_type {
 	RXE_LOOKUP_REMOTE,
 };
 
-#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
-
-struct rxe_phys_buf {
-	u64      addr;
-	u64      size;
-};
-
-struct rxe_map {
-	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
-};
-
 static inline int rkey_is_mw(u32 rkey)
 {
 	u32 index = rkey >> 8;
@@ -311,22 +300,16 @@ struct rxe_mr {
 	u32			rkey;
 	enum rxe_mr_state	state;
 	int			access;
+	atomic_t		num_mw;
 
 	unsigned int		page_offset;
 	unsigned int		page_shift;
 	u64			page_mask;
-	int			map_shift;
-	int			map_mask;
 
 	u32			num_buf;
 	u32			nbuf;
 
-	u32			max_buf;
-	u32			num_map;
-
-	atomic_t		num_mw;
-
-	struct rxe_map		**map;
+	struct xarray		page_list;
 };
 
 static inline unsigned int mr_page_size(struct rxe_mr *mr)
Signature · 1 line
2.37.2
maintainer
#7 lore raw
In reply to Bob Pearson
Quoted 10 lines · Bob Pearson

This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.

v7:

Made changes requested by Jason to return RESPST_ERR_XXX from rxe_mr.c to rxe_resp.c.

I took it to for-next, but I made these changes, please check:

drivers/infiniband/sw/rxe/rxe_mr.c +4 -12
index fe4049330c9f19..c80458634962c6 100644
@@ -114,7 +114,8 @@ static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
 
 			if (persistent && !is_pmem_page(page)) {
 				rxe_dbg_mr(mr, "Page can't be persistent\n");
-				return -EINVAL;
+				xas_set_err(&xas, -EINVAL);
+				break;
 			}
 
 			xas_store(&xas, page);
@@ -213,7 +214,6 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
 	struct page *page = virt_to_page(iova & mr->page_mask);
-	XA_STATE(xas, &mr->page_list, mr->nbuf);
 	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
 	int err;
 
@@ -225,13 +225,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
 	if (unlikely(mr->nbuf == mr->num_buf))
 		return -ENOMEM;
 
-	do {
-		xas_lock(&xas);
-		xas_store(&xas, page);
-		xas_unlock(&xas);
-	} while (xas_nomem(&xas, GFP_KERNEL));
-
-	err = xas_error(&xas);
+	err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));
 	if (err)
 		return err;
 
@@ -458,10 +452,8 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
 				mr_page_size(mr) - page_offset);
 
 		va = kmap_local_page(page);
-		if (!va)
-			return -EFAULT;
-
 		arch_wb_cache_pmem(va + page_offset, bytes);
+		kunmap_local(va);
 
 		length -= bytes;
 		iova += bytes;
#8 lore raw
Quoted 53 lines · Jason Gunthorpe
Quoted 10 lines · Bob Pearson

This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.

v7:

Made changes requested by Jason to return RESPST_ERR_XXX from rxe_mr.c to rxe_resp.c.

I took it to for-next, but I made these changes, please check:

drivers/infiniband/sw/rxe/rxe_mr.c +3 -9
index fe4049330c9f19..c80458634962c6 100644
@@ -114,7 +114,8 @@ static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
 
 			if (persistent && !is_pmem_page(page)) {
 				rxe_dbg_mr(mr, "Page can't be persistent\n");
-				return -EINVAL;
+				xas_set_err(&xas, -EINVAL);
+				break;
 			}
 
 			xas_store(&xas, page);
@@ -213,7 +214,6 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
 	struct page *page = virt_to_page(iova & mr->page_mask);
-	XA_STATE(xas, &mr->page_list, mr->nbuf);
 	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
 	int err;
 
@@ -225,13 +225,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
 	if (unlikely(mr->nbuf == mr->num_buf))
 		return -ENOMEM;
 
-	do {
-		xas_lock(&xas);
-		xas_store(&xas, page);
-		xas_unlock(&xas);
-	} while (xas_nomem(&xas, GFP_KERNEL));
-
-	err = xas_error(&xas);
+	err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));
 	if (err)
 		return err;
 
@@ -458,10 +452,8 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
 				mr_page_size(mr) - page_offset);

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) · line 459

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
 
 		va = kmap_local_page(page);
-		if (!va)
-			return -EFAULT;
-
 		arch_wb_cache_pmem(va + page_offset, bytes);
+		kunmap_local(va);
 
 		length -= bytes;
 		iova += bytes;

Thanks. I'll check these.

Bob

#9 lore raw
In reply to Jason Gunthorpe
Quoted 19 lines · Jason Gunthorpe
Quoted 10 lines · Bob Pearson

This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.

v7:

Made changes requested by Jason to return RESPST_ERR_XXX from rxe_mr.c to rxe_resp.c.

I took it to for-next, but I made these changes, please check:

drivers/infiniband/sw/rxe/rxe_mr.c +0 -0
index fe4049330c9f19..c80458634962c6 100644

drivers/infiniband/sw/rxe/rxe_mr.c static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) · line 121

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_mr.c +2 -1
@@ -114,7 +114,8 @@ static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
 
 			if (persistent && !is_pmem_page(page)) {
 				rxe_dbg_mr(mr, "Page can't be persistent\n");
-				return -EINVAL;
+				xas_set_err(&xas, -EINVAL);
+				break;
 			}
 
 			xas_store(&xas, page);

Looks good. Good catch.

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +0 -1
@@ -213,7 +214,6 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
 	struct page *page = virt_to_page(iova & mr->page_mask);
-	XA_STATE(xas, &mr->page_list, mr->nbuf);
 	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
 	int err;
 
@@ -225,13 +225,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 iova)
 	if (unlikely(mr->nbuf == mr->num_buf))

drivers/infiniband/sw/rxe/rxe_mr.c static int rxe_set_page(struct ib_mr *ibmr, u64 iova) · line 228

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +1 -7
 		return -ENOMEM;
 
-	do {
-		xas_lock(&xas);
-		xas_store(&xas, page);
-		xas_unlock(&xas);
-	} while (xas_nomem(&xas, GFP_KERNEL));
-
-	err = xas_error(&xas);
+	err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));

Looks good.

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_mr.c +0 -0
 	if (err)
 		return err;
 
@@ -458,10 +452,8 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
 				mr_page_size(mr) - page_offset);

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) · line 459

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
 
 		va = kmap_local_page(page);
-		if (!va)
-			return -EFAULT;
-
 		arch_wb_cache_pmem(va + page_offset, bytes);
+		kunmap_local(va);
 
 		length -= bytes;
 		iova += bytes;

Looks good. Good catch. I take it kmap_local_page shouldn't fail.

Thanks,

Bob