Skip to contentSkip to conversation

RDMA/rxe: Replace mr page map with an xarray

PATCH Superseded Bob Pearson 4 participants 16 messages Jan 13, 2023 – Jan 18, 2023

Subsystems (2)

v3 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v3 cover RDMA/rxe: Replace mr page map with an xarray

This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.

v3:
Fixed an error reported by kernel test robot
v2:
Rebased to 6.2.0-rc1+
Minor cleanups
Fixed error reported by Jason in 4/6 missing if after else.

Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

 drivers/infiniband/sw/rxe/rxe_loc.h   |   6 +-
 drivers/infiniband/sw/rxe/rxe_mr.c    | 574 +++++++++++++++-----------
 drivers/infiniband/sw/rxe/rxe_resp.c  | 107 ++---
 drivers/infiniband/sw/rxe/rxe_verbs.c |  36 --
 drivers/infiniband/sw/rxe/rxe_verbs.h |  32 +-
 5 files changed, 374 insertions(+), 381 deletions(-)

base-commit: bd99ede8ef2dc03e29a181b755ba4f78da2644e6

Signature · 1 line
2.37.2
#1 lore raw

PATCH v3 1/6 RDMA/rxe: Cleanup mr_check_range

Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.

---
 drivers/infiniband/sw/rxe/rxe_mr.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
index 072eac4b65d2..632ee1e516a1 100644
@@ -26,8 +26,6 @@ u8 rxe_get_next_key(u32 last_key)
 
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 {
-
-
 	switch (mr->ibmr.type) {
 	case IB_MR_TYPE_DMA:
 		return 0;
@@ -41,7 +39,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 
 	default:
 		rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);
-		return -EFAULT;
+		return -EINVAL;
 	}
 }
 
Signature · 1 line
2.37.2
#2 lore raw

PATCH v3 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c

In reply to Bob Pearson

Move rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   |  2 ++
 drivers/infiniband/sw/rxe/rxe_mr.c    | 36 +++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.c | 36 ---------------------------
 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 948ce4902b10..29b6c2143045 100644
@@ -69,6 +69,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset);
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
index 632ee1e516a1..229c7259644c 100644
@@ -223,6 +223,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
 	return err;
 }
 
+static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
+{
+	struct rxe_mr *mr = to_rmr(ibmr);
+	struct rxe_map *map;
+	struct rxe_phys_buf *buf;
+
+	if (unlikely(mr->nbuf == mr->num_buf))
+		return -ENOMEM;
+
+	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
+	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
+
+	buf->addr = addr;
+	buf->size = ibmr->page_size;
+	mr->nbuf++;
+
+	return 0;
+}
+
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset)
+{
+	struct rxe_mr *mr = to_rmr(ibmr);
+	int n;
+
+	mr->nbuf = 0;
+
+	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
+
+	mr->page_shift = ilog2(ibmr->page_size);
+	mr->page_mask = ibmr->page_size - 1;
+	mr->offset = ibmr->iova & mr->page_mask;
+
+	return n;
+}
+
 static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
index 025b35bf014e..7a902e0a0607 100644
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	return ERR_PTR(err);
 }
 
-static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	struct rxe_map *map;
-	struct rxe_phys_buf *buf;
-
-	if (unlikely(mr->nbuf == mr->num_buf))
-		return -ENOMEM;
-
-	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
-	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
-
-	buf->addr = addr;
-	buf->size = ibmr->page_size;
-	mr->nbuf++;
-
-	return 0;
-}
-
-static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-			 int sg_nents, unsigned int *sg_offset)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-
-	mr->nbuf = 0;
-
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
-
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
-
-	return n;
-}
-
 static ssize_t parent_show(struct device *device,
 			   struct device_attribute *attr, char *buf)
 {
Signature · 1 line
2.37.2
#3 lore raw

PATCH v3 3/6 RDMA-rxe: Isolate mr code from atomic_reply()

In reply to Bob Pearson

Isolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.

---
 drivers/infiniband/sw/rxe/rxe_loc.h  |  2 ++
 drivers/infiniband/sw/rxe/rxe_mr.c   | 30 +++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 49 ++++++++--------------------
 3 files changed, 45 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 29b6c2143045..bcb1bbcf50df 100644
@@ -72,6 +72,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			u64 compare, u64 swap_add, u64 *orig_val);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +30 -0
index 229c7259644c..791731be6067 100644
@@ -538,6 +538,36 @@ int copy_data(
 	return err;
 }
 
+static DEFINE_SPINLOCK(atomic_ops_lock);
+
+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			u64 compare, u64 swap_add, u64 *orig_val)
+{
+	u64 *vaddr = iova_to_vaddr(mr, iova, sizeof(u64));
+	u64 value;
+
+	/* needs to match rxe_resp.c */
+	if (mr->state != RXE_MR_STATE_VALID || !vaddr)
+		return -EFAULT;
+	if ((uintptr_t)vaddr & 0x7)
+		return -EINVAL;
+
+	spin_lock_bh(&atomic_ops_lock);
+	value = *orig_val = *vaddr;
+
+	if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
+		if (value == compare)
+			value = swap_add;
+	} else {
+		value += swap_add;
+	}
+
+	*vaddr = value;
+	spin_unlock_bh(&atomic_ops_lock);
+
+	return 0;
+}
+
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
 	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +13 -36

71 lines not on this page · raw message

Signature · 1 line
2.37.2
#4 lore raw

PATCH v3 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()

In reply to Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
 3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 			u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +50 -0

56 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +22 -36

77 lines not on this page · raw message

Signature · 1 line
2.37.2
#5 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 39 lines · Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

  drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
  drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
  drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
  3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
  void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
  int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
  			u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
  struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
  			 enum rxe_mr_lookup_type type);
  int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +1 -0
index 791731be6067..1e74f5e8e10b 100644
@@ -568,6 +568,56 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
  	return 0;
  }
 
+/**

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 581

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +10 -0
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write
+ *
+ * Returns:
+ *	 0 on success
+ *	-1 for misaligned address
+ *	-2 for access errors
+ *	-3 for cpu without native 64 bit support

Well, i'm not favor of such error code. especialy when it's not a staic subroutine. Any comments from other maintainers ?

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_mr.c +6 -0
+ */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr)
+{
+#if defined CONFIG_64BIT
+	u64 *vaddr;
+	u64 value;

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 597

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +10 -0
+	unsigned int length = 8;
+
+	/* See IBA oA19-28 */
+	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+		rxe_dbg_mr(mr, "mr not valid");
+		return -2;
+	}
+
+	/* See IBA A19.4.2 */
+	if (unlikely((uintptr_t)vaddr & 0x7 || iova & 0x7)) {

vaddr used before being initialized

Thanks
Zhijian

Quoted 109 lines
drivers/infiniband/sw/rxe/rxe_mr.c +23 -0
+		rxe_dbg_mr(mr, "misaligned address");
+		return -1;
+	}
+
+	vaddr = iova_to_vaddr(mr, iova, length);
+	if (unlikely(!vaddr)) {
+		rxe_dbg_mr(mr, "iova out of range");
+		return -2;
+	}
+
+	/* this makes no sense. What of payload is not 8? */
+	memcpy(&value, addr, length);
+
+	/* Do atomic write after all prior operations have completed */
+	smp_store_release(vaddr, value);
+
+	return 0;
+#else
+	rxe_dbg_mr(mr, "64 bit integers not atomic");
+	return -3;
+#endif
+}
+
  int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
  {
  	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +22 -36
index 02301e3f343c..1083cda22c65 100644
@@ -762,26 +762,33 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
  	return RESPST_ACKNOWLEDGE;
  }
 
-#ifdef CONFIG_64BIT
-static enum resp_states do_atomic_write(struct rxe_qp *qp,
-					struct rxe_pkt_info *pkt)
+static enum resp_states atomic_write_reply(struct rxe_qp *qp,
+					   struct rxe_pkt_info *pkt)
  {
+	u64 iova = qp->resp.va + qp->resp.offset;
+	struct resp_res *res = qp->resp.res;
  	struct rxe_mr *mr = qp->resp.mr;
+	void *addr = payload_addr(pkt);
  	int payload = payload_size(pkt);
-	u64 src, *dst;
-
-	if (mr->state != RXE_MR_STATE_VALID)
-		return RESPST_ERR_RKEY_VIOLATION;
+	int err;
 
-	memcpy(&src, payload_addr(pkt), payload);
+	if (!res) {
+		res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
+		qp->resp.res = res;
+	}
 
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	err = rxe_mr_do_atomic_write(mr, iova, addr);
+	if (unlikely(err)) {
+		if (err == -3)
+			return RESPST_ERR_UNSUPPORTED_OPCODE;
+		else if (err == -2)
+			return RESPST_ERR_RKEY_VIOLATION;
+		else
+			return RESPST_ERR_MISALIGNED_ATOMIC;
+	}
 
  	/* decrease resp.resid to zero */
  	qp->resp.resid -= sizeof(payload);
@@ -794,29 +801,8 @@ static enum resp_states do_atomic_write(struct rxe_qp *qp,
 
  	qp->resp.opcode = pkt->opcode;
  	qp->resp.status = IB_WC_SUCCESS;
-	return RESPST_ACKNOWLEDGE;
-}
-#else
-static enum resp_states do_atomic_write(struct rxe_qp *qp,
-					struct rxe_pkt_info *pkt)
-{
-	return RESPST_ERR_UNSUPPORTED_OPCODE;
-}
-#endif /* CONFIG_64BIT */
-
-static enum resp_states atomic_write_reply(struct rxe_qp *qp,
-					   struct rxe_pkt_info *pkt)
-{
-	struct resp_res *res = qp->resp.res;
 
-	if (!res) {
-		res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
-		qp->resp.res = res;
-	}
-
-	if (res->replay)
-		return RESPST_ACKNOWLEDGE;
-	return do_atomic_write(qp, pkt);
+	return RESPST_ACKNOWLEDGE;
  }
 
  static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
maintainer
#6 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 12 lines · On Mon, Jan 16, 2023 at 02:11:29AM +0000, [email protected] wrote:
Quoted 11 lines

+/**
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write
+ *
+ * Returns:
+ * 0 on success
+ * -1 for misaligned address
+ * -2 for access errors
+ * -3 for cpu without native 64 bit support

Quoted 2 lines

Well, i'm not favor of such error code. especialy when it's not a staic subroutine. Any comments from other maintainers ?

It should at least have proper constants, but can it be structured to avoid this in the first place?

Jason

maintainer
#7 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

In reply to Bob Pearson
Quoted 44 lines · Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
 3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
                        u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +5 -0
index 791731be6067..1e74f5e8e10b 100644
@@ -568,6 +568,56 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
        return 0;
 }
 
+/**
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 585

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +10 -0
+ *
+ * Returns:
+ *      0 on success
+ *     -1 for misaligned address
+ *     -2 for access errors
+ *     -3 for cpu without native 64 bit support
+ */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr)
+{
+#if defined CONFIG_64BIT

IS_ENABLED is better?

Zhu Yanjun

Quoted 124 lines
drivers/infiniband/sw/rxe/rxe_mr.c +35 -0
+       u64 *vaddr;
+       u64 value;
+       unsigned int length = 8;
+
+       /* See IBA oA19-28 */
+       if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+               rxe_dbg_mr(mr, "mr not valid");
+               return -2;
+       }
+
+       /* See IBA A19.4.2 */
+       if (unlikely((uintptr_t)vaddr & 0x7 || iova & 0x7)) {
+               rxe_dbg_mr(mr, "misaligned address");
+               return -1;
+       }
+
+       vaddr = iova_to_vaddr(mr, iova, length);
+       if (unlikely(!vaddr)) {
+               rxe_dbg_mr(mr, "iova out of range");
+               return -2;
+       }
+
+       /* this makes no sense. What of payload is not 8? */
+       memcpy(&value, addr, length);
+
+       /* Do atomic write after all prior operations have completed */
+       smp_store_release(vaddr, value);
+
+       return 0;
+#else
+       rxe_dbg_mr(mr, "64 bit integers not atomic");
+       return -3;
+#endif
+}
+
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
        struct rxe_sge          *sge    = &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +22 -37
index 02301e3f343c..1083cda22c65 100644
@@ -762,26 +762,33 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
        return RESPST_ACKNOWLEDGE;
 }
 
-#ifdef CONFIG_64BIT
-static enum resp_states do_atomic_write(struct rxe_qp *qp,
-                                       struct rxe_pkt_info *pkt)
+static enum resp_states atomic_write_reply(struct rxe_qp *qp,
+                                          struct rxe_pkt_info *pkt)
 {
+       u64 iova = qp->resp.va + qp->resp.offset;
+       struct resp_res *res = qp->resp.res;
        struct rxe_mr *mr = qp->resp.mr;
+       void *addr = payload_addr(pkt);
        int payload = payload_size(pkt);
-       u64 src, *dst;
-
-       if (mr->state != RXE_MR_STATE_VALID)
-               return RESPST_ERR_RKEY_VIOLATION;
+       int err;
 
-       memcpy(&src, payload_addr(pkt), payload);
+       if (!res) {
+               res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
+               qp->resp.res = res;
+       }
 
-       dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-       /* check vaddr is 8 bytes aligned. */
-       if (!dst || (uintptr_t)dst & 7)
-               return RESPST_ERR_MISALIGNED_ATOMIC;
+       if (res->replay)
+               return RESPST_ACKNOWLEDGE;
 
-       /* Do atomic write after all prior operations have completed */
-       smp_store_release(dst, src);
+       err = rxe_mr_do_atomic_write(mr, iova, addr);
+       if (unlikely(err)) {
+               if (err == -3)
+                       return RESPST_ERR_UNSUPPORTED_OPCODE;
+               else if (err == -2)
+                       return RESPST_ERR_RKEY_VIOLATION;
+               else
+                       return RESPST_ERR_MISALIGNED_ATOMIC;
+       }
 
        /* decrease resp.resid to zero */
        qp->resp.resid -= sizeof(payload);
@@ -794,29 +801,8 @@ static enum resp_states do_atomic_write(struct rxe_qp *qp,
 
        qp->resp.opcode = pkt->opcode;
        qp->resp.status = IB_WC_SUCCESS;
-       return RESPST_ACKNOWLEDGE;
-}
-#else
-static enum resp_states do_atomic_write(struct rxe_qp *qp,
-                                       struct rxe_pkt_info *pkt)
-{
-       return RESPST_ERR_UNSUPPORTED_OPCODE;
-}
-#endif /* CONFIG_64BIT */
-
-static enum resp_states atomic_write_reply(struct rxe_qp *qp,
-                                          struct rxe_pkt_info *pkt)
-{
-       struct resp_res *res = qp->resp.res;
 
-       if (!res) {
-               res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
-               qp->resp.res = res;
-       }
-
-       if (res->replay)
-               return RESPST_ACKNOWLEDGE;
-       return do_atomic_write(qp, pkt);
+       return RESPST_ACKNOWLEDGE;
 }
 
 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
--

2.37.2

maintainer
#8 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 49 lines · Zhu Yanjun
Quoted 48 lines · Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
 3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
                        u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +9 -0
index 791731be6067..1e74f5e8e10b 100644
@@ -568,6 +568,56 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
        return 0;
 }
 
+/**
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write
+ *
+ * Returns:
+ *      0 on success
+ *     -1 for misaligned address
Quoted 8 lines

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 585

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_mr.c +6 -0
+ *     -2 for access errors
+ *     -3 for cpu without native 64 bit support
+ */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr)
+{
+#if defined CONFIG_64BIT

IS_ENABLED is better?

is_enabled won't work here because the code doesn't compile.

Jason

#9 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 59 lines · Jason Gunthorpe
Quoted 49 lines · Zhu Yanjun
Quoted 48 lines · Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
 3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
                        u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +9 -0
index 791731be6067..1e74f5e8e10b 100644
@@ -568,6 +568,56 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
        return 0;
 }
 
+/**
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write
+ *
+ * Returns:
+ *      0 on success
+ *     -1 for misaligned address
Quoted 8 lines

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 585

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_mr.c +6 -0
+ *     -2 for access errors
+ *     -3 for cpu without native 64 bit support
+ */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr)
+{
+#if defined CONFIG_64BIT

IS_ENABLED is better?

Quoted 3 lines

is_enabled won't work here because the code doesn't compile.

Jason

exactly.

maintainer
#10 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 61 lines · Bob Pearson
Quoted 59 lines · Jason Gunthorpe
Quoted 57 lines · Zhu Yanjun
Quoted 44 lines · Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
 3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
                        u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +5 -0
index 791731be6067..1e74f5e8e10b 100644
@@ -568,6 +568,56 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
        return 0;
 }
 
+/**
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 585

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +10 -0
+ *
+ * Returns:
+ *      0 on success
+ *     -1 for misaligned address
+ *     -2 for access errors
+ *     -3 for cpu without native 64 bit support
+ */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr)
+{
+#if defined CONFIG_64BIT

IS_ENABLED is better?

Quoted 2 lines
Quoted 2 lines

is_enabled won't work here because the code doesn't compile.

drivers/infiniband/sw/rxe/rxe_net.c:

45
46 #if IS_ENABLED(CONFIG_IPV6)
47 static struct dst_entry *rxe_find_route6(struct rxe_qp *qp,

 48                                          struct net_device *ndev,
 49                                          struct in6_addr *saddr,
 50                                          struct in6_addr *daddr)

51 {
52 struct dst_entry *ndst;
53 struct flowi6 fl6 = { { 0 } };

Zhu Yanjun

Quoted 3 lines
Quoted 1 line

Jason

exactly.

maintainer
#11 lore raw

Re: [PATCH for-next v3 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 62 lines · Zhu Yanjun
Quoted 61 lines · Bob Pearson
Quoted 59 lines · Jason Gunthorpe
Quoted 57 lines · Zhu Yanjun
Quoted 44 lines · Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
v3:
Fixed bug reported by kernel test robot. Ifdef'ed out atomic 8 byte write if CONFIG_64BIT is not defined as orignally intended by the developers of the atomic write implementation.

 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 50 ++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 58 +++++++++++-----------------
 3 files changed, 73 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..fd70c71a9e4e 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
                        u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +5 -0
index 791731be6067..1e74f5e8e10b 100644
@@ -568,6 +568,56 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
        return 0;
 }
 
+/**
+ * rxe_mr_do_atomic_write() - write 64bit value to iova from addr
+ * @mr: memory region
+ * @iova: iova in mr
+ * @addr: source of data to write

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 585

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +10 -0
+ *
+ * Returns:
+ *      0 on success
+ *     -1 for misaligned address
+ *     -2 for access errors
+ *     -3 for cpu without native 64 bit support
+ */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr)
+{
+#if defined CONFIG_64BIT

IS_ENABLED is better?

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines

is_enabled won't work here because the code doesn't compile.

drivers/infiniband/sw/rxe/rxe_net.c:

45
46 #if IS_ENABLED(CONFIG_IPV6)

You only need this pattern if the config symbol is tristate

CONFIG_64BIT is a bool

Jason

#12 lore raw

PATCH v3 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c

In reply to Bob Pearson

Cleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.

---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 31 ++++++++++++---------------
 drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++++++---
 2 files changed, 22 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +14 -17
index 1e74f5e8e10b..fd5537ee7f04 100644
@@ -60,6 +60,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
 	mr->lkey = mr->ibmr.lkey = lkey;
 	mr->rkey = mr->ibmr.rkey = rkey;
 
+	mr->ibmr.page_size = PAGE_SIZE;
+	mr->page_mask = PAGE_MASK;
+	mr->page_shift = PAGE_SHIFT;
 	mr->state = RXE_MR_STATE_INVALID;
 }
 
@@ -149,9 +152,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		goto err_release_umem;
 	}
 
-	mr->page_shift = PAGE_SHIFT;
-	mr->page_mask = PAGE_SIZE - 1;
-
 	num_buf			= 0;
 	map = mr->map;
 	if (length > 0) {
@@ -180,7 +180,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 				goto err_release_umem;
 			}
 			buf->addr = (uintptr_t)vaddr;
-			buf->size = PAGE_SIZE;
+			buf->size = mr_page_size(mr);
 			num_buf++;
 			buf++;
 
@@ -189,10 +189,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 
 	mr->umem = umem;
 	mr->access = access;
-	mr->offset = ib_umem_offset(umem);
+	mr->page_offset = ib_umem_offset(umem);
 	mr->state = RXE_MR_STATE_VALID;
 	mr->ibmr.type = IB_MR_TYPE_USER;
-	mr->ibmr.page_size = PAGE_SIZE;
 
 	return 0;
 
@@ -246,29 +245,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-
-	mr->nbuf = 0;
+	unsigned int page_size = mr_page_size(mr);
 
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
+	mr->page_shift = ilog2(page_size);
+	mr->page_mask = ~((u64)page_size - 1);
+	mr->page_offset = ibmr->iova & (page_size - 1);
 
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
+	mr->nbuf = 0;
 
-	return n;
+	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
 }
 
 static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
-	size_t offset = iova - mr->ibmr.iova + mr->offset;
+	size_t offset = iova - mr->ibmr.iova + mr->page_offset;
 	int			map_index;
 	int			buf_index;
 	u64			length;
 
 	if (likely(mr->page_shift)) {
-		*offset_out = offset & mr->page_mask;
+		*offset_out = offset & (mr_page_size(mr) - 1);
 		offset >>= mr->page_shift;
 		*n_out = offset & mr->map_mask;
 		*m_out = offset >> mr->map_shift;
@@ -342,7 +339,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)
 	if (mr->ibmr.type == IB_MR_TYPE_DMA)
 		return -EFAULT;
 
-	offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;
+	offset = (iova - mr->ibmr.iova + mr->page_offset) & mr->page_mask;
 	while (length > 0) {
 		u8 *va;
 		int bytes;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
index 19ddfa890480..bfc94caaeec5 100644
@@ -310,11 +310,11 @@ struct rxe_mr {
 	u32			lkey;
 	u32			rkey;
 	enum rxe_mr_state	state;
-	u32			offset;
 	int			access;
 
-	int			page_shift;
-	int			page_mask;
+	unsigned int		page_offset;
+	unsigned int		page_shift;
+	u64			page_mask;
 	int			map_shift;
 	int			map_mask;
 
@@ -329,6 +329,11 @@ struct rxe_mr {
 	struct rxe_map		**map;
 };
 
+static inline unsigned int mr_page_size(struct rxe_mr *mr)
+{
+	return mr ? mr->ibmr.page_size : PAGE_SIZE;
+}
+
 enum rxe_mw_state {
 	RXE_MW_STATE_INVALID	= RXE_MR_STATE_INVALID,
 	RXE_MW_STATE_FREE	= RXE_MR_STATE_FREE,
Signature · 1 line
2.37.2
#13 lore raw

PATCH v3 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

In reply to Bob Pearson

Replace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.

This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   |   1 -
 drivers/infiniband/sw/rxe/rxe_mr.c    | 533 ++++++++++++--------------
 drivers/infiniband/sw/rxe/rxe_resp.c  |   2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  21 +-
 4 files changed, 251 insertions(+), 306 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +0 -1
index fd70c71a9e4e..0cf78f9bb27c 100644
@@ -71,7 +71,6 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 			u64 compare, u64 swap_add, u64 *orig_val);
 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
drivers/infiniband/sw/rxe/rxe_mr.c +248 -285

715 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
index 1083cda22c65..b0ff36f6dc4a 100644
@@ -744,7 +744,7 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 					  atmeth_comp(pkt),
 					  atmeth_swap_add(pkt),
 					  &res->atomic.orig_val);
-		if (err == -EINVAL)
+		if (err == -1)
 			return RESPST_ERR_MISALIGNED_ATOMIC;
 		else if (err)
 			return RESPST_ERR_RKEY_VIOLATION;
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
index bfc94caaeec5..64551cf354a6 100644
@@ -283,17 +283,6 @@ enum rxe_mr_lookup_type {
 	RXE_LOOKUP_REMOTE,
 };
 
-#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
-
-struct rxe_phys_buf {
-	u64      addr;
-	u64      size;
-};
-
-struct rxe_map {
-	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
-};
-
 static inline int rkey_is_mw(u32 rkey)
 {
 	u32 index = rkey >> 8;
@@ -311,22 +300,16 @@ struct rxe_mr {
 	u32			rkey;
 	enum rxe_mr_state	state;
 	int			access;
+	atomic_t		num_mw;
 
 	unsigned int		page_offset;
 	unsigned int		page_shift;
 	u64			page_mask;
-	int			map_shift;
-	int			map_mask;
 
 	u32			num_buf;
 	u32			nbuf;
 
-	u32			max_buf;
-	u32			num_map;
-
-	atomic_t		num_mw;
-
-	struct rxe_map		**map;
+	struct xarray		pages;
 };
 
 static inline unsigned int mr_page_size(struct rxe_mr *mr)
Signature · 1 line
2.37.2
#14 lore raw

Re: [PATCH for-next v3 6/6] RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

Quoted 35 lines · Bob Pearson

Replace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.

This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.

---
  drivers/infiniband/sw/rxe/rxe_loc.h   |   1 -
  drivers/infiniband/sw/rxe/rxe_mr.c    | 533 ++++++++++++--------------
  drivers/infiniband/sw/rxe/rxe_resp.c  |   2 +-
  drivers/infiniband/sw/rxe/rxe_verbs.h |  21 +-
  4 files changed, 251 insertions(+), 306 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +0 -1
index fd70c71a9e4e..0cf78f9bb27c 100644
@@ -71,7 +71,6 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
  	      void *addr, int length, enum rxe_mr_copy_dir dir);
  int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
  		  int sg_nents, unsigned int *sg_offset);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
  int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
  			u64 compare, u64 swap_add, u64 *orig_val);
  int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
drivers/infiniband/sw/rxe/rxe_mr.c +0 -0
index fd5537ee7f04..e4634279080a 100644

[snip...]

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -5
-static bool is_pmem_page(struct page *pg)
+static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
  {
-	unsigned long paddr = page_to_phys(pg);
+	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
+}
 
-	return REGION_INTERSECTS ==
-	       region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
-				 IORES_DESC_PERSISTENT_MEMORY);

[snip...]

Quoted 31 lines
drivers/infiniband/sw/rxe/rxe_mr.c +6 -16
+	rxe_mr_init(access, mr);
+
  	umem = ib_umem_get(&rxe->ib_dev, start, length, access);
  	if (IS_ERR(umem)) {
  		rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
  			(int)PTR_ERR(umem));
-		err = PTR_ERR(umem);
-		goto err_out;
+		return PTR_ERR(umem);
  	}
 
-	num_buf = ib_umem_num_pages(umem);
-
-	rxe_mr_init(access, mr);
-
-	err = rxe_mr_alloc(mr, num_buf);
+	err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
  	if (err) {
-		rxe_dbg_mr(mr, "Unable to allocate memory for map\n");
-		goto err_release_umem;
+		ib_umem_release(umem);
+		return err;
  	}
 
-	num_buf			= 0;
-	map = mr->map;
-	if (length > 0) {
-		bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT;
-
-		buf = map[0]->buf;
-		for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {

drivers/infiniband/sw/rxe/rxe_mr.c static void rxe_mr_init(int access, struct rxe_mr *mr) · line 168

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -6
-			struct page *pg = sg_page_iter_page(&sg_iter);
+	mr->umem = umem;
+	mr->ibmr.type = IB_MR_TYPE_USER;
+	mr->state = RXE_MR_STATE_VALID;
 
-			if (persistent_access && !is_pmem_page(pg)) {
-				rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n");
-				err = -EINVAL;
-				goto err_release_umem;
-			}

I read you removed is_pmem_page and its checking, but there is no description about this. IMO, it's required by IBA spec.

Thanks
Zhijian

#15 lore raw

Re: [PATCH for-next v3 6/6] RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

Quoted 98 lines · On 1/15/23 20:21, [email protected] wrote:
Quoted 35 lines · Bob Pearson

Replace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.

This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.

---
  drivers/infiniband/sw/rxe/rxe_loc.h   |   1 -
  drivers/infiniband/sw/rxe/rxe_mr.c    | 533 ++++++++++++--------------
  drivers/infiniband/sw/rxe/rxe_resp.c  |   2 +-
  drivers/infiniband/sw/rxe/rxe_verbs.h |  21 +-
  4 files changed, 251 insertions(+), 306 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +0 -1
index fd70c71a9e4e..0cf78f9bb27c 100644
@@ -71,7 +71,6 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
  	      void *addr, int length, enum rxe_mr_copy_dir dir);
  int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
  		  int sg_nents, unsigned int *sg_offset);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
  int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
  			u64 compare, u64 swap_add, u64 *orig_val);
  int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, void *addr);
drivers/infiniband/sw/rxe/rxe_mr.c +0 -0
index fd5537ee7f04..e4634279080a 100644

[snip...]

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -5
-static bool is_pmem_page(struct page *pg)
+static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
  {
-	unsigned long paddr = page_to_phys(pg);
+	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
+}
 
-	return REGION_INTERSECTS ==
-	       region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
-				 IORES_DESC_PERSISTENT_MEMORY);

[snip...]

Quoted 31 lines
drivers/infiniband/sw/rxe/rxe_mr.c +6 -16
+	rxe_mr_init(access, mr);
+
  	umem = ib_umem_get(&rxe->ib_dev, start, length, access);
  	if (IS_ERR(umem)) {
  		rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
  			(int)PTR_ERR(umem));
-		err = PTR_ERR(umem);
-		goto err_out;
+		return PTR_ERR(umem);
  	}
 
-	num_buf = ib_umem_num_pages(umem);
-
-	rxe_mr_init(access, mr);
-
-	err = rxe_mr_alloc(mr, num_buf);
+	err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
  	if (err) {
-		rxe_dbg_mr(mr, "Unable to allocate memory for map\n");
-		goto err_release_umem;
+		ib_umem_release(umem);
+		return err;
  	}
 
-	num_buf			= 0;
-	map = mr->map;
-	if (length > 0) {
-		bool persistent_access = access & IB_ACCESS_FLUSH_PERSISTENT;
-
-		buf = map[0]->buf;
-		for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {

drivers/infiniband/sw/rxe/rxe_mr.c static void rxe_mr_init(int access, struct rxe_mr *mr) · line 168

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -6
-			struct page *pg = sg_page_iter_page(&sg_iter);
+	mr->umem = umem;
+	mr->ibmr.type = IB_MR_TYPE_USER;
+	mr->state = RXE_MR_STATE_VALID;
 
-			if (persistent_access && !is_pmem_page(pg)) {
-				rxe_dbg_mr(mr, "Unable to register persistent access to non-pmem device\n");
-				err = -EINVAL;
-				goto err_release_umem;
-			}

I read you removed is_pmem_page and its checking, but there is no description about this. IMO, it's required by IBA spec.

Quoted 2 lines

Thanks
Zhijian

Zhijian,

It was dropped by accident. I just posted an update with the calls put back. Please take a look and if possible can you test to see if it doesn't regress your pmem changes. I have no way to test pmem. Also look at a comment I added to the pmem flush call. I am suspicious of the smp_store_release() call based on the original comment.

Regards,

Bob