Skip to contentSkip to conversation

RDMA/rxe: Replace mr page map with an xarray

PATCH Superseded Bob Pearson 2 participants 16 messages Jan 16, 2023 – Jan 18, 2023

Subsystems (2)

v5 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v5 cover RDMA/rxe: Replace mr page map with an xarray

This patch series replaces the page map carried in each memory region with a struct xarray. It is based on a sketch developed by Jason Gunthorpe. The first five patches are preparation that tries to cleanly isolate all the mr specific code into rxe_mr.c. The sixth patch is the actual change.

v5:
Responded to a note from [email protected] and restored calls to is_pmem_page() which were accidentally dropped in earlier versions.
v4:
Responded to a comment by Zhu and cleaned up error passing between rxe_mr.c and rxe_resp.c.
Other various cleanups including more careful use of unsigned ints. Rebased to current for-next.
v3:
Fixed an error reported by kernel test robot
v2:
Rebased to 6.2.0-rc1+
Minor cleanups
Fixed error reported by Jason in 4/6 missing if after else.

Bob Pearson (6):
RDMA/rxe: Cleanup mr_check_range
RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c
RDMA-rxe: Isolate mr code from atomic_reply()
RDMA-rxe: Isolate mr code from atomic_write_reply() RDMA/rxe: Cleanup page variables in rxe_mr.c RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

 drivers/infiniband/sw/rxe/rxe_loc.h   |  13 +-
 drivers/infiniband/sw/rxe/rxe_mr.c    | 625 +++++++++++++++-----------
 drivers/infiniband/sw/rxe/rxe_resp.c  | 118 ++---
 drivers/infiniband/sw/rxe/rxe_verbs.c |  36 --
 drivers/infiniband/sw/rxe/rxe_verbs.h |  37 +-
 5 files changed, 425 insertions(+), 404 deletions(-)

base-commit: 1ec82317a1daac78c04b0c15af89018ccf9fa2b7

Signature · 1 line
2.37.2
#1 lore raw

PATCH v5 1/6 RDMA/rxe: Cleanup mr_check_range

Remove blank lines and replace EFAULT by EINVAL when an invalid mr type is used.

---
 drivers/infiniband/sw/rxe/rxe_mr.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +1 -3
index 072eac4b65d2..632ee1e516a1 100644
@@ -26,8 +26,6 @@ u8 rxe_get_next_key(u32 last_key)
 
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 {
-
-
 	switch (mr->ibmr.type) {
 	case IB_MR_TYPE_DMA:
 		return 0;
@@ -41,7 +39,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 
 	default:
 		rxe_dbg_mr(mr, "type (%d) not supported\n", mr->ibmr.type);
-		return -EFAULT;
+		return -EINVAL;
 	}
 }
 
Signature · 1 line
2.37.2
#2 lore raw

PATCH v5 2/6 RDMA/rxe: Move rxe_map_mr_sg to rxe_mr.c

In reply to Bob Pearson

Move rxe_map_mr_sg() to rxe_mr.c where it makes a little more sense.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   |  2 ++
 drivers/infiniband/sw/rxe/rxe_mr.c    | 36 +++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.c | 36 ---------------------------
 3 files changed, 38 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 948ce4902b10..29b6c2143045 100644
@@ -69,6 +69,8 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset);
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
drivers/infiniband/sw/rxe/rxe_mr.c +36 -0
index 632ee1e516a1..229c7259644c 100644
@@ -223,6 +223,42 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
 	return err;
 }
 
+static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
+{
+	struct rxe_mr *mr = to_rmr(ibmr);
+	struct rxe_map *map;
+	struct rxe_phys_buf *buf;
+
+	if (unlikely(mr->nbuf == mr->num_buf))
+		return -ENOMEM;
+
+	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
+	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
+
+	buf->addr = addr;
+	buf->size = ibmr->page_size;
+	mr->nbuf++;
+
+	return 0;
+}
+
+int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		  int sg_nents, unsigned int *sg_offset)
+{
+	struct rxe_mr *mr = to_rmr(ibmr);
+	int n;
+
+	mr->nbuf = 0;
+
+	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
+
+	mr->page_shift = ilog2(ibmr->page_size);
+	mr->page_mask = ibmr->page_size - 1;
+	mr->offset = ibmr->iova & mr->page_mask;
+
+	return n;
+}
+
 static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -36
index 025b35bf014e..7a902e0a0607 100644
@@ -948,42 +948,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	return ERR_PTR(err);
 }
 
-static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	struct rxe_map *map;
-	struct rxe_phys_buf *buf;
-
-	if (unlikely(mr->nbuf == mr->num_buf))
-		return -ENOMEM;
-
-	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
-	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
-
-	buf->addr = addr;
-	buf->size = ibmr->page_size;
-	mr->nbuf++;
-
-	return 0;
-}
-
-static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-			 int sg_nents, unsigned int *sg_offset)
-{
-	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-
-	mr->nbuf = 0;
-
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
-
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
-
-	return n;
-}
-
 static ssize_t parent_show(struct device *device,
 			   struct device_attribute *attr, char *buf)
 {
Signature · 1 line
2.37.2
#3 lore raw

PATCH v5 3/6 RDMA-rxe: Isolate mr code from atomic_reply()

In reply to Bob Pearson

Isolate mr specific code from atomic_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_op() in rxe_mr.c.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   |  2 +
 drivers/infiniband/sw/rxe/rxe_mr.c    | 83 ++++++++++++++++++---------
 drivers/infiniband/sw/rxe/rxe_resp.c  | 49 +++++-----------
 drivers/infiniband/sw/rxe/rxe_verbs.h |  5 ++
 4 files changed, 76 insertions(+), 63 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 29b6c2143045..bcb1bbcf50df 100644
@@ -72,6 +72,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
+int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
+			u64 compare, u64 swap_add, u64 *orig_val);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +55 -28

111 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +14 -35

73 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.h +5 -0
index 19ddfa890480..691200d99d6b 100644
@@ -273,6 +273,11 @@ enum rxe_mr_state {
 	RXE_MR_STATE_VALID,
 };
 
+/* some rxe specific error conditions */
+enum rxe_mr_error {
+	RXE_ERR_NOT_ALIGNED = 0x1001,	/* misaligned address */
+};
+
 enum rxe_mr_copy_dir {
 	RXE_TO_MR_OBJ,
 	RXE_FROM_MR_OBJ,
Signature · 1 line
2.37.2
maintainer
#4 lore raw

Re: [PATCH for-next v5 3/6] RDMA-rxe: Isolate mr code from atomic_reply()

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 751

Quoted 9 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_resp.c +9 -0
+		err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
+					  atmeth_comp(pkt),
+					  atmeth_swap_add(pkt),
+					  &res->atomic.orig_val);
+		if (unlikely(err)) {
+			if (err == -RXE_ERR_NOT_ALIGNED)
+				return RESPST_ERR_MISALIGNED_ATOMIC;
+			else
+				return RESPST_ERR_RKEY_VIOLATION;

Why not just return these RESPST_ constants directly from rxe_mr_do_atomic_op ?

Jason

#5 lore raw

Re: [PATCH for-next v5 3/6] RDMA-rxe: Isolate mr code from atomic_reply()

Quoted 15 lines · Jason Gunthorpe

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 751

Quoted 9 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_resp.c +9 -0
+		err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
+					  atmeth_comp(pkt),
+					  atmeth_swap_add(pkt),
+					  &res->atomic.orig_val);
+		if (unlikely(err)) {
+			if (err == -RXE_ERR_NOT_ALIGNED)
+				return RESPST_ERR_MISALIGNED_ATOMIC;
+			else
+				return RESPST_ERR_RKEY_VIOLATION;

Why not just return these RESPST_ constants directly from rxe_mr_do_atomic_op ?

Quoted 1 line

Jason

The main reason is that the responder state machine is fairly complicated and hiding bits of it in other files just makes it more difficult to follow. Originally all the code was inline here but I felt it would be better to keep all the mr specific detail in one file.

Bob

#6 lore raw

PATCH v5 4/6 RDMA-rxe: Isolate mr code from atomic_write_reply()

In reply to Bob Pearson

Isolate mr specific code from atomic_write_reply() in rxe_resp.c into a subroutine rxe_mr_do_atomic_write() in rxe_mr.c.
Check length for atomic write operation.
Make iova_to_vaddr() static.

---
 drivers/infiniband/sw/rxe/rxe_loc.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_mr.c   | 34 ++++++++++++++
 drivers/infiniband/sw/rxe/rxe_resp.c | 69 ++++++++++++----------------
 3 files changed, 64 insertions(+), 40 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -0
index bcb1bbcf50df..b1dda0cf891b 100644
@@ -74,6 +74,7 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 			u64 compare, u64 swap_add, u64 *orig_val);
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
drivers/infiniband/sw/rxe/rxe_mr.c +34 -0
index 15a8d44daa35..10484f671977 100644
@@ -565,6 +565,40 @@ int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 	return 0;
 }
 
+/* only implemented for 64 bit architectures */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
+{
+#if defined CONFIG_64BIT
+	u64 *va;
+
+	/* See IBA oA19-28 */
+	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+		rxe_dbg_mr(mr, "mr not in valid state");
+		return -EINVAL;
+	}
+
+	va = iova_to_vaddr(mr, iova, sizeof(value));
+	if (unlikely(!va)) {
+		rxe_dbg_mr(mr, "iova out of range");
+		return -ERANGE;
+	}
+
+	/* See IBA A19.4.2 */
+	if (unlikely((uintptr_t)va & 0x7 || iova & 0x7)) {
+		rxe_dbg_mr(mr, "misaligned address");
+		return -RXE_ERR_NOT_ALIGNED;
+	}
+
+	/* Do atomic write after all prior operations have completed */
+	smp_store_release(va, value);
+
+	return 0;
+#else
+	WARN_ON(1);
+	return -EINVAL;
+#endif
+}
+
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
 	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +29 -40

88 lines not on this page · raw message

Signature · 1 line
2.37.2
maintainer
#7 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 571

Quoted 4 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_mr.c +4 -0
+/* only implemented for 64 bit architectures */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
+{
+#if defined CONFIG_64BIT

#ifdef

It is a little more typical style to provide an alternate version of the function when #ifdefing

Quoted 71 lines
drivers/infiniband/sw/rxe/rxe_mr.c +30 -0
+	u64 *va;
+
+	/* See IBA oA19-28 */
+	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+		rxe_dbg_mr(mr, "mr not in valid state");
+		return -EINVAL;
+	}
+
+	va = iova_to_vaddr(mr, iova, sizeof(value));
+	if (unlikely(!va)) {
+		rxe_dbg_mr(mr, "iova out of range");
+		return -ERANGE;
+	}
+
+	/* See IBA A19.4.2 */
+	if (unlikely((uintptr_t)va & 0x7 || iova & 0x7)) {
+		rxe_dbg_mr(mr, "misaligned address");
+		return -RXE_ERR_NOT_ALIGNED;
+	}
+
+	/* Do atomic write after all prior operations have completed */
+	smp_store_release(va, value);
+
+	return 0;
+#else
+	WARN_ON(1);
+	return -EINVAL;
+#endif
+}
+
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
 	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +13 -14
index 1e38e5da1f4c..49298ff88d25 100644
@@ -764,30 +764,40 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 	return RESPST_ACKNOWLEDGE;
 }
 
-#ifdef CONFIG_64BIT
-static enum resp_states do_atomic_write(struct rxe_qp *qp,
-					struct rxe_pkt_info *pkt)
+static enum resp_states atomic_write_reply(struct rxe_qp *qp,
+					   struct rxe_pkt_info *pkt)
 {
-	struct rxe_mr *mr = qp->resp.mr;
-	int payload = payload_size(pkt);
-	u64 src, *dst;
-
-	if (mr->state != RXE_MR_STATE_VALID)
-		return RESPST_ERR_RKEY_VIOLATION;
+	struct resp_res *res = qp->resp.res;
+	struct rxe_mr *mr;
+	u64 value;
+	u64 iova;
+	int err;
 
-	memcpy(&src, payload_addr(pkt), payload);
+	if (!res) {
+		res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
+		qp->resp.res = res;
+	}
 
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_resp.c +4 -4
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
 
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 794

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c +6 -0
+	err = rxe_mr_do_atomic_write(mr, iova, value);
+	if (unlikely(err)) {
+		if (err == -RXE_ERR_NOT_ALIGNED)
+			return RESPST_ERR_MISALIGNED_ATOMIC;
+		else
+			return RESPST_ERR_RKEY_VIOLATION;

Again why not return the RESPST directly then the stub can return RESPST_ERR_UNSUPPORTED_OPCODE?

Jason

#8 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 11 lines · Jason Gunthorpe

drivers/infiniband/sw/rxe/rxe_mr.c int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, · line 571

Quoted 4 lines · Bob Pearson
drivers/infiniband/sw/rxe/rxe_mr.c +4 -0
+/* only implemented for 64 bit architectures */
+int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
+{
+#if defined CONFIG_64BIT

#ifdef

It is a little more typical style to provide an alternate version of the function when #ifdefing

I will do that.

Quoted 79 lines
Quoted 79 lines
drivers/infiniband/sw/rxe/rxe_mr.c +30 -0
+	u64 *va;
+
+	/* See IBA oA19-28 */
+	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+		rxe_dbg_mr(mr, "mr not in valid state");
+		return -EINVAL;
+	}
+
+	va = iova_to_vaddr(mr, iova, sizeof(value));
+	if (unlikely(!va)) {
+		rxe_dbg_mr(mr, "iova out of range");
+		return -ERANGE;
+	}
+
+	/* See IBA A19.4.2 */
+	if (unlikely((uintptr_t)va & 0x7 || iova & 0x7)) {
+		rxe_dbg_mr(mr, "misaligned address");
+		return -RXE_ERR_NOT_ALIGNED;
+	}
+
+	/* Do atomic write after all prior operations have completed */
+	smp_store_release(va, value);
+
+	return 0;
+#else
+	WARN_ON(1);
+	return -EINVAL;
+#endif
+}
+
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
 	struct rxe_sge		*sge	= &dma->sge[dma->cur_sge];
drivers/infiniband/sw/rxe/rxe_resp.c +16 -16
index 1e38e5da1f4c..49298ff88d25 100644
@@ -764,30 +764,40 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,
 	return RESPST_ACKNOWLEDGE;
 }
 
-#ifdef CONFIG_64BIT
-static enum resp_states do_atomic_write(struct rxe_qp *qp,
-					struct rxe_pkt_info *pkt)
+static enum resp_states atomic_write_reply(struct rxe_qp *qp,
+					   struct rxe_pkt_info *pkt)
 {
-	struct rxe_mr *mr = qp->resp.mr;
-	int payload = payload_size(pkt);
-	u64 src, *dst;
-
-	if (mr->state != RXE_MR_STATE_VALID)
-		return RESPST_ERR_RKEY_VIOLATION;
+	struct resp_res *res = qp->resp.res;
+	struct rxe_mr *mr;
+	u64 value;
+	u64 iova;
+	int err;
 
-	memcpy(&src, payload_addr(pkt), payload);
+	if (!res) {
+		res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
+		qp->resp.res = res;
+	}
 
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
 
Quoted 5 lines

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788

Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1 -2
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.

I really don't understand why Fujitsu did it all this way instead of just using a spinlock for 32 bit architectures as a fallback. But if I want to keep to the spirit of their implementation this is fairly clear I think.

Bob

Quoted 12 lines

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 794

Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_resp.c +6 -0
+	err = rxe_mr_do_atomic_write(mr, iova, value);
+	if (unlikely(err)) {
+		if (err == -RXE_ERR_NOT_ALIGNED)
+			return RESPST_ERR_MISALIGNED_ATOMIC;
+		else
+			return RESPST_ERR_RKEY_VIOLATION;

Again why not return the RESPST directly then the stub can return RESPST_ERR_UNSUPPORTED_OPCODE?

Jason

maintainer
#9 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 13 lines · Bob Pearson
Quoted 13 lines
Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_resp.c +5 -6
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
Quoted 8 lines
Quoted 5 lines

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788

Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1 -2
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.

? That doesn't seem right

Quoted 3 lines

I really don't understand why Fujitsu did it all this way instead of just using a spinlock for 32 bit architectures as a fallback. But if I want to keep to the spirit of their implementation this is fairly clear I think.

IIRC the IBA definition is that this is supposed to be coherent with the host CPU, the spinlock version is for the non-atomic atomics which only has to be coherent with the "hca"

So a spinlock will not provide coherency with userspace that may be touching this same atomic memory.

Jason

#10 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 21 lines · Jason Gunthorpe
Quoted 19 lines · Bob Pearson
Quoted 18 lines
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c +2 -4
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_resp.c +4 -4
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
 
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

Quoted 4 lines
Quoted 2 lines

This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.

? That doesn't seem right

that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.

Quoted 11 lines
Quoted 4 lines

I really don't understand why Fujitsu did it all this way instead of just using a spinlock for 32 bit architectures as a fallback. But if I want to keep to the spirit of their implementation this is fairly clear I think.

IIRC the IBA definition is that this is supposed to be coherent with the host CPU, the spinlock version is for the non-atomic atomics which only has to be coherent with the "hca"

So a spinlock will not provide coherency with userspace that may be touching this same atomic memory.

Thanks. That makes sense.

Bob

Quoted 2 lines

Jason

maintainer
#11 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 22 lines · Bob Pearson
Quoted 21 lines · Jason Gunthorpe
Quoted 19 lines · Bob Pearson
Quoted 18 lines
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c +2 -4
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_resp.c +4 -4
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
 
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.

? That doesn't seem right

that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.

I feel like it should be solvable without this ifdef though

Jason

#12 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 28 lines · Jason Gunthorpe
Quoted 27 lines · Bob Pearson
Quoted 25 lines · Jason Gunthorpe
Quoted 13 lines · Bob Pearson
Quoted 13 lines
Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_resp.c +5 -6
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
Quoted 8 lines
Quoted 5 lines

drivers/infiniband/sw/rxe/rxe_resp.c static enum resp_states atomic_reply(struct rxe_qp *qp, · line 788

Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1 -2
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.

? That doesn't seem right

Quoted 7 lines
Quoted 3 lines

that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.

I feel like it should be solvable without this ifdef though

Jason

You could get rid of the ifdef in the atomic_write_reply() routine but then the rxe_mr_do_atomic_write() routine would have to have a second version in the #else case that would have to return something different so that the third exit could be taken i.e.
whatever replaces the original -3. I really think this is simpler.

Bob

maintainer
#13 lore raw

Re: [PATCH for-next v5 4/6] RDMA-rxe: Isolate mr code from atomic_write_reply()

Quoted 33 lines · Bob Pearson
Quoted 32 lines · Jason Gunthorpe
Quoted 30 lines · Bob Pearson
Quoted 21 lines · Jason Gunthorpe
Quoted 19 lines · Bob Pearson
Quoted 18 lines
Quoted 6 lines
(quoted diff) +2 -4
-	dst = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset, payload);
-	/* check vaddr is 8 bytes aligned. */
-	if (!dst || (uintptr_t)dst & 7)
-		return RESPST_ERR_MISALIGNED_ATOMIC;
+	if (res->replay)
+		return RESPST_ACKNOWLEDGE;

Review

Quoted 10 lines
(quoted diff) +4 -4
 
-	/* Do atomic write after all prior operations have completed */
-	smp_store_release(dst, src);
+	mr = qp->resp.mr;
+	value = *(u64 *)payload_addr(pkt);
+	iova = qp->resp.va + qp->resp.offset;
 
-	/* decrease resp.resid to zero */
-	qp->resp.resid -= sizeof(payload);
+#if defined CONFIG_64BIT

Shouldn't need a #ifdef here

Quoted 4 lines
Quoted 2 lines

This avoids a new special error (i.e. NOT_64_bit) and makes it clear we won't call the code in mr.

? That doesn't seem right

that was the -3 of the -1, -2, -3 that we just fixed. there are three error paths out of this state and we need a way to get to them. The #ifdef provides that third path.

Quoted 7 lines
Quoted 3 lines

I feel like it should be solvable without this ifdef though

Jason

You could get rid of the ifdef in the atomic_write_reply() routine but then the rxe_mr_do_atomic_write() routine would have to have a second version in the #else case that would have to return something different so that the third exit could be taken i.e.
whatever replaces the original -3. I really think this is simpler.

You really should just return the RESPST_* from these functions.

Jason

#14 lore raw

PATCH v5 5/6 RDMA/rxe: Cleanup page variables in rxe_mr.c

In reply to Bob Pearson

Cleanup usage of mr->page_shift and mr->page_mask and introduce an extractor for mr->ibmr.page_size. Normal usage in the kernel has page_mask masking out offset in page rather than masking out the page number. The rxe driver had reversed that which was confusing. Implicitly there can be a per mr page_size which was not uniformly supported.

---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 31 ++++++++++++---------------
 drivers/infiniband/sw/rxe/rxe_verbs.h | 11 +++++++---
 2 files changed, 22 insertions(+), 20 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +14 -17
index 10484f671977..fdf76df4cf3e 100644
@@ -62,6 +62,9 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
 	mr->lkey = mr->ibmr.lkey = lkey;
 	mr->rkey = mr->ibmr.rkey = rkey;
 
+	mr->ibmr.page_size = PAGE_SIZE;
+	mr->page_mask = PAGE_MASK;
+	mr->page_shift = PAGE_SHIFT;
 	mr->state = RXE_MR_STATE_INVALID;
 }
 
@@ -151,9 +154,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		goto err_release_umem;
 	}
 
-	mr->page_shift = PAGE_SHIFT;
-	mr->page_mask = PAGE_SIZE - 1;
-
 	num_buf			= 0;
 	map = mr->map;
 	if (length > 0) {
@@ -182,7 +182,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 				goto err_release_umem;
 			}
 			buf->addr = (uintptr_t)vaddr;
-			buf->size = PAGE_SIZE;
+			buf->size = mr_page_size(mr);
 			num_buf++;
 			buf++;
 
@@ -191,10 +191,9 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 
 	mr->umem = umem;
 	mr->access = access;
-	mr->offset = ib_umem_offset(umem);
+	mr->page_offset = ib_umem_offset(umem);
 	mr->state = RXE_MR_STATE_VALID;
 	mr->ibmr.type = IB_MR_TYPE_USER;
-	mr->ibmr.page_size = PAGE_SIZE;
 
 	return 0;
 
@@ -248,29 +247,27 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset)
 {
 	struct rxe_mr *mr = to_rmr(ibmr);
-	int n;
-
-	mr->nbuf = 0;
+	unsigned int page_size = mr_page_size(mr);
 
-	n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
+	mr->page_shift = ilog2(page_size);
+	mr->page_mask = ~((u64)page_size - 1);
+	mr->page_offset = ibmr->iova & (page_size - 1);
 
-	mr->page_shift = ilog2(ibmr->page_size);
-	mr->page_mask = ibmr->page_size - 1;
-	mr->offset = ibmr->iova & mr->page_mask;
+	mr->nbuf = 0;
 
-	return n;
+	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page);
 }
 
 static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
 			size_t *offset_out)
 {
-	size_t offset = iova - mr->ibmr.iova + mr->offset;
+	size_t offset = iova - mr->ibmr.iova + mr->page_offset;
 	int			map_index;
 	int			buf_index;
 	u64			length;
 
 	if (likely(mr->page_shift)) {
-		*offset_out = offset & mr->page_mask;
+		*offset_out = offset & (mr_page_size(mr) - 1);
 		offset >>= mr->page_shift;
 		*n_out = offset & mr->map_mask;
 		*m_out = offset >> mr->map_shift;
@@ -329,7 +326,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length)
 	if (mr->ibmr.type == IB_MR_TYPE_DMA)
 		return -EFAULT;
 
-	offset = (iova - mr->ibmr.iova + mr->offset) & mr->page_mask;
+	offset = (iova - mr->ibmr.iova + mr->page_offset) & mr->page_mask;
 	while (length > 0) {
 		u8 *va;
 		int bytes;
drivers/infiniband/sw/rxe/rxe_verbs.h +8 -3
index 691200d99d6b..5c3d1500ca68 100644
@@ -315,11 +315,11 @@ struct rxe_mr {
 	u32			lkey;
 	u32			rkey;
 	enum rxe_mr_state	state;
-	u32			offset;
 	int			access;
 
-	int			page_shift;
-	int			page_mask;
+	unsigned int		page_offset;
+	unsigned int		page_shift;
+	u64			page_mask;
 	int			map_shift;
 	int			map_mask;
 
@@ -334,6 +334,11 @@ struct rxe_mr {
 	struct rxe_map		**map;
 };
 
+static inline unsigned int mr_page_size(struct rxe_mr *mr)
+{
+	return mr ? mr->ibmr.page_size : PAGE_SIZE;
+}
+
 enum rxe_mw_state {
 	RXE_MW_STATE_INVALID	= RXE_MR_STATE_INVALID,
 	RXE_MW_STATE_FREE	= RXE_MR_STATE_FREE,
Signature · 1 line
2.37.2
#15 lore raw

PATCH v5 6/6 RDMA/rxe: Replace rxe_map and rxe_phys_buf by xarray

In reply to Bob Pearson

Replace struct rxe-phys_buf and struct rxe_map by struct xarray in rxe_verbs.h. This allows using rcu locking on reads for the memory maps stored in each mr.

This is based off of a sketch of a patch from Jason Gunthorpe in the link below. Some changes were needed to make this work. It applies cleanly to the current for-next and passes the pyverbs, perftest and the same blktests test cases which run today.

---
v5:

Responded to a note from [email protected] and restored calls to is_pmem_page() which were accidentally dropped in earlier versions.

 drivers/infiniband/sw/rxe/rxe_loc.h   |   8 +-
 drivers/infiniband/sw/rxe/rxe_mr.c    | 559 +++++++++++++-------------
 drivers/infiniband/sw/rxe/rxe_verbs.h |  21 +-
 3 files changed, 285 insertions(+), 303 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +4 -4
index b1dda0cf891b..c1f78ed0f991 100644
@@ -64,14 +64,14 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr);
 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr);
 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
-int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, int length);
-int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
-		enum rxe_mr_copy_dir dir);
+int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova,
+			unsigned int length);
+int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
+		unsigned int length, enum rxe_mr_copy_dir dir);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
-void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
 			u64 compare, u64 swap_add, u64 *orig_val);
 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
drivers/infiniband/sw/rxe/rxe_mr.c +279 -280

737 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.h +2 -19
index 5c3d1500ca68..09437077ceee 100644
@@ -288,17 +288,6 @@ enum rxe_mr_lookup_type {
 	RXE_LOOKUP_REMOTE,
 };
 
-#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
-
-struct rxe_phys_buf {
-	u64      addr;
-	u64      size;
-};
-
-struct rxe_map {
-	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
-};
-
 static inline int rkey_is_mw(u32 rkey)
 {
 	u32 index = rkey >> 8;
@@ -316,22 +305,16 @@ struct rxe_mr {
 	u32			rkey;
 	enum rxe_mr_state	state;
 	int			access;
+	atomic_t		num_mw;
 
 	unsigned int		page_offset;
 	unsigned int		page_shift;
 	u64			page_mask;
-	int			map_shift;
-	int			map_mask;
 
 	u32			num_buf;
 	u32			nbuf;
 
-	u32			max_buf;
-	u32			num_map;
-
-	atomic_t		num_mw;
-
-	struct rxe_map		**map;
+	struct xarray		page_list;
 };
 
 static inline unsigned int mr_page_size(struct rxe_mr *mr)
Signature · 1 line
2.37.2