Skip to contentSkip to conversation

RDMA/mana_ib: Extend MANA RC QP support for UAPI

PATCH Applied Konstantin Taranov 3 participants 8 messages Aug 27 – Sep 1

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

v2 cover RDMA/mana_ib: Extend MANA RC QP support for UAPI

From: Konstantin Taranov <[email protected]>

Extend MANA RC QP support with an updated userspace ABI and optional MMQ and fixed-size send WQE features.

The series first extends the RC QP creation request and response while preserving compatibility with existing userspace. It then allows userspace to create an MMQ and receive its queue ID. Finally, it adds fixed-size send WQE support, including MSN-in-WQE when supported by the adapter.

Existing applications continue to use RC QPs without these optional features.

v1->v2: use << instead of BIT() in the mana-abi.h

Konstantin Taranov (3):
RDMA/mana_ib: Extend RC QP udata request and response RDMA/mana_ib: MMQ creation for RC QPs RDMA/mana_ib: fixed send wqe size support for RC QPs

 drivers/infiniband/hw/mana/main.c    |  4 +++
 drivers/infiniband/hw/mana/mana_ib.h | 11 ++++++--
 drivers/infiniband/hw/mana/qp.c      | 42 +++++++++++++++++++++-------
 include/net/mana/gdma.h              |  1 +
 include/uapi/rdma/mana-abi.h         | 11 ++++++++
 5 files changed, 56 insertions(+), 13 deletions(-)
Signature · 1 line
2.43.0
#1 lore raw

PATCH v2 1/3 RDMA/mana_ib: Extend RC QP udata request and response

From: Konstantin Taranov <[email protected]>

Extend the RC QP creation UAPI to support optional fixed-size WQEs and an MMQ. Add a compatibility mask to the request, return the MMQ ID in the response, and advertise the extended ABI through ucontext capabilities.

Use compatibility-aware udata validation so existing userspace remains supported. Rename the legacy FMR queue definitions to MMQ and add the corresponding hardware creation flags.

---
v1->v2: use << instead of BIT() in the mana-abi.h

 drivers/infiniband/hw/mana/main.c    |  1 +
 drivers/infiniband/hw/mana/mana_ib.h |  6 ++++--
 drivers/infiniband/hw/mana/qp.c      | 10 +++++-----
 include/uapi/rdma/mana-abi.h         | 11 +++++++++++
 4 files changed, 21 insertions(+), 7 deletions(-)
drivers/infiniband/hw/mana/main.c +1 -0
index e4414d208..9d043f48e 100644
@@ -247,6 +247,7 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
 
 	ucontext->doorbell = doorbell_page;
 	ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT;
+	ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT;
 	ret = ib_respond_udata(udata, ucmd_resp);
 	if (ret)
 		return ret;
drivers/infiniband/hw/mana/mana_ib.h +4 -2
index f69866696..547ddf9dd 100644
@@ -172,7 +172,7 @@ struct mana_ib_cq {
 enum mana_rc_queue_type {
 	MANA_RC_SEND_QUEUE_REQUESTER = 0,
 	MANA_RC_SEND_QUEUE_RESPONDER,
-	MANA_RC_SEND_QUEUE_FMR,
+	MANA_RC_SEND_QUEUE_MMQ,
 	MANA_RC_RECV_QUEUE_REQUESTER,
 	MANA_RC_RECV_QUEUE_RESPONDER,
 	MANA_RC_QUEUE_TYPE_MAX,
@@ -372,7 +372,9 @@ struct mana_rnic_destroy_cq_resp {
 }; /* HW Data */
 
 enum mana_rnic_create_rc_flags {
-	MANA_RC_FLAG_NO_FMR = 2,
+	MANA_RC_FLAG_NO_MMQ = BIT(1),
+	MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3),
+	MANA_RC_FLAG_MSN_IN_WQE = BIT(4),
 };
 
 struct mana_rnic_create_qp_req {
drivers/infiniband/hw/mana/qp.c +5 -5
index 8b7be1255..aa5f8c607 100644
@@ -549,14 +549,14 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
 	doorbell = mana_ucontext->doorbell;
-	flags = MANA_RC_FLAG_NO_FMR;
-	err = ib_copy_validate_udata_in(udata, ucmd, queue_size);
+	flags = MANA_RC_FLAG_NO_MMQ;
+	err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size,
+					   MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE);
 	if (err)
 		return err;
 
 	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-		/* skip FMR for user-level RC QPs */
-		if (i == MANA_RC_SEND_QUEUE_FMR) {
+		if (i == MANA_RC_SEND_QUEUE_MMQ) {
 			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
 			qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
 			continue;
@@ -580,7 +580,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	if (udata) {
 		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-			if (i == MANA_RC_SEND_QUEUE_FMR)
+			if (i == MANA_RC_SEND_QUEUE_MMQ)
 				continue;
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
 			j++;
include/uapi/rdma/mana-abi.h +11 -0
index 32cbbfc80..c9a241766 100644
@@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp {
 	__u32 reserved;
 };
 
+enum mana_ib_create_rc_qp_flags {
+	MANA_IB_RC_QP_FIXED_WQE = 1 << 0,
+	MANA_IB_RC_MMQ_CREATE = 1 << 1,
+};
+
 struct mana_ib_create_rc_qp {
 	__aligned_u64 queue_buf[4];
 	__u32 queue_size[4];
+	__aligned_u64 mmq_buf;
+	__u32 mmq_size;
+	__u32 comp_mask;
 };
 
 struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
+	__u32 mmq_id;
+	__u32 reserved;
 };
 
 struct mana_ib_create_uc_qp {
@@ -100,6 +110,7 @@ struct mana_ib_create_qp_rss_resp {
 
 enum mana_ib_ucontext_support {
 	MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+	MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1,
 };
 
 struct mana_ib_alloc_ucontext_resp {
Signature · 1 line
2.43.0
Reviewed-by on v2 1/3
#2 lore raw

RE: [PATCH rdma-next v2 1/3] RDMA/mana_ib: Extend RC QP udata request and response

Quoted 11 lines

From: Konstantin Taranov <[email protected]>

Extend the RC QP creation UAPI to support optional fixed-size WQEs and an MMQ. Add a compatibility mask to the request, return the MMQ ID in the response, and advertise the extended ABI through ucontext capabilities.

Use compatibility-aware udata validation so existing userspace remains supported. Rename the legacy FMR queue definitions to MMQ and add the corresponding hardware creation flags.

Quoted 121 lines

---
v1->v2: use << instead of BIT() in the mana-abi.h

 drivers/infiniband/hw/mana/main.c    |  1 +
 drivers/infiniband/hw/mana/mana_ib.h |  6 ++++--
 drivers/infiniband/hw/mana/qp.c      | 10 +++++-----
 include/uapi/rdma/mana-abi.h         | 11 +++++++++++
 4 files changed, 21 insertions(+), 7 deletions(-)
drivers/infiniband/hw/mana/main.c +1 -0
index e4414d208..9d043f48e 100644
@@ -247,6 +247,7 @@ int mana_ib_alloc_ucontext(struct ib_ucontext
 *ibcontext,
 
 	ucontext->doorbell = doorbell_page;
 	ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT;
+	ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT;
 	ret = ib_respond_udata(udata, ucmd_resp);
 	if (ret)
drivers/infiniband/hw/mana/main.c +0 -0
 		return ret;
drivers/infiniband/hw/mana/mana_ib.h +4 -2
index f69866696..547ddf9dd 100644
@@ -172,7 +172,7 @@ struct mana_ib_cq {
 enum mana_rc_queue_type {
 	MANA_RC_SEND_QUEUE_REQUESTER = 0,
 	MANA_RC_SEND_QUEUE_RESPONDER,
-	MANA_RC_SEND_QUEUE_FMR,
+	MANA_RC_SEND_QUEUE_MMQ,
 	MANA_RC_RECV_QUEUE_REQUESTER,
 	MANA_RC_RECV_QUEUE_RESPONDER,
 	MANA_RC_QUEUE_TYPE_MAX,
@@ -372,7 +372,9 @@ struct mana_rnic_destroy_cq_resp {  }; /* HW Data */
 
 enum mana_rnic_create_rc_flags {
-	MANA_RC_FLAG_NO_FMR = 2,
+	MANA_RC_FLAG_NO_MMQ = BIT(1),
+	MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3),
+	MANA_RC_FLAG_MSN_IN_WQE = BIT(4),
 };
 
 struct mana_rnic_create_qp_req {
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index 8b7be1255..aa5f8c607 100644

drivers/infiniband/hw/mana/qp.c +4 -4
@@ -549,14 +549,14 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp,
 struct ib_pd *ibpd,
 
 	mana_ucontext = rdma_udata_to_drv_context(udata, struct
 mana_ib_ucontext, ibucontext);
 	doorbell = mana_ucontext->doorbell;
-	flags = MANA_RC_FLAG_NO_FMR;
-	err = ib_copy_validate_udata_in(udata, ucmd, queue_size);
+	flags = MANA_RC_FLAG_NO_MMQ;
+	err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size,
+					   MANA_IB_RC_QP_FIXED_WQE |
 MANA_IB_RC_MMQ_CREATE);
 	if (err)
 		return err;
 
 	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-		/* skip FMR for user-level RC QPs */
-		if (i == MANA_RC_SEND_QUEUE_FMR) {
+		if (i == MANA_RC_SEND_QUEUE_MMQ) {
drivers/infiniband/hw/mana/qp.c +0 -0
 			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
 			qp->rc_qp.queues[i].gdma_region =

GDMA_INVALID_DMA_REGION;

drivers/infiniband/hw/mana/qp.c +1 -1
 			continue;
@@ -580,7 +580,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp,
 struct ib_pd *ibpd,
 
 	if (udata) {
 		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-			if (i == MANA_RC_SEND_QUEUE_FMR)
+			if (i == MANA_RC_SEND_QUEUE_MMQ)
 				continue;
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
drivers/infiniband/hw/mana/qp.c +0 -0
 			j++;
include/uapi/rdma/mana-abi.h +11 -1
index 32cbbfc80..c9a241766 100644
@@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp {
 	__u32 reserved;
 };
 
+enum mana_ib_create_rc_qp_flags {
+	MANA_IB_RC_QP_FIXED_WQE = 1 << 0,
+	MANA_IB_RC_MMQ_CREATE = 1 << 1,
+};
+
 struct mana_ib_create_rc_qp {
 	__aligned_u64 queue_buf[4];
 	__u32 queue_size[4];
+	__aligned_u64 mmq_buf;
+	__u32 mmq_size;
+	__u32 comp_mask;
 };
 
 struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
+	__u32 mmq_id;
+	__u32 reserved;
 };
 
 struct mana_ib_create_uc_qp {
@@ -100,6 +110,7 @@ struct mana_ib_create_qp_rss_resp {
 
 enum mana_ib_ucontext_support {
 	MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+	MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1,
 };
 
 struct mana_ib_alloc_ucontext_resp {
--

2.43.0

#3 lore raw

PATCH v2 2/3 RDMA/mana_ib: MMQ creation for RC QPs

In reply to Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Create an MMQ for an RC QP when userspace sets the corresponding compatibility-mask flag. Register the userspace MMQ buffer and pass the queue to hardware instead of setting the NO_MMQ creation flag.

Keep the MMQ disabled for applications that do not request it, preserving the existing behavior. Return the created MMQ ID to userspace in the extended RC QP response.

---
 drivers/infiniband/hw/mana/qp.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
drivers/infiniband/hw/mana/qp.c +14 -6
index aa5f8c607..c6821ae5f 100644
@@ -557,16 +557,22 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
 		if (i == MANA_RC_SEND_QUEUE_MMQ) {
-			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
-			qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+			if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) {
+				flags &= ~MANA_RC_FLAG_NO_MMQ;
+				err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size,
+							   &qp->rc_qp.queues[i]);
+				if (err)
+					goto destroy_queues;
+			} else {
+				qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
+				qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+			}
 			continue;
 		}
 		err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
 					   &qp->rc_qp.queues[i]);
-		if (err) {
-			ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
+		if (err)
 			goto destroy_queues;
-		}
 		j++;
 	}
 
@@ -580,8 +586,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	if (udata) {
 		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-			if (i == MANA_RC_SEND_QUEUE_MMQ)
+			if (i == MANA_RC_SEND_QUEUE_MMQ) {
+				resp.mmq_id = qp->rc_qp.queues[i].id;
 				continue;
+			}
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
 			j++;
 		}
Signature · 1 line
2.43.0
Reviewed-by on v2 2/3
#4 lore raw

RE: [PATCH rdma-next v2 2/3] RDMA/mana_ib: MMQ creation for RC QPs

Quoted 11 lines

From: Konstantin Taranov <[email protected]>

Create an MMQ for an RC QP when userspace sets the corresponding compatibility-mask flag. Register the userspace MMQ buffer and pass the queue to hardware instead of setting the NO_MMQ creation flag.

Keep the MMQ disabled for applications that do not request it, preserving the existing behavior. Return the created MMQ ID to userspace in the extended RC QP response.

Quoted 59 lines
---
 drivers/infiniband/hw/mana/qp.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index aa5f8c607..c6821ae5f 100644

drivers/infiniband/hw/mana/qp.c +10 -2
@@ -557,16 +557,22 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp,
 struct ib_pd *ibpd,
 
 	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
 		if (i == MANA_RC_SEND_QUEUE_MMQ) {
-			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
-			qp->rc_qp.queues[i].gdma_region =
 GDMA_INVALID_DMA_REGION;
+			if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE)
 {
+				flags &= ~MANA_RC_FLAG_NO_MMQ;
+				err = mana_ib_create_queue(mdev,
 ucmd.mmq_buf, ucmd.mmq_size,
+							   &qp-
 >rc_qp.queues[i]);
+				if (err)
+					goto destroy_queues;
+			} else {
+				qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
+				qp->rc_qp.queues[i].gdma_region =
 GDMA_INVALID_DMA_REGION;
+			}
 			continue;
 		}
 		err = mana_ib_create_queue(mdev, ucmd.queue_buf[j],

ucmd.queue_size[j],

drivers/infiniband/hw/mana/qp.c +0 -2
 					   &qp->rc_qp.queues[i]);
-		if (err) {
-			ibdev_err(&mdev->ib_dev, "Failed to create queue %d,

err %d\n", i, err);

drivers/infiniband/hw/mana/qp.c +4 -2
+		if (err)
 			goto destroy_queues;
-		}
 		j++;
 	}
@@ -580,8 +586,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp,
 struct ib_pd *ibpd,
 
 	if (udata) {
 		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-			if (i == MANA_RC_SEND_QUEUE_MMQ)
+			if (i == MANA_RC_SEND_QUEUE_MMQ) {
+				resp.mmq_id = qp->rc_qp.queues[i].id;
 				continue;
+			}
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
 			j++;

}
--
2.43.0

#5 lore raw

PATCH v2 3/3 RDMA/mana_ib: fixed send wqe size support for RC QPs

In reply to Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Support fixed-size send WQEs for RC QPs when requested by userspace. Calculate the WQE size from the maximum send SGE count and the 32-byte inline OOB size, round it up to a power of two, and pass the size to hardware in basic units.

When supported by the adapter, also enable placing the message sequence number in the WQE. Preserve the existing variable-size WQE behavior when userspace does not request fixed-size WQEs.

---
 drivers/infiniband/hw/mana/main.c    |  3 +++
 drivers/infiniband/hw/mana/mana_ib.h |  5 ++++-
 drivers/infiniband/hw/mana/qp.c      | 14 ++++++++++++++
 include/net/mana/gdma.h              |  1 +
 4 files changed, 22 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/main.c +3 -0
index 9d043f48e..57c6d9faa 100644
@@ -1060,6 +1060,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	req.max_recv_sge = attr->cap.max_recv_sge;
 	req.flags = flags;
 
+	if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE)
+		req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
+
 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
 	if (err)
 		return err;
drivers/infiniband/hw/mana/mana_ib.h +4 -1
index 547ddf9dd..828ca75fd 100644
@@ -180,6 +180,7 @@ enum mana_rc_queue_type {
 
 struct mana_ib_rc_qp {
 	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
+	u32 wqe_size_in_bu;
 };
 
 enum mana_uc_queue_type {
@@ -262,6 +263,7 @@ enum mana_ib_adapter_features {
 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
 	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
 	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
+	MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
 };
 
 struct mana_ib_query_adapter_caps_resp {
@@ -391,7 +393,8 @@ struct mana_rnic_create_qp_req {
 	u32 max_recv_wr;
 	u32 max_send_sge;
 	u32 max_recv_sge;
-	u32 reserved;
+	u8 wqe_size_in_bu;
+	u8 reserved[3];
 }; /* HW Data */
 
 struct mana_rnic_create_qp_resp {
drivers/infiniband/hw/mana/qp.c +14 -0
index c6821ae5f..29bfee9ac 100644
@@ -422,6 +422,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
 	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);
 }
 
+static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size)
+{
+	return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size));
+}
+
 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
 {
 	u32 queue_size;
@@ -576,6 +581,15 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 		j++;
 	}
 
+	if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) {
+		u32 wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge,
+						      INLINE_OOB_EXTRA_LARGE_SIZE);
+		flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
+		if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)
+			flags |= MANA_RC_FLAG_MSN_IN_WQE;
+		qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
+	}
+
 	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create rc qp  %d\n", err);
include/net/mana/gdma.h +1 -0
index de17c9bba..f3601ca78 100644
@@ -522,6 +522,7 @@ struct gdma_wqe {
 
 #define INLINE_OOB_SMALL_SIZE 8
 #define INLINE_OOB_LARGE_SIZE 24
+#define INLINE_OOB_EXTRA_LARGE_SIZE 32
 
 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30
 
Signature · 1 line
2.43.0
Reviewed-by on v2 3/3
#6 lore raw

RE: [PATCH rdma-next v2 3/3] RDMA/mana_ib: fixed send wqe size support for RC QPs

Quoted 7 lines

From: Konstantin Taranov <[email protected]>

Support fixed-size send WQEs for RC QPs when requested by userspace. Calculate the WQE size from the maximum send SGE count and the 32-byte inline OOB size, round it up to a power of two, and pass the size to hardware in basic units.

Quoted 5 lines

When supported by the adapter, also enable placing the message sequence number in the WQE. Preserve the existing variable-size WQE behavior when userspace does not request fixed-size WQEs.

Quoted 102 lines
---
 drivers/infiniband/hw/mana/main.c    |  3 +++
 drivers/infiniband/hw/mana/mana_ib.h |  5 ++++-
 drivers/infiniband/hw/mana/qp.c      | 14 ++++++++++++++
 include/net/mana/gdma.h              |  1 +
 4 files changed, 22 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/main.c +3 -0
index 9d043f48e..57c6d9faa 100644
@@ -1060,6 +1060,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev
 *mdev, struct mana_ib_qp *qp,
 	req.max_recv_sge = attr->cap.max_recv_sge;
 	req.flags = flags;
 
+	if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE)
+		req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
+
 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),
 &resp);
drivers/infiniband/hw/mana/main.c +0 -0
 	if (err)
 		return err;
drivers/infiniband/hw/mana/mana_ib.h +2 -0
index 547ddf9dd..828ca75fd 100644
@@ -180,6 +180,7 @@ enum mana_rc_queue_type {
 
 struct mana_ib_rc_qp {
 	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
+	u32 wqe_size_in_bu;
 };
 
 enum mana_uc_queue_type {
@@ -262,6 +263,7 @@ enum mana_ib_adapter_features {
 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
 	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
 	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
+	MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
 };
 
 struct mana_ib_query_adapter_caps_resp { @@ -391,7 +393,8 @@ struct

mana_rnic_create_qp_req {

drivers/infiniband/hw/mana/mana_ib.h +2 -1
 	u32 max_recv_wr;
 	u32 max_send_sge;
 	u32 max_recv_sge;
-	u32 reserved;
+	u8 wqe_size_in_bu;
+	u8 reserved[3];
 }; /* HW Data */
 
 struct mana_rnic_create_qp_resp {
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index c6821ae5f..29bfee9ac 100644

drivers/infiniband/hw/mana/qp.c +8 -0
@@ -422,6 +422,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
 	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);  }
 
+static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size) {
+	return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size)); }
+
 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)  {
 	u32 queue_size;
@@ -576,6 +581,15 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp,
 struct ib_pd *ibpd,
 		j++;
 	}
 
+	if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) {
+		u32 wqe_size = mana_ib_fixed_wqe_size(attr-
 >cap.max_send_sge,
+
 INLINE_OOB_EXTRA_LARGE_SIZE);
+		flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
+		if (mdev->adapter_caps.feature_flags &

MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)

drivers/infiniband/hw/mana/qp.c +2 -0
+			flags |= MANA_RC_FLAG_MSN_IN_WQE;
+		qp->rc_qp.wqe_size_in_bu = wqe_size /

GDMA_WQE_BU_SIZE;

drivers/infiniband/hw/mana/qp.c +2 -0
+	}
+
 	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create rc qp  %d\n", err);
include/net/mana/gdma.h index renamed from include/net/mana/gdma.h +0 -0

de17c9bba..f3601ca78 100644

include/net/mana/gdma.h +1 -1
@@ -522,6 +522,7 @@ struct gdma_wqe {
 
 #define INLINE_OOB_SMALL_SIZE 8
 #define INLINE_OOB_LARGE_SIZE 24
+#define INLINE_OOB_EXTRA_LARGE_SIZE 32
 
 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30
 
--

2.43.0

#7 lore raw
In reply to Konstantin Taranov
Quoted 5 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Extend MANA RC QP support with an updated userspace ABI and optional MMQ and fixed-size send WQE features.

Quoted 7 lines

The series first extends the RC QP creation request and response while preserving compatibility with existing userspace. It then allows userspace to create an MMQ and receive its queue ID. Finally, it adds fixed-size send WQE support, including MSN-in-WQE when supported by the adapter.

[...]

Applied, thanks!

[1/3] RDMA/mana_ib: Extend RC QP udata request and response

      https://git.kernel.org/rdma/rdma/c/71fc2d52047e79
[2/3] RDMA/mana_ib: MMQ creation for RC QPs
      https://git.kernel.org/rdma/rdma/c/4309bec53b2496
[3/3] RDMA/mana_ib: fixed send wqe size support for RC QPs
      https://git.kernel.org/rdma/rdma/c/23d5ff8fd62f27

Best regards,

Signature · 1 line
Leon Romanovsky <[email protected]>