Skip to contentSkip to conversation

RDMA/mana_ib: UC QP support

PATCH Superseded Konstantin Taranov 1 participant 3 messages Jul 17

Subsystems (2)

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v4 cover RDMA/mana_ib: UC QP support

From: Konstantin Taranov <[email protected]>

Implement UC QP creation in the RNIC HW for user API.

v4: renamed rc_qp destrot helper to rnic_qp. Use the latest robust udata API. Unified QP lookup table to simplify code for inserting UC QPs. That is why the changes were split into 2 patches. The first patch is about the changes related to qp lookup/store.
v3: use new udata helpers
v2: removed udata check and removed enum from mana-abi.h

Konstantin Taranov (2):
RDMA/mana_ib: unify QP lookup table
RDMA/mana_ib: UC QP support for UAPI

 drivers/infiniband/hw/mana/main.c    |  49 ++++++++-
 drivers/infiniband/hw/mana/mana_ib.h |  83 +++++++++++++--
 drivers/infiniband/hw/mana/qp.c      | 154 +++++++++++++++++++--------
 include/uapi/rdma/mana-abi.h         |  11 ++
 4 files changed, 243 insertions(+), 54 deletions(-)
Signature · 1 line
2.43.0
#1 lore raw

PATCH v4 1/2 RDMA/mana_ib: unify QP lookup table

From: Konstantin Taranov <[email protected]>

Add helpers to retrieve the send and receive queues of a QP. Use these helpers when storing queue IDs in the lookup table. MANA queue IDs are 2-bit aligned, allowing the two least significant bits to be omitted when storing and looking up queue IDs.

---
 drivers/infiniband/hw/mana/mana_ib.h | 30 ++++++++++++
 drivers/infiniband/hw/mana/qp.c      | 70 ++++++++++++----------------
 2 files changed, 59 insertions(+), 41 deletions(-)
drivers/infiniband/hw/mana/mana_ib.h +30 -0
index da05966af..18688072f 100644
@@ -26,6 +26,8 @@
 
 /* Send queue ID mask */
 #define MANA_SENDQ_MASK	BIT(31)
+/* Queue ID encodes type in the lower 2 bits */
+#define MANA_QID_SUBTYPE_MASK 0x3
 
 /*
  * The hardware limit of number of MRs is greater than maximum number of MRs
@@ -582,12 +584,40 @@ static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
 	return mdev->gdma_dev->gdma_context;
 }
 
+static inline struct mana_ib_queue *mana_qp_get_sq(struct mana_ib_qp *qp)
+{
+	switch (qp->ibqp.qp_type) {
+	case IB_QPT_RC:
+		return &qp->rc_qp.queues[MANA_RC_SEND_QUEUE_REQUESTER];
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return &qp->ud_qp.queues[MANA_UD_SEND_QUEUE];
+	default:
+		return NULL;
+	}
+}
+
+static inline struct mana_ib_queue *mana_qp_get_rq(struct mana_ib_qp *qp)
+{
+	switch (qp->ibqp.qp_type) {
+	case IB_QPT_RC:
+		return &qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER];
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return &qp->ud_qp.queues[MANA_UD_RECV_QUEUE];
+	default:
+		return NULL;
+	}
+}
+
 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,
 						 u32 qid, bool is_sq)
 {
 	struct mana_ib_qp *qp;
 	unsigned long flag;
 
+	/* Remove subtype bits */
+	qid &= ~MANA_QID_SUBTYPE_MASK;
 	if (is_sq)
 		qid |= MANA_SENDQ_MASK;
 
drivers/infiniband/hw/mana/qp.c +29 -41

97 lines not on this page · raw message

Signature · 1 line
2.43.0
#2 lore raw

PATCH v4 2/2 RDMA/mana_ib: UC QP support for UAPI

In reply to Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Implement UC QP creation in the RNIC HW for user API. An UC QP is exposed as three work queues: send, receive, and memory management. The latter is used for bind and invalidate WQEs to support memory windows.

---
 drivers/infiniband/hw/mana/main.c    | 49 ++++++++++++++--
 drivers/infiniband/hw/mana/mana_ib.h | 53 ++++++++++++++++--
 drivers/infiniband/hw/mana/qp.c      | 84 +++++++++++++++++++++++++++-
 include/uapi/rdma/mana-abi.h         | 11 ++++
 4 files changed, 184 insertions(+), 13 deletions(-)
drivers/infiniband/hw/mana/main.c +44 -5
index 0bfb1883a..52feb474b 100644
@@ -1046,16 +1046,55 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	return 0;
 }
 
-int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+int mana_ib_gd_create_uc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
+			    struct ib_qp_init_attr *attr, u32 doorbell, u64 flags)
 {
-	struct mana_rnic_destroy_rc_qp_resp resp = {0};
-	struct mana_rnic_destroy_rc_qp_req req = {0};
+	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
+	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
+	struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd);
 	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct mana_rnic_create_uc_qp_resp resp = {};
+	struct mana_rnic_create_uc_qp_req req = {};
+	int err, i;
 
-	mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof(resp));
+	mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UC_QP, sizeof(req), sizeof(resp));
 	req.hdr.dev_id = mdev->gdma_dev->dev_id;
 	req.adapter = mdev->adapter_handle;
-	req.rc_qp_handle = qp->qp_handle;
+	req.pd_handle = pd->pd_handle;
+	req.send_cq_handle = send_cq->cq_handle;
+	req.recv_cq_handle = recv_cq->cq_handle;
+	for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; i++)
+		req.dma_region[i] = qp->uc_qp.queues[i].gdma_region;
+	req.doorbell_page = doorbell;
+	req.max_send_wr = attr->cap.max_send_wr;
+	req.max_recv_wr = attr->cap.max_recv_wr;
+	req.max_send_sge = attr->cap.max_send_sge;
+	req.max_recv_sge = attr->cap.max_recv_sge;
+	req.flags = flags;
+
+	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
+	if (err)
+		return err;
+
+	qp->qp_handle = resp.qp_handle;
+	for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; i++) {
+		qp->uc_qp.queues[i].id = resp.queue_ids[i];
+		/* The GDMA regions are now owned by the RNIC QP handle */
+		qp->uc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+	}
+	return 0;
+}
+
+int mana_ib_gd_destroy_rnic_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+{
+	struct mana_rnic_destroy_rnic_qp_resp resp = {0};
+	struct mana_rnic_destroy_rnic_qp_req req = {0};
+	struct gdma_context *gc = mdev_to_gc(mdev);
+
+	mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RNIC_QP, sizeof(req), sizeof(resp));
+	req.hdr.dev_id = mdev->gdma_dev->dev_id;
+	req.adapter = mdev->adapter_handle;
+	req.qp_handle = qp->qp_handle;
 
 	return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
 }
drivers/infiniband/hw/mana/mana_ib.h +47 -6

103 lines not on this page · raw message

drivers/infiniband/hw/mana/qp.c +82 -2

126 lines not on this page · raw message

include/uapi/rdma/mana-abi.h +11 -0
index 8336bf51b..169bf91b3 100644
@@ -57,6 +57,17 @@ struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
 };
 
+struct mana_ib_create_uc_qp {
+	__aligned_u64 queue_buf[3];
+	__u32 queue_size[3];
+	__u32 comp_mask;
+};
+
+struct mana_ib_create_uc_qp_resp {
+	__u32 queue_id[3];
+	__u32 reserved;
+};
+
 struct mana_ib_create_wq {
 	__aligned_u64 wq_buf_addr;
 	__u32 wq_buf_size;
Signature · 1 line
2.43.0