Skip to contentSkip to conversation

RDMA/mana_ib: UC QP support for UAPI

PATCH Superseded Konstantin Taranov 2 participants 2 messages Mar 19

Subsystems (2)

v1 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 RDMA/mana_ib: UC QP support for UAPI

From: Konstantin Taranov <[email protected]>

Implement UC QP creation in the RNIC HW for user API. An UC QP is exposed as three work queues: send, receive, and memory management. The latter is used for bind and invalidate WQEs to support memory windows.

---
 drivers/infiniband/hw/mana/main.c    | 41 ++++++++++++-
 drivers/infiniband/hw/mana/mana_ib.h | 41 ++++++++++++-
 drivers/infiniband/hw/mana/qp.c      | 92 ++++++++++++++++++++++++++--
 include/uapi/rdma/mana-abi.h         | 18 ++++++
 4 files changed, 183 insertions(+), 9 deletions(-)
drivers/infiniband/hw/mana/main.c +40 -1
index 8d99cd00f..8d96151ca 100644
@@ -1056,7 +1056,46 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	return 0;
 }
 
-int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+int mana_ib_gd_create_uc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
+			    struct ib_qp_init_attr *attr, u32 doorbell, u64 flags)
+{
+	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
+	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
+	struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd);
+	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct mana_rnic_create_uc_qp_resp resp = {};
+	struct mana_rnic_create_uc_qp_req req = {};
+	int err, i;
+
+	mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UC_QP, sizeof(req), sizeof(resp));
+	req.hdr.dev_id = mdev->gdma_dev->dev_id;
+	req.adapter = mdev->adapter_handle;
+	req.pd_handle = pd->pd_handle;
+	req.send_cq_handle = send_cq->cq_handle;
+	req.recv_cq_handle = recv_cq->cq_handle;
+	for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; i++)
+		req.dma_region[i] = qp->uc_qp.queues[i].gdma_region;
+	req.doorbell_page = doorbell;
+	req.max_send_wr = attr->cap.max_send_wr;
+	req.max_recv_wr = attr->cap.max_recv_wr;
+	req.max_send_sge = attr->cap.max_send_sge;
+	req.max_recv_sge = attr->cap.max_recv_sge;
+	req.flags = flags;
+
+	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
+	if (err)
+		return err;
+
+	qp->qp_handle = resp.qp_handle;
+	for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; i++) {
+		qp->uc_qp.queues[i].id = resp.queue_ids[i];
+		/* The GDMA regions are now owned by the RNIC QP handle */
+		qp->uc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+	}
+	return 0;
+}
+
+int mana_ib_gd_destroy_rnic_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
 {
 	struct mana_rnic_destroy_rc_qp_resp resp = {0};
 	struct mana_rnic_destroy_rc_qp_req req = {0};
drivers/infiniband/hw/mana/mana_ib.h +39 -2

71 lines not on this page · raw message

drivers/infiniband/hw/mana/qp.c +86 -6

157 lines not on this page · raw message

include/uapi/rdma/mana-abi.h +18 -0
index a75bf32b8..a50d4c012 100644
@@ -57,6 +57,24 @@ struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
 };
 
+enum {
+	MANA_UC_UDATA_SQR = 0,
+	MANA_UC_UDATA_RQR = 1,
+	MANA_UC_UDATA_SMQ = 2,
+	MANA_UC_UDATA_MAX = 3,
+};
+
+struct mana_ib_create_uc_qp {
+	__aligned_u64 queue_buf[MANA_UC_UDATA_MAX];
+	__u32 queue_size[MANA_UC_UDATA_MAX];
+	__u32 reserved;
+};
+
+struct mana_ib_create_uc_qp_resp {
+	__u32 queue_id[MANA_UC_UDATA_MAX];
+	__u32 reserved;
+};
+
 struct mana_ib_create_wq {
 	__aligned_u64 wq_buf_addr;
 	__u32 wq_buf_size;
Signature · 1 line
2.43.0
#1 lore raw
Quoted 13 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Implement UC QP creation in the RNIC HW for user API. An UC QP is exposed as three work queues: send, receive, and memory management. The latter is used for bind and invalidate WQEs to support memory windows.

---
 drivers/infiniband/hw/mana/main.c    | 41 ++++++++++++-
 drivers/infiniband/hw/mana/mana_ib.h | 41 ++++++++++++-
 drivers/infiniband/hw/mana/qp.c      | 92 ++++++++++++++++++++++++++--
 include/uapi/rdma/mana-abi.h         | 18 ++++++
 4 files changed, 183 insertions(+), 9 deletions(-)

<...>

Quoted 3 lines
drivers/infiniband/hw/mana/qp.c +3 -0
+static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
+				struct ib_qp_init_attr *attr, struct ib_udata *udata)
+{

<...>

Quoted 2 lines
drivers/infiniband/hw/mana/qp.c +2 -0
+	if (!udata || udata->inlen < sizeof(ucmd))
+		return -EINVAL;

<...>

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, · line 640

Quoted 1 line
drivers/infiniband/hw/mana/qp.c +1 -0
+	if (udata) {

udata is always true.

Quoted 8 lines
include/uapi/rdma/mana-abi.h +0 -0
index a75bf32b8..a50d4c012 100644
@@ -57,6 +57,24 @@ struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
 };
 

include/uapi/rdma/mana-abi.h struct mana_ib_create_rc_qp_resp { · line 69

Quoted 10 lines
include/uapi/rdma/mana-abi.h +10 -0
+enum {
+	MANA_UC_UDATA_SQR = 0,
+	MANA_UC_UDATA_RQR = 1,
+	MANA_UC_UDATA_SMQ = 2,
+	MANA_UC_UDATA_MAX = 3,
+};
+
+struct mana_ib_create_uc_qp {
+	__aligned_u64 queue_buf[MANA_UC_UDATA_MAX];
+	__u32 queue_size[MANA_UC_UDATA_MAX];

This array is not possible to extend. Any addition to enum above will break ABI. You shouldn't add enum and use 3 directly here.

include/uapi/rdma/mana-abi.h struct mana_ib_create_rc_qp_resp { · line 74

Quoted 5 lines
include/uapi/rdma/mana-abi.h +5 -0
+	__u32 reserved;
+};
+
+struct mana_ib_create_uc_qp_resp {
+	__u32 queue_id[MANA_UC_UDATA_MAX];

Same.

Thanks

Quoted 9 lines
include/uapi/rdma/mana-abi.h +3 -0
+	__u32 reserved;
+};
+
 struct mana_ib_create_wq {
 	__aligned_u64 wq_buf_addr;
 	__u32 wq_buf_size;
Signature · 1 line
2.43.0