Skip to contentSkip to conversation

qedr: Add support for user mode XRC-SRQ's

PATCH Applied Yuval Basson 3 participants 4 messages Jul 22, 2020 – Sep 15, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 qedr: Add support for user mode XRC-SRQ's

Implement the XRC specific verbs.
The additional QP type introduced new logic to the rest of the verbs that now require distinguishing whether a QP has an "RQ" or an "SQ" or both.

---
 drivers/infiniband/hw/qedr/main.c  |  19 +++
 drivers/infiniband/hw/qedr/qedr.h  |  33 +++++
 drivers/infiniband/hw/qedr/verbs.c | 291 +++++++++++++++++++++++++------------
 drivers/infiniband/hw/qedr/verbs.h |   4 +-
 4 files changed, 255 insertions(+), 92 deletions(-)
drivers/infiniband/hw/qedr/main.c +19 -0
index c9eeed2..b35d2e3 100644
@@ -179,6 +179,8 @@ static int qedr_iw_register_device(struct qedr_dev *dev)
 static const struct ib_device_ops qedr_roce_dev_ops = {
 	.get_port_immutable = qedr_roce_port_immutable,
 	.query_pkey = qedr_query_pkey,
+	.alloc_xrcd = qedr_alloc_xrcd,
+	.dealloc_xrcd = qedr_dealloc_xrcd,
 };
 
 static void qedr_roce_register_device(struct qedr_dev *dev)
@@ -186,6 +188,10 @@ static void qedr_roce_register_device(struct qedr_dev *dev)
 	dev->ibdev.node_type = RDMA_NODE_IB_CA;
 
 	ib_set_device_ops(&dev->ibdev, &qedr_roce_dev_ops);
+
+	dev->ibdev.uverbs_cmd_mask |= QEDR_UVERBS(OPEN_XRCD) |
+		QEDR_UVERBS(CLOSE_XRCD) |
+		QEDR_UVERBS(CREATE_XSRQ);
 }
 
 static const struct ib_device_ops qedr_dev_ops = {
@@ -232,6 +238,7 @@ static void qedr_roce_register_device(struct qedr_dev *dev)
 	INIT_RDMA_OBJ_SIZE(ib_cq, qedr_cq, ibcq),
 	INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq),
+	INIT_RDMA_OBJ_SIZE(ib_xrcd, qedr_xrcd, ibxrcd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext),
 };
 
@@ -703,6 +710,18 @@ static void qedr_affiliated_event(void *context, u8 e_code, void *fw_handle)
 			event.event = IB_EVENT_SRQ_ERR;
 			event_type = EVENT_TYPE_SRQ;
 			break;
+		case ROCE_ASYNC_EVENT_XRC_DOMAIN_ERR:
+			event.event = IB_EVENT_QP_ACCESS_ERR;
+			event_type = EVENT_TYPE_QP;
+			break;
+		case ROCE_ASYNC_EVENT_INVALID_XRCETH_ERR:
+			event.event = IB_EVENT_QP_ACCESS_ERR;
+			event_type = EVENT_TYPE_QP;
+			break;
+		case ROCE_ASYNC_EVENT_XRC_SRQ_CATASTROPHIC_ERR:
+			event.event = IB_EVENT_CQ_ERR;
+			event_type = EVENT_TYPE_CQ;
+			break;
 		default:
 			DP_ERR(dev, "unsupported event %d on handle=%llx\n",
 			       e_code, roce_handle64);
drivers/infiniband/hw/qedr/qedr.h +33 -0
index 4602921..d0b3a7a 100644
@@ -310,6 +310,11 @@ struct qedr_pd {
 	struct qedr_ucontext *uctx;
 };
 
+struct qedr_xrcd {
+	struct ib_xrcd ibxrcd;
+	u16 xrcd_id;
+};
+
 struct qedr_qp_hwq_info {
 	/* WQE Elements */
 	struct qed_chain pbl;
@@ -361,6 +366,7 @@ struct qedr_srq {
 	struct ib_umem *prod_umem;
 	u16 srq_id;
 	u32 srq_limit;
+	bool is_xrc;
 	/* lock to protect srq recv post */
 	spinlock_t lock;
 };
@@ -573,6 +579,11 @@ static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
 	return container_of(ibpd, struct qedr_pd, ibpd);
 }
 
+static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd)
+{
+	return container_of(ibxrcd, struct qedr_xrcd, ibxrcd);
+}
+
 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
 {
 	return container_of(ibcq, struct qedr_cq, ibcq);
@@ -598,6 +609,28 @@ static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
 	return container_of(ibsrq, struct qedr_srq, ibsrq);
 }
 
+static inline bool qedr_qp_has_srq(struct qedr_qp *qp)
+{
+	return !!qp->srq;
+}
+
+static inline bool qedr_qp_has_sq(struct qedr_qp *qp)
+{
+	if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT)
+		return 0;
+
+	return 1;
+}
+
+static inline bool qedr_qp_has_rq(struct qedr_qp *qp)
+{
+	if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI ||
+	    qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp))
+		return 0;
+
+	return 1;
+}
+
 static inline struct qedr_user_mmap_entry *
 get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
 {
drivers/infiniband/hw/qedr/verbs.c +200 -91

452 lines not on this page · raw message

drivers/infiniband/hw/qedr/verbs.h +3 -1
index 39dd628..bb09c64 100644
@@ -48,7 +48,9 @@ int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
 int qedr_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
 void qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
-
+int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd,
+				struct ib_udata *udata);
+void qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata);
 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		   struct ib_udata *udata);
 int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *);
Signature · 1 line
1.8.3.1
maintainer
#1 lore raw
Quoted 4 lines · Yuval Basson

Implement the XRC specific verbs.
The additional QP type introduced new logic to the rest of the verbs that now require distinguishing whether a QP has an "RQ" or an "SQ" or both.

Quoted 8 lines
---
 drivers/infiniband/hw/qedr/main.c  |  19 +++
 drivers/infiniband/hw/qedr/qedr.h  |  33 +++++
 drivers/infiniband/hw/qedr/verbs.c | 291 +++++++++++++++++++++++++------------
 drivers/infiniband/hw/qedr/verbs.h |   4 +-
 4 files changed, 255 insertions(+), 92 deletions(-)

Where is the rdma-core part of this?

Jason

maintainer
#2 lore raw
Quoted 14 lines

From: Jason Gunthorpe <[email protected]>
Sent: Monday, July 27, 2020 9:49 PM

Quoted 3 lines · Yuval Basson

Implement the XRC specific verbs.
The additional QP type introduced new logic to the rest of the verbs that now require distinguishing whether a QP has an "RQ" or an "SQ" or

both.

Quoted 7 lines
---
 drivers/infiniband/hw/qedr/main.c  |  19 +++

drivers/infiniband/hw/qedr/qedr.h | 33 +++++
drivers/infiniband/hw/qedr/verbs.c | 291

Quoted 5 lines

+++++++++++++++++++++++++------------

Quoted 2 lines
 drivers/infiniband/hw/qedr/verbs.h |   4 +-
 4 files changed, 255 insertions(+), 92 deletions(-)

Where is the rdma-core part of this?

Sorry for the delay on this one, it slipped.
Related pull request: https://github.com/linux-rdma/rdma-core/pull/821

Thanks,
Michal

Quoted 2 lines

Jason

maintainer Applied
#3 lore raw
In reply to Yuval Basson
Quoted 4 lines · Yuval Basson

Implement the XRC specific verbs.
The additional QP type introduced new logic to the rest of the verbs that now require distinguishing whether a QP has an "RQ" or an "SQ" or both.

Quoted 8 lines
---
 drivers/infiniband/hw/qedr/main.c  |  19 +++
 drivers/infiniband/hw/qedr/qedr.h  |  33 +++++
 drivers/infiniband/hw/qedr/verbs.c | 291 +++++++++++++++++++++++++------------
 drivers/infiniband/hw/qedr/verbs.h |   4 +-
 4 files changed, 255 insertions(+), 92 deletions(-)

Applied to for-next

Thanks,
Jason