Skip to contentSkip to conversation

RDMA/mana_ib: memory windows

PATCH Applied Konstantin Taranov 4 participants 4 messages Mar 31 – Apr 9

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 RDMA/mana_ib: memory windows

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

---
v3: Use v2 request
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

 drivers/infiniband/hw/mana/device.c  |  3 ++
 drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 54 +++++++++++++++++++++++++++-
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 69 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +3 -0
index ccc2279ca..9811570ab 100644
@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 	.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
 
 	.add_gid = mana_ib_gd_add_gid,
+	.alloc_mw = mana_ib_alloc_mw,
 	.alloc_pd = mana_ib_alloc_pd,
 	.alloc_ucontext = mana_ib_alloc_ucontext,
 	.create_ah = mana_ib_create_ah,
@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 	.create_qp = mana_ib_create_qp,
 	.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
 	.create_wq = mana_ib_create_wq,
+	.dealloc_mw = mana_ib_dealloc_mw,
 	.dealloc_pd = mana_ib_dealloc_pd,
 	.dealloc_ucontext = mana_ib_dealloc_ucontext,
 	.del_gid = mana_ib_gd_del_gid,
@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 
 	INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
+	INIT_RDMA_OBJ_SIZE(ib_mw, mana_ib_mw, ibmw),
 	INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
drivers/infiniband/hw/mana/mana_ib.h +8 -0
index a7c8c0fd7..c9c94e86a 100644
@@ -125,6 +125,11 @@ struct mana_ib_ah {
 	dma_addr_t dma_handle;
 };
 
+struct mana_ib_mw {
+	struct ib_mw ibmw;
+	mana_handle_t mw_handle;
+};
+
 struct mana_ib_mr {
 	struct ib_mr ibmr;
 	struct ib_umem *umem;
@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);
 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 
+int mana_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);
+int mana_ib_dealloc_mw(struct ib_mw *mw);
+
 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
 					 u64 iova, int fd, int mr_access_flags,
 					 struct ib_dmah *dmah,
drivers/infiniband/hw/mana/mr.c +53 -1

72 lines not on this page · raw message

include/net/mana/gdma.h +5 -0
index 766f4fb25..fc6468ac7 100644
@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {
 	GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
 	GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
 	GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
+	GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5),
 };
 
 /* GDMA_CREATE_DMA_REGION */
@@ -870,6 +871,10 @@ enum gdma_mr_type {
 	GDMA_MR_TYPE_ZBVA = 4,
 	/* Device address MRs */
 	GDMA_MR_TYPE_DM = 5,
+	/* Memory Window type 1 */
+	GDMA_MR_TYPE_MW1 = 6,
+	/* Memory Window type 2 */
+	GDMA_MR_TYPE_MW2 = 7,
 };
 
 struct gdma_create_mr_params {
Signature · 1 line
2.43.0
#1 lore raw
Quoted 6 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

Quoted 8 lines

---
v3: Use v2 request
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

 drivers/infiniband/hw/mana/device.c  |  3 ++
 drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 54 +++++++++++++++++++++++++++-
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 69 insertions(+), 1 deletion(-)

How did you test this patch? How can applications know that alloc_mw is supported now if you didn't set IB_DEVICE_MEM_WINDOW and props->max_mw?

Thanks

#2 lore raw
Quoted 13 lines · Konstantin Taranov
Quoted 4 lines

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

Quoted 8 lines

---
v3: Use v2 request
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

drivers/infiniband/hw/mana/device.c  |  3 ++
drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 54
Quoted 7 lines

+++++++++++++++++++++++++++-

Quoted 2 lines
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 69 insertions(+), 1 deletion(-)

How did you test this patch? How can applications know that alloc_mw is supported now if you didn't set IB_DEVICE_MEM_WINDOW and props-

Quoted 1 line

max_mw?

Our mana HW supports MWs and I tested against it. That V1 and V2 helps the HW correctly detect request struct size and for MWs it does not matter as its fields fit into v1 size. The application can just try to create an MW and see if it works. prop->max_mw will be the same as max_mr, since they are modelled similarly in the HW. IB_DEVICE_MEM_WINDOW is an informative and not enforced, so nothing prevents to test it.

My plan was to upstream MWs, UC, RC support of MW, and then upstream a patch that correctly indicates that it is all available. As now even if we report MW caps, there is no QPs that can take advantage of it. In MANA, WQEs that perform memory operations should be submitted to special queue, which is different from the send queue. So UC patch includes the special queue, and an RC patch will expose an extra queue.

If you say, I must add that caps now, I can send v4 with the max_mw and IB_DEVICE_MEM_WINDOW.

Thanks

Quoted 2 lines

Thanks

maintainer Applied
#3 lore raw
In reply to Konstantin Taranov
Quoted 6 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

Quoted 8 lines

---
v3: Use v2 request
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

 drivers/infiniband/hw/mana/device.c  |  3 ++
 drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 54 +++++++++++++++++++++++++++-
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 69 insertions(+), 1 deletion(-)

Applied to for-next, please don't delay getting the other parts completed

Jason