Skip to contentSkip to conversation

RDMA/mana_ib: memory windows

PATCH Superseded Konstantin Taranov 4 participants 4 messages Mar 18 – Mar 31

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 RDMA/mana_ib: memory windows

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

---
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

 drivers/infiniband/hw/mana/device.c  |  3 ++
 drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 53 +++++++++++++++++++++++++++-
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 68 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +3 -0
index ccc2279ca..9811570ab 100644
@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 	.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
 
 	.add_gid = mana_ib_gd_add_gid,
+	.alloc_mw = mana_ib_alloc_mw,
 	.alloc_pd = mana_ib_alloc_pd,
 	.alloc_ucontext = mana_ib_alloc_ucontext,
 	.create_ah = mana_ib_create_ah,
@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 	.create_qp = mana_ib_create_qp,
 	.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
 	.create_wq = mana_ib_create_wq,
+	.dealloc_mw = mana_ib_dealloc_mw,
 	.dealloc_pd = mana_ib_dealloc_pd,
 	.dealloc_ucontext = mana_ib_dealloc_ucontext,
 	.del_gid = mana_ib_gd_del_gid,
@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 
 	INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
+	INIT_RDMA_OBJ_SIZE(ib_mw, mana_ib_mw, ibmw),
 	INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
drivers/infiniband/hw/mana/mana_ib.h +8 -0
index a7c8c0fd7..c9c94e86a 100644
@@ -125,6 +125,11 @@ struct mana_ib_ah {
 	dma_addr_t dma_handle;
 };
 
+struct mana_ib_mw {
+	struct ib_mw ibmw;
+	mana_handle_t mw_handle;
+};
+
 struct mana_ib_mr {
 	struct ib_mr ibmr;
 	struct ib_umem *umem;
@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);
 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 
+int mana_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);
+int mana_ib_dealloc_mw(struct ib_mw *mw);
+
 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
 					 u64 iova, int fd, int mr_access_flags,
 					 struct ib_dmah *dmah,
drivers/infiniband/hw/mana/mr.c +52 -1

71 lines not on this page · raw message

include/net/mana/gdma.h +5 -0
index 766f4fb25..fc6468ac7 100644
@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {
 	GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
 	GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
 	GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
+	GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5),
 };
 
 /* GDMA_CREATE_DMA_REGION */
@@ -870,6 +871,10 @@ enum gdma_mr_type {
 	GDMA_MR_TYPE_ZBVA = 4,
 	/* Device address MRs */
 	GDMA_MR_TYPE_DM = 5,
+	/* Memory Window type 1 */
+	GDMA_MR_TYPE_MW1 = 6,
+	/* Memory Window type 2 */
+	GDMA_MR_TYPE_MW2 = 7,
 };
 
 struct gdma_create_mr_params {
Signature · 1 line
2.43.0
Reviewed-by on v2 1/1
#1 lore raw
Quoted 5 lines

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

Quoted 174 lines

---
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

drivers/infiniband/hw/mana/device.c  |  3 ++
drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 53
+++++++++++++++++++++++++++-
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 68 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +3 -0
index ccc2279ca..9811570ab 100644
@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 	.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
 
 	.add_gid = mana_ib_gd_add_gid,
+	.alloc_mw = mana_ib_alloc_mw,
 	.alloc_pd = mana_ib_alloc_pd,
 	.alloc_ucontext = mana_ib_alloc_ucontext,
 	.create_ah = mana_ib_create_ah,
@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 	.create_qp = mana_ib_create_qp,
 	.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
 	.create_wq = mana_ib_create_wq,
+	.dealloc_mw = mana_ib_dealloc_mw,
 	.dealloc_pd = mana_ib_dealloc_pd,
 	.dealloc_ucontext = mana_ib_dealloc_ucontext,
 	.del_gid = mana_ib_gd_del_gid,
@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {
 
 	INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
+	INIT_RDMA_OBJ_SIZE(ib_mw, mana_ib_mw, ibmw),
 	INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
drivers/infiniband/hw/mana/mana_ib.h +8 -0
index a7c8c0fd7..c9c94e86a 100644
@@ -125,6 +125,11 @@ struct mana_ib_ah {
 	dma_addr_t dma_handle;
 };
 
+struct mana_ib_mw {
+	struct ib_mw ibmw;
+	mana_handle_t mw_handle;
+};
+
 struct mana_ib_mr {
 	struct ib_mr ibmr;
 	struct ib_umem *umem;
@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev
 *mdev);  int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc
 *wc);  int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 
+int mana_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); int
+mana_ib_dealloc_mw(struct ib_mw *mw);
+
 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
 u64 length,
 					 u64 iova, int fd, int mr_access_flags,
drivers/infiniband/hw/mana/mana_ib.h +0 -0
 					 struct ib_dmah *dmah,
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 9613b225d..02236488f 100644

drivers/infiniband/hw/mana/mr.c +1 -1
@@ -6,7 +6,7 @@
 #include "mana_ib.h"
 
 #define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE |
 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |\
-			IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED)
+			IB_ACCESS_REMOTE_ATOMIC |
 IB_ACCESS_MW_BIND | IB_ZERO_BASED)
 

#define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE)

drivers/infiniband/hw/mana/mr.c +26 -0
@@ -27,6 +27,9 @@ mana_ib_verbs_to_gdma_access_flags(int access_flags)
 	if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
 		flags |= GDMA_ACCESS_FLAG_REMOTE_ATOMIC;
 
+	if (access_flags & IB_ACCESS_MW_BIND)
+		flags |= GDMA_ACCESS_FLAG_BIND_MW;
+
 	return flags;
 }
 
@@ -304,6 +307,54 @@ struct ib_mr *mana_ib_get_dma_mr(struct ib_pd
 *ibpd, int access_flags)
 	return ERR_PTR(err);
 }
 
+static int mana_ib_gd_create_mw(struct mana_ib_dev *dev, struct
+mana_ib_pd *pd, struct ib_mw *ibmw) {
+	struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw,
 ibmw);
+	struct gdma_context *gc = mdev_to_gc(dev);
+	struct gdma_create_mr_response resp = {};
+	struct gdma_create_mr_request req = {};
+	int err;
+
+	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
 sizeof(resp));
+	req.pd_handle = pd->pd_handle;
+
+	switch (mw->ibmw.type) {
+	case IB_MW_TYPE_1:
+		req.mr_type = GDMA_MR_TYPE_MW1;
+		break;
+	case IB_MW_TYPE_2:
+		req.mr_type = GDMA_MR_TYPE_MW2;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),

&resp);

drivers/infiniband/hw/mana/mr.c +11 -0
+	if (err)
+		return err;
+
+	mw->ibmw.rkey = resp.rkey;
+	mw->mw_handle = resp.mr_handle;
+
+	return 0;
+}
+
+int mana_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) {
+	struct mana_ib_dev *mdev = container_of(ibmw->device, struct

mana_ib_dev, ib_dev);

drivers/infiniband/hw/mana/mr.c +7 -0
+	struct mana_ib_pd *pd = container_of(ibmw->pd, struct mana_ib_pd,
+ibpd);
+
+	return mana_ib_gd_create_mw(mdev, pd, ibmw); }
+
+int mana_ib_dealloc_mw(struct ib_mw *ibmw) {
+	struct mana_ib_dev *dev = container_of(ibmw->device, struct

mana_ib_dev, ib_dev);

drivers/infiniband/hw/mana/mr.c +1 -0
+	struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw,

ibmw);

drivers/infiniband/hw/mana/mr.c +3 -0
+
+	return mana_ib_gd_destroy_mr(dev, mw->mw_handle); }
+
 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)  {
 	struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr,

ibmr); diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 766f4fb25..fc6468ac7 100644

include/net/mana/gdma.h +5 -1
@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {
 	GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
 	GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
 	GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
+	GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5),
 };
 
 /* GDMA_CREATE_DMA_REGION */
@@ -870,6 +871,10 @@ enum gdma_mr_type {
 	GDMA_MR_TYPE_ZBVA = 4,
 	/* Device address MRs */
 	GDMA_MR_TYPE_DM = 5,
+	/* Memory Window type 1 */
+	GDMA_MR_TYPE_MW1 = 6,
+	/* Memory Window type 2 */
+	GDMA_MR_TYPE_MW2 = 7,
 };
 
 struct gdma_create_mr_params {
--

2.43.0

#2 lore raw
In reply to Konstantin Taranov
Quoted 12 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

---
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

 drivers/infiniband/hw/mana/device.c  |  3 ++
 drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 53 +++++++++++++++++++++++++++-
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 68 insertions(+), 1 deletion(-)

<...>

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) · line 319

Quoted 10 lines
drivers/infiniband/hw/mana/mr.c +10 -0
+static int mana_ib_gd_create_mw(struct mana_ib_dev *dev, struct mana_ib_pd *pd, struct ib_mw *ibmw)
+{
+	struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw);
+	struct gdma_context *gc = mdev_to_gc(dev);
+	struct gdma_create_mr_response resp = {};
+	struct gdma_create_mr_request req = {};
+	int err;
+
+	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req), sizeof(resp));
+	req.pd_handle = pd->pd_handle;

Both sashiko, which runs on Gemnini https://sashiko.dev/#/patchset/[email protected]
and my local claude checks pointed to the same area of the code:

   25 > +     mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req), sizeof(resp));
   26 > +     req.pd_handle = pd->pd_handle;
   27
   28 mana_gd_init_req_hdr() sets msg_version to GDMA_MESSAGE_V1.
   29 mana_ib_gd_create_mr() explicitly upgrades this to GDMA_MESSAGE_V2
   30 for all GDMA_CREATE_MR requests, regardless of mr_type:
   31
   32 mana_ib_gd_create_mr() {
   33         mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, ...);
   34         req.hdr.req.msg_version = GDMA_MESSAGE_V2;
   35         ...
   36 }
   37
   38 Is it intentional that mana_ib_gd_create_mw() sends GDMA_CREATE_MR
   39 at GDMA_MESSAGE_V1 rather than GDMA_MESSAGE_V2?

Thanks

#3 lore raw
Quoted 57 lines · Konstantin Taranov
Quoted 10 lines

From: Konstantin Taranov <[email protected]>

Implement .alloc_mw() and .dealloc_mw() for mana device.

---
v2: fixed comments. Cleaned up the use of mana_gd_send_request()

drivers/infiniband/hw/mana/device.c  |  3 ++
drivers/infiniband/hw/mana/mana_ib.h |  8 +++++
 drivers/infiniband/hw/mana/mr.c      | 53

+++++++++++++++++++++++++++-

Quoted 2 lines
 include/net/mana/gdma.h              |  5 +++
 4 files changed, 68 insertions(+), 1 deletion(-)

<...>

Review

Quoted 3 lines
(quoted diff) +3 -0
+static int mana_ib_gd_create_mw(struct mana_ib_dev *dev, struct
+mana_ib_pd *pd, struct ib_mw *ibmw) {
+	struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw,

ibmw);

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)

Quoted 6 lines
drivers/infiniband/hw/mana/mr.c +6 -0
+	struct gdma_context *gc = mdev_to_gc(dev);
+	struct gdma_create_mr_response resp = {};
+	struct gdma_create_mr_request req = {};
+	int err;
+
+	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),

sizeof(resp));

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) · line 319

Quoted 1 line
drivers/infiniband/hw/mana/mr.c +1 -0
+	req.pd_handle = pd->pd_handle;

Both sashiko, which runs on Gemnini
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsashi
ko.dev%2F%23%2Fpatchset%2F20260318172323.1416803-1- kotaranov%40linux.microsoft.com&data=05%7C02%7Ckotaranov%40microso ft.com%7Ce8888ce68913479d8db508de8e53a186%7C72f988bf86f141af91ab 2d7cd011db47%7C1%7C0%7C639104687038043303%7CUnknown%7CTWFp bGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4z MiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=r9n1Jmd Ke5smdwDQto80%2BHWosk3DrRa1z4jdNmzjLjk%3D&reserved=0 and my local claude checks pointed to the same area of the code:

   25 > +     mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),

sizeof(resp));
26 > + req.pd_handle = pd->pd_handle;
27
28 mana_gd_init_req_hdr() sets msg_version to GDMA_MESSAGE_V1. 29 mana_ib_gd_create_mr() explicitly upgrades this to
GDMA_MESSAGE_V2
30 for all GDMA_CREATE_MR requests, regardless of mr_type:

   31
   32 mana_ib_gd_create_mr() {
   33         mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, ...);
   34         req.hdr.req.msg_version = GDMA_MESSAGE_V2;
   35         ...
   36 }
   37
   38 Is it intentional that mana_ib_gd_create_mw() sends
Quoted 3 lines

GDMA_CREATE_MR
39 at GDMA_MESSAGE_V1 rather than GDMA_MESSAGE_V2?

Thanks. I sent v3. It does not matter v1 vs v2 as HW supports both, I just wrote this patch long time ago. I changed to v2 to make code more unified.

Quoted 1 line

Thanks