RDMA/mana_ib: memory windows
PATCH v2 1/1 RDMA/mana_ib: memory windows
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
---
v2: fixed comments. Cleaned up the use of mana_gd_send_request()
drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 +++++ drivers/infiniband/hw/mana/mr.c | 53 +++++++++++++++++++++++++++- include/net/mana/gdma.h | 5 +++ 4 files changed, 68 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +3 -0
@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, .add_gid = mana_ib_gd_add_gid, + .alloc_mw = mana_ib_alloc_mw, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_ah = mana_ib_create_ah,
@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_qp = mana_ib_create_qp, .create_rwq_ind_table = mana_ib_create_rwq_ind_table, .create_wq = mana_ib_create_wq, + .dealloc_mw = mana_ib_dealloc_mw, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, .del_gid = mana_ib_gd_del_gid,
@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, mana_ib_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, mana_ib_cq, ibcq), +INIT_RDMA_OBJ_SIZE (ib_mw, mana_ib_mw, ibmw),INIT_RDMA_OBJ_SIZE (ib_pd, mana_ib_pd, ibpd),INIT_RDMA_OBJ_SIZE (ib_qp, mana_ib_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
drivers/infiniband/hw/mana/mana_ib.h +8 -0
@@ -125,6 +125,11 @@ struct mana_ib_ah {dma_addr_t dma_handle; }; +struct mana_ib_mw { +struct ib_mw ibmw; +mana_handle_t mw_handle; +}; +struct mana_ib_mr {struct ib_mr ibmr;struct ib_umem *umem;
@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);int mana_ib_poll_cq (struct ib_cq *ibcq,int num_entries,struct ib_wc *wc);int mana_ib_arm_cq (struct ib_cq *ibcq,enum ib_cq_notify_flags flags); +int mana_ib_alloc_mw (struct ib_mw *mw,struct ib_udata *udata); +int mana_ib_dealloc_mw (struct ib_mw *mw); + struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length, u64 iova, int fd, int mr_access_flags, struct ib_dmah *dmah,
drivers/infiniband/hw/mana/mr.c +52 -1
71 lines not on this page · raw message
include/net/mana/gdma.h +5 -0
@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {GDMA_ACCESS_FLAG_REMOTE_READ =BIT_ULL (2 ),GDMA_ACCESS_FLAG_REMOTE_WRITE =BIT_ULL (3 ),GDMA_ACCESS_FLAG_REMOTE_ATOMIC =BIT_ULL (4 ), + GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5), };/* GDMA_CREATE_DMA_REGION */
@@ -870,6 +871,10 @@ enum gdma_mr_type {GDMA_MR_TYPE_ZBVA =4 ,/* Device address MRs */ GDMA_MR_TYPE_DM =5 , +/* Memory Window type 1 */ +GDMA_MR_TYPE_MW1 =6 , +/* Memory Window type 2 */ + GDMA_MR_TYPE_MW2 = 7, }; struct gdma_create_mr_params {
Signature · 1 line
2.43.0
Quoted 5 lines
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
Signed-off-by Konstantin Taranov <[email protected]>
Quoted 174 lines
---
v2: fixed comments. Cleaned up the use ofmana_gd_send_request()drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 +++++ drivers/infiniband/hw/mana/mr.c | 53 +++++++++++++++++++++++++++- include/net/mana/gdma.h | 5 +++ 4 files changed, 68 insertions(+), 1 deletion(-)drivers/infiniband/hw/mana/device.c +3
-0@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, .add_gid = mana_ib_gd_add_gid, + .alloc_mw = mana_ib_alloc_mw, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_ah = mana_ib_create_ah,@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_qp = mana_ib_create_qp, .create_rwq_ind_table = mana_ib_create_rwq_ind_table, .create_wq = mana_ib_create_wq, + .dealloc_mw = mana_ib_dealloc_mw, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, .del_gid = mana_ib_gd_del_gid,@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, mana_ib_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, mana_ib_cq, ibcq), +INIT_RDMA_OBJ_SIZE (ib_mw, mana_ib_mw, ibmw),INIT_RDMA_OBJ_SIZE (ib_pd, mana_ib_pd, ibpd),INIT_RDMA_OBJ_SIZE (ib_qp, mana_ib_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),drivers/infiniband/hw/mana/mana_ib.h +8
-0@@ -125,6 +125,11 @@ struct mana_ib_ah {dma_addr_t dma_handle; }; +struct mana_ib_mw { +struct ib_mw ibmw; +mana_handle_t mw_handle; +}; +struct mana_ib_mr {struct ib_mr ibmr;struct ib_umem *umem;@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev); int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);int mana_ib_arm_cq (struct ib_cq *ibcq,enum ib_cq_notify_flags flags); +int mana_ib_alloc_mw (struct ib_mw *mw,struct ib_udata *udata);int +mana_ib_dealloc_mw (struct ib_mw *mw); + struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length, u64 iova, int fd, int mr_access_flags,drivers/infiniband/hw/mana/mana_ib.h +0
-0struct ib_dmah *dmah,(quoted diff) +0
-0
b/drivers/infiniband/hw/mana/mr.cindex9613b225d..02236488f100644drivers/infiniband/hw/mana/mr.c +1
-1@@ -6,7 +6,7 @@#include "mana_ib.h" #define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE |IB_ACCESS_REMOTE_WRITE |IB_ACCESS_REMOTE_READ |\ - IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED) +IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND | IB_ZERO_BASED)#define
VALID_DMA_MR_FLAGS(IB_ACCESS_LOCAL_WRITE)drivers/infiniband/hw/mana/mr.c +26
-0@@ -27,6 +27,9 @@ mana_ib_verbs_to_gdma_access_flags(int access_flags)if (access_flags &IB_ACCESS_REMOTE_ATOMIC ) flags |=GDMA_ACCESS_FLAG_REMOTE_ATOMIC ; +if (access_flags &IB_ACCESS_MW_BIND ) + flags |=GDMA_ACCESS_FLAG_BIND_MW ; +return flags; }@@ -304,6 +307,54 @@ struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)return ERR_PTR (err); } +static int mana_ib_gd_create_mw (struct mana_ib_dev *dev,struct +mana_ib_pd *pd,struct ib_mw *ibmw) { + struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw); +struct gdma_context *gc =mdev_to_gc (dev); +struct gdma_create_mr_response resp = {}; +struct gdma_create_mr_request req = {}; +int err; + +mana_gd_init_req_hdr (&req.hdr,GDMA_CREATE_MR ,sizeof (req),sizeof (resp)); + req.pd_handle = pd->pd_handle; + +switch (mw->ibmw.type) { +case IB_MW_TYPE_1 : + req.mr_type =GDMA_MR_TYPE_MW1 ; +break ; +case IB_MW_TYPE_2 : + req.mr_type =GDMA_MR_TYPE_MW2 ; +break ; +default : +return -EINVAL ; + } + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),&resp);
drivers/infiniband/hw/mana/mr.c +11
-0+if (err) +return err; + + mw->ibmw.rkey = resp.rkey; + mw->mw_handle = resp.mr_handle; + +return 0 ; +} + +int mana_ib_alloc_mw (struct ib_mw *ibmw,struct ib_udata *udata) { + struct mana_ib_dev *mdev = container_of(ibmw->device, struct
mana_ib_dev,ib_dev);drivers/infiniband/hw/mana/mr.c +7
-0+ struct mana_ib_pd *pd = container_of(ibmw->pd, struct mana_ib_pd, +ibpd); + +return mana_ib_gd_create_mw (mdev, pd, ibmw); } + +int mana_ib_dealloc_mw (struct ib_mw *ibmw) { + struct mana_ib_dev *dev = container_of(ibmw->device, struct
mana_ib_dev,ib_dev);drivers/infiniband/hw/mana/mr.c +1
-0+ struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw,ibmw);
drivers/infiniband/hw/mana/mr.c +3
-0+ +return mana_ib_gd_destroy_mr (dev, mw->mw_handle); } +int mana_ib_dereg_mr (struct ib_mr *ibmr,struct ib_udata *udata) { struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr,ibmr); diff --git
a/include/net/mana/gdma.hb/include/net/mana/gdma.hindex766f4fb25..fc6468ac7100644include/net/mana/gdma.h +5
-1@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {GDMA_ACCESS_FLAG_REMOTE_READ =BIT_ULL (2 ),GDMA_ACCESS_FLAG_REMOTE_WRITE =BIT_ULL (3 ),GDMA_ACCESS_FLAG_REMOTE_ATOMIC =BIT_ULL (4 ), + GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5), };/* GDMA_CREATE_DMA_REGION */ @@ -870,6 +871,10 @@ enum gdma_mr_type {GDMA_MR_TYPE_ZBVA =4 ,/* Device address MRs */ GDMA_MR_TYPE_DM =5 , +/* Memory Window type 1 */ +GDMA_MR_TYPE_MW1 =6 , +/* Memory Window type 2 */ + GDMA_MR_TYPE_MW2 = 7, }; struct gdma_create_mr_params { --2.43.0
Quoted 12 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
Signed-off-by Konstantin Taranov <[email protected]>---
v2: fixed comments. Cleaned up the use ofmana_gd_send_request()drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 +++++ drivers/infiniband/hw/mana/mr.c | 53 +++++++++++++++++++++++++++- include/net/mana/gdma.h | 5 +++ 4 files changed, 68 insertions(+), 1 deletion(-)
<...>
drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) · line 319
Quoted 10 lines
drivers/infiniband/hw/mana/mr.c +10
-0+static int mana_ib_gd_create_mw (struct mana_ib_dev *dev,struct mana_ib_pd *pd,struct ib_mw *ibmw) +{ + struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw); +struct gdma_context *gc =mdev_to_gc (dev); +struct gdma_create_mr_response resp = {}; +struct gdma_create_mr_request req = {}; +int err; + +mana_gd_init_req_hdr (&req.hdr,GDMA_CREATE_MR ,sizeof (req),sizeof (resp)); + req.pd_handle = pd->pd_handle;
Both sashiko, which runs on Gemnini https://sashiko.dev/#/patchset/[email protected]
and my local claude checks pointed to the same area of the code:
25 > + mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req), sizeof(resp));
26 > + req.pd_handle = pd->pd_handle;
27
28 mana_gd_init_req_hdr() sets msg_version to GDMA_MESSAGE_V1.
29 mana_ib_gd_create_mr() explicitly upgrades this to GDMA_MESSAGE_V2
30 for all GDMA_CREATE_MR requests, regardless of mr_type:
31
32 mana_ib_gd_create_mr() {
33 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, ...);
34 req.hdr.req.msg_version = GDMA_MESSAGE_V2;
35 ...
36 }
37
38 Is it intentional that mana_ib_gd_create_mw() sends GDMA_CREATE_MR
39 at GDMA_MESSAGE_V1 rather than GDMA_MESSAGE_V2?Thanks
Konstantin Taranov
maintainerQuoted 57 lines · Konstantin Taranov
Quoted 10 lines
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
Signed-off-by Konstantin Taranov <[email protected]>---
v2: fixed comments. Cleaned up the use ofmana_gd_send_request()drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 +++++ drivers/infiniband/hw/mana/mr.c | 53+++++++++++++++++++++++++++-
Quoted 2 lines
include/net/mana/gdma.h | 5 +++ 4 files changed, 68 insertions(+), 1 deletion(-)<...>
Review
Quoted 3 lines
(quoted diff) +3
-0+static int mana_ib_gd_create_mw(struct mana_ib_dev *dev, struct +mana_ib_pd *pd, struct ib_mw *ibmw) { + struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw,ibmw);
drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)
Quoted 6 lines
drivers/infiniband/hw/mana/mr.c +6
-0+struct gdma_context *gc =mdev_to_gc (dev); +struct gdma_create_mr_response resp = {}; +struct gdma_create_mr_request req = {}; +int err; + + mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),sizeof(resp));
drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) · line 319
Quoted 1 line
drivers/infiniband/hw/mana/mr.c +1
-0+ req.pd_handle = pd->pd_handle;Both sashiko, which runs on Gemnini
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsashi
ko.dev%2F%23%2Fpatchset%2F20260318172323.1416803-1- kotaranov%40linux.microsoft.com&data=05%7C02%7Ckotaranov%40microso ft.com%7Ce8888ce68913479d8db508de8e53a186%7C72f988bf86f141af91ab 2d7cd011db47%7C1%7C0%7C639104687038043303%7CUnknown%7CTWFp bGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4z MiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=r9n1Jmd Ke5smdwDQto80%2BHWosk3DrRa1z4jdNmzjLjk%3D&reserved=0 and my local claude checks pointed to the same area of the code:25 > + mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),sizeof(resp));
26 > + req.pd_handle =pd->pd_handle;
27
28mana_gd_init_req_hdr()setsmsg_versiontoGDMA_MESSAGE_V1. 29mana_ib_gd_create_mr()explicitly upgrades this toGDMA_MESSAGE_V2
30 for allGDMA_CREATE_MRrequests, regardless ofmr_type:31 32 mana_ib_gd_create_mr() { 33 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, ...); 34 req.hdr.req.msg_version = GDMA_MESSAGE_V2; 35 ... 36 } 37 38 Is it intentional that mana_ib_gd_create_mw() sends
Quoted 3 lines
GDMA_CREATE_MR
39 atGDMA_MESSAGE_V1rather thanGDMA_MESSAGE_V2?
Thanks. I sent v3. It does not matter v1 vs v2 as HW supports both, I just wrote this patch long time ago. I changed to v2 to make code more unified.
Quoted 1 line
Thanks