RDMA/mana_ib: memory windows
PATCH 1/1 RDMA/mana_ib: memory windows
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
---
As I see that Jason's rdma_uapi is not in the next yet. I will make a patch adding his helpers (e.g., ib_is_udata_in_empty() for mw) with all other api calls.
drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 ++++ drivers/infiniband/hw/mana/mr.c | 57 +++++++++++++++++++++++++++- include/net/mana/gdma.h | 5 +++ 4 files changed, 72 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mana/device.c +3 -0
@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, .add_gid = mana_ib_gd_add_gid, + .alloc_mw = mana_ib_alloc_mw, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_ah = mana_ib_create_ah,
@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_qp = mana_ib_create_qp, .create_rwq_ind_table = mana_ib_create_rwq_ind_table, .create_wq = mana_ib_create_wq, + .dealloc_mw = mana_ib_dealloc_mw, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, .del_gid = mana_ib_gd_del_gid,
@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, mana_ib_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, mana_ib_cq, ibcq), +INIT_RDMA_OBJ_SIZE (ib_mw, mana_ib_mw, ibmw),INIT_RDMA_OBJ_SIZE (ib_pd, mana_ib_pd, ibpd),INIT_RDMA_OBJ_SIZE (ib_qp, mana_ib_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
drivers/infiniband/hw/mana/mana_ib.h +8 -0
@@ -125,6 +125,11 @@ struct mana_ib_ah {dma_addr_t dma_handle; }; +struct mana_ib_mw { +struct ib_mw ibmw; +mana_handle_t mw_handle; +}; +struct mana_ib_mr {struct ib_mr ibmr;struct ib_umem *umem;
@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);int mana_ib_poll_cq (struct ib_cq *ibcq,int num_entries,struct ib_wc *wc);int mana_ib_arm_cq (struct ib_cq *ibcq,enum ib_cq_notify_flags flags); +int mana_ib_alloc_mw (struct ib_mw *mw,struct ib_udata *udata); +int mana_ib_dealloc_mw (struct ib_mw *mw); + struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length, u64 iova, int fd, int mr_access_flags, struct ib_dmah *dmah,
drivers/infiniband/hw/mana/mr.c +56 -1
75 lines not on this page · raw message
include/net/mana/gdma.h +5 -0
@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {GDMA_ACCESS_FLAG_REMOTE_READ =BIT_ULL (2 ),GDMA_ACCESS_FLAG_REMOTE_WRITE =BIT_ULL (3 ),GDMA_ACCESS_FLAG_REMOTE_ATOMIC =BIT_ULL (4 ), + GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5), };/* GDMA_CREATE_DMA_REGION */
@@ -870,6 +871,10 @@ enum gdma_mr_type {GDMA_MR_TYPE_ZBVA =4 ,/* Device address MRs */ GDMA_MR_TYPE_DM =5 , +/* Device address MRs */ +GDMA_MR_TYPE_MW1 =6 , +/* Device address MRs */ + GDMA_MR_TYPE_MW2 = 7, }; struct gdma_create_mr_params {
Signature · 1 line
2.43.0
Quoted 181 lines
-----Original Message-----
From: Konstantin Taranov <[email protected]>
Sent: Friday, March 6, 2026 2:58 AM
To: Konstantin Taranov <[email protected]>; Shiraz Saleem <[email protected]>; Long Li <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]
Subject: [PATCH rdma-next 1/1] RDMA/mana_ib: memory windowsFrom: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
Signed-off-by Konstantin Taranov <[email protected]>---
As I see that Jason'srdma_uapiis not in the next yet. I will make a patch adding his helpers (e.g.,ib_is_udata_in_empty()for mw) with all other api calls.drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 ++++ drivers/infiniband/hw/mana/mr.c | 57 +++++++++++++++++++++++++++- include/net/mana/gdma.h | 5 +++ 4 files changed, 72 insertions(+), 1 deletion(-)drivers/infiniband/hw/mana/device.c +3
-1@@ -17,6 +17,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, .add_gid = mana_ib_gd_add_gid, + .alloc_mw = mana_ib_alloc_mw, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_ah = mana_ib_create_ah,@@ -24,6 +25,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_qp = mana_ib_create_qp, .create_rwq_ind_table = mana_ib_create_rwq_ind_table, .create_wq = mana_ib_create_wq, + .dealloc_mw = mana_ib_dealloc_mw, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, .del_gid = mana_ib_gd_del_gid,@@ -53,6 +55,7 @@ static const struct ib_device_ops mana_ib_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, mana_ib_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, mana_ib_cq, ibcq), +INIT_RDMA_OBJ_SIZE (ib_mw, mana_ib_mw, ibmw),INIT_RDMA_OBJ_SIZE (ib_pd, mana_ib_pd, ibpd),INIT_RDMA_OBJ_SIZE (ib_qp, mana_ib_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext), diff - -git a/drivers/infiniband/hw/mana/mana_ib.h
b/drivers/infiniband/hw/mana/mana_ib.h
indexa7c8c0fd7..c9c94e86a100644drivers/infiniband/hw/mana/mana_ib.h +5
-0@@ -125,6 +125,11 @@ struct mana_ib_ah {dma_addr_t dma_handle; }; +struct mana_ib_mw { +struct ib_mw ibmw; +mana_handle_t mw_handle; +}; +struct mana_ib_mr {struct ib_mr ibmr;struct ib_umem *umem;@@ -736,6 +741,9 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);int
mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); intmana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);drivers/infiniband/hw/mana/mana_ib.h +3
-0+int mana_ib_alloc_mw (struct ib_mw *mw,struct ib_udata *udata);int +mana_ib_dealloc_mw (struct ib_mw *mw); + struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64length,
drivers/infiniband/hw/mana/mana_ib.h +0
-0u64 iova, int fd, int mr_access_flags, struct ib_dmah *dmah,drivers/infiniband/hw/mana/mr.c +1
-1@@ -6,7 +6,7 @@#include "mana_ib.h" #define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE |IB_ACCESS_REMOTE_WRITE |IB_ACCESS_REMOTE_READ |\ - IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED) +IB_ACCESS_REMOTE_ATOMIC |IB_ACCESS_MW_BIND | IB_ZERO_BASED)#define
VALID_DMA_MR_FLAGS(IB_ACCESS_LOCAL_WRITE)drivers/infiniband/hw/mana/mr.c +41
-0@@ -27,6 +27,9 @@ mana_ib_verbs_to_gdma_access_flags(int access_flags)if (access_flags &IB_ACCESS_REMOTE_ATOMIC ) flags |=GDMA_ACCESS_FLAG_REMOTE_ATOMIC ; +if (access_flags &IB_ACCESS_MW_BIND ) + flags |=GDMA_ACCESS_FLAG_BIND_MW ; +return flags; }@@ -304,6 +307,58 @@ struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)return ERR_PTR (err); } +static int mana_ib_gd_create_mw (struct mana_ib_dev *dev,struct +mana_ib_pd *pd,struct ib_mw *ibmw) { + struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw); +struct gdma_context *gc =mdev_to_gc (dev); +struct gdma_create_mr_response resp = {}; +struct gdma_create_mr_request req = {}; +int err; + +mana_gd_init_req_hdr (&req.hdr,GDMA_CREATE_MR ,sizeof (req),sizeof (resp)); + req.pd_handle = pd->pd_handle; + +switch (mw->ibmw.type) { +case IB_MW_TYPE_1 : + req.mr_type =GDMA_MR_TYPE_MW1 ; +break ; +case IB_MW_TYPE_2 : + req.mr_type =GDMA_MR_TYPE_MW2 ; +break ; +default : +return -EINVAL ; + } + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err || resp.hdr.status) { +if (!err) + err = -EPROTO ; + +return err; + } + + mw->ibmw.rkey = resp.rkey; + mw->mw_handle = resp.mr_handle; + +return 0 ; +} + +int mana_ib_alloc_mw (struct ib_mw *ibmw,struct ib_udata *udata) { + struct mana_ib_dev *mdev = container_of(ibmw->device, struct
mana_ib_dev,ib_dev);drivers/infiniband/hw/mana/mr.c +7
-0+ struct mana_ib_pd *pd = container_of(ibmw->pd, struct mana_ib_pd, +ibpd); + +return mana_ib_gd_create_mw (mdev, pd, ibmw); } + +int mana_ib_dealloc_mw (struct ib_mw *ibmw) { + struct mana_ib_dev *dev = container_of(ibmw->device, struct
mana_ib_dev,ib_dev);drivers/infiniband/hw/mana/mr.c +1
-0+ struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw,ibmw);
drivers/infiniband/hw/mana/mr.c +3
-0+ +return mana_ib_gd_destroy_mr (dev, mw->mw_handle); } +int mana_ib_dereg_mr (struct ib_mr *ibmr,struct ib_udata *udata) { struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);include/net/mana/gdma.h index renamed from include/net/mana/gdma.h +0
-0
766f4fb25..948f62bb8100644include/net/mana/gdma.h +1
-0@@ -778,6 +778,7 @@ enum gdma_mr_access_flags {GDMA_ACCESS_FLAG_REMOTE_READ =BIT_ULL (2 ),GDMA_ACCESS_FLAG_REMOTE_WRITE =BIT_ULL (3 ),GDMA_ACCESS_FLAG_REMOTE_ATOMIC =BIT_ULL (4 ), + GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5), };/* GDMA_CREATE_DMA_REGION */
include/net/mana/gdma.h enum gdma_mr_type { · line 874
Quoted 4 lines
include/net/mana/gdma.h +1
-0@@ -870,6 +871,10 @@ enum gdma_mr_type {GDMA_MR_TYPE_ZBVA =4 ,/* Device address MRs */ GDMA_MR_TYPE_DM = 5, +/* Device address MRs */
It seems a copy/paste for the wrong comment
Quoted 8 lines
include/net/mana/gdma.h +3
-1+GDMA_MR_TYPE_MW1 =6 , +/* Device address MRs */ + GDMA_MR_TYPE_MW2 = 7, }; struct gdma_create_mr_params { --2.43.0
Also, there is a mw_count in struct mana_ib_adapter_caps, should we do sanity check on it?
Quoted 14 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
Signed-off-by Konstantin Taranov <[email protected]>---
As I see that Jason'srdma_uapiis not in the next yet. I will make a patch adding his helpers (e.g.,ib_is_udata_in_empty()for mw) with all other api calls.drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 ++++ drivers/infiniband/hw/mana/mr.c | 57 +++++++++++++++++++++++++++- include/net/mana/gdma.h | 5 +++ 4 files changed, 72 insertions(+), 1 deletion(-)
<...>
drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) · line 338
Quoted 7 lines
drivers/infiniband/hw/mana/mr.c +7
-0+ err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err || resp.hdr.status) { +if (!err) + err = -EPROTO ; + +return err; + }
We already had this discussion about this specific pattern.
https://lore.kernel.org/linux-rdma/20260127141929.GV13967@unreal/
Please fix it first.
Thanks
Konstantin Taranov
maintainerQuoted 40 lines · Konstantin Taranov
Quoted 4 lines
From: Konstantin Taranov <[email protected]>
Implement .alloc_mw() and .dealloc_mw() for mana device.
Quoted 8 lines
Signed-off-by Konstantin Taranov <[email protected]>---
As I see that Jason'srdma_uapiis not in the next yet. I will make a patch adding his helpers (e.g.,ib_is_udata_in_empty()for mw) with all other api calls.drivers/infiniband/hw/mana/device.c | 3 ++ drivers/infiniband/hw/mana/mana_ib.h | 8 ++++ drivers/infiniband/hw/mana/mr.c | 57+++++++++++++++++++++++++++-
Quoted 2 lines
include/net/mana/gdma.h | 5 +++ 4 files changed, 72 insertions(+), 1 deletion(-)<...>
Quoted 1 line
+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp),
&resp);
drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags) · line 338
Quoted 6 lines
drivers/infiniband/hw/mana/mr.c +6
-0+if (err || resp.hdr.status) { +if (!err) + err = -EPROTO ; + +return err; + }We already had this discussion about this specific pattern.
https://lore.k/
ernel.org%2Flinux- rdma%2F20260127141929.GV13967%40unreal%2F&data=05%7C02%7Ckotar anov%40microsoft.com%7C25731fedf33543d30a1008de83771e19%7C72f988 bf86f141af91ab2d7cd011db47%7C1%7C0%7C639092744818898960%7CUnk nown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwM CIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C& sdata=LjG%2B6fXmfART5jJrlV%2FawXB2QpwG9GzHj%2BfTmk0vuf0%3D&rese rved=0
Quoted 1 line
Please fix it first.
Sorry, I missed it. Thanks! I will send it soon.
- Konstantin
Quoted 2 lines
Thanks