RDMA/mana_ib: device memory support
PATCH 1/1 RDMA/mana_ib: device memory support
From: Konstantin Taranov <[email protected]>
Basic implementation of DM allowing to create and register DM memory and use its memory keys for networking.
--- drivers/infiniband/hw/mana/device.c | 7 ++ drivers/infiniband/hw/mana/mana_ib.h | 12 +++ drivers/infiniband/hw/mana/mr.c | 138 +++++++++++++++++++++++++++ include/net/mana/gdma.h | 47 ++++++++- 4 files changed, 201 insertions(+), 3 deletions(-)
drivers/infiniband/hw/mana/device.c +7 -0
@@ -69,6 +69,12 @@ static const struct ib_device_ops mana_ib_device_stats_ops = { .alloc_hw_device_stats = mana_ib_alloc_hw_device_stats, }; +const struct ib_device_ops mana_ib_dev_dm_ops = { + .alloc_dm = mana_ib_alloc_dm, + .dealloc_dm = mana_ib_dealloc_dm, + .reg_dm_mr = mana_ib_reg_dm_mr, +}; +static int mana_ib_netdev_event (struct notifier_block *this,unsigned long event,void *ptr) {
@@ -139,6 +145,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,ib_set_device_ops (&dev->ib_dev, &mana_ib_stats_ops);if (dev->adapter_caps.feature_flags &MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT )ib_set_device_ops (&dev->ib_dev, &mana_ib_device_stats_ops); +ib_set_device_ops (&dev->ib_dev, &mana_ib_dev_dm_ops); ret =mana_ib_create_eqs (dev);if (ret) {
drivers/infiniband/hw/mana/mana_ib.h +12 -0
@@ -131,6 +131,11 @@ struct mana_ib_mr {mana_handle_t mr_handle; }; +struct mana_ib_dm { +struct ib_dm ibdm; +mana_handle_t dm_handle; +}; +struct mana_ib_cq {struct ib_cq ibcq;struct mana_ib_queue queue;
@@ -735,4 +740,11 @@ struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 leng u64 iova, int fd, int mr_access_flags, struct ib_dmah *dmah, struct uverbs_attr_bundle *attrs); + +struct ib_dm *mana_ib_alloc_dm (struct ib_device *dev,struct ib_ucontext *context, +struct ib_dm_alloc_attr *attr,struct uverbs_attr_bundle *attrs); +int mana_ib_dealloc_dm (struct ib_dm *dm,struct uverbs_attr_bundle *attrs); +struct ib_mr *mana_ib_reg_dm_mr (struct ib_pd *pd,struct ib_dm *dm,struct ib_dm_mr_attr *attr, +struct uverbs_attr_bundle *attrs); +#endif
drivers/infiniband/hw/mana/mr.c +138 -0
153 lines not on this page · raw message
include/net/mana/gdma.h +44 -3
81 lines not on this page · raw message
Signature · 1 line
2.43.0
Quoted 20 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
Basic implementation of DM allowing to create and register DM memory and use its memory keys for networking.
Signed-off-by Konstantin Taranov <[email protected]>--- drivers/infiniband/hw/mana/device.c | 7 ++ drivers/infiniband/hw/mana/mana_ib.h | 12 +++ drivers/infiniband/hw/mana/mr.c | 138 +++++++++++++++++++++++++++ include/net/mana/gdma.h | 47 ++++++++- 4 files changed, 201 insertions(+), 3 deletions(-)drivers/infiniband/hw/mana/device.c +0
-0@@ -69,6 +69,12 @@ static const struct ib_device_ops mana_ib_device_stats_ops = { .alloc_hw_device_stats = mana_ib_alloc_hw_device_stats, };
<...>
Quoted 34 lines
drivers/infiniband/hw/mana/device.c +34
-0+static int mana_ib_gd_alloc_dm (struct mana_ib_dev *mdev,struct mana_ib_dm *dm, +struct ib_dm_alloc_attr *attr) +{ +struct gdma_context *gc =mdev_to_gc (mdev); +struct gdma_alloc_dm_resp resp = {}; +struct gdma_alloc_dm_req req = {}; +int err; + +mana_gd_init_req_hdr (&req.hdr,GDMA_ALLOC_DM ,sizeof (req),sizeof (resp)); + req.length = attr->length; + req.alignment = attr->alignment; + req.flags = attr->flags; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err || resp.hdr.status) { +ibdev_dbg (&mdev->ib_dev,"Failed to alloc dm err %d, %u" , err, + resp.hdr.status); +if (!err) + err = -EPROTO ; + +return err; + } + + dm->dm_handle = resp.dm_handle; + +return 0 ; +} + +struct ib_dm *mana_ib_alloc_dm (struct ib_device *ibdev, +struct ib_ucontext *context, +struct ib_dm_alloc_attr *attr, +struct uverbs_attr_bundle *attrs) +{ + struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/device.c
Quoted 10 lines
drivers/infiniband/hw/mana/device.c +10
-0+struct mana_ib_dm *dm; +int err; + + dm =kzalloc (sizeof (*dm),GFP_KERNEL ); +if (!dm) +return ERR_PTR (-ENOMEM ); + + err =mana_ib_gd_alloc_dm (dev, dm, attr); +if (err) { +ibdev_dbg (ibdev,"Failed allocate dm memory, %d\n" , err);
I intended to apply this patch, but encountered too many minor issues that needed local fixes.
You already printed debug message when mana_ib_gd_alloc_dm() failed.
Quoted 32 lines
drivers/infiniband/hw/mana/device.c +32
-0+ goto err_free; + } + +return &dm->ibdm; + +err_free: +kfree (dm); +return ERR_PTR (err); +} + +static int mana_ib_gd_destroy_dm (struct mana_ib_dev *mdev,struct mana_ib_dm *dm) +{ +struct gdma_context *gc =mdev_to_gc (mdev); +struct gdma_destroy_dm_resp resp = {}; +struct gdma_destroy_dm_req req = {}; +int err; + +mana_gd_init_req_hdr (&req.hdr,GDMA_DESTROY_DM ,sizeof (req),sizeof (resp)); + req.dm_handle = dm->dm_handle; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err || resp.hdr.status) { +ibdev_dbg (&mdev->ib_dev,"Failed to destroy dm err %d, %u" , err, + resp.hdr.status); +if (!err) + err = -EPROTO ; + +return err; + } + +return 0 ; +}
drivers/infiniband/hw/mana/device.c
Quoted 10 lines
drivers/infiniband/hw/mana/device.c +10
-0+ +int mana_ib_dealloc_dm (struct ib_dm *ibdm,struct uverbs_attr_bundle *attrs) +{ + struct mana_ib_dev *dev = container_of(ibdm->device, struct mana_ib_dev, ib_dev); + struct mana_ib_dm *dm = container_of(ibdm, struct mana_ib_dm, ibdm); +int err; + + err =mana_ib_gd_destroy_dm (dev, dm); +if (err) +ibdev_dbg (ibdm->device,"Failed destroy dm memory, %d\n" , err);
Same error print as for alloc.
In order to simplify things for you: unless you can clearly justify why this print is required and why you cannot proceed without it, I must ask you to stop adding any new debug or error messages to the mana_ib driver. There is a wide range of existing tools and well‑established practices for debugging the kernel, and none of them require spamming dmesg.
drivers/infiniband/hw/mana/device.c
Quoted 5 lines
drivers/infiniband/hw/mana/device.c +5
-0+/* Free dm even on HW errors as we do not use the DM anymore. +* Since DM is a purely HW resource, faulty HW cannot harm the kernel. +*/ +kfree (dm); +return err;
You have already freed the DM, so you should return 0 here and make mana_ib_gd_destroy_dm() return void, as no one uses its return value.
Thanks