Skip to contentSkip to conversation

RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

PATCH Superseded Konstantin Taranov 4 participants 5 messages Jan 10, 2024 – Jan 14, 2024

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context
2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc
2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
drivers/infiniband/hw/mana/cq.c +21 -2
index 83ebd0705..255e74ab7 100644
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	int err;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	if (udata->inlen < sizeof(ucmd))
 		return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	int err;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
 	if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
 	if (cq->ibcq.comp_handler)
 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
+{
+	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct gdma_queue *gdma_cq;
+
+	/* Create CQ table entry */
+	WARN_ON(gc->cq_table[cq->id]);
+	gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+	if (!gdma_cq)
+		return -ENOMEM;
+
+	gdma_cq->cq.context = cq;
+	gdma_cq->type = GDMA_CQ;
+	gdma_cq->cq.callback = mana_ib_cq_handler;
+	gdma_cq->id = cq->id;
+	gc->cq_table[cq->id] = gdma_cq;
+	return 0;
+}
drivers/infiniband/hw/mana/main.c +14 -26
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 			 u32 port)
 {
-	struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
 	struct mana_port_context *mpc;
 	struct net_device *ndev;
-	struct mana_context *mc;
 
-	mc = gd->driver_data;
-	ndev = mc->ports[port];
+	ndev = mana_ib_get_netdev(&dev->ib_dev, port);
 	mpc = netdev_priv(ndev);
 
 	mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
 		      u32 doorbell_id)
 {
-	struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
 	struct mana_port_context *mpc;
-	struct mana_context *mc;
 	struct net_device *ndev;
 	int err;
 
-	mc = mdev->driver_data;
-	ndev = mc->ports[port];
+	ndev = mana_ib_get_netdev(&dev->ib_dev, port);
 	mpc = netdev_priv(ndev);
 
 	mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct gdma_create_pd_req req = {};
 	enum gdma_pd_flags flags = 0;
 	struct mana_ib_dev *dev;
-	struct gdma_dev *mdev;
+	struct gdma_context *gc;
 	int err;
 
 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	mdev = dev->gdma_dev;
+	gc = mdev_to_gc(dev);
 
 	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
 			     sizeof(resp));
 
 	req.flags = flags;
-	err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+	err = mana_gd_send_request(gc, sizeof(req), &req,
 				   sizeof(resp), &resp);
 
 	if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct gdma_destory_pd_resp resp = {};
 	struct gdma_destroy_pd_req req = {};
 	struct mana_ib_dev *dev;
-	struct gdma_dev *mdev;
+	struct gdma_context *gc;
 	int err;
 
 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	mdev = dev->gdma_dev;
+	gc = mdev_to_gc(dev);
 
 	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
 			     sizeof(resp));
 
 	req.pd_handle = pd->pd_handle;
-	err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+	err = mana_gd_send_request(gc, sizeof(req), &req,
 				   sizeof(resp), &resp);
 
 	if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
 	struct ib_device *ibdev = ibcontext->device;
 	struct mana_ib_dev *mdev;
 	struct gdma_context *gc;
-	struct gdma_dev *dev;
 	int doorbell_page;
 	int ret;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	dev = mdev->gdma_dev;
-	gc = dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	/* Allocate a doorbell page index */
 	ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
@@ -238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 	int ret;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
 	if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 	size_t max_pgs_create_cmd;
 	struct gdma_context *gc;
 	size_t num_pages_total;
-	struct gdma_dev *mdev;
 	unsigned long page_sz;
 	unsigned int tail = 0;
 	u64 *page_addr_list;
 	void *request_buf;
 	int err;
 
-	mdev = dev->gdma_dev;
-	gc = mdev->gdma_context;
+	gc = mdev_to_gc(dev);
 	hwc = gc->hwc.driver_data;
 
 	/* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
 {
-	struct gdma_dev *mdev = dev->gdma_dev;
-	struct gdma_context *gc;
+	struct gdma_context *gc = mdev_to_gc(dev);
 
-	gc = mdev->gdma_context;
 	ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
 
 	return mana_gd_destroy_dma_region(gc, gdma_region);
@@ -447,7 +435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
 	int ret;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	if (vma->vm_pgoff != 0) {
 		ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
@@ -531,7 +519,7 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
 	req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
 	req.hdr.dev_id = dev->gdma_dev->dev_id;
 
-	err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+	err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
 				   &req, sizeof(resp), &resp);
 
 	if (err) {
drivers/infiniband/hw/mana/mana_ib.h +17 -0
index 6bdc0f549..6b15b2ab5 100644
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
 	u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
+{
+	return mdev->gdma_dev->gdma_context;
+}
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
+{
+	struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct mana_context *mc = gc->mana.driver_data;
+
+	if (port < 1 || port > mc->num_ports)
+		return NULL;
+	return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 				 mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
drivers/infiniband/hw/mana/mr.c +3 -10
index 351207c60..ee4d4f834 100644
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
 {
 	struct gdma_create_mr_response resp = {};
 	struct gdma_create_mr_request req = {};
-	struct gdma_dev *mdev = dev->gdma_dev;
-	struct gdma_context *gc;
+	struct gdma_context *gc = mdev_to_gc(dev);
 	int err;
 
-	gc = mdev->gdma_context;
-
 	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
 			     sizeof(resp));
 	req.pd_handle = mr_params->pd_handle;
@@ -77,12 +74,9 @@ static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
 {
 	struct gdma_destroy_mr_response resp = {};
 	struct gdma_destroy_mr_request req = {};
-	struct gdma_dev *mdev = dev->gdma_dev;
-	struct gdma_context *gc;
+	struct gdma_context *gc = mdev_to_gc(dev);
 	int err;
 
-	gc = mdev->gdma_context;
-
 	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
 			     sizeof(resp));
 
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
 	return &mr->ibmr;
 
 err_dma_region:
-	mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-				   dma_region_handle);
+	mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
 	ib_umem_release(mr->umem);
drivers/infiniband/hw/mana/qp.c +8 -28
index e6d063d45..e889c798f 100644
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
 	struct mana_cfg_rx_steer_resp resp = {};
 	mana_handle_t *req_indir_tab;
 	struct gdma_context *gc;
-	struct gdma_dev *mdev;
 	u32 req_buf_size;
 	int i, err;
 
-	gc = dev->gdma_dev->gdma_context;
-	mdev = &gc->mana;
+	gc = mdev_to_gc(dev);
 
 	req_buf_size =
 		sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
@@ -39,7 +37,7 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
 	req->rx_enable = 1;
 	req->update_default_rxobj = 1;
 	req->default_rxobj = default_rxobj;
-	req->hdr.dev_id = mdev->dev_id;
+	req->hdr.dev_id = gc->mana.dev_id;
 
 	/* If there are more than 1 entries in indirection table, enable RSS */
 	if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 	struct gdma_queue **gdma_cq_allocated;
 	mana_handle_t *mana_ind_table;
 	struct mana_port_context *mpc;
-	struct gdma_queue *gdma_cq;
 	unsigned int ind_tbl_size;
 	struct net_device *ndev;
 	struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		mana_ind_table[i] = wq->rx_object;
 
 		/* Create CQ table entry */
-		WARN_ON(gc->cq_table[cq->id]);
-		gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-		if (!gdma_cq) {
-			ret = -ENOMEM;
+		ret = mana_ib_install_cq_cb(mdev, cq);
+		if (!ret)
 			goto fail;
-		}
-		gdma_cq_allocated[i] = gdma_cq;
 
-		gdma_cq->cq.context = cq;
-		gdma_cq->type = GDMA_CQ;
-		gdma_cq->cq.callback = mana_ib_cq_handler;
-		gdma_cq->id = cq->id;
-		gc->cq_table[cq->id] = gdma_cq;
+		gdma_cq_allocated[i] = gc->cq_table[cq->id];
 	}
 	resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	send_cq->id = cq_spec.queue_index;
 
 	/* Create CQ table entry */
-	WARN_ON(gc->cq_table[send_cq->id]);
-	gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-	if (!gdma_cq) {
-		err = -ENOMEM;
+	err = mana_ib_install_cq_cb(mdev, send_cq);
+	if (err)
 		goto err_destroy_wq_obj;
-	}
-
-	gdma_cq->cq.context = send_cq;
-	gdma_cq->type = GDMA_CQ;
-	gdma_cq->cq.callback = mana_ib_cq_handler;
-	gdma_cq->id = send_cq->id;
-	gc->cq_table[send_cq->id] = gdma_cq;
 
 	ibdev_dbg(&mdev->ib_dev,
 		  "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
@@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 err_release_gdma_cq:
 	kfree(gdma_cq);
-	gd->gdma_context->cq_table[send_cq->id] = NULL;
+	gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
 	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03

Signature · 1 line
2.43.0
#1 lore raw
Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

Quoted 375 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
drivers/infiniband/hw/mana/cq.c +21 -2
index 83ebd0705..255e74ab7 100644
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	int err;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	if (udata->inlen < sizeof(ucmd))
 		return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	int err;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
 	if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
 	if (cq->ibcq.comp_handler)
 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
+{
+	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct gdma_queue *gdma_cq;
+
+	/* Create CQ table entry */
+	WARN_ON(gc->cq_table[cq->id]);
+	gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+	if (!gdma_cq)
+		return -ENOMEM;
+
+	gdma_cq->cq.context = cq;
+	gdma_cq->type = GDMA_CQ;
+	gdma_cq->cq.callback = mana_ib_cq_handler;
+	gdma_cq->id = cq->id;
+	gc->cq_table[cq->id] = gdma_cq;
+	return 0;
+}
drivers/infiniband/hw/mana/main.c +14 -26
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 			 u32 port)
 {
-	struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
 	struct mana_port_context *mpc;
 	struct net_device *ndev;
-	struct mana_context *mc;
 
-	mc = gd->driver_data;
-	ndev = mc->ports[port];
+	ndev = mana_ib_get_netdev(&dev->ib_dev, port);
 	mpc = netdev_priv(ndev);
 
 	mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
 		      u32 doorbell_id)
 {
-	struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
 	struct mana_port_context *mpc;
-	struct mana_context *mc;
 	struct net_device *ndev;
 	int err;
 
-	mc = mdev->driver_data;
-	ndev = mc->ports[port];
+	ndev = mana_ib_get_netdev(&dev->ib_dev, port);
 	mpc = netdev_priv(ndev);
 
 	mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct gdma_create_pd_req req = {};
 	enum gdma_pd_flags flags = 0;
 	struct mana_ib_dev *dev;
-	struct gdma_dev *mdev;
+	struct gdma_context *gc;
 	int err;
 
 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	mdev = dev->gdma_dev;
+	gc = mdev_to_gc(dev);
 
 	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
 			     sizeof(resp));
 
 	req.flags = flags;
-	err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+	err = mana_gd_send_request(gc, sizeof(req), &req,
 				   sizeof(resp), &resp);
 
 	if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct gdma_destory_pd_resp resp = {};
 	struct gdma_destroy_pd_req req = {};
 	struct mana_ib_dev *dev;
-	struct gdma_dev *mdev;
+	struct gdma_context *gc;
 	int err;
 
 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	mdev = dev->gdma_dev;
+	gc = mdev_to_gc(dev);
 
 	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
 			     sizeof(resp));
 
 	req.pd_handle = pd->pd_handle;
-	err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+	err = mana_gd_send_request(gc, sizeof(req), &req,
 				   sizeof(resp), &resp);
 
 	if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
 	struct ib_device *ibdev = ibcontext->device;
 	struct mana_ib_dev *mdev;
 	struct gdma_context *gc;
-	struct gdma_dev *dev;
 	int doorbell_page;
 	int ret;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	dev = mdev->gdma_dev;
-	gc = dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	/* Allocate a doorbell page index */
 	ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
@@ -238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 	int ret;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
 	if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 	size_t max_pgs_create_cmd;
 	struct gdma_context *gc;
 	size_t num_pages_total;
-	struct gdma_dev *mdev;
 	unsigned long page_sz;
 	unsigned int tail = 0;
 	u64 *page_addr_list;
 	void *request_buf;
 	int err;
 
-	mdev = dev->gdma_dev;
-	gc = mdev->gdma_context;
+	gc = mdev_to_gc(dev);
 	hwc = gc->hwc.driver_data;
 
 	/* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
 {
-	struct gdma_dev *mdev = dev->gdma_dev;
-	struct gdma_context *gc;
+	struct gdma_context *gc = mdev_to_gc(dev);
 
-	gc = mdev->gdma_context;
 	ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
 
 	return mana_gd_destroy_dma_region(gc, gdma_region);
@@ -447,7 +435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
 	int ret;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-	gc = mdev->gdma_dev->gdma_context;
+	gc = mdev_to_gc(mdev);
 
 	if (vma->vm_pgoff != 0) {
 		ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
@@ -531,7 +519,7 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
 	req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
 	req.hdr.dev_id = dev->gdma_dev->dev_id;
 
-	err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+	err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
 				   &req, sizeof(resp), &resp);
 
 	if (err) {
drivers/infiniband/hw/mana/mana_ib.h +17 -0
index 6bdc0f549..6b15b2ab5 100644
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
 	u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
+{
+	return mdev->gdma_dev->gdma_context;
+}
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
+{
+	struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct mana_context *mc = gc->mana.driver_data;
+
+	if (port < 1 || port > mc->num_ports)
+		return NULL;
+	return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 				 mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
drivers/infiniband/hw/mana/mr.c +3 -10
index 351207c60..ee4d4f834 100644
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
 {
 	struct gdma_create_mr_response resp = {};
 	struct gdma_create_mr_request req = {};
-	struct gdma_dev *mdev = dev->gdma_dev;
-	struct gdma_context *gc;
+	struct gdma_context *gc = mdev_to_gc(dev);
 	int err;
 
-	gc = mdev->gdma_context;
-
 	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
 			     sizeof(resp));
 	req.pd_handle = mr_params->pd_handle;
@@ -77,12 +74,9 @@ static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
 {
 	struct gdma_destroy_mr_response resp = {};
 	struct gdma_destroy_mr_request req = {};
-	struct gdma_dev *mdev = dev->gdma_dev;
-	struct gdma_context *gc;
+	struct gdma_context *gc = mdev_to_gc(dev);
 	int err;
 
-	gc = mdev->gdma_context;
-
 	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
 			     sizeof(resp));
 
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
 	return &mr->ibmr;
 
 err_dma_region:
-	mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-				   dma_region_handle);
+	mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
 	ib_umem_release(mr->umem);
drivers/infiniband/hw/mana/qp.c +8 -28
index e6d063d45..e889c798f 100644
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
 	struct mana_cfg_rx_steer_resp resp = {};
 	mana_handle_t *req_indir_tab;
 	struct gdma_context *gc;
-	struct gdma_dev *mdev;
 	u32 req_buf_size;
 	int i, err;
 
-	gc = dev->gdma_dev->gdma_context;
-	mdev = &gc->mana;
+	gc = mdev_to_gc(dev);
 
 	req_buf_size =
 		sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
@@ -39,7 +37,7 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
 	req->rx_enable = 1;
 	req->update_default_rxobj = 1;
 	req->default_rxobj = default_rxobj;
-	req->hdr.dev_id = mdev->dev_id;
+	req->hdr.dev_id = gc->mana.dev_id;
 
 	/* If there are more than 1 entries in indirection table, enable RSS */
 	if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 	struct gdma_queue **gdma_cq_allocated;
 	mana_handle_t *mana_ind_table;
 	struct mana_port_context *mpc;
-	struct gdma_queue *gdma_cq;
 	unsigned int ind_tbl_size;
 	struct net_device *ndev;
 	struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		mana_ind_table[i] = wq->rx_object;
 
 		/* Create CQ table entry */
-		WARN_ON(gc->cq_table[cq->id]);
-		gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-		if (!gdma_cq) {
-			ret = -ENOMEM;
+		ret = mana_ib_install_cq_cb(mdev, cq);
+		if (!ret)
 			goto fail;
-		}
-		gdma_cq_allocated[i] = gdma_cq;
 
-		gdma_cq->cq.context = cq;
-		gdma_cq->type = GDMA_CQ;
-		gdma_cq->cq.callback = mana_ib_cq_handler;
-		gdma_cq->id = cq->id;
-		gc->cq_table[cq->id] = gdma_cq;
+		gdma_cq_allocated[i] = gc->cq_table[cq->id];
 	}
 	resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	send_cq->id = cq_spec.queue_index;
 
 	/* Create CQ table entry */
-	WARN_ON(gc->cq_table[send_cq->id]);
-	gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-	if (!gdma_cq) {
-		err = -ENOMEM;
+	err = mana_ib_install_cq_cb(mdev, send_cq);
+	if (err)
 		goto err_destroy_wq_obj;
-	}
-
-	gdma_cq->cq.context = send_cq;
-	gdma_cq->type = GDMA_CQ;
-	gdma_cq->cq.callback = mana_ib_cq_handler;
-	gdma_cq->id = send_cq->id;
-	gc->cq_table[send_cq->id] = gdma_cq;
 
 	ibdev_dbg(&mdev->ib_dev,
 		  "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
@@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 err_release_gdma_cq:
 	kfree(gdma_cq);
-	gd->gdma_context->cq_table[send_cq->id] = NULL;
+	gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
 	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03

Signature · 1 line
2.43.0
#2 lore raw

Hi Leon,

// Sorry for re-sending the email. I did not know that I had to enable plain-text messages in outlook.

thanks for pointing out the merge window. I will resend after it.

I can also split it into 2 commits (mdev_to_gc and mana_ib_get_netdev in one, and CQ cb in the other), if it is required. I cannot separate "mdev_to_gc" and "mana_ib_get_netdev " into 2 commits as it will double the changes and obstruct the reader.

Quoted 1 line

And how is it safe now? What is unsafe here?

The issue with the gdma_dev is the following.
Our mana HW device has two gdma_devs: one for ethernet and one for rdma.
And in the code of mana_ib.ko it is not clearly indicated what gdma_dev is used. So it means in some functions it is ethernet and in some it is rdma.
The use of a wrong device in the hardware channel leads to errors and waste time of developers.

This problem is addressed as follows:
We avoid using gdma_dev explicitly in functions of mana_ib.ko except the init.
As the gdma_context is only one, we abstract it into mdev_to_gc, to prevent the user to declare gdma_dev.
When a function actually needs to use gdma_dev of the ethernet, it uses netdev, which encapsulates the correct gdma_dev. Getting netdev is implemented with mana_ib_get_netdev, to prevent the user to declare gdma_dev.
When a function needs to use gdma_dev of the rdma, it will use struct mana_ib_dev (a child of ib_dev) in its hardware channel requests.

I hope it explains what becomes safer. If it was useful, I can add this explanation to the commit message.

Thanks,
Konstantin

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

Quoted 370 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
drivers/infiniband/hw/mana/cq.c +21 -2
index 83ebd0705..255e74ab7 100644
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
+{
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +14 -26
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
                    u32 doorbell_id)
 {
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
@@ -238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
 {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
 
      return mana_gd_destroy_dma_region(gc, gdma_region);
@@ -447,7 +435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
@@ -531,7 +519,7 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
 
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +17 -0
index 6bdc0f549..6b15b2ab5 100644
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
+{
+     return mdev->gdma_dev->gdma_context;
+}
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
+{
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
drivers/infiniband/hw/mana/mr.c +3 -10
index 351207c60..ee4d4f834 100644
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
 {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle;
@@ -77,12 +74,9 @@ static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
 {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
 
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
drivers/infiniband/hw/mana/qp.c +8 -28
index e6d063d45..e889c798f 100644
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
@@ -39,7 +37,7 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
@@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 err_release_gdma_cq:
      kfree(gdma_cq);
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Quoted 5 lines

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

Quoted 370 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
drivers/infiniband/hw/mana/cq.c +21 -2
index 83ebd0705..255e74ab7 100644
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
+{
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +14 -26
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
                    u32 doorbell_id)
 {
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
@@ -238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
 {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
 
      return mana_gd_destroy_dma_region(gc, gdma_region);
@@ -447,7 +435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
@@ -531,7 +519,7 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
 
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +17 -0
index 6bdc0f549..6b15b2ab5 100644
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
+{
+     return mdev->gdma_dev->gdma_context;
+}
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
+{
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
drivers/infiniband/hw/mana/mr.c +3 -10
index 351207c60..ee4d4f834 100644
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
 {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle;
@@ -77,12 +74,9 @@ static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
 {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
 
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
drivers/infiniband/hw/mana/qp.c +8 -28
index e6d063d45..e889c798f 100644
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
@@ -39,7 +37,7 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
@@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 err_release_gdma_cq:
      kfree(gdma_cq);
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Quoted 5 lines

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

Quoted 375 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
drivers/infiniband/hw/mana/cq.c +21 -2
index 83ebd0705..255e74ab7 100644
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
+{
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +14 -26
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
                    u32 doorbell_id)
 {
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
@@ -238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
 {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
 
      return mana_gd_destroy_dma_region(gc, gdma_region);
@@ -447,7 +435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
@@ -531,7 +519,7 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
 
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +17 -0
index 6bdc0f549..6b15b2ab5 100644
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
+{
+     return mdev->gdma_dev->gdma_context;
+}
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
+{
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
drivers/infiniband/hw/mana/mr.c +3 -10
index 351207c60..ee4d4f834 100644
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
 {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle;
@@ -77,12 +74,9 @@ static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
 {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
 
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
drivers/infiniband/hw/mana/qp.c +8 -28
index e6d063d45..e889c798f 100644
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
@@ -39,7 +37,7 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
@@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 err_release_gdma_cq:
      kfree(gdma_cq);
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

#3 lore raw
Quoted 35 lines

Subject: Re: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

Hi Leon,

// Sorry for re-sending the email. I did not know that I had to enable plain-text messages in outlook.

thanks for pointing out the merge window. I will resend after it.

I can also split it into 2 commits (mdev_to_gc and mana_ib_get_netdev in one, and CQ cb in the other), if it is required.
I cannot separate "mdev_to_gc" and "mana_ib_get_netdev " into 2 commits as it will double the changes and obstruct the reader.

Quoted 1 line

And how is it safe now? What is unsafe here?

The issue with the gdma_dev is the following.
Our mana HW device has two gdma_devs: one for ethernet and one for rdma. And in the code of mana_ib.ko it is not clearly indicated what gdma_dev is used. So it means in some functions it is ethernet and in some it is rdma. The use of a wrong device in the hardware channel leads to errors and waste time of developers.

This problem is addressed as follows:
We avoid using gdma_dev explicitly in functions of mana_ib.ko except the init. As the gdma_context is only one, we abstract it into mdev_to_gc, to prevent the user to declare gdma_dev.
When a function actually needs to use gdma_dev of the ethernet, it uses netdev, which encapsulates the correct gdma_dev.
Getting netdev is implemented with mana_ib_get_netdev, to prevent the user to declare gdma_dev.
When a function needs to use gdma_dev of the rdma, it will use struct mana_ib_dev (a child of ib_dev) in its hardware channel requests.

Quoted 2 lines

I hope it explains what becomes safer. If it was useful, I can add this explanation to the commit message.

You patch doesn't make it safer, but rather makes it easier to understand. The wording "Unsafe and inconsistent" should be removed.

Long

Quoted 1,319 lines

Thanks,
Konstantin

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; linux- [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at
https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

drivers/infiniband/hw/mana/cq.c int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 16 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/cq.c index 83ebd0705..255e74ab7 100644

drivers/infiniband/hw/mana/cq.c +0 -0
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

ib_udata *udata)

drivers/infiniband/hw/mana/cq.c void mana_ib_cq_handler(void *ctx, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct

gdma_queue *gdma_cq)

Quoted 43 lines
drivers/infiniband/hw/mana/cq.c +19 -0
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq) {
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +1 -4
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev,
Quoted 1 line

struct mana_ib_pd *pd, int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32

port, struct mana_ib_pd *pd,

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

Quoted 15 lines

u32 doorbell_id)
{

drivers/infiniband/hw/mana/main.c +1 -4
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext

*ibcontext,

Quoted 5 lines
drivers/infiniband/hw/mana/main.c +0 -1
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +1 -3
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); @@
-238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext

*ibcontext)

drivers/infiniband/hw/mana/main.c int mana_ib_gd_create_dma_region(struct

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

Quoted 29 lines
drivers/infiniband/hw/mana/main.c +2 -6
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct
 mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64
 gdma_region)  {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n",
 gdma_region);
Quoted 2 lines
      return mana_gd_destroy_dma_region(gc, gdma_region); @@ -447,7

+435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct

vm_area_struct *vma)

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n",

vma->vm_pgoff); @@ -531,7 +519,7 @@ int

mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)

Quoted 14 lines
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
drivers/infiniband/hw/mana/main.c +1 -1
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +0 -0
index 6bdc0f549..6b15b2ab5 100644

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp { · line 151

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +7 -0
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev
+*mdev) {
+     return mdev->gdma_dev->gdma_context; }
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device
+*ibdev, u32 port) {
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,

ib_dev);

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp {

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +8 -0
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem

*umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 3 lines
drivers/infiniband/hw/mana/mana_ib.h +0 -0
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

Quoted 40 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 351207c60..ee4d4f834 100644

drivers/infiniband/hw/mana/mr.c +1 -4
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev
 *dev, struct mana_ib_mr *mr,  {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle; @@ -77,12 +74,9 @@ static
int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)  {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
drivers/infiniband/hw/mana/mr.c +1 -4
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

*ibpd, u64 start, u64 length,

drivers/infiniband/hw/mana/qp.c static int mana_ib_cfg_vport_steering(struct

Quoted 14 lines
drivers/infiniband/hw/mana/mr.c +1 -2
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index e6d063d45..e889c798f 100644

drivers/infiniband/hw/mana/qp.c +0 -0
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct

mana_ib_dev *dev,

Quoted 13 lines
drivers/infiniband/hw/mana/qp.c +1 -3
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) *
Quoted 1 line

MANA_INDIRECT_TABLE_SIZE; @@ -39,7 +37,7 @@ static int

mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 9 lines
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
drivers/infiniband/hw/mana/qp.c +1 -1
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 8 lines
drivers/infiniband/hw/mana/qp.c +0 -1
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

Quoted 23 lines
drivers/infiniband/hw/mana/qp.c +3 -11
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

struct ib_pd *ibpd,

Quoted 31 lines
drivers/infiniband/hw/mana/qp.c +2 -11
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n",

err, @@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,

err_release_gdma_cq:

      kfree(gdma_cq);
drivers/infiniband/hw/mana/qp.c +1 -1
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Quoted 5 lines

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; linux- [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at
https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

drivers/infiniband/hw/mana/cq.c int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 16 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/cq.c index 83ebd0705..255e74ab7 100644

drivers/infiniband/hw/mana/cq.c +0 -0
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

ib_udata *udata)

drivers/infiniband/hw/mana/cq.c void mana_ib_cq_handler(void *ctx, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct

gdma_queue *gdma_cq)

Quoted 43 lines
drivers/infiniband/hw/mana/cq.c +19 -0
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq) {
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +1 -4
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev,
Quoted 1 line

struct mana_ib_pd *pd, int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32

port, struct mana_ib_pd *pd,

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

Quoted 15 lines

u32 doorbell_id)
{

drivers/infiniband/hw/mana/main.c +1 -4
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext

*ibcontext,

Quoted 5 lines
drivers/infiniband/hw/mana/main.c +0 -1
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +1 -3
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); @@
-238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext

*ibcontext)

drivers/infiniband/hw/mana/main.c int mana_ib_gd_create_dma_region(struct

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

Quoted 29 lines
drivers/infiniband/hw/mana/main.c +2 -6
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct
 mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64
 gdma_region)  {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n",
 gdma_region);
Quoted 2 lines
      return mana_gd_destroy_dma_region(gc, gdma_region); @@ -447,7

+435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct

vm_area_struct *vma)

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n",

vma->vm_pgoff); @@ -531,7 +519,7 @@ int

mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)

Quoted 14 lines
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
drivers/infiniband/hw/mana/main.c +1 -1
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +0 -0
index 6bdc0f549..6b15b2ab5 100644

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp { · line 151

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +7 -0
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev
+*mdev) {
+     return mdev->gdma_dev->gdma_context; }
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device
+*ibdev, u32 port) {
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,

ib_dev);

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp {

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +8 -0
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem

*umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 3 lines
drivers/infiniband/hw/mana/mana_ib.h +0 -0
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

Quoted 40 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 351207c60..ee4d4f834 100644

drivers/infiniband/hw/mana/mr.c +1 -4
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev
 *dev, struct mana_ib_mr *mr,  {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle; @@ -77,12 +74,9 @@ static
int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)  {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
drivers/infiniband/hw/mana/mr.c +1 -4
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

*ibpd, u64 start, u64 length,

drivers/infiniband/hw/mana/qp.c static int mana_ib_cfg_vport_steering(struct

Quoted 14 lines
drivers/infiniband/hw/mana/mr.c +1 -2
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index e6d063d45..e889c798f 100644

drivers/infiniband/hw/mana/qp.c +0 -0
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct

mana_ib_dev *dev,

Quoted 13 lines
drivers/infiniband/hw/mana/qp.c +1 -3
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) *
Quoted 1 line

MANA_INDIRECT_TABLE_SIZE; @@ -39,7 +37,7 @@ static int

mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 9 lines
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
drivers/infiniband/hw/mana/qp.c +1 -1
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 8 lines
drivers/infiniband/hw/mana/qp.c +0 -1
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

Quoted 23 lines
drivers/infiniband/hw/mana/qp.c +3 -11
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

struct ib_pd *ibpd,

Quoted 31 lines
drivers/infiniband/hw/mana/qp.c +2 -11
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n",

err, @@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,

err_release_gdma_cq:

      kfree(gdma_cq);
drivers/infiniband/hw/mana/qp.c +1 -1
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Quoted 5 lines

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; linux- [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at
https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

drivers/infiniband/hw/mana/cq.c int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 16 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/cq.c index 83ebd0705..255e74ab7 100644

drivers/infiniband/hw/mana/cq.c +0 -0
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

ib_udata *udata)

drivers/infiniband/hw/mana/cq.c void mana_ib_cq_handler(void *ctx, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct

gdma_queue *gdma_cq)

Quoted 43 lines
drivers/infiniband/hw/mana/cq.c +19 -0
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq) {
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +1 -4
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev,
Quoted 1 line

struct mana_ib_pd *pd, int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32

port, struct mana_ib_pd *pd,

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

Quoted 15 lines

u32 doorbell_id)
{

drivers/infiniband/hw/mana/main.c +1 -4
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext

*ibcontext,

Quoted 5 lines
drivers/infiniband/hw/mana/main.c +0 -1
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +1 -3
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); @@
-238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext

*ibcontext)

drivers/infiniband/hw/mana/main.c int mana_ib_gd_create_dma_region(struct

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

Quoted 29 lines
drivers/infiniband/hw/mana/main.c +2 -6
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct
 mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64
 gdma_region)  {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n",
 gdma_region);
Quoted 2 lines
      return mana_gd_destroy_dma_region(gc, gdma_region); @@ -447,7

+435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct

vm_area_struct *vma)

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n",

vma->vm_pgoff); @@ -531,7 +519,7 @@ int

mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)

Quoted 14 lines
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
drivers/infiniband/hw/mana/main.c +1 -1
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +0 -0
index 6bdc0f549..6b15b2ab5 100644

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp { · line 151

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +7 -0
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev
+*mdev) {
+     return mdev->gdma_dev->gdma_context; }
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device
+*ibdev, u32 port) {
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,

ib_dev);

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp {

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +8 -0
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem

*umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 3 lines
drivers/infiniband/hw/mana/mana_ib.h +0 -0
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

Quoted 40 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 351207c60..ee4d4f834 100644

drivers/infiniband/hw/mana/mr.c +1 -4
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev
 *dev, struct mana_ib_mr *mr,  {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle; @@ -77,12 +74,9 @@ static
int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)  {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
drivers/infiniband/hw/mana/mr.c +1 -4
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

*ibpd, u64 start, u64 length,

drivers/infiniband/hw/mana/qp.c static int mana_ib_cfg_vport_steering(struct

Quoted 14 lines
drivers/infiniband/hw/mana/mr.c +1 -2
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index e6d063d45..e889c798f 100644

drivers/infiniband/hw/mana/qp.c +0 -0
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct

mana_ib_dev *dev,

Quoted 13 lines
drivers/infiniband/hw/mana/qp.c +1 -3
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) *
Quoted 1 line

MANA_INDIRECT_TABLE_SIZE; @@ -39,7 +37,7 @@ static int

mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 9 lines
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
drivers/infiniband/hw/mana/qp.c +1 -1
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 8 lines
drivers/infiniband/hw/mana/qp.c +0 -1
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

Quoted 23 lines
drivers/infiniband/hw/mana/qp.c +3 -11
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

struct ib_pd *ibpd,

Quoted 36 lines
drivers/infiniband/hw/mana/qp.c +2 -11
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n",

err, @@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,

err_release_gdma_cq:

      kfree(gdma_cq);
drivers/infiniband/hw/mana/qp.c +1 -1
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

#4 lore raw
Quoted 35 lines · Long Li
Quoted 35 lines

Subject: Re: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

Hi Leon,

// Sorry for re-sending the email. I did not know that I had to enable plain-text messages in outlook.

thanks for pointing out the merge window. I will resend after it.

I can also split it into 2 commits (mdev_to_gc and mana_ib_get_netdev in one, and CQ cb in the other), if it is required.
I cannot separate "mdev_to_gc" and "mana_ib_get_netdev " into 2 commits as it will double the changes and obstruct the reader.

Quoted 1 line

And how is it safe now? What is unsafe here?

The issue with the gdma_dev is the following.
Our mana HW device has two gdma_devs: one for ethernet and one for rdma. And in the code of mana_ib.ko it is not clearly indicated what gdma_dev is used. So it means in some functions it is ethernet and in some it is rdma. The use of a wrong device in the hardware channel leads to errors and waste time of developers.

This problem is addressed as follows:
We avoid using gdma_dev explicitly in functions of mana_ib.ko except the init. As the gdma_context is only one, we abstract it into mdev_to_gc, to prevent the user to declare gdma_dev.
When a function actually needs to use gdma_dev of the ethernet, it uses netdev, which encapsulates the correct gdma_dev.
Getting netdev is implemented with mana_ib_get_netdev, to prevent the user to declare gdma_dev.
When a function needs to use gdma_dev of the rdma, it will use struct mana_ib_dev (a child of ib_dev) in its hardware channel requests.

Quoted 4 lines
Quoted 2 lines

I hope it explains what becomes safer. If it was useful, I can add this explanation to the commit message.

You patch doesn't make it safer, but rather makes it easier to understand. The wording "Unsafe and inconsistent" should be removed.

Right, I have no problems with cleanup patch, just don't want to see wrong/inaccurate description.

Thanks

Quoted 1,322 lines

Long

Quoted 1,319 lines

Thanks,
Konstantin

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; linux- [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at
https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

drivers/infiniband/hw/mana/cq.c int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 16 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/cq.c index 83ebd0705..255e74ab7 100644

drivers/infiniband/hw/mana/cq.c +0 -0
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

ib_udata *udata)

drivers/infiniband/hw/mana/cq.c void mana_ib_cq_handler(void *ctx, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct

gdma_queue *gdma_cq)

Quoted 43 lines
drivers/infiniband/hw/mana/cq.c +19 -0
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq) {
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +1 -4
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev,
Quoted 1 line

struct mana_ib_pd *pd, int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32

port, struct mana_ib_pd *pd,

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

Quoted 15 lines

u32 doorbell_id)
{

drivers/infiniband/hw/mana/main.c +1 -4
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext

*ibcontext,

Quoted 5 lines
drivers/infiniband/hw/mana/main.c +0 -1
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +1 -3
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); @@
-238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext

*ibcontext)

drivers/infiniband/hw/mana/main.c int mana_ib_gd_create_dma_region(struct

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

Quoted 29 lines
drivers/infiniband/hw/mana/main.c +2 -6
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct
 mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64
 gdma_region)  {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n",
 gdma_region);
Quoted 2 lines
      return mana_gd_destroy_dma_region(gc, gdma_region); @@ -447,7

+435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct

vm_area_struct *vma)

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n",

vma->vm_pgoff); @@ -531,7 +519,7 @@ int

mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)

Quoted 14 lines
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
drivers/infiniband/hw/mana/main.c +1 -1
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +0 -0
index 6bdc0f549..6b15b2ab5 100644

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp { · line 151

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +7 -0
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev
+*mdev) {
+     return mdev->gdma_dev->gdma_context; }
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device
+*ibdev, u32 port) {
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,

ib_dev);

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp {

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +8 -0
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem

*umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 3 lines
drivers/infiniband/hw/mana/mana_ib.h +0 -0
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

Quoted 40 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 351207c60..ee4d4f834 100644

drivers/infiniband/hw/mana/mr.c +1 -4
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev
 *dev, struct mana_ib_mr *mr,  {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle; @@ -77,12 +74,9 @@ static
int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)  {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
drivers/infiniband/hw/mana/mr.c +1 -4
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

*ibpd, u64 start, u64 length,

drivers/infiniband/hw/mana/qp.c static int mana_ib_cfg_vport_steering(struct

Quoted 14 lines
drivers/infiniband/hw/mana/mr.c +1 -2
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index e6d063d45..e889c798f 100644

drivers/infiniband/hw/mana/qp.c +0 -0
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct

mana_ib_dev *dev,

Quoted 13 lines
drivers/infiniband/hw/mana/qp.c +1 -3
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) *
Quoted 1 line

MANA_INDIRECT_TABLE_SIZE; @@ -39,7 +37,7 @@ static int

mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 9 lines
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
drivers/infiniband/hw/mana/qp.c +1 -1
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 8 lines
drivers/infiniband/hw/mana/qp.c +0 -1
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

Quoted 23 lines
drivers/infiniband/hw/mana/qp.c +3 -11
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

struct ib_pd *ibpd,

Quoted 31 lines
drivers/infiniband/hw/mana/qp.c +2 -11
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n",

err, @@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,

err_release_gdma_cq:

      kfree(gdma_cq);
drivers/infiniband/hw/mana/qp.c +1 -1
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Quoted 5 lines

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; linux- [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at
https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

drivers/infiniband/hw/mana/cq.c int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 16 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/cq.c index 83ebd0705..255e74ab7 100644

drivers/infiniband/hw/mana/cq.c +0 -0
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

ib_udata *udata)

drivers/infiniband/hw/mana/cq.c void mana_ib_cq_handler(void *ctx, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct

gdma_queue *gdma_cq)

Quoted 43 lines
drivers/infiniband/hw/mana/cq.c +19 -0
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq) {
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +1 -4
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev,
Quoted 1 line

struct mana_ib_pd *pd, int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32

port, struct mana_ib_pd *pd,

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

Quoted 15 lines

u32 doorbell_id)
{

drivers/infiniband/hw/mana/main.c +1 -4
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext

*ibcontext,

Quoted 5 lines
drivers/infiniband/hw/mana/main.c +0 -1
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +1 -3
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); @@
-238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext

*ibcontext)

drivers/infiniband/hw/mana/main.c int mana_ib_gd_create_dma_region(struct

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

Quoted 29 lines
drivers/infiniband/hw/mana/main.c +2 -6
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct
 mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64
 gdma_region)  {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n",
 gdma_region);
Quoted 2 lines
      return mana_gd_destroy_dma_region(gc, gdma_region); @@ -447,7

+435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct

vm_area_struct *vma)

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n",

vma->vm_pgoff); @@ -531,7 +519,7 @@ int

mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)

Quoted 14 lines
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
drivers/infiniband/hw/mana/main.c +1 -1
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +0 -0
index 6bdc0f549..6b15b2ab5 100644

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp { · line 151

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +7 -0
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev
+*mdev) {
+     return mdev->gdma_dev->gdma_context; }
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device
+*ibdev, u32 port) {
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,

ib_dev);

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp {

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +8 -0
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem

*umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 3 lines
drivers/infiniband/hw/mana/mana_ib.h +0 -0
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

Quoted 40 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 351207c60..ee4d4f834 100644

drivers/infiniband/hw/mana/mr.c +1 -4
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev
 *dev, struct mana_ib_mr *mr,  {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle; @@ -77,12 +74,9 @@ static
int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)  {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
drivers/infiniband/hw/mana/mr.c +1 -4
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

*ibpd, u64 start, u64 length,

drivers/infiniband/hw/mana/qp.c static int mana_ib_cfg_vport_steering(struct

Quoted 14 lines
drivers/infiniband/hw/mana/mr.c +1 -2
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index e6d063d45..e889c798f 100644

drivers/infiniband/hw/mana/qp.c +0 -0
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct

mana_ib_dev *dev,

Quoted 13 lines
drivers/infiniband/hw/mana/qp.c +1 -3
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) *
Quoted 1 line

MANA_INDIRECT_TABLE_SIZE; @@ -39,7 +37,7 @@ static int

mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 9 lines
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
drivers/infiniband/hw/mana/qp.c +1 -1
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 8 lines
drivers/infiniband/hw/mana/qp.c +0 -1
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

Quoted 23 lines
drivers/infiniband/hw/mana/qp.c +3 -11
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

struct ib_pd *ibpd,

Quoted 31 lines
drivers/infiniband/hw/mana/qp.c +2 -11
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n",

err, @@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,

err_release_gdma_cq:

      kfree(gdma_cq);
drivers/infiniband/hw/mana/qp.c +1 -1
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Quoted 5 lines

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0

________________________________________
From: Leon Romanovsky <[email protected]>
Sent: Thursday, January 11, 2024 12:14 PM
To: Konstantin Taranov
Cc: Konstantin Taranov; Long Li; [email protected]; [email protected]; linux- [email protected]
Subject: [EXTERNAL] Re: [PATCH next v2 1/1] RDMA/mana_ib: Introduce three helper functions to clean mana_ib code

[You don't often get email from [email protected]. Learn why this is important at
https://aka.ms/LearnAboutSenderIdentification ]

Quoted 4 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

This patch aims to address two issues in mana_ib:
1) Unsafe and inconsistent access to gdma_dev and gdma_context

And how is it safe now? What is unsafe here?

Quoted 6 lines

2) Code repetitions

As a rule of thumb, functions should not access gdma_dev directly

Introduced functions:
1) mdev_to_gc

Which is exactly "mdev->gdma_dev->gdma_context" as before.

Quoted 4 lines

2) mana_ib_get_netdev
3) mana_ib_install_cq_cb

We are in merge window and cleanup patch will need to wait till it ends.

Thanks

drivers/infiniband/hw/mana/cq.c int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 16 lines

---
Sorry that was sent again, I forgot to add RDMA/mana_ib to the subject

---
 drivers/infiniband/hw/mana/cq.c      | 23 ++++++++++++++--
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++------------------
 drivers/infiniband/hw/mana/mana_ib.h | 17 ++++++++++++
 drivers/infiniband/hw/mana/mr.c      | 13 +++------
 drivers/infiniband/hw/mana/qp.c      | 36 ++++++-------------------
 5 files changed, 63 insertions(+), 66 deletions(-)
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/cq.c index 83ebd0705..255e74ab7 100644

drivers/infiniband/hw/mana/cq.c +0 -0
@@ -16,7 +16,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (udata->inlen < sizeof(ucmd))
              return -EINVAL;
@@ -81,7 +81,7 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct

ib_udata *udata)

drivers/infiniband/hw/mana/cq.c void mana_ib_cq_handler(void *ctx, struct

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +1 -1
      int err;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
      if (err) {
@@ -107,3 +107,22 @@ void mana_ib_cq_handler(void *ctx, struct

gdma_queue *gdma_cq)

Quoted 43 lines
drivers/infiniband/hw/mana/cq.c +19 -0
      if (cq->ibcq.comp_handler)
              cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
+
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq) {
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct gdma_queue *gdma_cq;
+
+     /* Create CQ table entry */
+     WARN_ON(gc->cq_table[cq->id]);
+     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+     if (!gdma_cq)
+             return -ENOMEM;
+
+     gdma_cq->cq.context = cq;
+     gdma_cq->type = GDMA_CQ;
+     gdma_cq->cq.callback = mana_ib_cq_handler;
+     gdma_cq->id = cq->id;
+     gc->cq_table[cq->id] = gdma_cq;
+     return 0;
+}
drivers/infiniband/hw/mana/main.c +1 -4
index faca09245..29dd2438d 100644
@@ -8,13 +8,10 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                       u32 port)
 {
-     struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
      struct net_device *ndev;
-     struct mana_context *mc;
 
-     mc = gd->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -31,14 +28,11 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev,
Quoted 1 line

struct mana_ib_pd *pd, int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32

port, struct mana_ib_pd *pd,

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

Quoted 15 lines

u32 doorbell_id)
{

drivers/infiniband/hw/mana/main.c +1 -4
-     struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
      struct mana_port_context *mpc;
-     struct mana_context *mc;
      struct net_device *ndev;
      int err;
 
-     mc = mdev->driver_data;
-     ndev = mc->ports[port];
+     ndev = mana_ib_get_netdev(&dev->ib_dev, port);
      mpc = netdev_priv(ndev);
 
      mutex_lock(&pd->vport_mutex);
@@ -79,17 +73,17 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_create_pd_req req = {};
      enum gdma_pd_flags flags = 0;
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
                           sizeof(resp));
 
      req.flags = flags;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -119,17 +113,17 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct

ib_udata *udata)

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext

Quoted 21 lines
drivers/infiniband/hw/mana/main.c +3 -3
      struct gdma_destory_pd_resp resp = {};
      struct gdma_destroy_pd_req req = {};
      struct mana_ib_dev *dev;
-     struct gdma_dev *mdev;
+     struct gdma_context *gc;
      int err;
 
      dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     mdev = dev->gdma_dev;
+     gc = mdev_to_gc(dev);
 
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
                           sizeof(resp));
 
      req.pd_handle = pd->pd_handle;
-     err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
+     err = mana_gd_send_request(gc, sizeof(req), &req,
                                 sizeof(resp), &resp);
 
      if (err || resp.hdr.status) {
@@ -206,13 +200,11 @@ int mana_ib_alloc_ucontext(struct ib_ucontext

*ibcontext,

Quoted 5 lines
drivers/infiniband/hw/mana/main.c +0 -1
      struct ib_device *ibdev = ibcontext->device;
      struct mana_ib_dev *mdev;
      struct gdma_context *gc;
-     struct gdma_dev *dev;
      int doorbell_page;

drivers/infiniband/hw/mana/main.c int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +1 -3
      int ret;
 
      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
-     dev = mdev->gdma_dev;
-     gc = dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      /* Allocate a doorbell page index */
      ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); @@
-238,7 +230,7 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext

*ibcontext)

drivers/infiniband/hw/mana/main.c int mana_ib_gd_create_dma_region(struct

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
      if (ret)
@@ -322,15 +314,13 @@ int mana_ib_gd_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

Quoted 29 lines
drivers/infiniband/hw/mana/main.c +2 -6
      size_t max_pgs_create_cmd;
      struct gdma_context *gc;
      size_t num_pages_total;
-     struct gdma_dev *mdev;
      unsigned long page_sz;
      unsigned int tail = 0;
      u64 *page_addr_list;
      void *request_buf;
      int err;
 
-     mdev = dev->gdma_dev;
-     gc = mdev->gdma_context;
+     gc = mdev_to_gc(dev);
      hwc = gc->hwc.driver_data;
 
      /* Hardware requires dma region to align to chosen page size */
@@ -426,10 +416,8 @@ int mana_ib_gd_create_dma_region(struct
 mana_ib_dev *dev, struct ib_umem *umem,
 
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64
 gdma_region)  {
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
 
-     gc = mdev->gdma_context;
      ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n",
 gdma_region);
Quoted 2 lines
      return mana_gd_destroy_dma_region(gc, gdma_region); @@ -447,7

+435,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct

vm_area_struct *vma)

Quoted 9 lines
      int ret;

      mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
drivers/infiniband/hw/mana/main.c +1 -1
-     gc = mdev->gdma_dev->gdma_context;
+     gc = mdev_to_gc(mdev);
 
      if (vma->vm_pgoff != 0) {
              ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n",

vma->vm_pgoff); @@ -531,7 +519,7 @@ int

mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)

Quoted 14 lines
      req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
      req.hdr.dev_id = dev->gdma_dev->dev_id;
drivers/infiniband/hw/mana/main.c +1 -1
-     err = mana_gd_send_request(dev->gdma_dev->gdma_context, sizeof(req),
+     err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
                                 &req, sizeof(resp), &resp);
 
      if (err) {
drivers/infiniband/hw/mana/mana_ib.h +0 -0
index 6bdc0f549..6b15b2ab5 100644

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp { · line 151

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +7 -0
@@ -142,6 +142,22 @@ struct mana_ib_query_adapter_caps_resp {
      u32 max_inline_data_size;
 }; /* HW Data */
 
+static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev
+*mdev) {
+     return mdev->gdma_dev->gdma_context; }
+
+static inline struct net_device *mana_ib_get_netdev(struct ib_device
+*ibdev, u32 port) {
+     struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,

ib_dev);

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_query_adapter_caps_resp {

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +8 -0
+     struct gdma_context *gc = mdev_to_gc(mdev);
+     struct mana_context *mc = gc->mana.driver_data;
+
+     if (port < 1 || port > mc->num_ports)
+             return NULL;
+     return mc->ports[port - 1];
+}
+
 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem

*umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_cq(struct ib_cq *ibcq, const struct

Quoted 3 lines
drivers/infiniband/hw/mana/mana_ib.h +0 -0
                               mana_handle_t *gdma_region);
 
@@ -188,6 +204,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct

ib_cq_init_attr *attr,

drivers/infiniband/hw/mana/mr.c struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

Quoted 40 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
                    struct ib_udata *udata);
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq
+*cq);
 
 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);

(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/mr.c index 351207c60..ee4d4f834 100644

drivers/infiniband/hw/mana/mr.c +1 -4
@@ -30,12 +30,9 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev
 *dev, struct mana_ib_mr *mr,  {
      struct gdma_create_mr_response resp = {};
      struct gdma_create_mr_request req = {};
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
                           sizeof(resp));
      req.pd_handle = mr_params->pd_handle; @@ -77,12 +74,9 @@ static
int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)  {
      struct gdma_destroy_mr_response resp = {};
      struct gdma_destroy_mr_request req = {};
drivers/infiniband/hw/mana/mr.c +1 -4
-     struct gdma_dev *mdev = dev->gdma_dev;
-     struct gdma_context *gc;
+     struct gdma_context *gc = mdev_to_gc(dev);
      int err;
 
-     gc = mdev->gdma_context;
-
      mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
                           sizeof(resp));
@@ -164,8 +158,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd

*ibpd, u64 start, u64 length,

drivers/infiniband/hw/mana/qp.c static int mana_ib_cfg_vport_steering(struct

Quoted 14 lines
drivers/infiniband/hw/mana/mr.c +1 -2
      return &mr->ibmr;
 
 err_dma_region:
-     mana_gd_destroy_dma_region(dev->gdma_dev->gdma_context,
-                                dma_region_handle);
+     mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
 
 err_umem:
      ib_umem_release(mr->umem);
(quoted diff) +0 -0

b/drivers/infiniband/hw/mana/qp.c index e6d063d45..e889c798f 100644

drivers/infiniband/hw/mana/qp.c +0 -0
@@ -17,12 +17,10 @@ static int mana_ib_cfg_vport_steering(struct

mana_ib_dev *dev,

Quoted 13 lines
drivers/infiniband/hw/mana/qp.c +1 -3
      struct mana_cfg_rx_steer_resp resp = {};
      mana_handle_t *req_indir_tab;
      struct gdma_context *gc;
-     struct gdma_dev *mdev;
      u32 req_buf_size;
      int i, err;
 
-     gc = dev->gdma_dev->gdma_context;
-     mdev = &gc->mana;
+     gc = mdev_to_gc(dev);
 
      req_buf_size =
              sizeof(*req) + sizeof(mana_handle_t) *
Quoted 1 line

MANA_INDIRECT_TABLE_SIZE; @@ -39,7 +37,7 @@ static int

mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 9 lines
      req->rx_enable = 1;
      req->update_default_rxobj = 1;
      req->default_rxobj = default_rxobj;
drivers/infiniband/hw/mana/qp.c +1 -1
-     req->hdr.dev_id = mdev->dev_id;
+     req->hdr.dev_id = gc->mana.dev_id;
 
      /* If there are more than 1 entries in indirection table, enable RSS */
      if (log_ind_tbl_size)
@@ -106,7 +104,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

Quoted 8 lines
drivers/infiniband/hw/mana/qp.c +0 -1
      struct gdma_queue **gdma_cq_allocated;
      mana_handle_t *mana_ind_table;
      struct mana_port_context *mpc;
-     struct gdma_queue *gdma_cq;
      unsigned int ind_tbl_size;
      struct net_device *ndev;
      struct mana_ib_cq *cq;
@@ -231,19 +228,11 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp,

struct ib_pd *pd,

drivers/infiniband/hw/mana/qp.c static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

Quoted 23 lines
drivers/infiniband/hw/mana/qp.c +3 -11
              mana_ind_table[i] = wq->rx_object;
 
              /* Create CQ table entry */
-             WARN_ON(gc->cq_table[cq->id]);
-             gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-             if (!gdma_cq) {
-                     ret = -ENOMEM;
+             ret = mana_ib_install_cq_cb(mdev, cq);
+             if (!ret)
                      goto fail;
-             }
-             gdma_cq_allocated[i] = gdma_cq;
 
-             gdma_cq->cq.context = cq;
-             gdma_cq->type = GDMA_CQ;
-             gdma_cq->cq.callback = mana_ib_cq_handler;
-             gdma_cq->id = cq->id;
-             gc->cq_table[cq->id] = gdma_cq;
+             gdma_cq_allocated[i] = gc->cq_table[cq->id];
      }
      resp.num_entries = i;
 
@@ -409,18 +398,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp,

struct ib_pd *ibpd,

Quoted 36 lines
drivers/infiniband/hw/mana/qp.c +2 -11
      send_cq->id = cq_spec.queue_index;
 
      /* Create CQ table entry */
-     WARN_ON(gc->cq_table[send_cq->id]);
-     gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
-     if (!gdma_cq) {
-             err = -ENOMEM;
+     err = mana_ib_install_cq_cb(mdev, send_cq);
+     if (err)
              goto err_destroy_wq_obj;
-     }
-
-     gdma_cq->cq.context = send_cq;
-     gdma_cq->type = GDMA_CQ;
-     gdma_cq->cq.callback = mana_ib_cq_handler;
-     gdma_cq->id = send_cq->id;
-     gc->cq_table[send_cq->id] = gdma_cq;
 
      ibdev_dbg(&mdev->ib_dev,
                "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n",

err, @@ -442,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,

err_release_gdma_cq:

      kfree(gdma_cq);
drivers/infiniband/hw/mana/qp.c +1 -1
-     gd->gdma_context->cq_table[send_cq->id] = NULL;
+     gc->cq_table[send_cq->id] = NULL;
 
 err_destroy_wq_obj:
      mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);

base-commit: d24b923f1d696ddacb09f0f2d1b1f4f045cfe65e
prerequisite-patch-id: 1b5d35ba40b675d080bfbe6a0e73b0dd163f4a03 --
2.43.0