Define and use mana queues for CQs and WQs
Subsystems (2)
v3 cover Define and use mana queues for CQs and WQs
From: Konstantin Taranov <[email protected]>
This patch series aims to reduce code duplication by introducing a notion of mana ib queues and corresponding helpers to create and destroy them.
v2->v3:
* [in 4/4] Do not define an additional struct for a raw qp
v1->v2:
* [in 1/4] Added a comment about the ignored return value
* [in 2/4] Replaced RDMA:mana_ib to RDMA/mana_ib in the subject
* [in 4/4] Renamed mana_ib_raw_qp to mana_ib_raw_sq
Konstantin Taranov (4):
RDMA/mana_ib: Introduce helpers to create and destroy mana queues RDMA/mana_ib: Use struct mana_ib_queue for CQs RDMA/mana_ib: Use struct mana_ib_queue for WQs RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs
drivers/infiniband/hw/mana/cq.c | 52 ++++------------ drivers/infiniband/hw/mana/main.c | 43 ++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 26 ++++---- drivers/infiniband/hw/mana/qp.c | 88 ++++++++++------------------ drivers/infiniband/hw/mana/wq.c | 31 ++-------- 5 files changed, 103 insertions(+), 137 deletions(-)
base-commit: 96d9cbe2f2ff7abde021bac75eafaceabe9a51fa
Signature · 1 line
2.43.0
PATCH v3 1/4 RDMA/mana_ib: Introduce helpers to create and destroy mana queues
From: Konstantin Taranov <[email protected]>
Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id.
A queue can become a WQ or a CQ.
--- drivers/infiniband/hw/mana/main.c | 43 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++ 2 files changed, 53 insertions(+)
drivers/infiniband/hw/mana/main.c +43 -0
@@ -237,6 +237,49 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)ibdev_dbg (ibdev,"Failed to destroy doorbell page %d\n" , ret); } +int mana_ib_create_queue (struct mana_ib_dev *mdev,u64 addr,u32 size, +struct mana_ib_queue *queue) +{ +struct ib_umem *umem; +int err; + + queue->umem =NULL ; + queue->id =INVALID_QUEUE_ID ; + queue->gdma_region =GDMA_INVALID_DMA_REGION ; + + umem =ib_umem_get (&mdev->ib_dev, addr, size,IB_ACCESS_LOCAL_WRITE ); +if (IS_ERR (umem)) { + err =PTR_ERR (umem); +ibdev_dbg (&mdev->ib_dev,"Failed to get umem, %d\n" , err); +return err; + } + + err =mana_ib_create_zero_offset_dma_region (mdev, umem, &queue->gdma_region); +if (err) { +ibdev_dbg (&mdev->ib_dev,"Failed to create dma region, %d\n" , err); + goto free_umem; + } + queue->umem = umem; + +ibdev_dbg (&mdev->ib_dev, +"create_dma_region ret %d gdma_region 0x%llx\n" , + err, queue->gdma_region); + +return 0 ; +free_umem: +ib_umem_release (umem); +return err; +} + +void mana_ib_destroy_queue (struct mana_ib_dev *mdev,struct mana_ib_queue *queue) +{ +/* Ignore return code as there is not much we can do about it. +* The error message is printed inside. +*/ +mana_ib_gd_destroy_dma_region (mdev, queue->gdma_region); +ib_umem_release (queue->umem); +} + static int mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc,
drivers/infiniband/hw/mana/mana_ib.h +10 -0
@@ -45,6 +45,12 @@ struct mana_ib_adapter_caps {u32 max_inline_data_size; }; +struct mana_ib_queue { +struct ib_umem *umem; +u64 gdma_region; +u64 id; +}; +struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev;
@@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,int mana_ib_gd_destroy_dma_region (struct mana_ib_dev *dev,mana_handle_t gdma_region); +int mana_ib_create_queue (struct mana_ib_dev *mdev,u64 addr,u32 size, +struct mana_ib_queue *queue); +void mana_ib_destroy_queue (struct mana_ib_dev *mdev,struct mana_ib_queue *queue); +struct ib_wq *mana_ib_create_wq (struct ib_pd *pd,struct ib_wq_init_attr *init_attr,struct ib_udata *udata);
Signature · 1 line
2.43.0
Re: [PATCH rdma-next v3 1/4] RDMA/mana_ib: Introduce helpers to create and destroy mana queues
Quoted 38 lines · Konstantin Taranov
From: Konstantin Taranov <[email protected]>
Intoduce helpers to work with mana ib queues (struct
mana_ib_queue). A queue always consists of umem,gdma_region, and id.
A queue can become a WQ or a CQ.Signed-off-by Konstantin Taranov <[email protected]>Reviewed-by Long Li <[email protected]>--- drivers/infiniband/hw/mana/main.c | 43 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++ 2 files changed, 53 insertions(+)drivers/infiniband/hw/mana/main.c +17
-0@@ -237,6 +237,49 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)ibdev_dbg (ibdev,"Failed to destroy doorbell page %d\n" , ret); } +int mana_ib_create_queue (struct mana_ib_dev *mdev,u64 addr,u32 size, +struct mana_ib_queue *queue) +{ +struct ib_umem *umem; +int err; + + queue->umem =NULL ; + queue->id =INVALID_QUEUE_ID ; + queue->gdma_region =GDMA_INVALID_DMA_REGION ; + + umem =ib_umem_get (&mdev->ib_dev, addr, size,IB_ACCESS_LOCAL_WRITE ); +if (IS_ERR (umem)) { + err =PTR_ERR (umem); +ibdev_dbg (&mdev->ib_dev,"Failed to get umem, %d\n" , err); +return err; + } +
drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) · line 266
Quoted 10 lines
drivers/infiniband/hw/mana/main.c +10
-0+ err =mana_ib_create_zero_offset_dma_region (mdev, umem, &queue->gdma_region); +if (err) { +ibdev_dbg (&mdev->ib_dev,"Failed to create dma region, %d\n" , err); + goto free_umem; + } + queue->umem = umem; + +ibdev_dbg (&mdev->ib_dev, +"create_dma_region ret %d gdma_region 0x%llx\n" , + err, queue->gdma_region);
I see such prints in all code and patches, please remove them. We assume that the code works and you don't need to print about success.
Also you are printing "err" too which is always 0 in this case.
Thanks
Quoted 51 lines
drivers/infiniband/hw/mana/main.c +16
-0+ +return 0 ; +free_umem: +ib_umem_release (umem); +return err; +} + +void mana_ib_destroy_queue (struct mana_ib_dev *mdev,struct mana_ib_queue *queue) +{ +/* Ignore return code as there is not much we can do about it. +* The error message is printed inside. +*/ +mana_ib_gd_destroy_dma_region (mdev, queue->gdma_region); +ib_umem_release (queue->umem); +} + static int mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc,drivers/infiniband/hw/mana/mana_ib.h +10
-0@@ -45,6 +45,12 @@ struct mana_ib_adapter_caps {u32 max_inline_data_size; }; +struct mana_ib_queue { +struct ib_umem *umem; +u64 gdma_region; +u64 id; +}; +struct mana_ib_dev {struct ib_device ib_dev;struct gdma_dev *gdma_dev;@@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,int mana_ib_gd_destroy_dma_region (struct mana_ib_dev *dev,mana_handle_t gdma_region); +int mana_ib_create_queue (struct mana_ib_dev *mdev,u64 addr,u32 size, +struct mana_ib_queue *queue); +void mana_ib_destroy_queue (struct mana_ib_dev *mdev,struct mana_ib_queue *queue); +struct ib_wq *mana_ib_create_wq (struct ib_pd *pd,struct ib_wq_init_attr *init_attr,struct ib_udata *udata);Signature · 1 line
2.43.0
PATCH v3 2/4 RDMA/mana_ib: Use struct mana_ib_queue for CQs
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Use struct mana_ib_queue and its helpers for CQs
--- drivers/infiniband/hw/mana/cq.c | 52 ++++++---------------------- drivers/infiniband/hw/mana/mana_ib.h | 4 +-- drivers/infiniband/hw/mana/qp.c | 26 +++++++------- 3 files changed, 24 insertions(+), 58 deletions(-)
drivers/infiniband/hw/mana/cq.c +10 -42
@@ -39,37 +39,13 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } cq->cqe = attr->cqe; - cq->umem =ib_umem_get (ibdev, ucmd.buf_addr, cq->cqe *COMP_ENTRY_SIZE , -IB_ACCESS_LOCAL_WRITE ); -if (IS_ERR (cq->umem)) { - err =PTR_ERR (cq->umem); -ibdev_dbg (ibdev,"Failed to get umem for create cq, err %d\n" , - err); -return err; - } - - err =mana_ib_create_zero_offset_dma_region (mdev, cq->umem, &cq->gdma_region); + err =mana_ib_create_queue (mdev, ucmd.buf_addr, cq->cqe *COMP_ENTRY_SIZE , &cq->queue);if (err) { -ibdev_dbg (ibdev, -"Failed to create dma region for create cq, %d\n" , - err); - goto err_release_umem; +ibdev_dbg (ibdev,"Failed to create queue for create cq, %d\n" , err); +return err; } -ibdev_dbg (ibdev, -"create_dma_region ret %d gdma_region 0x%llx\n" , - err, cq->gdma_region); - -/* -* The CQ ID is not known at this time. The ID is generated at create_qp -*/ - cq->id =INVALID_QUEUE_ID ; -return 0 ; - -err_release_umem: -ib_umem_release (cq->umem); -return err; }int mana_ib_destroy_cq (struct ib_cq *ibcq,struct ib_udata *udata)
@@ -78,24 +54,16 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)struct ib_device *ibdev = ibcq->device;struct mana_ib_dev *mdev;struct gdma_context *gc; -int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); gc =mdev_to_gc (mdev); - err =mana_ib_gd_destroy_dma_region (mdev, cq->gdma_region); -if (err) { -ibdev_dbg (ibdev, -"Failed to destroy dma region, %d\n" , err); -return err; - } - -if (cq->id !=INVALID_QUEUE_ID ) { -kfree (gc->cq_table[cq->id]); - gc->cq_table[cq->id] =NULL ; +if (cq->queue.id !=INVALID_QUEUE_ID ) { +kfree (gc->cq_table[cq->queue.id]); + gc->cq_table[cq->queue.id] =NULL ; } -ib_umem_release (cq->umem); +mana_ib_destroy_queue (mdev, &cq->queue);return 0 ; }
@@ -114,7 +82,7 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)struct gdma_queue *gdma_cq;/* Create CQ table entry */ -WARN_ON (gc->cq_table[cq->id]); +WARN_ON (gc->cq_table[cq->queue.id]); gdma_cq =kzalloc (sizeof (*gdma_cq),GFP_KERNEL );if (!gdma_cq)return -ENOMEM ;
@@ -122,7 +90,7 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) gdma_cq->cq.context = cq; gdma_cq->type =GDMA_CQ ; gdma_cq->cq.callback = mana_ib_cq_handler; - gdma_cq->id = cq->id; - gc->cq_table[cq->id] = gdma_cq; + gdma_cq->id = cq->queue.id; + gc->cq_table[cq->queue.id] = gdma_cq;return 0 ; }
drivers/infiniband/hw/mana/mana_ib.h +1 -3
@@ -88,10 +88,8 @@ struct mana_ib_mr {struct mana_ib_cq {struct ib_cq ibcq; -struct ib_umem *umem; +struct mana_ib_queue queue;int cqe; -u64 gdma_region; -u64 id;u32 comp_vector; };
drivers/infiniband/hw/mana/qp.c +13 -13
@@ -197,7 +197,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, wq_spec.gdma_region = wq->gdma_region; wq_spec.queue_size = wq->wq_buf_size; - cq_spec.gdma_region = cq->gdma_region; + cq_spec.gdma_region = cq->queue.gdma_region; cq_spec.queue_size = cq->cqe *COMP_ENTRY_SIZE ; cq_spec.modr_ctx_id =0 ; eq = &mpc->ac->eqs[cq->comp_vector % gc->max_num_queues];
@@ -213,16 +213,16 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,/* The GDMA regions are now owned by the WQ object */ wq->gdma_region =GDMA_INVALID_DMA_REGION ; - cq->gdma_region =GDMA_INVALID_DMA_REGION ; + cq->queue.gdma_region =GDMA_INVALID_DMA_REGION ; wq->id = wq_spec.queue_index; - cq->id = cq_spec.queue_index; + cq->queue.id = cq_spec.queue_index;ibdev_dbg (&mdev->ib_dev,"ret %d rx_object 0x%llx wq id %llu cq id %llu\n" , - ret, wq->rx_object, wq->id, cq->id); + ret, wq->rx_object, wq->id, cq->queue.id); - resp.entries[i].cqid = cq->id; + resp.entries[i].cqid = cq->queue.id; resp.entries[i].wqid = wq->id; mana_ind_table[i] = wq->rx_object;
@@ -232,7 +232,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,if (ret) goto fail; - gdma_cq_allocated[i] = gc->cq_table[cq->id]; + gdma_cq_allocated[i] = gc->cq_table[cq->queue.id]; } resp.num_entries = i;
@@ -264,7 +264,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, wq = container_of(ibwq, struct mana_ib_wq, ibwq); cq = container_of(ibcq, struct mana_ib_cq, ibcq); - gc->cq_table[cq->id] =NULL ; + gc->cq_table[cq->queue.id] =NULL ;kfree (gdma_cq_allocated[i]);mana_destroy_wq_obj (mpc,GDMA_RQ , wq->rx_object);
@@ -374,7 +374,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, wq_spec.gdma_region = qp->sq_gdma_region; wq_spec.queue_size = ucmd.sq_buf_size; - cq_spec.gdma_region = send_cq->gdma_region; + cq_spec.gdma_region = send_cq->queue.gdma_region; cq_spec.queue_size = send_cq->cqe *COMP_ENTRY_SIZE ; cq_spec.modr_ctx_id =0 ; eq_vec = send_cq->comp_vector % gc->max_num_queues;
@@ -392,10 +392,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,/* The GDMA regions are now owned by the WQ object */ qp->sq_gdma_region =GDMA_INVALID_DMA_REGION ; - send_cq->gdma_region =GDMA_INVALID_DMA_REGION ; + send_cq->queue.gdma_region =GDMA_INVALID_DMA_REGION ; qp->sq_id = wq_spec.queue_index; - send_cq->id = cq_spec.queue_index; + send_cq->queue.id = cq_spec.queue_index;/* Create CQ table entry */ err =mana_ib_install_cq_cb (mdev, send_cq);
@@ -404,10 +404,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,ibdev_dbg (&mdev->ib_dev,"ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n" , err, - qp->tx_object, qp->sq_id, send_cq->id); + qp->tx_object, qp->sq_id, send_cq->queue.id); resp.sqid = qp->sq_id; - resp.cqid = send_cq->id; + resp.cqid = send_cq->queue.id; resp.tx_vp_offset = pd->tx_vp_offset; err =ib_copy_to_udata (udata, &resp,sizeof (resp));
@@ -422,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, err_release_gdma_cq:kfree (gdma_cq); - gc->cq_table[send_cq->id] =NULL ; + gc->cq_table[send_cq->queue.id] =NULL ; err_destroy_wq_obj:mana_destroy_wq_obj (mpc,GDMA_SQ , qp->tx_object);
Signature · 1 line
2.43.0
PATCH v3 3/4 RDMA/mana_ib: Use struct mana_ib_queue for WQs
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Use struct mana_ib_queue and its helpers for WQs
--- drivers/infiniband/hw/mana/mana_ib.h | 4 +--- drivers/infiniband/hw/mana/qp.c | 10 ++++----- drivers/infiniband/hw/mana/wq.c | 31 ++++------------------------ 3 files changed, 10 insertions(+), 35 deletions(-)
drivers/infiniband/hw/mana/mana_ib.h +1 -3
@@ -59,11 +59,9 @@ struct mana_ib_dev {struct mana_ib_wq {struct ib_wq ibwq; -struct ib_umem *umem; +struct mana_ib_queue queue;int wqe;u32 wq_buf_size; -u64 gdma_region; -u64 id;mana_handle_t rx_object; };
drivers/infiniband/hw/mana/qp.c +5 -5
@@ -194,7 +194,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ibcq = ibwq->cq; cq = container_of(ibcq, struct mana_ib_cq, ibcq); - wq_spec.gdma_region = wq->gdma_region; + wq_spec.gdma_region = wq->queue.gdma_region; wq_spec.queue_size = wq->wq_buf_size; cq_spec.gdma_region = cq->queue.gdma_region;
@@ -212,18 +212,18 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, }/* The GDMA regions are now owned by the WQ object */ - wq->gdma_region =GDMA_INVALID_DMA_REGION ; + wq->queue.gdma_region =GDMA_INVALID_DMA_REGION ; cq->queue.gdma_region =GDMA_INVALID_DMA_REGION ; - wq->id = wq_spec.queue_index; + wq->queue.id = wq_spec.queue_index; cq->queue.id = cq_spec.queue_index;ibdev_dbg (&mdev->ib_dev,"ret %d rx_object 0x%llx wq id %llu cq id %llu\n" , - ret, wq->rx_object, wq->id, cq->queue.id); + ret, wq->rx_object, wq->queue.id, cq->queue.id); resp.entries[i].cqid = cq->queue.id; - resp.entries[i].wqid = wq->id; + resp.entries[i].wqid = wq->queue.id; mana_ind_table[i] = wq->rx_object;
drivers/infiniband/hw/mana/wq.c +4 -27
@@ -13,7 +13,6 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, container_of(pd->device, struct mana_ib_dev, ib_dev);struct mana_ib_create_wq ucmd = {};struct mana_ib_wq *wq; -struct ib_umem *umem;int err;if (udata->inlen <sizeof (ucmd))
@@ -32,39 +31,18 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,ibdev_dbg (&mdev->ib_dev,"ucmd wq_buf_addr 0x%llx\n" , ucmd.wq_buf_addr); - umem =ib_umem_get (pd->device, ucmd.wq_buf_addr, ucmd.wq_buf_size, -IB_ACCESS_LOCAL_WRITE ); -if (IS_ERR (umem)) { - err =PTR_ERR (umem); + err =mana_ib_create_queue (mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue); +if (err) {ibdev_dbg (&mdev->ib_dev, -"Failed to get umem for create wq, err %d\n" , err); +"Failed to create queue for create wq, %d\n" , err); goto err_free_wq; } - wq->umem = umem; wq->wqe = init_attr->max_wr; wq->wq_buf_size = ucmd.wq_buf_size; wq->rx_object =INVALID_MANA_HANDLE ; - - err =mana_ib_create_zero_offset_dma_region (mdev, wq->umem, &wq->gdma_region); -if (err) { -ibdev_dbg (&mdev->ib_dev, -"Failed to create dma region for create wq, %d\n" , - err); - goto err_release_umem; - } - -ibdev_dbg (&mdev->ib_dev, -"create_dma_region ret %d gdma_region 0x%llx\n" , - err, wq->gdma_region); - -/* WQ ID is returned at wq_create time, doesn't know the value yet */ -return &wq->ibwq; -err_release_umem: -ib_umem_release (umem); - err_free_wq:kfree (wq);
@@ -86,8 +64,7 @@ int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev); -mana_ib_gd_destroy_dma_region (mdev, wq->gdma_region); -ib_umem_release (wq->umem); +mana_ib_destroy_queue (mdev, &wq->queue);kfree (wq);
Signature · 1 line
2.43.0
PATCH v3 4/4 RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Use struct mana_ib_queue and its helpers for RAW QPs
--- drivers/infiniband/hw/mana/mana_ib.h | 8 +--- drivers/infiniband/hw/mana/qp.c | 56 ++++++++-------------------- 2 files changed, 18 insertions(+), 46 deletions(-)
drivers/infiniband/hw/mana/mana_ib.h +2 -6
@@ -94,12 +94,8 @@ struct mana_ib_cq {struct mana_ib_qp {struct ib_qp ibqp; -/* Work queue info */ -struct ib_umem *sq_umem; -int sqe; -u64 sq_gdma_region; -u64 sq_id; -mana_handle_t tx_object; +mana_handle_t qp_handle; +struct mana_ib_queue raw_sq;/* The port on the IB device, starting with 1 */ u32 port;
drivers/infiniband/hw/mana/qp.c +16 -40
109 lines not on this page · raw message
Signature · 1 line
2.43.0