Skip to contentSkip to conversation

Define and use mana queues for CQs and WQs

PATCH Superseded Konstantin Taranov 4 participants 13 messages Mar 13, 2024 – Mar 18, 2024

Subsystems (2)

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

cover Define and use mana queues for CQs and WQs

From: Konstantin Taranov <[email protected]>

This patch series aims to reduce code duplication by introducing a notion of mana ib queues and corresponding helpers to create and destroy them.

Konstantin Taranov (4):
RDMA/mana_ib: Introduce helpers to create and destroy mana queues RDMA:mana_ib: Use struct mana_ib_queue for CQs RDMA/mana_ib: Use struct mana_ib_queue for WQs RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs

 drivers/infiniband/hw/mana/cq.c      | 52 ++++-------------
 drivers/infiniband/hw/mana/main.c    | 40 +++++++++++++
 drivers/infiniband/hw/mana/mana_ib.h | 31 ++++++----
 drivers/infiniband/hw/mana/qp.c      | 86 ++++++++++------------------
 drivers/infiniband/hw/mana/wq.c      | 31 ++--------
 5 files changed, 104 insertions(+), 136 deletions(-)

base-commit: 96d9cbe2f2ff7abde021bac75eafaceabe9a51fa

Signature · 1 line
2.43.0
#1 lore raw

PATCH 1/4 RDMA/mana_ib: Introduce helpers to create and destroy mana queues

From: Konstantin Taranov <[email protected]>

Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id.
A queue can be used for a WQ or a CQ.

---
 drivers/infiniband/hw/mana/main.c    | 40 ++++++++++++++++++++++++++++
 drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++
 2 files changed, 50 insertions(+)
drivers/infiniband/hw/mana/main.c +40 -0
index 71e33feee..0ec940b97 100644
@@ -237,6 +237,46 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 		ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
 }
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+			 struct mana_ib_queue *queue)
+{
+	struct ib_umem *umem;
+	int err;
+
+	queue->umem = NULL;
+	queue->id = INVALID_QUEUE_ID;
+	queue->gdma_region = GDMA_INVALID_DMA_REGION;
+
+	umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
+	if (IS_ERR(umem)) {
+		err = PTR_ERR(umem);
+		ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err);
+		return err;
+	}
+
+	err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region);
+	if (err) {
+		ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err);
+		goto free_umem;
+	}
+	queue->umem = umem;
+
+	ibdev_dbg(&mdev->ib_dev,
+		  "create_dma_region ret %d gdma_region 0x%llx\n",
+		  err, queue->gdma_region);
+
+	return 0;
+free_umem:
+	ib_umem_release(umem);
+	return err;
+}
+
+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue)
+{
+	mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region);
+	ib_umem_release(queue->umem);
+}
+
 static int
 mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
 			    struct gdma_context *gc,
drivers/infiniband/hw/mana/mana_ib.h +10 -0
index f83390eeb..859fd3bfc 100644
@@ -45,6 +45,12 @@ struct mana_ib_adapter_caps {
 	u32 max_inline_data_size;
 };
 
+struct mana_ib_queue {
+	struct ib_umem *umem;
+	u64 gdma_region;
+	u64 id;
+};
+
 struct mana_ib_dev {
 	struct ib_device ib_dev;
 	struct gdma_dev *gdma_dev;
@@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
 				  mana_handle_t gdma_region);
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+			 struct mana_ib_queue *queue);
+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
+
 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
 				struct ib_wq_init_attr *init_attr,
 				struct ib_udata *udata);
Signature · 1 line
2.43.0
Reviewed-by on v1 1/4
#2 lore raw

RE: [PATCH rdma-next 1/4] RDMA/mana_ib: Introduce helpers to create and destroy mana queues

Quoted 12 lines

-----Original Message-----
From: Konstantin Taranov <[email protected]>
Sent: Wednesday, March 13, 2024 6:25 AM
To: Konstantin Taranov <[email protected]>; [email protected]; Long Li <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]
Subject: [PATCH rdma-next 1/4] RDMA/mana_ib: Introduce helpers to create and destroy mana queues

Quoted 7 lines

From: Konstantin Taranov <[email protected]>

Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id.
A queue can be used for a WQ or a CQ.

#3 lore raw

Re: [PATCH rdma-next 1/4] RDMA/mana_ib: Introduce helpers to create and destroy mana queues

In reply to Konstantin Taranov
Quoted 47 lines · Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id.
A queue can be used for a WQ or a CQ.

---
  drivers/infiniband/hw/mana/main.c    | 40 ++++++++++++++++++++++++++++
  drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++
  2 files changed, 50 insertions(+)
drivers/infiniband/hw/mana/main.c +27 -0
index 71e33feee..0ec940b97 100644
@@ -237,6 +237,46 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
  		ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
  }
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+			 struct mana_ib_queue *queue)
+{
+	struct ib_umem *umem;
+	int err;
+
+	queue->umem = NULL;
+	queue->id = INVALID_QUEUE_ID;
+	queue->gdma_region = GDMA_INVALID_DMA_REGION;
+
+	umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
+	if (IS_ERR(umem)) {
+		err = PTR_ERR(umem);
+		ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err);
+		return err;
+	}
+
+	err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region);
+	if (err) {
+		ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err);
+		goto free_umem;
+	}
+	queue->umem = umem;
+
+	ibdev_dbg(&mdev->ib_dev,
+		  "create_dma_region ret %d gdma_region 0x%llx\n",
+		  err, queue->gdma_region);

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) · line 276

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +10 -0
+
+	return 0;
+free_umem:
+	ib_umem_release(umem);
+	return err;
+}
+
+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue)
+{
+	mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region);

The function mana_ib_gd_destroy_dma_region will call mana_gd_destroy_dma_region. In the function mana_gd_destroy_dma_region, the function mana_gd_send_request will return the error -EPROTO. The procedure is as below. So the function mana_ib_destroy_queue should also handle this error?

mana_ib_gd_destroy_dma_region --- > mana_gd_destroy_dma_region

693 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64
dma_region_handle)
694 {

...

706 err = mana_gd_send_request(gc, sizeof(req), &req,
sizeof(resp), &resp);
707 if (err || resp.hdr.status) {
708 dev_err(gc->dev, "Failed to destroy DMA region:
%d, 0x%x\n",
709 err, resp.hdr.status);
710 return -EPROTO;
711 }

...

714 }

Zhu Yanjun

Quoted 34 lines
drivers/infiniband/hw/mana/main.c +3 -0
+	ib_umem_release(queue->umem);
+}
+
  static int
  mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
  			    struct gdma_context *gc,
drivers/infiniband/hw/mana/mana_ib.h +10 -0
index f83390eeb..859fd3bfc 100644
@@ -45,6 +45,12 @@ struct mana_ib_adapter_caps {
  	u32 max_inline_data_size;
  };
 
+struct mana_ib_queue {
+	struct ib_umem *umem;
+	u64 gdma_region;
+	u64 id;
+};
+
  struct mana_ib_dev {
  	struct ib_device ib_dev;
  	struct gdma_dev *gdma_dev;
@@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
  int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
  				  mana_handle_t gdma_region);
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+			 struct mana_ib_queue *queue);
+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
+
  struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
  				struct ib_wq_init_attr *init_attr,
  				struct ib_udata *udata);
#4 lore raw

RE: [EXTERNAL] Re: [PATCH rdma-next 1/4] RDMA/mana_ib: Introduce helpers to create and destroy mana queues

Quoted 61 lines
Quoted 9 lines

From: Konstantin Taranov <[email protected]>

Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id.
A queue can be used for a WQ or a CQ.

---
  drivers/infiniband/hw/mana/main.c    | 40

++++++++++++++++++++++++++++

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct

Quoted 9 lines
  drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++
  2 files changed, 50 insertions(+)
drivers/infiniband/hw/mana/main.c +0 -0
index 71e33feee..0ec940b97 100644
@@ -237,6 +237,46 @@ void mana_ib_dealloc_ucontext(struct

ib_ucontext *ibcontext)

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)

Quoted 13 lines
drivers/infiniband/hw/mana/main.c +10 -0
              ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
  }
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+                      struct mana_ib_queue *queue) {
+     struct ib_umem *umem;
+     int err;
+
+     queue->umem = NULL;
+     queue->id = INVALID_QUEUE_ID;
+     queue->gdma_region = GDMA_INVALID_DMA_REGION;
+
+     umem = ib_umem_get(&mdev->ib_dev, addr, size,

IB_ACCESS_LOCAL_WRITE);

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +9 -0
+     if (IS_ERR(umem)) {
+             err = PTR_ERR(umem);
+             ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err);
+             return err;
+     }
+
+     err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue-
 gdma_region);
+     if (err) {
+             ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n",

err);

Quoted 16 lines
drivers/infiniband/hw/mana/main.c +16 -0
+             goto free_umem;
+     }
+     queue->umem = umem;
+
+     ibdev_dbg(&mdev->ib_dev,
+               "create_dma_region ret %d gdma_region 0x%llx\n",
+               err, queue->gdma_region);
+
+     return 0;
+free_umem:
+     ib_umem_release(umem);
+     return err;
+}
+
+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct
+mana_ib_queue *queue) {
Quoted 8 lines

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) · line 276

Quoted 1 line
drivers/infiniband/hw/mana/main.c +1 -0
+     mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region);

The function mana_ib_gd_destroy_dma_region will call mana_gd_destroy_dma_region. In the function mana_gd_destroy_dma_region, the function mana_gd_send_request will return the error -EPROTO.
The procedure is as below. So the function mana_ib_destroy_queue should also handle this error?

Thanks for the comment!
This error can be ignored and it was ignored before this commit.
I checked the corresponding Windows driver code, and it is also intentionally ignored there. I can add a comment that the error is ignored intentionally if you want.

Quoted 62 lines

mana_ib_gd_destroy_dma_region --- > mana_gd_destroy_dma_region

693 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64
dma_region_handle)
694 {

...

706 err = mana_gd_send_request(gc, sizeof(req), &req,
sizeof(resp), &resp);
707 if (err || resp.hdr.status) {
708 dev_err(gc->dev, "Failed to destroy DMA region:
%d, 0x%x\n",
709 err, resp.hdr.status);
710 return -EPROTO;
711 }

...

714 }

Zhu Yanjun

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_dma_region(struct

Quoted 25 lines
drivers/infiniband/hw/mana/main.c +3 -0
+     ib_umem_release(queue->umem);
+}
+
  static int
  mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
                          struct gdma_context *gc, diff --git

a/drivers/infiniband/hw/mana/mana_ib.h
b/drivers/infiniband/hw/mana/mana_ib.h
index f83390eeb..859fd3bfc 100644

drivers/infiniband/hw/mana/mana_ib.h +6 -0
@@ -45,6 +45,12 @@ struct mana_ib_adapter_caps {
      u32 max_inline_data_size;
  };
 
+struct mana_ib_queue {
+     struct ib_umem *umem;
+     u64 gdma_region;
+     u64 id;
+};
+
  struct mana_ib_dev {
      struct ib_device ib_dev;
      struct gdma_dev *gdma_dev;
@@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,

Quoted 6 lines
drivers/infiniband/hw/mana/mana_ib.h +3 -0
  int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
                                mana_handle_t gdma_region);
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+                      struct mana_ib_queue *queue); void
+mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct

mana_ib_queue

Quoted 5 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
+*queue);
+
  struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
                              struct ib_wq_init_attr *init_attr,
                              struct ib_udata *udata);
#5 lore raw

Re: [EXTERNAL] Re: [PATCH rdma-next 1/4] RDMA/mana_ib: Introduce helpers to create and destroy mana queues

Quoted 68 lines · Konstantin Taranov
Quoted 60 lines
Quoted 9 lines

From: Konstantin Taranov <[email protected]>

Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id.
A queue can be used for a WQ or a CQ.

---
   drivers/infiniband/hw/mana/main.c    | 40

++++++++++++++++++++++++++++

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct

Quoted 9 lines
   drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++
   2 files changed, 50 insertions(+)
drivers/infiniband/hw/mana/main.c +0 -0
index 71e33feee..0ec940b97 100644
@@ -237,6 +237,46 @@ void mana_ib_dealloc_ucontext(struct

ib_ucontext *ibcontext)

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)

Quoted 13 lines
drivers/infiniband/hw/mana/main.c +10 -0
               ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
   }
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+                      struct mana_ib_queue *queue) {
+     struct ib_umem *umem;
+     int err;
+
+     queue->umem = NULL;
+     queue->id = INVALID_QUEUE_ID;
+     queue->gdma_region = GDMA_INVALID_DMA_REGION;
+
+     umem = ib_umem_get(&mdev->ib_dev, addr, size,

IB_ACCESS_LOCAL_WRITE);

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)

Quoted 10 lines
drivers/infiniband/hw/mana/main.c +9 -0
+     if (IS_ERR(umem)) {
+             err = PTR_ERR(umem);
+             ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err);
+             return err;
+     }
+
+     err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue-
 gdma_region);
+     if (err) {
+             ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n",

err);

Quoted 15 lines
drivers/infiniband/hw/mana/main.c +15 -0
+             goto free_umem;
+     }
+     queue->umem = umem;
+
+     ibdev_dbg(&mdev->ib_dev,
+               "create_dma_region ret %d gdma_region 0x%llx\n",
+               err, queue->gdma_region);
+
+     return 0;
+free_umem:
+     ib_umem_release(umem);
+     return err;
+}
+
+void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct
Quoted 8 lines

drivers/infiniband/hw/mana/main.c void mana_ib_dealloc_ucontext(struct · line 276

Quoted 2 lines
drivers/infiniband/hw/mana/main.c +2 -0
+mana_ib_queue *queue) {
+     mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region);

The function mana_ib_gd_destroy_dma_region will call mana_gd_destroy_dma_region. In the function mana_gd_destroy_dma_region, the function mana_gd_send_request will return the error -EPROTO.
The procedure is as below. So the function mana_ib_destroy_queue should also handle this error?

Quoted 4 lines

Thanks for the comment!
This error can be ignored and it was ignored before this commit.
I checked the corresponding Windows driver code, and it is also intentionally ignored there. I can add a comment that the error is ignored intentionally if you want.

Sure. Thanks a lot.

Zhu Yanjun

Quoted 62 lines
Quoted 62 lines

mana_ib_gd_destroy_dma_region --- > mana_gd_destroy_dma_region

693 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64
dma_region_handle)
694 {

...

706 err = mana_gd_send_request(gc, sizeof(req), &req,
sizeof(resp), &resp);
707 if (err || resp.hdr.status) {
708 dev_err(gc->dev, "Failed to destroy DMA region:
%d, 0x%x\n",

   709                         err, resp.hdr.status);
   710                 return -EPROTO;
   711         }

...

   714 }

Zhu Yanjun

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_dma_region(struct

Quoted 25 lines
drivers/infiniband/hw/mana/main.c +3 -0
+     ib_umem_release(queue->umem);
+}
+
   static int
   mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
                           struct gdma_context *gc, diff --git

a/drivers/infiniband/hw/mana/mana_ib.h
b/drivers/infiniband/hw/mana/mana_ib.h
index f83390eeb..859fd3bfc 100644

drivers/infiniband/hw/mana/mana_ib.h +6 -0
@@ -45,6 +45,12 @@ struct mana_ib_adapter_caps {
       u32 max_inline_data_size;
   };
 
+struct mana_ib_queue {
+     struct ib_umem *umem;
+     u64 gdma_region;
+     u64 id;
+};
+
   struct mana_ib_dev {
       struct ib_device ib_dev;
       struct gdma_dev *gdma_dev;
@@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct

mana_ib_dev *dev, struct ib_umem *umem,

drivers/infiniband/hw/mana/mana_ib.h int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,

Quoted 6 lines
drivers/infiniband/hw/mana/mana_ib.h +3 -0
   int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
                                 mana_handle_t gdma_region);
 
+int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
+                      struct mana_ib_queue *queue); void
+mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct

mana_ib_queue

Quoted 5 lines
drivers/infiniband/hw/mana/mana_ib.h +2 -0
+*queue);
+
   struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
                               struct ib_wq_init_attr *init_attr,
                               struct ib_udata *udata);
#6 lore raw

PATCH 2/4 RDMA:mana_ib: Use struct mana_ib_queue for CQs

In reply to Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Use struct mana_ib_queue and its helpers for CQs

---
 drivers/infiniband/hw/mana/cq.c      | 52 ++++++----------------------
 drivers/infiniband/hw/mana/mana_ib.h |  4 +--
 drivers/infiniband/hw/mana/qp.c      | 26 +++++++-------
 3 files changed, 24 insertions(+), 58 deletions(-)
drivers/infiniband/hw/mana/cq.c +10 -42
index 4a71e678d..c9129218f 100644
@@ -39,37 +39,13 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	}
 
 	cq->cqe = attr->cqe;
-	cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
-			       IB_ACCESS_LOCAL_WRITE);
-	if (IS_ERR(cq->umem)) {
-		err = PTR_ERR(cq->umem);
-		ibdev_dbg(ibdev, "Failed to get umem for create cq, err %d\n",
-			  err);
-		return err;
-	}
-
-	err = mana_ib_create_zero_offset_dma_region(mdev, cq->umem, &cq->gdma_region);
+	err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, &cq->queue);
 	if (err) {
-		ibdev_dbg(ibdev,
-			  "Failed to create dma region for create cq, %d\n",
-			  err);
-		goto err_release_umem;
+		ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
+		return err;
 	}
 
-	ibdev_dbg(ibdev,
-		  "create_dma_region ret %d gdma_region 0x%llx\n",
-		  err, cq->gdma_region);
-
-	/*
-	 * The CQ ID is not known at this time. The ID is generated at create_qp
-	 */
-	cq->id = INVALID_QUEUE_ID;
-
 	return 0;
-
-err_release_umem:
-	ib_umem_release(cq->umem);
-	return err;
 }
 
 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
@@ -78,24 +54,16 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	struct ib_device *ibdev = ibcq->device;
 	struct mana_ib_dev *mdev;
 	struct gdma_context *gc;
-	int err;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
 	gc = mdev_to_gc(mdev);
 
-	err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
-	if (err) {
-		ibdev_dbg(ibdev,
-			  "Failed to destroy dma region, %d\n", err);
-		return err;
-	}
-
-	if (cq->id != INVALID_QUEUE_ID) {
-		kfree(gc->cq_table[cq->id]);
-		gc->cq_table[cq->id] = NULL;
+	if (cq->queue.id != INVALID_QUEUE_ID) {
+		kfree(gc->cq_table[cq->queue.id]);
+		gc->cq_table[cq->queue.id] = NULL;
 	}
 
-	ib_umem_release(cq->umem);
+	mana_ib_destroy_queue(mdev, &cq->queue);
 
 	return 0;
 }
@@ -114,7 +82,7 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 	struct gdma_queue *gdma_cq;
 
 	/* Create CQ table entry */
-	WARN_ON(gc->cq_table[cq->id]);
+	WARN_ON(gc->cq_table[cq->queue.id]);
 	gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
 	if (!gdma_cq)
 		return -ENOMEM;
@@ -122,7 +90,7 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 	gdma_cq->cq.context = cq;
 	gdma_cq->type = GDMA_CQ;
 	gdma_cq->cq.callback = mana_ib_cq_handler;
-	gdma_cq->id = cq->id;
-	gc->cq_table[cq->id] = gdma_cq;
+	gdma_cq->id = cq->queue.id;
+	gc->cq_table[cq->queue.id] = gdma_cq;
 	return 0;
 }
drivers/infiniband/hw/mana/mana_ib.h +1 -3
index 859fd3bfc..6acb5c281 100644
@@ -88,10 +88,8 @@ struct mana_ib_mr {
 
 struct mana_ib_cq {
 	struct ib_cq ibcq;
-	struct ib_umem *umem;
+	struct mana_ib_queue queue;
 	int cqe;
-	u64 gdma_region;
-	u64 id;
 	u32 comp_vector;
 };
 
drivers/infiniband/hw/mana/qp.c +13 -13
index 6e7627745..d7485ee6a 100644
@@ -197,7 +197,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		wq_spec.gdma_region = wq->gdma_region;
 		wq_spec.queue_size = wq->wq_buf_size;
 
-		cq_spec.gdma_region = cq->gdma_region;
+		cq_spec.gdma_region = cq->queue.gdma_region;
 		cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
 		cq_spec.modr_ctx_id = 0;
 		eq = &mpc->ac->eqs[cq->comp_vector % gc->max_num_queues];
@@ -213,16 +213,16 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 
 		/* The GDMA regions are now owned by the WQ object */
 		wq->gdma_region = GDMA_INVALID_DMA_REGION;
-		cq->gdma_region = GDMA_INVALID_DMA_REGION;
+		cq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
 
 		wq->id = wq_spec.queue_index;
-		cq->id = cq_spec.queue_index;
+		cq->queue.id = cq_spec.queue_index;
 
 		ibdev_dbg(&mdev->ib_dev,
 			  "ret %d rx_object 0x%llx wq id %llu cq id %llu\n",
-			  ret, wq->rx_object, wq->id, cq->id);
+			  ret, wq->rx_object, wq->id, cq->queue.id);
 
-		resp.entries[i].cqid = cq->id;
+		resp.entries[i].cqid = cq->queue.id;
 		resp.entries[i].wqid = wq->id;
 
 		mana_ind_table[i] = wq->rx_object;
@@ -232,7 +232,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		if (ret)
 			goto fail;
 
-		gdma_cq_allocated[i] = gc->cq_table[cq->id];
+		gdma_cq_allocated[i] = gc->cq_table[cq->queue.id];
 	}
 	resp.num_entries = i;
 
@@ -264,7 +264,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
 		cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 
-		gc->cq_table[cq->id] = NULL;
+		gc->cq_table[cq->queue.id] = NULL;
 		kfree(gdma_cq_allocated[i]);
 
 		mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
@@ -374,7 +374,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	wq_spec.gdma_region = qp->sq_gdma_region;
 	wq_spec.queue_size = ucmd.sq_buf_size;
 
-	cq_spec.gdma_region = send_cq->gdma_region;
+	cq_spec.gdma_region = send_cq->queue.gdma_region;
 	cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
 	cq_spec.modr_ctx_id = 0;
 	eq_vec = send_cq->comp_vector % gc->max_num_queues;
@@ -392,10 +392,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	/* The GDMA regions are now owned by the WQ object */
 	qp->sq_gdma_region = GDMA_INVALID_DMA_REGION;
-	send_cq->gdma_region = GDMA_INVALID_DMA_REGION;
+	send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
 
 	qp->sq_id = wq_spec.queue_index;
-	send_cq->id = cq_spec.queue_index;
+	send_cq->queue.id = cq_spec.queue_index;
 
 	/* Create CQ table entry */
 	err = mana_ib_install_cq_cb(mdev, send_cq);
@@ -404,10 +404,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	ibdev_dbg(&mdev->ib_dev,
 		  "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
-		  qp->tx_object, qp->sq_id, send_cq->id);
+		  qp->tx_object, qp->sq_id, send_cq->queue.id);
 
 	resp.sqid = qp->sq_id;
-	resp.cqid = send_cq->id;
+	resp.cqid = send_cq->queue.id;
 	resp.tx_vp_offset = pd->tx_vp_offset;
 
 	err = ib_copy_to_udata(udata, &resp, sizeof(resp));
@@ -422,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 err_release_gdma_cq:
 	kfree(gdma_cq);
-	gc->cq_table[send_cq->id] = NULL;
+	gc->cq_table[send_cq->queue.id] = NULL;
 
 err_destroy_wq_obj:
 	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
Signature · 1 line
2.43.0
Reviewed-by on v1 2/4
#7 lore raw

RE: [PATCH rdma-next 2/4] RDMA:mana_ib: Use struct mana_ib_queue for CQs

Quoted 12 lines

-----Original Message-----
From: Konstantin Taranov <[email protected]>
Sent: Wednesday, March 13, 2024 6:25 AM
To: Konstantin Taranov <[email protected]>; [email protected]; Long Li <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]
Subject: [PATCH rdma-next 2/4] RDMA:mana_ib: Use struct mana_ib_queue for CQs

Quoted 5 lines

From: Konstantin Taranov <[email protected]>

Use struct mana_ib_queue and its helpers for CQs

#8 lore raw

PATCH 3/4 RDMA/mana_ib: Use struct mana_ib_queue for WQs

In reply to Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Use struct mana_ib_queue and its helpers for WQs

---
 drivers/infiniband/hw/mana/mana_ib.h |  4 +---
 drivers/infiniband/hw/mana/qp.c      | 10 ++++-----
 drivers/infiniband/hw/mana/wq.c      | 31 ++++------------------------
 3 files changed, 10 insertions(+), 35 deletions(-)
drivers/infiniband/hw/mana/mana_ib.h +1 -3
index 6acb5c281..a8953ee80 100644
@@ -59,11 +59,9 @@ struct mana_ib_dev {
 
 struct mana_ib_wq {
 	struct ib_wq ibwq;
-	struct ib_umem *umem;
+	struct mana_ib_queue queue;
 	int wqe;
 	u32 wq_buf_size;
-	u64 gdma_region;
-	u64 id;
 	mana_handle_t rx_object;
 };
 
drivers/infiniband/hw/mana/qp.c +5 -5
index d7485ee6a..f606caa75 100644
@@ -194,7 +194,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		ibcq = ibwq->cq;
 		cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 
-		wq_spec.gdma_region = wq->gdma_region;
+		wq_spec.gdma_region = wq->queue.gdma_region;
 		wq_spec.queue_size = wq->wq_buf_size;
 
 		cq_spec.gdma_region = cq->queue.gdma_region;
@@ -212,18 +212,18 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 		}
 
 		/* The GDMA regions are now owned by the WQ object */
-		wq->gdma_region = GDMA_INVALID_DMA_REGION;
+		wq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
 		cq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
 
-		wq->id = wq_spec.queue_index;
+		wq->queue.id = wq_spec.queue_index;
 		cq->queue.id = cq_spec.queue_index;
 
 		ibdev_dbg(&mdev->ib_dev,
 			  "ret %d rx_object 0x%llx wq id %llu cq id %llu\n",
-			  ret, wq->rx_object, wq->id, cq->queue.id);
+			  ret, wq->rx_object, wq->queue.id, cq->queue.id);
 
 		resp.entries[i].cqid = cq->queue.id;
-		resp.entries[i].wqid = wq->id;
+		resp.entries[i].wqid = wq->queue.id;
 
 		mana_ind_table[i] = wq->rx_object;
 
drivers/infiniband/hw/mana/wq.c +4 -27
index 7c9c69962..f959f4b92 100644
@@ -13,7 +13,6 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
 		container_of(pd->device, struct mana_ib_dev, ib_dev);
 	struct mana_ib_create_wq ucmd = {};
 	struct mana_ib_wq *wq;
-	struct ib_umem *umem;
 	int err;
 
 	if (udata->inlen < sizeof(ucmd))
@@ -32,39 +31,18 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
 
 	ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
 
-	umem = ib_umem_get(pd->device, ucmd.wq_buf_addr, ucmd.wq_buf_size,
-			   IB_ACCESS_LOCAL_WRITE);
-	if (IS_ERR(umem)) {
-		err = PTR_ERR(umem);
+	err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue);
+	if (err) {
 		ibdev_dbg(&mdev->ib_dev,
-			  "Failed to get umem for create wq, err %d\n", err);
+			  "Failed to create queue for create wq, %d\n", err);
 		goto err_free_wq;
 	}
 
-	wq->umem = umem;
 	wq->wqe = init_attr->max_wr;
 	wq->wq_buf_size = ucmd.wq_buf_size;
 	wq->rx_object = INVALID_MANA_HANDLE;
-
-	err = mana_ib_create_zero_offset_dma_region(mdev, wq->umem, &wq->gdma_region);
-	if (err) {
-		ibdev_dbg(&mdev->ib_dev,
-			  "Failed to create dma region for create wq, %d\n",
-			  err);
-		goto err_release_umem;
-	}
-
-	ibdev_dbg(&mdev->ib_dev,
-		  "create_dma_region ret %d gdma_region 0x%llx\n",
-		  err, wq->gdma_region);
-
-	/* WQ ID is returned at wq_create time, doesn't know the value yet */
-
 	return &wq->ibwq;
 
-err_release_umem:
-	ib_umem_release(umem);
-
 err_free_wq:
 	kfree(wq);
 
@@ -86,8 +64,7 @@ int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)
 
 	mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev);
 
-	mana_ib_gd_destroy_dma_region(mdev, wq->gdma_region);
-	ib_umem_release(wq->umem);
+	mana_ib_destroy_queue(mdev, &wq->queue);
 
 	kfree(wq);
 
Signature · 1 line
2.43.0
Reviewed-by on v1 3/4
#9 lore raw

RE: [PATCH rdma-next 3/4] RDMA/mana_ib: Use struct mana_ib_queue for WQs

Quoted 12 lines

-----Original Message-----
From: Konstantin Taranov <[email protected]>
Sent: Wednesday, March 13, 2024 6:25 AM
To: Konstantin Taranov <[email protected]>; [email protected]; Long Li <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]
Subject: [PATCH rdma-next 3/4] RDMA/mana_ib: Use struct mana_ib_queue for WQs

Quoted 5 lines

From: Konstantin Taranov <[email protected]>

Use struct mana_ib_queue and its helpers for WQs

#10 lore raw

PATCH 4/4 RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs

In reply to Konstantin Taranov

From: Konstantin Taranov <[email protected]>

Use struct mana_ib_queue and its helpers for RAW QPs

---
 drivers/infiniband/hw/mana/mana_ib.h | 13 +++----
 drivers/infiniband/hw/mana/qp.c      | 54 ++++++++--------------------
 2 files changed, 22 insertions(+), 45 deletions(-)
drivers/infiniband/hw/mana/mana_ib.h +7 -6
index a8953ee80..ddb6e73e3 100644
@@ -91,15 +91,16 @@ struct mana_ib_cq {
 	u32 comp_vector;
 };
 
+struct mana_ib_raw_qp {
+	/* Work queue info */
+	struct mana_ib_queue queue;
+	mana_handle_t tx_object;
+};
+
 struct mana_ib_qp {
 	struct ib_qp ibqp;
 
-	/* Work queue info */
-	struct ib_umem *sq_umem;
-	int sqe;
-	u64 sq_gdma_region;
-	u64 sq_id;
-	mana_handle_t tx_object;
+	struct mana_ib_raw_qp sq;
 
 	/* The port on the IB device, starting with 1 */
 	u32 port;
drivers/infiniband/hw/mana/qp.c +15 -39

107 lines not on this page · raw message

Signature · 1 line
2.43.0
#11 lore raw

RE: [PATCH rdma-next 4/4] RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs

Quoted 6 lines
drivers/infiniband/hw/mana/mana_ib.h +6 -0
+struct mana_ib_raw_qp {
+	/* Work queue info */
+	struct mana_ib_queue queue;
+	mana_handle_t tx_object;
+};
+

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_cq { · line 103

Quoted 10 lines
drivers/infiniband/hw/mana/mana_ib.h +1 -6
 struct mana_ib_qp {
 	struct ib_qp ibqp;
 
-	/* Work queue info */
-	struct ib_umem *sq_umem;
-	int sqe;
-	u64 sq_gdma_region;
-	u64 sq_id;
-	mana_handle_t tx_object;
+	struct mana_ib_raw_qp sq;

What's the naming scheme for RC? Maybe use raw_sq here?

#12 lore raw

RE: [PATCH rdma-next 4/4] RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs

Quoted 18 lines

From: Long Li <[email protected]>
Sent: Friday, 15 March 2024 18:04
To: Konstantin Taranov <[email protected]>; Konstantin Taranov <[email protected]>; [email protected]; [email protected]; [email protected]
Cc: [email protected]; [email protected]
Subject: RE: [PATCH rdma-next 4/4] RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs

Quoted 9 lines
drivers/infiniband/hw/mana/mana_ib.h +6 -0
+struct mana_ib_raw_qp {
+	/* Work queue info */
+	struct mana_ib_queue queue;
+	mana_handle_t tx_object;
+};
+
 struct mana_ib_qp {
 	struct ib_qp ibqp;
Quoted 9 lines

drivers/infiniband/hw/mana/mana_ib.h struct mana_ib_cq { · line 103

Quoted 7 lines
drivers/infiniband/hw/mana/mana_ib.h +1 -6
-	/* Work queue info */
-	struct ib_umem *sq_umem;
-	int sqe;
-	u64 sq_gdma_region;
-	u64 sq_id;
-	mana_handle_t tx_object;
+	struct mana_ib_raw_qp sq;

What's the naming scheme for RC? Maybe use raw_sq here?

The plan is to use struct mana_ib_rc_qp for RC QP.
But I think mana_ib_raw_sq is a good proposal for RAW packets. I will make it in v2. Thanks!