Skip to contentSkip to conversation

RDMA/mana_ib: Add EQ interrupt support to mana ib driver.

PATCH Superseded Wei Hu 3 participants 4 messages Jun 15, 2023 – Jun 30, 2023

Subsystems (5)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Wei Hu

author maintainer
#0 lore raw

PATCH v3 1/1 RDMA/mana_ib: Add EQ interrupt support to mana ib driver.

Add EQ interrupt support for mana ib driver. Allocate EQs per ucontext to receive interrupt. Attach EQ when CQ is created. Call CQ interrupt handler when completion interrupt happens. EQs are destroyed when ucontext is deallocated.

The change calls some public APIs in mana ethernet driver to allocate EQs and other resources. Ehe EQ process routine is also shared by mana ethernet and mana ib drivers.

---

v2: Use ibdev_dbg to print error messages and return -ENOMEN
when kzalloc fails.

v3: Check return value on mana_ib_gd_destroy_dma_region(). Remove most
debug prints.

 drivers/infiniband/hw/mana/cq.c               |  35 ++++-
 drivers/infiniband/hw/mana/main.c             |  85 ++++++++++++
 drivers/infiniband/hw/mana/mana_ib.h          |   4 +
 drivers/infiniband/hw/mana/qp.c               |  79 ++++++++++-
 .../net/ethernet/microsoft/mana/gdma_main.c   | 131 ++++++++++--------
 drivers/net/ethernet/microsoft/mana/mana_en.c |   1 +
 include/net/mana/gdma.h                       |   9 +-
 7 files changed, 280 insertions(+), 64 deletions(-)
drivers/infiniband/hw/mana/cq.c +33 -2
index d141cab8a1e6..b6f61cd2d5eb 100644
@@ -12,13 +12,20 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	struct ib_device *ibdev = ibcq->device;
 	struct mana_ib_create_cq ucmd = {};
 	struct mana_ib_dev *mdev;
+	struct gdma_context *gc;
+	struct gdma_dev *gd;
 	int err;
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+	gd = mdev->gdma_dev;
+	gc = gd->gdma_context;
 
 	if (udata->inlen < sizeof(ucmd))
 		return -EINVAL;
 
+	cq->comp_vector = attr->comp_vector > gc->max_num_queues ?
+				0 : attr->comp_vector;
+
 	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
 	if (err) {
 		ibdev_dbg(ibdev,
@@ -69,11 +76,35 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 	struct ib_device *ibdev = ibcq->device;
 	struct mana_ib_dev *mdev;
+	struct gdma_context *gc;
+	struct gdma_dev *gd;
+	int err;
+
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+	gd = mdev->gdma_dev;
+	gc = gd->gdma_context;
+
 
-	mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
-	ib_umem_release(cq->umem);
+
+	if (atomic_read(&ibcq->usecnt) == 0) {
+		err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
+		if (err) {
+			ibdev_dbg(ibdev,
+				  "Faile to destroy dma region, %d\n", err);
+			return err;
+		}
+		kfree(gc->cq_table[cq->id]);
+		gc->cq_table[cq->id] = NULL;
+		ib_umem_release(cq->umem);
+	}
 
 	return 0;
 }
+
+void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
+{
+	struct mana_ib_cq *cq = ctx;
+
+	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
drivers/infiniband/hw/mana/main.c +85 -0

104 lines not on this page · raw message

drivers/infiniband/hw/mana/mana_ib.h +4 -0
index 502cc8672eef..9672fa1670a5 100644
@@ -67,6 +67,7 @@ struct mana_ib_cq {
 	int cqe;
 	u64 gdma_region;
 	u64 id;
+	u32 comp_vector;
 };
 
 struct mana_ib_qp {
@@ -86,6 +87,7 @@ struct mana_ib_qp {
 struct mana_ib_ucontext {
 	struct ib_ucontext ibucontext;
 	u32 doorbell;
+	struct mana_eq *eqs;
 };
 
 struct mana_ib_rwq_ind_table {
@@ -159,4 +161,6 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
 
 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
 
+void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq);
+
 #endif
drivers/infiniband/hw/mana/qp.c +73 -6

162 lines not on this page · raw message

drivers/net/ethernet/microsoft/mana/gdma_main.c +77 -54

227 lines not on this page · raw message

drivers/net/ethernet/microsoft/mana/mana_en.c +1 -0
index 06d6292e09b3..85345225813f 100644
@@ -1156,6 +1156,7 @@ static int mana_create_eq(struct mana_context *ac)
 	spec.eq.callback = NULL;
 	spec.eq.context = ac->eqs;
 	spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
+	spec.eq.msix_allocated = false;
 
 	for (i = 0; i < gc->max_num_queues; i++) {
 		err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq);
include/net/mana/gdma.h +7 -2
index 96c120160f15..cc728fc42043 100644
@@ -6,6 +6,7 @@
 
 #include <linux/dma-mapping.h>
 #include <linux/netdevice.h>
+#include <linux/list.h>
 
 #include "shm_channel.h"
 
@@ -291,6 +292,8 @@ struct gdma_queue {
 	u32 head;
 	u32 tail;
 
+	struct list_head entry;
+
 	/* Extra fields specific to EQ/CQ. */
 	union {
 		struct {
@@ -325,6 +328,8 @@ struct gdma_queue_spec {
 			void *context;
 
 			unsigned long log2_throttle_limit;
+			bool msix_allocated;
+			unsigned int msix_index;
 		} eq;
 
 		struct {
@@ -340,8 +345,8 @@ struct gdma_queue_spec {
 #define MANA_IRQ_NAME_SZ 32
 
 struct gdma_irq_context {
-	void (*handler)(void *arg);
-	void *arg;
+	void (*handler)(struct list_head *arg);
+	struct list_head eq_list;
 	char name[MANA_IRQ_NAME_SZ];
 };
 
Signature · 1 line
2.25.1
#1 lore raw

On Thu, Jun 15, 2023 at 11:14:12AM +0000, Wei Hu wrote:

Hi Wei Hu,

some minor nits from my side.

...

Quoted 12 lines
drivers/infiniband/hw/mana/cq.c +6 -0
@@ -69,11 +76,35 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 	struct ib_device *ibdev = ibcq->device;
 	struct mana_ib_dev *mdev;
+	struct gdma_context *gc;
+	struct gdma_dev *gd;
+	int err;
+
 
 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+	gd = mdev->gdma_dev;
+	gc = gd->gdma_context;

drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) · line 94

Quoted 10 lines
drivers/infiniband/hw/mana/cq.c +7 -2
+
 
-	mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
-	ib_umem_release(cq->umem);
+
+	if (atomic_read(&ibcq->usecnt) == 0) {
+		err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
+		if (err) {
+			ibdev_dbg(ibdev,
+				  "Faile to destroy dma region, %d\n", err);

nit: Faile -> Failed

Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +6 -0
+			return err;
+		}
+		kfree(gc->cq_table[cq->id]);
+		gc->cq_table[cq->id] = NULL;
+		ib_umem_release(cq->umem);
+	}
 
 	return 0;
 }

...

drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) · line 150

Quoted 13 lines
drivers/infiniband/hw/mana/main.c +5 -0
index 7be4c3adb4e2..e2affb6ae5ad 100644
@@ -143,6 +143,79 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	return err;
 }
 
+static void mana_ib_destroy_eq(struct mana_ib_ucontext *ucontext,
+			       struct mana_ib_dev *mdev)
+{
+	struct gdma_context *gc = mdev->gdma_dev->gdma_context;
+	struct ib_device *ibdev = ucontext->ibucontext.device;

nit: ibdev is set but unused.

GCC 12.3.0 with W=1 says:

drivers/infiniband/hw/mana/main.c: In function 'mana_ib_destroy_eq':
drivers/infiniband/hw/mana/main.c:150:27: warning: unused variable 'ibdev' [-Wunused-variable]
150 | struct ib_device *ibdev = ucontext->ibucontext.device;

Quoted 17 lines
drivers/infiniband/hw/mana/main.c +17 -0
+	struct gdma_queue *eq;
+	int i;
+
+	if (!ucontext->eqs)
+		return;
+
+	for (i = 0; i < gc->max_num_queues; i++) {
+		eq = ucontext->eqs[i].eq;
+		if (!eq)
+			continue;
+
+		mana_gd_destroy_queue(gc, eq);
+	}
+
+	kfree(ucontext->eqs);
+	ucontext->eqs = NULL;
+}

...

#2 lore raw
In reply to Wei Hu
kernel test robot noticed the following build warnings:

Hi Wei,

kernel test robot noticed the following build warnings:

[auto build test WARNING on linus/master]
[also build test WARNING on horms-ipvs/master v6.4-rc6 next-20230615] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Wei-Hu/RDMA-mana_ib-Add-EQ-interrupt-support-to-mana-ib-driver/20230615-191709
base:   linus/master
patch link:    https://lore.kernel.org/r/20230615111412.1687573-1-weh%40microsoft.com
patch subject: [PATCH v3 1/1] RDMA/mana_ib: Add EQ interrupt support to mana ib driver.
config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20230616/[email protected]/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0

reproduce (this is a W=1 build):
git checkout linus/master
b4 shazam https://lore.kernel.org/r/[email protected]
# save the config file
mkdir build_dir && cp config build_dir/.config
make W=1 O=build_dir ARCH=x86_64 olddefconfig
make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash drivers/infiniband/hw/mana/

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

drivers/infiniband/hw/mana/main.c: In function 'mana_ib_destroy_eq':

>> drivers/infiniband/hw/mana/main.c:150:27: warning: unused variable 'ibdev' [-Wunused-variable]
     150 |         struct ib_device *ibdev = ucontext->ibucontext.device;
         |                           ^~~~~

vim +/ibdev +150 drivers/infiniband/hw/mana/main.c

145
146 static void mana_ib_destroy_eq(struct mana_ib_ucontext *ucontext,
147 struct mana_ib_dev *mdev)
148 {
149 struct gdma_context *gc = mdev->gdma_dev->gdma_context;

 > 150		struct ib_device *ibdev = ucontext->ibucontext.device;

151 struct gdma_queue *eq;
152 int i;
153
154 if (!ucontext->eqs)
155 return;
156
157 for (i = 0; i < gc->max_num_queues; i++) {
158 eq = ucontext->eqs[i].eq;
159 if (!eq)
160 continue;
161
162 mana_gd_destroy_queue(gc, eq);
163 }
164
165 kfree(ucontext->eqs);
166 ucontext->eqs = NULL;
167 }
168

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#3 lore raw
In reply to Wei Hu
kernel test robot noticed the following build warnings:

Hi Wei,

kernel test robot noticed the following build warnings:

[auto build test WARNING on linus/master]
[also build test WARNING on horms-ipvs/master v6.4 next-20230629] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Wei-Hu/RDMA-mana_ib-Add-EQ-interrupt-support-to-mana-ib-driver/20230615-191709
base:   linus/master
patch link:    https://lore.kernel.org/r/20230615111412.1687573-1-weh%40microsoft.com
patch subject: [PATCH v3 1/1] RDMA/mana_ib: Add EQ interrupt support to mana ib driver.
config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20230630/[email protected]/config)
compiler: clang version 15.0.7 (https://github.com/llvm/llvm-project.git 8dfdcc7b7bf66834a761bd8de445840ef68e4d1a)
reproduce: (https://download.01.org/0day-ci/archive/20230630/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> drivers/infiniband/hw/mana/main.c:150:20: warning: unused variable 'ibdev' [-Wunused-variable]
           struct ib_device *ibdev = ucontext->ibucontext.device;
                             ^

1 warning generated.

vim +/ibdev +150 drivers/infiniband/hw/mana/main.c

145
146 static void mana_ib_destroy_eq(struct mana_ib_ucontext *ucontext,
147 struct mana_ib_dev *mdev)
148 {
149 struct gdma_context *gc = mdev->gdma_dev->gdma_context;

 > 150		struct ib_device *ibdev = ucontext->ibucontext.device;

151 struct gdma_queue *eq;
152 int i;
153
154 if (!ucontext->eqs)
155 return;
156
157 for (i = 0; i < gc->max_num_queues; i++) {
158 eq = ucontext->eqs[i].eq;
159 if (!eq)
160 continue;
161
162 mana_gd_destroy_queue(gc, eq);
163 }
164
165 kfree(ucontext->eqs);
166 ucontext->eqs = NULL;
167 }
168

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki