RDMA/mana_ib: Add EQ interrupt support to mana ib driver.
Subsystems (5)
Wei Hu
maintainerPATCH v3 1/1 RDMA/mana_ib: Add EQ interrupt support to mana ib driver.
Add EQ interrupt support for mana ib driver. Allocate EQs per ucontext to receive interrupt. Attach EQ when CQ is created. Call CQ interrupt handler when completion interrupt happens. EQs are destroyed when ucontext is deallocated.
The change calls some public APIs in mana ethernet driver to allocate EQs and other resources. Ehe EQ process routine is also shared by mana ethernet and mana ib drivers.
---
v2: Use ibdev_dbg to print error messages and return -ENOMEN
when kzalloc fails.
v3: Check return value on mana_ib_gd_destroy_dma_region(). Remove most
debug prints.
drivers/infiniband/hw/mana/cq.c | 35 ++++- drivers/infiniband/hw/mana/main.c | 85 ++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 4 + drivers/infiniband/hw/mana/qp.c | 79 ++++++++++- .../net/ethernet/microsoft/mana/gdma_main.c | 131 ++++++++++-------- drivers/net/ethernet/microsoft/mana/mana_en.c | 1 + include/net/mana/gdma.h | 9 +- 7 files changed, 280 insertions(+), 64 deletions(-)
drivers/infiniband/hw/mana/cq.c +33 -2
@@ -12,13 +12,20 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,struct ib_device *ibdev = ibcq->device;struct mana_ib_create_cq ucmd = {};struct mana_ib_dev *mdev; +struct gdma_context *gc; +struct gdma_dev *gd;int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); + gd = mdev->gdma_dev; + gc = gd->gdma_context;if (udata->inlen <sizeof (ucmd))return -EINVAL ; + cq->comp_vector = attr->comp_vector > gc->max_num_queues ? +0 : attr->comp_vector; + err =ib_copy_from_udata (&ucmd, udata,min (sizeof (ucmd), udata->inlen));if (err) { ibdev_dbg(ibdev,
@@ -69,11 +76,35 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);struct ib_device *ibdev = ibcq->device;struct mana_ib_dev *mdev; +struct gdma_context *gc; +struct gdma_dev *gd; +int err; + mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); + gd = mdev->gdma_dev; + gc = gd->gdma_context; + -mana_ib_gd_destroy_dma_region (mdev, cq->gdma_region); -ib_umem_release (cq->umem); + +if (atomic_read (&ibcq->usecnt) ==0 ) { + err =mana_ib_gd_destroy_dma_region (mdev, cq->gdma_region); +if (err) { +ibdev_dbg (ibdev, +"Faile to destroy dma region, %d\n" , err); +return err; + } +kfree (gc->cq_table[cq->id]); + gc->cq_table[cq->id] =NULL ; +ib_umem_release (cq->umem); + }return 0 ; } + +void mana_ib_cq_handler (void *ctx,struct gdma_queue *gdma_cq) +{ +struct mana_ib_cq *cq = ctx; + + cq->ibcq.comp_handler (&cq->ibcq, cq->ibcq.cq_context); +}
drivers/infiniband/hw/mana/main.c +85 -0
104 lines not on this page · raw message
drivers/infiniband/hw/mana/mana_ib.h +4 -0
@@ -67,6 +67,7 @@ struct mana_ib_cq {int cqe;u64 gdma_region;u64 id; +u32 comp_vector; };struct mana_ib_qp {
@@ -86,6 +87,7 @@ struct mana_ib_qp {struct mana_ib_ucontext {struct ib_ucontext ibucontext;u32 doorbell; +struct mana_eq *eqs; };struct mana_ib_rwq_ind_table {
@@ -159,4 +161,6 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,void mana_ib_disassociate_ucontext (struct ib_ucontext *ibcontext); +void mana_ib_cq_handler (void *ctx,struct gdma_queue *gdma_cq); +#endif
drivers/infiniband/hw/mana/qp.c +73 -6
162 lines not on this page · raw message
drivers/net/ethernet/microsoft/mana/gdma_main.c +77 -54
227 lines not on this page · raw message
drivers/net/ethernet/microsoft/mana/mana_en.c +1 -0
@@ -1156,6 +1156,7 @@ static int mana_create_eq(struct mana_context *ac) spec.eq.callback =NULL ; spec.eq.context = ac->eqs; spec.eq.log2_throttle_limit =LOG2_EQ_THROTTLE ; + spec.eq.msix_allocated = false;for (i =0 ; i < gc->max_num_queues; i++) { err =mana_gd_create_mana_eq (gd, &spec, &ac->eqs[i].eq);
include/net/mana/gdma.h +7 -2
@@ -6,6 +6,7 @@#include <linux/dma-mapping.h> #include <linux/netdevice.h> +#include <linux/list.h> #include "shm_channel.h"
@@ -291,6 +292,8 @@ struct gdma_queue {u32 head;u32 tail; +struct list_head entry; +/* Extra fields specific to EQ/CQ. */ union { struct {
@@ -325,6 +328,8 @@ struct gdma_queue_spec {void *context;unsigned long log2_throttle_limit; +bool msix_allocated; +unsigned int msix_index; } eq;struct {
@@ -340,8 +345,8 @@ struct gdma_queue_spec {#define MANA_IRQ_NAME_SZ 32struct gdma_irq_context { -void (*handler)(void *arg); -void *arg; +void (*handler)(struct list_head *arg); +struct list_head eq_list;char name[MANA_IRQ_NAME_SZ ]; };
Signature · 1 line
2.25.1
On Thu, Jun 15, 2023 at 11:14:12AM +0000, Wei Hu wrote:
Hi Wei Hu,
some minor nits from my side.
...
Quoted 12 lines
drivers/infiniband/hw/mana/cq.c +6
-0@@ -69,11 +76,35 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);struct ib_device *ibdev = ibcq->device;struct mana_ib_dev *mdev; +struct gdma_context *gc; +struct gdma_dev *gd; +int err; + mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); + gd = mdev->gdma_dev; + gc = gd->gdma_context;
drivers/infiniband/hw/mana/cq.c int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) · line 94
Quoted 10 lines
drivers/infiniband/hw/mana/cq.c +7
-2+ -mana_ib_gd_destroy_dma_region (mdev, cq->gdma_region); -ib_umem_release (cq->umem); + +if (atomic_read (&ibcq->usecnt) ==0 ) { + err =mana_ib_gd_destroy_dma_region (mdev, cq->gdma_region); +if (err) { +ibdev_dbg (ibdev, +"Faile to destroy dma region, %d\n" , err);
nit: Faile -> Failed
Quoted 9 lines
drivers/infiniband/hw/mana/cq.c +6
-0+return err; + } +kfree (gc->cq_table[cq->id]); + gc->cq_table[cq->id] =NULL ; +ib_umem_release (cq->umem); + }return 0 ; }
...
drivers/infiniband/hw/mana/main.c int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) · line 150
Quoted 13 lines
drivers/infiniband/hw/mana/main.c +5
-0@@ -143,6 +143,79 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)return err; } +static void mana_ib_destroy_eq (struct mana_ib_ucontext *ucontext, +struct mana_ib_dev *mdev) +{ +struct gdma_context *gc = mdev->gdma_dev->gdma_context; +struct ib_device *ibdev = ucontext->ibucontext.device;
nit: ibdev is set but unused.
GCC 12.3.0 with W=1 says:
drivers/infiniband/hw/mana/main.c: In function 'mana_ib_destroy_eq':
drivers/infiniband/hw/mana/main.c:150:27: warning: unused variable 'ibdev' [-Wunused-variable]
150 | struct ib_device *ibdev = ucontext->ibucontext.device;
Quoted 17 lines
drivers/infiniband/hw/mana/main.c +17
-0+struct gdma_queue *eq; +int i; + +if (!ucontext->eqs) +return ; + +for (i =0 ; i < gc->max_num_queues; i++) { + eq = ucontext->eqs[i].eq; +if (!eq) +continue ; + +mana_gd_destroy_queue (gc, eq); + } + +kfree (ucontext->eqs); + ucontext->eqs =NULL ; +}
...
kernel test robot
build botkernel test robot noticed the following build warnings:
Hi Wei,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on horms-ipvs/master v6.4-rc6 next-20230615] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Wei-Hu/RDMA-mana_ib-Add-EQ-interrupt-support-to-mana-ib-driver/20230615-191709 base: linus/master patch link: https://lore.kernel.org/r/20230615111412.1687573-1-weh%40microsoft.com patch subject: [PATCH v3 1/1] RDMA/mana_ib: Add EQ interrupt support to mana ib driver. config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20230616/[email protected]/config) compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build):
git checkout linus/master
b4 shazam https://lore.kernel.org/r/[email protected]
# save the config file
mkdir build_dir && cp config build_dir/.config
make W=1 O=build_dir ARCH=x86_64 olddefconfig
make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash drivers/infiniband/hw/mana/
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
drivers/infiniband/hw/mana/main.c: In function 'mana_ib_destroy_eq':
>> drivers/infiniband/hw/mana/main.c:150:27: warning: unused variable 'ibdev' [-Wunused-variable]
150 | struct ib_device *ibdev = ucontext->ibucontext.device;
| ^~~~~vim +/ibdev +150 drivers/infiniband/hw/mana/main.c
145
146 static void mana_ib_destroy_eq(struct mana_ib_ucontext *ucontext,
147 struct mana_ib_dev *mdev)
148 {
149 struct gdma_context *gc = mdev->gdma_dev->gdma_context;
> 150 struct ib_device *ibdev = ucontext->ibucontext.device;
151 struct gdma_queue *eq;
152 int i;
153
154 if (!ucontext->eqs)
155 return;
156
157 for (i = 0; i < gc->max_num_queues; i++) {
158 eq = ucontext->eqs[i].eq;
159 if (!eq)
160 continue;
161
162 mana_gd_destroy_queue(gc, eq);
163 }
164
165 kfree(ucontext->eqs);
166 ucontext->eqs = NULL;
167 }
168
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botkernel test robot noticed the following build warnings:
Hi Wei,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on horms-ipvs/master v6.4 next-20230629] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Wei-Hu/RDMA-mana_ib-Add-EQ-interrupt-support-to-mana-ib-driver/20230615-191709 base: linus/master patch link: https://lore.kernel.org/r/20230615111412.1687573-1-weh%40microsoft.com patch subject: [PATCH v3 1/1] RDMA/mana_ib: Add EQ interrupt support to mana ib driver. config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20230630/[email protected]/config) compiler: clang version 15.0.7 (https://github.com/llvm/llvm-project.git 8dfdcc7b7bf66834a761bd8de445840ef68e4d1a) reproduce: (https://download.01.org/0day-ci/archive/20230630/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> drivers/infiniband/hw/mana/main.c:150:20: warning: unused variable 'ibdev' [-Wunused-variable]
struct ib_device *ibdev = ucontext->ibucontext.device;
^1 warning generated.
vim +/ibdev +150 drivers/infiniband/hw/mana/main.c
145
146 static void mana_ib_destroy_eq(struct mana_ib_ucontext *ucontext,
147 struct mana_ib_dev *mdev)
148 {
149 struct gdma_context *gc = mdev->gdma_dev->gdma_context;
> 150 struct ib_device *ibdev = ucontext->ibucontext.device;
151 struct gdma_queue *eq;
152 int i;
153
154 if (!ucontext->eqs)
155 return;
156
157 for (i = 0; i < gc->max_num_queues; i++) {
158 eq = ucontext->eqs[i].eq;
159 if (!eq)
160 continue;
161
162 mana_gd_destroy_queue(gc, eq);
163 }
164
165 kfree(ucontext->eqs);
166 ucontext->eqs = NULL;
167 }
168
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki