Skip to contentSkip to conversation

RDMA/bnxt_re: Device re-initialization after Firmware error

PATCH Applied Selvin Xavier 2 participants 6 messages Sep 11, 2024

Subsystems (2)
lore
Share & tools
lore

Selvin Xavier

author maintainer
#0 lore raw

v2 cover RDMA/bnxt_re: Device re-initialization after Firmware error

Add support for complete re-initialization of the device when driver detects a firmware reset. Code reorg that updates the device handles stored with Auxiliary bus and the bnxt_en driver. bnxt_en driver calls suspend and resume hooks upon error recovery. Driver destroys and recreates the roce device instance upon receiving these calls.

Please review and apply these changes for 6.12.

Thanks,
Selvin Xavier

v1 -> v2:
- Fix the kbuild warning by removing the variable set but not used

Chandramohan Akula (2):
RDMA/bnxt_re: Change aux driver data to en_info to hold more
information
RDMA/bnxt_re: Use the aux device for L2 ULP callbacks

Selvin Xavier (2):
RDMA/bnxt_re: Group all operations under add_device and remove_device RDMA/bnxt_re: Recover the device when FW error is detected

 drivers/infiniband/hw/bnxt_re/bnxt_re.h   |  21 ++++
 drivers/infiniband/hw/bnxt_re/main.c      | 198 ++++++++++++++++++++----------
 drivers/infiniband/hw/bnxt_re/qplib_res.h |   1 +
 3 files changed, 157 insertions(+), 63 deletions(-)
Signature · 1 line
2.5.5
author maintainer
#1 lore raw

PATCH v2 1/4 RDMA/bnxt_re: Change aux driver data to en_info to hold more information

From: Chandramohan Akula <[email protected]>

rdev will be destroyed and recreated during the FW error recovery scenarios. So to keep the state, if any, use an en_info structure which gets created/freed based on auxiliary device initialization/de-initialization.

---
 drivers/infiniband/hw/bnxt_re/bnxt_re.h |  6 +++
 drivers/infiniband/hw/bnxt_re/main.c    | 71 ++++++++++++++++++++++++++++-----
 2 files changed, 66 insertions(+), 11 deletions(-)
drivers/infiniband/hw/bnxt_re/bnxt_re.h +6 -0
index 2be9a62..5df3ce1 100644
@@ -107,6 +107,11 @@ struct bnxt_re_gsi_context {
 	struct	bnxt_re_sqp_entries *sqp_tbl;
 };
 
+struct bnxt_re_en_dev_info {
+	struct bnxt_en_dev *en_dev;
+	struct bnxt_re_dev *rdev;
+};
+
 #define BNXT_RE_AEQ_IDX			0
 #define BNXT_RE_NQ_IDX			1
 #define BNXT_RE_GEN_P5_MAX_VF		64
@@ -155,6 +160,7 @@ struct bnxt_re_dev {
 #define BNXT_RE_FLAG_ERR_DEVICE_DETACHED       17
 #define BNXT_RE_FLAG_ISSUE_ROCE_STATS          29
 	struct net_device		*netdev;
+	struct auxiliary_device         *adev;
 	struct notifier_block		nb;
 	unsigned int			version, major, minor;
 	struct bnxt_qplib_chip_ctx	*chip_ctx;
drivers/infiniband/hw/bnxt_re/main.c +60 -11

158 lines not on this page · raw message

Signature · 1 line
2.5.5
author maintainer
#2 lore raw

PATCH v2 2/4 RDMA/bnxt_re: Use the aux device for L2 ULP callbacks

In reply to Selvin Xavier

From: Chandramohan Akula <[email protected]>

While registering with the L2 for ULP operations, use the aux device pointer as the handle. Aux device has the data bnxt_re_en_dev_info, which is used to store required information for the bnxt_re_suspend and bnxt_re_resume functions.

---
 drivers/infiniband/hw/bnxt_re/main.c | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)
drivers/infiniband/hw/bnxt_re/main.c +20 -6
index 085a03c..2a91699 100644
@@ -305,11 +305,18 @@ static void bnxt_re_shutdown(struct auxiliary_device *adev)
 
 static void bnxt_re_stop_irq(void *handle)
 {
-	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
-	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+	struct bnxt_qplib_rcfw *rcfw;
+	struct bnxt_re_dev *rdev;
 	struct bnxt_qplib_nq *nq;
 	int indx;
 
+	if (!en_info)
+		return;
+
+	rdev = en_info->rdev;
+	rcfw = &rdev->rcfw;
+
 	for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) {
 		nq = &rdev->nq[indx - 1];
 		bnxt_qplib_nq_stop_irq(nq, false);
@@ -320,12 +327,19 @@ static void bnxt_re_stop_irq(void *handle)
 
 static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
 {
-	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
-	struct bnxt_msix_entry *msix_ent = rdev->en_dev->msix_entries;
-	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+	struct bnxt_msix_entry *msix_ent;
+	struct bnxt_qplib_rcfw *rcfw;
+	struct bnxt_re_dev *rdev;
 	struct bnxt_qplib_nq *nq;
 	int indx, rc;
 
+	if (!en_info)
+		return;
+
+	rdev = en_info->rdev;
+	msix_ent = rdev->en_dev->msix_entries;
+	rcfw = &rdev->rcfw;
 	if (!ent) {
 		/* Not setting the f/w timeout bit in rcfw.
 		 * During the driver unload the first command
@@ -374,7 +388,7 @@ static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
 
 	en_dev = rdev->en_dev;
 
-	rc = bnxt_register_dev(en_dev, &bnxt_re_ulp_ops, rdev);
+	rc = bnxt_register_dev(en_dev, &bnxt_re_ulp_ops, rdev->adev);
 	if (!rc)
 		rdev->qplib_res.pdev = rdev->en_dev->pdev;
 	return rc;
Signature · 1 line
2.5.5
author maintainer
#3 lore raw

PATCH v2 3/4 RDMA/bnxt_re: Group all operations under add_device and remove_device

In reply to Selvin Xavier

Adding and removing device need to be handled from multiple contexts when Firmware error recovery is supported. So group all the add and remove operations to add_device and remove_device function.

---
 drivers/infiniband/hw/bnxt_re/main.c | 65 ++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 32 deletions(-)
drivers/infiniband/hw/bnxt_re/main.c +33 -32
index 2a91699..dc63ad0 100644
@@ -88,6 +88,7 @@ static int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);
 
 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
 			     u32 *offset);
+static void bnxt_re_setup_cc(struct bnxt_re_dev *rdev, bool enable);
 static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
 {
 	struct bnxt_qplib_chip_ctx *cctx;
@@ -1860,6 +1861,16 @@ static int bnxt_re_add_device(struct auxiliary_device *adev)
 		goto re_dev_uninit;
 	}
 
+	rdev->nb.notifier_call = bnxt_re_netdev_event;
+	rc = register_netdevice_notifier(&rdev->nb);
+	if (rc) {
+		rdev->nb.notifier_call = NULL;
+		pr_err("%s: Cannot register to netdevice_notifier",
+		       ROCE_DRV_MODULE_NAME);
+		return rc;
+	}
+	bnxt_re_setup_cc(rdev, true);
+
 	return 0;
 
 re_dev_uninit:
@@ -1947,20 +1958,9 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
 
 #define BNXT_ADEV_NAME "bnxt_en"
 
-static void bnxt_re_remove(struct auxiliary_device *adev)
+static void bnxt_re_remove_device(struct bnxt_re_dev *rdev,
+				  struct auxiliary_device *aux_dev)
 {
-	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
-	struct bnxt_re_dev *rdev;
-
-	mutex_lock(&bnxt_re_mutex);
-	if (!en_info) {
-		mutex_unlock(&bnxt_re_mutex);
-		return;
-	}
-	rdev = en_info->rdev;
-	if (!rdev)
-		goto skip_remove;
-
 	if (rdev->nb.notifier_call) {
 		unregister_netdevice_notifier(&rdev->nb);
 		rdev->nb.notifier_call = NULL;
@@ -1968,13 +1968,30 @@ static void bnxt_re_remove(struct auxiliary_device *adev)
 		/* If notifier is null, we should have already done a
 		 * clean up before coming here.
 		 */
-		goto skip_remove;
+		return;
 	}
 	bnxt_re_setup_cc(rdev, false);
 	ib_unregister_device(&rdev->ibdev);
 	bnxt_re_dev_uninit(rdev);
 	ib_dealloc_device(&rdev->ibdev);
-skip_remove:
+}
+
+static void bnxt_re_remove(struct auxiliary_device *adev)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
+	struct bnxt_en_dev *en_dev;
+	struct bnxt_re_dev *rdev;
+
+	mutex_lock(&bnxt_re_mutex);
+	if (!en_info) {
+		mutex_unlock(&bnxt_re_mutex);
+		return;
+	}
+	en_dev = en_info->en_dev;
+	rdev = en_info->rdev;
+
+	if (rdev)
+		bnxt_re_remove_device(rdev, adev);
 	kfree(en_info);
 	mutex_unlock(&bnxt_re_mutex);
 }
@@ -1986,7 +2003,6 @@ static int bnxt_re_probe(struct auxiliary_device *adev,
 		container_of(adev, struct bnxt_aux_priv, aux_dev);
 	struct bnxt_re_en_dev_info *en_info;
 	struct bnxt_en_dev *en_dev;
-	struct bnxt_re_dev *rdev;
 	int rc;
 
 	en_dev = aux_priv->edev;
@@ -2002,23 +2018,8 @@ static int bnxt_re_probe(struct auxiliary_device *adev,
 	auxiliary_set_drvdata(adev, en_info);
 
 	rc = bnxt_re_add_device(adev);
-	if (rc) {
-		mutex_unlock(&bnxt_re_mutex);
-		return rc;
-	}
-
-	rdev = en_info->rdev;
-
-	rdev->nb.notifier_call = bnxt_re_netdev_event;
-	rc = register_netdevice_notifier(&rdev->nb);
-	if (rc) {
-		rdev->nb.notifier_call = NULL;
-		pr_err("%s: Cannot register to netdevice_notifier",
-		       ROCE_DRV_MODULE_NAME);
+	if (rc)
 		goto err;
-	}
-
-	bnxt_re_setup_cc(rdev, true);
 	mutex_unlock(&bnxt_re_mutex);
 	return 0;
 
Signature · 1 line
2.5.5
author maintainer
#4 lore raw

PATCH v2 4/4 RDMA/bnxt_re: Recover the device when FW error is detected

In reply to Selvin Xavier

If the FW crashes, L2 driver gets notified and it notifies the RoCE driver. Currently driver doesn't re-initialize the device. Add support for re-initialize the RoCE device.

RoCE device is removed and re-attached in the ulp_stop and ulp_start respectively. The recovery logic expects the RoCE driver to be registered with L2 driver while its being removed. So the driver avoids unregistering with L2 driver in the recovery path.

---
 drivers/infiniband/hw/bnxt_re/bnxt_re.h   | 15 +++++++
 drivers/infiniband/hw/bnxt_re/main.c      | 70 +++++++++++++++++--------------
 drivers/infiniband/hw/bnxt_re/qplib_res.h |  1 +
 3 files changed, 55 insertions(+), 31 deletions(-)
drivers/infiniband/hw/bnxt_re/bnxt_re.h +15 -0
index 5df3ce1..e94518b 100644
@@ -91,6 +91,15 @@ struct bnxt_re_ring_attr {
 	u8		mode;
 };
 
+/*
+ * Data structure and defines to handle
+ * recovery
+ */
+#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1
+#define BNXT_RE_COMPLETE_REMOVE 0x2
+#define BNXT_RE_POST_RECOVERY_INIT 0x4
+#define BNXT_RE_COMPLETE_INIT 0x8
+
 struct bnxt_re_sqp_entries {
 	struct bnxt_qplib_sge sge;
 	u64 wrid;
@@ -224,4 +233,10 @@ static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)
 }
 
 extern const struct uapi_definition bnxt_re_uapi_defs[];
+
+static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev)
+{
+	rdev->qplib_res.pacing_data->dev_err_state =
+		test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+}
 #endif
drivers/infiniband/hw/bnxt_re/main.c +39 -31

210 lines not on this page · raw message

drivers/infiniband/hw/bnxt_re/qplib_res.h +1 -0
index 049805a..c2f7103 100644
@@ -82,6 +82,7 @@ struct bnxt_qplib_db_pacing_data {
 	u32 fifo_room_mask;
 	u32 fifo_room_shift;
 	u32 grc_reg_offset;
+	u32 dev_err_state;
 };
 
 #define BNXT_QPLIB_DBR_PF_DB_OFFSET     0x10000
Signature · 1 line
2.5.5
#5 lore raw
In reply to Selvin Xavier
Quoted 8 lines · Selvin Xavier

Add support for complete re-initialization of the device when driver detects a firmware reset. Code reorg that updates the device handles stored with Auxiliary bus and the bnxt_en driver. bnxt_en driver calls suspend and resume hooks upon error recovery. Driver destroys and recreates the roce device instance upon receiving these calls.

[...]

Applied, thanks!

[1/4] RDMA/bnxt_re: Change aux driver data to en_info to hold more information

      https://git.kernel.org/rdma/rdma/c/6656fda9d744bd
[2/4] RDMA/bnxt_re: Use the aux device for L2 ULP callbacks
      https://git.kernel.org/rdma/rdma/c/5b93677f520908
[3/4] RDMA/bnxt_re: Group all operations under add_device and remove_device
      https://git.kernel.org/rdma/rdma/c/c1fcf8aff481b6
[4/4] RDMA/bnxt_re: Recover the device when FW error is detected
      https://git.kernel.org/rdma/rdma/c/5d13aa44d9aec1

Best regards,

Signature · 1 line
Leon Romanovsky <[email protected]>