Skip to contentSkip to conversation

RDMA/mlx5: quiesce CQ polling before device shutdown on reboot

PATCH In review Chenguang Zhao 3 participants 5 messages Jul 15 – Aug 7

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 RDMA/mlx5: quiesce CQ polling before device shutdown on reboot

From: Chenguang Zhao <[email protected]>

On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in wr_cqe->done().

Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip SYS_ERROR completion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.

---
changelog:
- Fix the race on MLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp / mlx5_ib event workqueues via an mlx5_ib quiesce hook before fast_unload/teardown.
- Check shutting-down under cq->lock in mlx5_ib_poll_cq/arm_cq.
- Export ib_comp_wq and ib_comp_unbound_wq so modular mlx5_ib can flush them.

v2:
https://lore.kernel.org/all/[email protected]/

v1:
https://lore.kernel.org/all/[email protected]/

 drivers/infiniband/core/device.c              |  2 ++
 drivers/infiniband/hw/mlx5/cq.c               | 11 ++++++++++
 drivers/infiniband/hw/mlx5/main.c             | 20 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/health.c  |  3 +++
 .../net/ethernet/mellanox/mlx5/core/main.c    | 10 ++++++++++
 .../mellanox/mlx5/core/sf/dev/driver.c        |  3 +++
 include/linux/mlx5/driver.h                   | 11 ++++++++++
 7 files changed, 60 insertions(+)
drivers/infiniband/core/device.c +2 -0
index b8193e077a74..6c3377bd53df 100644
@@ -56,7 +56,9 @@ MODULE_DESCRIPTION("core kernel InfiniBand API");
 MODULE_LICENSE("Dual BSD/GPL");
 
 struct workqueue_struct *ib_comp_wq;
+EXPORT_SYMBOL_GPL(ib_comp_wq);
 struct workqueue_struct *ib_comp_unbound_wq;
+EXPORT_SYMBOL_GPL(ib_comp_unbound_wq);
 struct workqueue_struct *ib_wq;
 EXPORT_SYMBOL_GPL(ib_wq);
 static struct workqueue_struct *ib_unreg_wq;
drivers/infiniband/hw/mlx5/cq.c +11 -0
index 49b4bf148a4a..5d951c186cb3 100644
@@ -619,6 +619,10 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	int npolled;
 
 	spin_lock_irqsave(&cq->lock, flags);
+	if (mlx5_core_is_shutting_down(mdev)) {
+		spin_unlock_irqrestore(&cq->lock, flags);
+		return 0;
+	}
 	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 		/* make sure no soft wqe's are waiting */
 		if (unlikely(!list_empty(&cq->wc_list)))
@@ -654,6 +658,10 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 	int ret = 0;
 
 	spin_lock_irqsave(&cq->lock, irq_flags);
+	if (mlx5_core_is_shutting_down(mdev)) {
+		spin_unlock_irqrestore(&cq->lock, irq_flags);
+		return 0;
+	}
 	if (cq->notify_flags != IB_CQ_NEXT_COMP)
 		cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
 
@@ -661,6 +669,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 		ret = 1;
 	spin_unlock_irqrestore(&cq->lock, irq_flags);
 
+	if (mlx5_core_is_shutting_down(mdev))
+		return ret;
+
 	mlx5_cq_arm(&cq->mcq,
 		    (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
 		    MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
drivers/infiniband/hw/mlx5/main.c +20 -0
index 02809114fc79..dfc805892684 100644
@@ -84,6 +84,13 @@ static LIST_HEAD(mlx5_ib_dev_list);
  */
 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
 
+static void mlx5_ib_shutdown_quiesce(void)
+{
+	flush_workqueue(ib_comp_wq);
+	flush_workqueue(ib_comp_unbound_wq);
+	flush_workqueue(mlx5_ib_event_wq);
+}
+
 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
 {
 	struct mlx5_ib_dev *dev;
@@ -2976,6 +2983,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
 	unsigned long flags_cq;
 	unsigned long flags;
 
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		return;
+
 	INIT_LIST_HEAD(&cq_armed_list);
 
 	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
@@ -3200,6 +3210,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)
 	struct mlx5_ib_dev *ibdev = work->dev;
 	struct ib_event ibev;
 
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		goto out;
+
 	ibev.event = IB_EVENT_DEVICE_FATAL;
 	mlx5_ib_handle_internal_error(ibdev);
 	ibev.element.port_num = (u8)(unsigned long)work->param;
@@ -3222,10 +3235,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb,
 				   unsigned long event, void *param)
 {
 	struct mlx5_ib_event_work *work;
+	struct mlx5_ib_dev *ibdev;
 
 	if (event != MLX5_DEV_EVENT_SYS_ERROR)
 		return NOTIFY_DONE;
 
+	ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events);
+	if (mlx5_core_is_shutting_down(ibdev->mdev))
+		return NOTIFY_OK;
+
 	work = kmalloc_obj(*work, GFP_ATOMIC);
 	if (!work)
 		return NOTIFY_DONE;
@@ -5529,6 +5547,7 @@ static int __init mlx5_ib_init(void)
 	if (ret)
 		goto drv_err;
 
+	mlx5_rdma_shutdown_quiesce = mlx5_ib_shutdown_quiesce;
 	return 0;
 
 drv_err:
@@ -5547,6 +5566,7 @@ static int __init mlx5_ib_init(void)
 
 static void __exit mlx5_ib_cleanup(void)
 {
+	mlx5_rdma_shutdown_quiesce = NULL;
 	mlx5_data_direct_driver_unregister();
 	auxiliary_driver_unregister(&mlx5r_driver);
 	auxiliary_driver_unregister(&mlx5r_mp_driver);
drivers/net/ethernet/mellanox/mlx5/core/health.c +3 -0
index aeeb136f5ebc..d9cc42c4c310 100644
@@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)
 		mlx5_cmd_flush(dev);
 	}
 
+	if (mlx5_core_is_shutting_down(dev))
+		return;
+
 	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
 }
 
drivers/net/ethernet/mellanox/mlx5/core/main.c +10 -0
index 643b4aac2033..51de0b4570aa 100644
@@ -2186,12 +2186,22 @@ static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
 	return 0;
 }
 
+void (*mlx5_rdma_shutdown_quiesce)(void);
+EXPORT_SYMBOL_GPL(mlx5_rdma_shutdown_quiesce);
+
 static void shutdown(struct pci_dev *pdev)
 {
 	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
 	int err;
 
 	mlx5_core_info(dev, "Shutdown was called\n");
+	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
+	/*
+	 * Ensure in-flight CQ pollers observe SHUTTING_DOWN before
+	 * fast_unload tears the device down.
+	 */
+	if (mlx5_rdma_shutdown_quiesce)
+		mlx5_rdma_shutdown_quiesce();
 	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
 	mlx5_drain_fw_reset(dev);
 	mlx5_drain_health_wq(dev);
drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c +3 -0
index 4391ef0bab5d..d9735e492971 100644
@@ -111,6 +111,9 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev)
 	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
 	struct mlx5_core_dev *mdev = sf_dev->mdev;
 
+	set_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &mdev->intf_state);
+	if (mlx5_rdma_shutdown_quiesce)
+		mlx5_rdma_shutdown_quiesce();
 	set_bit(MLX5_BREAK_FW_WAIT, &mdev->intf_state);
 	mlx5_drain_health_wq(mdev);
 	mlx5_unload_one(mdev, false);
include/linux/mlx5/driver.h +11 -0
index b1871c0821d0..d81d13169828 100644
@@ -653,8 +653,14 @@ enum mlx5_device_state {
 enum mlx5_interface_state {
 	MLX5_INTERFACE_STATE_UP = BIT(0),
 	MLX5_BREAK_FW_WAIT = BIT(1),
+	MLX5_INTERFACE_STATE_SHUTTING_DOWN = BIT(2),
 };
 
+/* Optional hook installed by mlx5_ib to flush CQ poll work after the
+ * SHUTTING_DOWN flag is set and before teardown continues.
+ */
+extern void (*mlx5_rdma_shutdown_quiesce)(void);
+
 enum mlx5_pci_status {
 	MLX5_PCI_STATUS_DISABLED,
 	MLX5_PCI_STATUS_ENABLED,
@@ -1220,6 +1226,11 @@ static inline bool mlx5_core_is_pf(const struct mlx5_core_dev *dev)
 	return dev->coredev_type == MLX5_COREDEV_PF;
 }
 
+static inline bool mlx5_core_is_shutting_down(struct mlx5_core_dev *dev)
+{
+	return test_bit(MLX5_INTERFACE_STATE_SHUTTING_DOWN, &dev->intf_state);
+}
+
 static inline bool mlx5_core_is_vf(const struct mlx5_core_dev *dev)
 {
 	return dev->coredev_type == MLX5_COREDEV_VF;
Signature · 1 line
2.25.1
#1 lore raw
Quoted 35 lines · Chenguang Zhao

From: Chenguang Zhao <[email protected]>

On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in wr_cqe->done().

Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip SYS_ERROR completion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.

---
changelog:
- Fix the race on MLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp / mlx5_ib event workqueues via an mlx5_ib quiesce hook before fast_unload/teardown.
- Check shutting-down under cq->lock in mlx5_ib_poll_cq/arm_cq.
- Export ib_comp_wq and ib_comp_unbound_wq so modular mlx5_ib can flush them.

v2:
https://lore.kernel.org/all/[email protected]/

v1:
https://lore.kernel.org/all/[email protected]/

 drivers/infiniband/core/device.c              |  2 ++
 drivers/infiniband/hw/mlx5/cq.c               | 11 ++++++++++
 drivers/infiniband/hw/mlx5/main.c             | 20 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/health.c  |  3 +++
 .../net/ethernet/mellanox/mlx5/core/main.c    | 10 ++++++++++
 .../mellanox/mlx5/core/sf/dev/driver.c        |  3 +++
 include/linux/mlx5/driver.h                   | 11 ++++++++++
 7 files changed, 60 insertions(+)

<...>

drivers/infiniband/hw/mlx5/main.c static LIST_HEAD(mlx5_ib_dev_list); · line 92

Quoted 6 lines
drivers/infiniband/hw/mlx5/main.c +6 -0
+static void mlx5_ib_shutdown_quiesce(void)
+{
+	flush_workqueue(ib_comp_wq);
+	flush_workqueue(ib_comp_unbound_wq);
+	flush_workqueue(mlx5_ib_event_wq);
+}

These workqueues are shared by all IB drivers and the core. Drivers must not flush or destroy them.

Why this is not FW issue?

Thanks

#2 lore raw

Hi, Leon
reboot -f skips orderly shutdown and goes directly to:

kernel_restart_prepare() -> device_shutdown() -> mlx5 shutdown Upper layers may still hold live CQs, while ib-comp-wq keeps polling — a use-after-free race.

Normal reboot usually works because userspace has already torn down RDMA and called ib_free_cq().

Thanks

Quoted 50 lines · Leon Romanovsky
Quoted 35 lines · Chenguang Zhao

From: Chenguang Zhao <[email protected]>

On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in wr_cqe->done().

Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip SYS_ERROR completion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.

---
changelog:
- Fix the race on MLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp / mlx5_ib event workqueues via an mlx5_ib quiesce hook before fast_unload/teardown.
- Check shutting-down under cq->lock in mlx5_ib_poll_cq/arm_cq.
- Export ib_comp_wq and ib_comp_unbound_wq so modular mlx5_ib can flush them.

v2:
https://lore.kernel.org/all/[email protected]/

v1:
https://lore.kernel.org/all/[email protected]/

 drivers/infiniband/core/device.c              |  2 ++
 drivers/infiniband/hw/mlx5/cq.c               | 11 ++++++++++
 drivers/infiniband/hw/mlx5/main.c             | 20 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/health.c  |  3 +++
 .../net/ethernet/mellanox/mlx5/core/main.c    | 10 ++++++++++
 .../mellanox/mlx5/core/sf/dev/driver.c        |  3 +++
 include/linux/mlx5/driver.h                   | 11 ++++++++++
 7 files changed, 60 insertions(+)

<...>

drivers/infiniband/hw/mlx5/main.c static LIST_HEAD(mlx5_ib_dev_list); · line 92

Quoted 6 lines
drivers/infiniband/hw/mlx5/main.c +6 -0
+static void mlx5_ib_shutdown_quiesce(void)
+{
+	flush_workqueue(ib_comp_wq);
+	flush_workqueue(ib_comp_unbound_wq);
+	flush_workqueue(mlx5_ib_event_wq);
+}

These workqueues are shared by all IB drivers and the core. Drivers must not flush or destroy them.

Why this is not FW issue?

Thanks

#3 lore raw
Quoted 6 lines · Chenguang Zhao

Hi, Leon
reboot -f skips orderly shutdown and goes directly to:

kernel_restart_prepare() -> device_shutdown() -> mlx5 shutdown Upper layers may still hold live CQs, while ib-comp-wq keeps polling — a use-after-free race.

The point is that this flow is neither RDMA- nor mlx5-specific, and it works as expected. mlx5 shutdown() stops the FW/HW, while the kernel stops and tears down the running threads.

Quoted 3 lines

Normal reboot usually works because userspace has already torn down RDMA and called ib_free_cq().

There is no difference, from the kernel's perspective, between a normal and a forced reboot, except that the former bypasses userspace shutdown.

Thanks

Quoted 54 lines

Thanks

Quoted 50 lines · Leon Romanovsky
Quoted 35 lines · Chenguang Zhao

From: Chenguang Zhao <[email protected]>

On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in wr_cqe->done().

Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip SYS_ERROR completion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.

---
changelog:
- Fix the race on MLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp / mlx5_ib event workqueues via an mlx5_ib quiesce hook before fast_unload/teardown.
- Check shutting-down under cq->lock in mlx5_ib_poll_cq/arm_cq.
- Export ib_comp_wq and ib_comp_unbound_wq so modular mlx5_ib can flush them.

v2:
https://lore.kernel.org/all/[email protected]/

v1:
https://lore.kernel.org/all/[email protected]/

 drivers/infiniband/core/device.c              |  2 ++
 drivers/infiniband/hw/mlx5/cq.c               | 11 ++++++++++
 drivers/infiniband/hw/mlx5/main.c             | 20 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/health.c  |  3 +++
 .../net/ethernet/mellanox/mlx5/core/main.c    | 10 ++++++++++
 .../mellanox/mlx5/core/sf/dev/driver.c        |  3 +++
 include/linux/mlx5/driver.h                   | 11 ++++++++++
 7 files changed, 60 insertions(+)

<...>

drivers/infiniband/hw/mlx5/main.c static LIST_HEAD(mlx5_ib_dev_list); · line 92

Quoted 6 lines
drivers/infiniband/hw/mlx5/main.c +6 -0
+static void mlx5_ib_shutdown_quiesce(void)
+{
+	flush_workqueue(ib_comp_wq);
+	flush_workqueue(ib_comp_unbound_wq);
+	flush_workqueue(mlx5_ib_event_wq);
+}

These workqueues are shared by all IB drivers and the core. Drivers must not flush or destroy them.

Why this is not FW issue?

Thanks

#4 lore raw
Quoted 11 lines · Leon Romanovsky
Quoted 6 lines · Chenguang Zhao

Hi, Leon
reboot -f skips orderly shutdown and goes directly to:

kernel_restart_prepare() -> device_shutdown() -> mlx5 shutdown Upper layers may still hold live CQs, while ib-comp-wq keeps polling — a use-after-free race.

The point is that this flow is neither RDMA- nor mlx5-specific, and it works as expected. mlx5 shutdown() stops the FW/HW, while the kernel stops and tears down the running threads.

I'm pretty sure we are missing stuff in the driver to synchronize everything while it is doing a tear down or health recovery.

Ripping a driver out from under a still-running subsystem is hard, this does seem like a real bug, but I also don't think the shutdown handler is the right place to fix it..

What was the actual UAF scenario, we should take a look at that more directly?

Jason