RDMA/mlx5: quiesce CQ polling before device shutdown on reboot
Subsystems (5)
PATCH v3 1/1 RDMA/mlx5: quiesce CQ polling before device shutdown on reboot
From: Chenguang Zhao <[email protected]>
On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in wr_cqe->done().
Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip SYS_ERROR completion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.
---
changelog:
- Fix the race on MLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp / mlx5_ib event workqueues via an mlx5_ib quiesce hook before fast_unload/teardown.
- Check shutting-down under cq->lock in mlx5_ib_poll_cq/arm_cq.
- Export ib_comp_wq and ib_comp_unbound_wq so modular mlx5_ib can flush them.
v2:
https://lore.kernel.org/all/[email protected]/
v1:
https://lore.kernel.org/all/[email protected]/
drivers/infiniband/core/device.c | 2 ++ drivers/infiniband/hw/mlx5/cq.c | 11 ++++++++++ drivers/infiniband/hw/mlx5/main.c | 20 +++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/health.c | 3 +++ .../net/ethernet/mellanox/mlx5/core/main.c | 10 ++++++++++ .../mellanox/mlx5/core/sf/dev/driver.c | 3 +++ include/linux/mlx5/driver.h | 11 ++++++++++ 7 files changed, 60 insertions(+)
drivers/infiniband/core/device.c +2 -0
@@ -56,7 +56,9 @@ MODULE_DESCRIPTION("core kernel InfiniBand API");MODULE_LICENSE ("Dual BSD/GPL" );struct workqueue_struct *ib_comp_wq; +EXPORT_SYMBOL_GPL (ib_comp_wq);struct workqueue_struct *ib_comp_unbound_wq; +EXPORT_SYMBOL_GPL (ib_comp_unbound_wq);struct workqueue_struct *ib_wq;EXPORT_SYMBOL_GPL (ib_wq);static struct workqueue_struct *ib_unreg_wq;
drivers/infiniband/hw/mlx5/cq.c +11 -0
@@ -619,6 +619,10 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)int npolled;spin_lock_irqsave (&cq->lock, flags); +if (mlx5_core_is_shutting_down (mdev)) { +spin_unlock_irqrestore (&cq->lock, flags); +return 0 ; + }if (mdev->state ==MLX5_DEVICE_STATE_INTERNAL_ERROR ) {/* make sure no soft wqe's are waiting */ if (unlikely (!list_empty (&cq->wc_list)))
@@ -654,6 +658,10 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)int ret =0 ;spin_lock_irqsave (&cq->lock, irq_flags); +if (mlx5_core_is_shutting_down (mdev)) { +spin_unlock_irqrestore (&cq->lock, irq_flags); +return 0 ; + }if (cq->notify_flags !=IB_CQ_NEXT_COMP ) cq->notify_flags = flags &IB_CQ_SOLICITED_MASK ;
@@ -661,6 +669,9 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) ret =1 ;spin_unlock_irqrestore (&cq->lock, irq_flags); +if (mlx5_core_is_shutting_down (mdev)) +return ret; + mlx5_cq_arm(&cq->mcq, (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
drivers/infiniband/hw/mlx5/main.c +20 -0
@@ -84,6 +84,13 @@ static LIST_HEAD(mlx5_ib_dev_list);*/ static DEFINE_MUTEX (mlx5_ib_multiport_mutex); +static void mlx5_ib_shutdown_quiesce (void ) +{ +flush_workqueue (ib_comp_wq); +flush_workqueue (ib_comp_unbound_wq); +flush_workqueue (mlx5_ib_event_wq); +} +struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi (struct mlx5_ib_multiport_info *mpi) {struct mlx5_ib_dev *dev;
@@ -2976,6 +2983,9 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)unsigned long flags_cq;unsigned long flags; +if (mlx5_core_is_shutting_down (ibdev->mdev)) +return ; +INIT_LIST_HEAD (&cq_armed_list);/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
@@ -3200,6 +3210,9 @@ static void mlx5_ib_handle_sys_error_event(struct work_struct *_work)struct mlx5_ib_dev *ibdev = work->dev;struct ib_event ibev; +if (mlx5_core_is_shutting_down (ibdev->mdev)) + goto out; + ibev.event =IB_EVENT_DEVICE_FATAL ;mlx5_ib_handle_internal_error (ibdev); ibev.element.port_num = (u8 )(unsigned long )work->param;
@@ -3222,10 +3235,15 @@ static int mlx5_ib_sys_error_event(struct notifier_block *nb, unsigned long event, void *param) { struct mlx5_ib_event_work *work; +struct mlx5_ib_dev *ibdev;if (event !=MLX5_DEV_EVENT_SYS_ERROR )return NOTIFY_DONE ; + ibdev = container_of(nb, struct mlx5_ib_dev, sys_error_events); +if (mlx5_core_is_shutting_down (ibdev->mdev)) +return NOTIFY_OK ; + work =kmalloc_obj (*work,GFP_ATOMIC );if (!work)return NOTIFY_DONE ;
@@ -5529,6 +5547,7 @@ static int __init mlx5_ib_init(void)if (ret) goto drv_err; + mlx5_rdma_shutdown_quiesce = mlx5_ib_shutdown_quiesce;return 0 ; drv_err:
@@ -5547,6 +5566,7 @@ static int __init mlx5_ib_init(void) static void __exit mlx5_ib_cleanup(void) { + mlx5_rdma_shutdown_quiesce =NULL ;mlx5_data_direct_driver_unregister ();auxiliary_driver_unregister (&mlx5r_driver);auxiliary_driver_unregister (&mlx5r_mp_driver);
drivers/net/ethernet/mellanox/mlx5/core/health.c +3 -0
@@ -202,6 +202,9 @@ static void enter_error_state(struct mlx5_core_dev *dev, bool force)mlx5_cmd_flush (dev); } +if (mlx5_core_is_shutting_down (dev)) +return ; +mlx5_notifier_call_chain (dev->priv.events,MLX5_DEV_EVENT_SYS_ERROR , (void *)1 ); }
drivers/net/ethernet/mellanox/mlx5/core/main.c +10 -0
@@ -2186,12 +2186,22 @@ static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)return 0 ; } +void (*mlx5_rdma_shutdown_quiesce)(void ); +EXPORT_SYMBOL_GPL (mlx5_rdma_shutdown_quiesce); +static void shutdown (struct pci_dev *pdev) {struct mlx5_core_dev *dev =pci_get_drvdata (pdev);int err;mlx5_core_info (dev,"Shutdown was called\n" ); +set_bit (MLX5_INTERFACE_STATE_SHUTTING_DOWN , &dev->intf_state); +/* +* Ensure in-flight CQ pollers observe SHUTTING_DOWN before +* fast_unload tears the device down. +*/ +if (mlx5_rdma_shutdown_quiesce) +mlx5_rdma_shutdown_quiesce ();set_bit (MLX5_BREAK_FW_WAIT , &dev->intf_state);mlx5_drain_fw_reset (dev);mlx5_drain_health_wq (dev);
drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c +3 -0
@@ -111,6 +111,9 @@ static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev) struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);struct mlx5_core_dev *mdev = sf_dev->mdev; +set_bit (MLX5_INTERFACE_STATE_SHUTTING_DOWN , &mdev->intf_state); +if (mlx5_rdma_shutdown_quiesce) +mlx5_rdma_shutdown_quiesce ();set_bit (MLX5_BREAK_FW_WAIT , &mdev->intf_state);mlx5_drain_health_wq (mdev);mlx5_unload_one (mdev, false);
include/linux/mlx5/driver.h +11 -0
@@ -653,8 +653,14 @@ enum mlx5_device_state {enum mlx5_interface_state {MLX5_INTERFACE_STATE_UP =BIT (0 ),MLX5_BREAK_FW_WAIT =BIT (1 ), +MLX5_INTERFACE_STATE_SHUTTING_DOWN =BIT (2 ), }; +/* Optional hook installed by mlx5_ib to flush CQ poll work after the +* SHUTTING_DOWN flag is set and before teardown continues. +*/ +extern void (*mlx5_rdma_shutdown_quiesce)(void ); +enum mlx5_pci_status {MLX5_PCI_STATUS_DISABLED ,MLX5_PCI_STATUS_ENABLED ,
@@ -1220,6 +1226,11 @@ static inline bool mlx5_core_is_pf(const struct mlx5_core_dev *dev)return dev->coredev_type ==MLX5_COREDEV_PF ; } +static inline bool mlx5_core_is_shutting_down (struct mlx5_core_dev *dev) +{ +return test_bit (MLX5_INTERFACE_STATE_SHUTTING_DOWN , &dev->intf_state); +} +static inline bool mlx5_core_is_vf (const struct mlx5_core_dev *dev) {return dev->coredev_type ==MLX5_COREDEV_VF ;
Signature · 1 line
2.25.1
Quoted 35 lines · Chenguang Zhao
From: Chenguang Zhao <[email protected]>
On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in
wr_cqe->done().Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip
SYS_ERRORcompletion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.Signed-off-by Chenguang Zhao <[email protected]>---
changelog:
- Fix the race onMLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp /mlx5_ibevent workqueues via anmlx5_ibquiesce hook before fast_unload/teardown.
- Check shutting-down undercq->lockin mlx5_ib_poll_cq/arm_cq.
- Exportib_comp_wqandib_comp_unbound_wqso modularmlx5_ibcan flush them.v2:
https://lore.kernel.org/all/[email protected]/v1:
https://lore.kernel.org/all/[email protected]/drivers/infiniband/core/device.c | 2 ++ drivers/infiniband/hw/mlx5/cq.c | 11 ++++++++++ drivers/infiniband/hw/mlx5/main.c | 20 +++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/health.c | 3 +++ .../net/ethernet/mellanox/mlx5/core/main.c | 10 ++++++++++ .../mellanox/mlx5/core/sf/dev/driver.c | 3 +++ include/linux/mlx5/driver.h | 11 ++++++++++ 7 files changed, 60 insertions(+)
<...>
drivers/infiniband/hw/mlx5/main.c static LIST_HEAD(mlx5_ib_dev_list); · line 92
Quoted 6 lines
drivers/infiniband/hw/mlx5/main.c +6
-0+static void mlx5_ib_shutdown_quiesce (void ) +{ +flush_workqueue (ib_comp_wq); +flush_workqueue (ib_comp_unbound_wq); +flush_workqueue (mlx5_ib_event_wq); +}
These workqueues are shared by all IB drivers and the core. Drivers must not flush or destroy them.
Why this is not FW issue?
Thanks
Hi, Leon
reboot -f skips orderly shutdown and goes directly to:
kernel_restart_prepare() -> device_shutdown() -> mlx5 shutdown Upper layers may still hold live CQs, while ib-comp-wq keeps polling — a use-after-free race.
Normal reboot usually works because userspace has already torn down RDMA and called ib_free_cq().
Thanks
Quoted 50 lines · Leon Romanovsky
Quoted 35 lines · Chenguang Zhao
From: Chenguang Zhao <[email protected]>
On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in
wr_cqe->done().Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip
SYS_ERRORcompletion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.Signed-off-by Chenguang Zhao <[email protected]>---
changelog:
- Fix the race onMLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp /mlx5_ibevent workqueues via anmlx5_ibquiesce hook before fast_unload/teardown.
- Check shutting-down undercq->lockin mlx5_ib_poll_cq/arm_cq.
- Exportib_comp_wqandib_comp_unbound_wqso modularmlx5_ibcan flush them.v2:
https://lore.kernel.org/all/[email protected]/v1:
https://lore.kernel.org/all/[email protected]/drivers/infiniband/core/device.c | 2 ++ drivers/infiniband/hw/mlx5/cq.c | 11 ++++++++++ drivers/infiniband/hw/mlx5/main.c | 20 +++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/health.c | 3 +++ .../net/ethernet/mellanox/mlx5/core/main.c | 10 ++++++++++ .../mellanox/mlx5/core/sf/dev/driver.c | 3 +++ include/linux/mlx5/driver.h | 11 ++++++++++ 7 files changed, 60 insertions(+)<...>
drivers/infiniband/hw/mlx5/main.c static LIST_HEAD(mlx5_ib_dev_list); · line 92
Quoted 6 lines
drivers/infiniband/hw/mlx5/main.c +6
-0+static void mlx5_ib_shutdown_quiesce (void ) +{ +flush_workqueue (ib_comp_wq); +flush_workqueue (ib_comp_unbound_wq); +flush_workqueue (mlx5_ib_event_wq); +}These workqueues are shared by all IB drivers and the core. Drivers must not flush or destroy them.
Why this is not FW issue?
Thanks
Quoted 6 lines · Chenguang Zhao
Hi, Leon
reboot -f skips orderly shutdown and goes directly to:
kernel_restart_prepare()->device_shutdown()-> mlx5 shutdown Upper layers may still hold live CQs, while ib-comp-wq keeps polling — a use-after-free race.
The point is that this flow is neither RDMA- nor mlx5-specific, and it works as expected. mlx5 shutdown() stops the FW/HW, while the kernel stops and tears down the running threads.
Quoted 3 lines
Normal reboot usually works because userspace has already torn down RDMA and called
ib_free_cq().
There is no difference, from the kernel's perspective, between a normal and a forced reboot, except that the former bypasses userspace shutdown.
Thanks
Quoted 54 lines
Thanks
Quoted 50 lines · Leon Romanovsky
Quoted 35 lines · Chenguang Zhao
From: Chenguang Zhao <[email protected]>
On reboot -f with NFS over RDMA, mlx5 shutdown can tear the device down while ib-comp-wq still polls live CQs, leading to UAF in
wr_cqe->done().Mark the device shutting down before teardown, flush completion workqueues so in-flight pollers observe the flag, skip
SYS_ERRORcompletion delivery, and make poll/arm CQ a no-op under the CQ lock while shutting down.Signed-off-by Chenguang Zhao <[email protected]>---
changelog:
- Fix the race onMLX5_INTERFACE_STATE_SHUTTING_DOWN: set the flag, then flush ib-comp /mlx5_ibevent workqueues via anmlx5_ibquiesce hook before fast_unload/teardown.
- Check shutting-down undercq->lockin mlx5_ib_poll_cq/arm_cq.
- Exportib_comp_wqandib_comp_unbound_wqso modularmlx5_ibcan flush them.v2:
https://lore.kernel.org/all/[email protected]/v1:
https://lore.kernel.org/all/[email protected]/drivers/infiniband/core/device.c | 2 ++ drivers/infiniband/hw/mlx5/cq.c | 11 ++++++++++ drivers/infiniband/hw/mlx5/main.c | 20 +++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/health.c | 3 +++ .../net/ethernet/mellanox/mlx5/core/main.c | 10 ++++++++++ .../mellanox/mlx5/core/sf/dev/driver.c | 3 +++ include/linux/mlx5/driver.h | 11 ++++++++++ 7 files changed, 60 insertions(+)<...>
drivers/infiniband/hw/mlx5/main.c static LIST_HEAD(mlx5_ib_dev_list); · line 92
Quoted 6 lines
drivers/infiniband/hw/mlx5/main.c +6
-0+static void mlx5_ib_shutdown_quiesce (void ) +{ +flush_workqueue (ib_comp_wq); +flush_workqueue (ib_comp_unbound_wq); +flush_workqueue (mlx5_ib_event_wq); +}These workqueues are shared by all IB drivers and the core. Drivers must not flush or destroy them.
Why this is not FW issue?
Thanks
Quoted 11 lines · Leon Romanovsky
Quoted 6 lines · Chenguang Zhao
Hi, Leon
reboot -f skips orderly shutdown and goes directly to:
kernel_restart_prepare()->device_shutdown()-> mlx5 shutdown Upper layers may still hold live CQs, while ib-comp-wq keeps polling — a use-after-free race.The point is that this flow is neither RDMA- nor mlx5-specific, and it works as expected. mlx5
shutdown()stops the FW/HW, while the kernel stops and tears down the running threads.
I'm pretty sure we are missing stuff in the driver to synchronize everything while it is doing a tear down or health recovery.
Ripping a driver out from under a still-running subsystem is hard, this does seem like a real bug, but I also don't think the shutdown handler is the right place to fix it..
What was the actual UAF scenario, we should take a look at that more directly?
Jason