Skip to contentSkip to conversation

devlink: disallow reload operation during device cleanup

PATCH In review Jiri Pirko 2 participants 2 messages Nov 9, 2019 – Nov 10, 2019

Subsystems (6)
lore
Share & tools
lore

Jiri Pirko

author maintainer
#0 lore raw

PATCH 1/1 devlink: disallow reload operation during device cleanup

From: Jiri Pirko <[email protected]>

There is a race between driver code that does setup/cleanup of device and devlink reload operation that in some drivers works with the same code. Use after free could we easily obtained by running:

while true; do

        echo "0000:00:10.0" >/sys/bus/pci/drivers/mlxsw_spectrum2/bind
        devlink dev reload pci/0000:00:10.0 &
        echo "0000:00:10.0" >/sys/bus/pci/drivers/mlxsw_spectrum2/unbind

done

Fix this by enabling reload only after setup of device is complete and disabling it at the beginning of the cleanup process.

Fixes 2d8dc5bbf4e7 ("devlink: Add support for reload")

---
This is -net version of fix, net-next fix was sent and is already applied. Also note that unlike net-next, in -net this is not reproducible with netdevsim, so the reproducer uses mlxsw driver instead. That is the only difference in the patch desctiption.

---
 drivers/net/ethernet/mellanox/mlx4/main.c  |  3 ++
 drivers/net/ethernet/mellanox/mlxsw/core.c |  6 +++-
 drivers/net/netdevsim/dev.c                |  2 ++
 include/net/devlink.h                      |  5 ++-
 net/core/devlink.c                         | 39 +++++++++++++++++++++-
 5 files changed, 52 insertions(+), 3 deletions(-)
drivers/net/ethernet/mellanox/mlx4/main.c +3 -0
index 69bb6bb06e76..d44ac666e730 100644
@@ -4010,6 +4010,7 @@ static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_params_unregister;
 
 	devlink_params_publish(devlink);
+	devlink_reload_enable(devlink);
 	pci_save_state(pdev);
 	return 0;
 
@@ -4121,6 +4122,8 @@ static void mlx4_remove_one(struct pci_dev *pdev)
 	struct devlink *devlink = priv_to_devlink(priv);
 	int active_vfs = 0;
 
+	devlink_reload_disable(devlink);
+
 	if (mlx4_is_slave(dev))
 		persist->interface_state |= MLX4_INTERFACE_STATE_NOWAIT;
 
drivers/net/ethernet/mellanox/mlxsw/core.c +5 -1
index 4421ab22182f..20e9dc46cacd 100644
@@ -1186,8 +1186,10 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	if (err)
 		goto err_thermal_init;
 
-	if (mlxsw_driver->params_register)
+	if (mlxsw_driver->params_register) {
 		devlink_params_publish(devlink);
+		devlink_reload_enable(devlink);
+	}
 
 	return 0;
 
@@ -1249,6 +1251,8 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
 
+	if (!reload)
+		devlink_reload_disable(devlink);
 	if (devlink_is_reload_failed(devlink)) {
 		if (!reload)
 			/* Only the parts that were not de-initialized in the
drivers/net/netdevsim/dev.c +2 -0
index 54ca6681ba31..44c2d857a7fa 100644
@@ -708,6 +708,7 @@ nsim_dev_create(struct nsim_bus_dev *nsim_bus_dev, unsigned int port_count)
 		goto err_debugfs_exit;
 
 	devlink_params_publish(devlink);
+	devlink_reload_enable(devlink);
 	return nsim_dev;
 
 err_debugfs_exit:
@@ -732,6 +733,7 @@ static void nsim_dev_destroy(struct nsim_dev *nsim_dev)
 {
 	struct devlink *devlink = priv_to_devlink(nsim_dev);
 
+	devlink_reload_disable(devlink);
 	nsim_bpf_dev_exit(nsim_dev);
 	nsim_dev_debugfs_exit(nsim_dev);
 	nsim_dev_traps_exit(devlink);
include/net/devlink.h +4 -1
index 23e4b65ec9df..2116c88663a1 100644
@@ -38,7 +38,8 @@ struct devlink {
 	struct device *dev;
 	possible_net_t _net;
 	struct mutex lock;
-	bool reload_failed;
+	u8 reload_failed:1,
+	   reload_enabled:1;
 	char priv[0] __aligned(NETDEV_ALIGN);
 };
 
@@ -774,6 +775,8 @@ struct ib_device;
 struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size);
 int devlink_register(struct devlink *devlink, struct device *dev);
 void devlink_unregister(struct devlink *devlink);
+void devlink_reload_enable(struct devlink *devlink);
+void devlink_reload_disable(struct devlink *devlink);
 void devlink_free(struct devlink *devlink);
 int devlink_port_register(struct devlink *devlink,
 			  struct devlink_port *devlink_port,
net/core/devlink.c +38 -1
index f80151eeaf51..7d64660a72fc 100644
@@ -2699,7 +2699,7 @@ static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
 	struct devlink *devlink = info->user_ptr[0];
 	int err;
 
-	if (!devlink_reload_supported(devlink))
+	if (!devlink_reload_supported(devlink) || !devlink->reload_enabled)
 		return -EOPNOTSUPP;
 
 	err = devlink_resources_validate(devlink, NULL, info);
@@ -6196,12 +6196,49 @@ EXPORT_SYMBOL_GPL(devlink_register);
 void devlink_unregister(struct devlink *devlink)
 {
 	mutex_lock(&devlink_mutex);
+	WARN_ON(devlink_reload_supported(devlink) &&
+		devlink->reload_enabled);
 	devlink_notify(devlink, DEVLINK_CMD_DEL);
 	list_del(&devlink->list);
 	mutex_unlock(&devlink_mutex);
 }
 EXPORT_SYMBOL_GPL(devlink_unregister);
 
+/**
+ *	devlink_reload_enable - Enable reload of devlink instance
+ *
+ *	@devlink: devlink
+ *
+ *	Should be called at end of device initialization
+ *	process when reload operation is supported.
+ */
+void devlink_reload_enable(struct devlink *devlink)
+{
+	mutex_lock(&devlink_mutex);
+	devlink->reload_enabled = true;
+	mutex_unlock(&devlink_mutex);
+}
+EXPORT_SYMBOL_GPL(devlink_reload_enable);
+
+/**
+ *	devlink_reload_disable - Disable reload of devlink instance
+ *
+ *	@devlink: devlink
+ *
+ *	Should be called at the beginning of device cleanup
+ *	process when reload operation is supported.
+ */
+void devlink_reload_disable(struct devlink *devlink)
+{
+	mutex_lock(&devlink_mutex);
+	/* Mutex is taken which ensures that no reload operation is in
+	 * progress while setting up forbidded flag.
+	 */
+	devlink->reload_enabled = false;
+	mutex_unlock(&devlink_mutex);
+}
+EXPORT_SYMBOL_GPL(devlink_reload_disable);
+
 /**
  *	devlink_free - Free devlink instance resources
  *
Signature · 1 line
2.21.0
maintainer
#1 lore raw

From: Jiri Pirko <[email protected]>
Date: Sat, 9 Nov 2019 11:29:46 +0100

Quoted 15 lines

From: Jiri Pirko <[email protected]>

There is a race between driver code that does setup/cleanup of device and devlink reload operation that in some drivers works with the same code. Use after free could we easily obtained by running:

while true; do

        echo "0000:00:10.0" >/sys/bus/pci/drivers/mlxsw_spectrum2/bind
        devlink dev reload pci/0000:00:10.0 &
        echo "0000:00:10.0" >/sys/bus/pci/drivers/mlxsw_spectrum2/unbind

done

Fix this by enabling reload only after setup of device is complete and disabling it at the beginning of the cleanup process.

Quoted 8 lines
Fixes 2d8dc5bbf4e7 ("devlink: Add support for reload")

---
This is -net version of fix, net-next fix was sent and is already applied. Also note that unlike net-next, in -net this is not reproducible with netdevsim, so the reproducer uses mlxsw driver instead. That is the only difference in the patch desctiption.

Applied and queued up for -stable.