Skip to contentSkip to conversation

block: loop: fix deadlock between open and remove

PATCH Applied Christoph Hellwig 4 participants 4 messages Jun 5, 2021 – Jun 11, 2021 Block layer

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 block: loop: fix deadlock between open and remove

Commit c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk") adds disk->part0->bd_mutex in del_gendisk(), this way causes the following AB/BA deadlock between removing loop and opening loop:

1) loop_control_ioctl(LOOP_CTL_REMOVE)

     -> mutex_lock(&loop_ctl_mutex)
     -> del_gendisk
         -> mutex_lock(&disk->part0->bd_mutex)

2) blkdev_get_by_dev

     -> mutex_lock(&disk->part0->bd_mutex)
     -> lo_open
         -> mutex_lock(&loop_ctl_mutex)

Add a new Lo_deleting state to remove the need for clearing ->private_data and thus holding loop_ctl_mutex in the ioctl LOOP_CTL_REMOVE path.

Based on an analysis and earlier patch from
Ming Lei <[email protected]>.

Fixes c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk")
---
 drivers/block/loop.c | 25 +++++++------------------
 drivers/block/loop.h |  1 +
 2 files changed, 8 insertions(+), 18 deletions(-)
drivers/block/loop.c +7 -18
index d58d68f3c7cd..76e12f3482a9 100644
@@ -1879,29 +1879,18 @@ static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
 
 static int lo_open(struct block_device *bdev, fmode_t mode)
 {
-	struct loop_device *lo;
+	struct loop_device *lo = bdev->bd_disk->private_data;
 	int err;
 
-	/*
-	 * take loop_ctl_mutex to protect lo pointer from race with
-	 * loop_control_ioctl(LOOP_CTL_REMOVE), however, to reduce contention
-	 * release it prior to updating lo->lo_refcnt.
-	 */
-	err = mutex_lock_killable(&loop_ctl_mutex);
-	if (err)
-		return err;
-	lo = bdev->bd_disk->private_data;
-	if (!lo) {
-		mutex_unlock(&loop_ctl_mutex);
-		return -ENXIO;
-	}
 	err = mutex_lock_killable(&lo->lo_mutex);
-	mutex_unlock(&loop_ctl_mutex);
 	if (err)
 		return err;
-	atomic_inc(&lo->lo_refcnt);
+	if (lo->lo_state == Lo_deleting)
+		err = -ENXIO;
+	else
+		atomic_inc(&lo->lo_refcnt);
 	mutex_unlock(&lo->lo_mutex);
-	return 0;
+	return err;
 }
 
 static void lo_release(struct gendisk *disk, fmode_t mode)
@@ -2285,7 +2274,7 @@ static long loop_control_ioctl(struct file *file, unsigned int cmd,
 			mutex_unlock(&lo->lo_mutex);
 			break;
 		}
-		lo->lo_disk->private_data = NULL;
+		lo->lo_state = Lo_deleting;
 		mutex_unlock(&lo->lo_mutex);
 		idr_remove(&loop_index_idr, lo->lo_number);
 		loop_remove(lo);
drivers/block/loop.h +1 -0
index a3c04f310672..5beb959b94d3 100644
@@ -22,6 +22,7 @@ enum {
 	Lo_unbound,
 	Lo_bound,
 	Lo_rundown,
+	Lo_deleting,
 };
 
 struct loop_func_table;
Signature · 1 line
2.30.2
Tested-by on v1 1/1
#1 lore raw
Quoted 85 lines · Christoph Hellwig

Commit c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk") adds disk->part0->bd_mutex in del_gendisk(), this way causes the following AB/BA deadlock between removing loop and opening loop:

1) loop_control_ioctl(LOOP_CTL_REMOVE)

     -> mutex_lock(&loop_ctl_mutex)
     -> del_gendisk
         -> mutex_lock(&disk->part0->bd_mutex)

2) blkdev_get_by_dev

     -> mutex_lock(&disk->part0->bd_mutex)
     -> lo_open
         -> mutex_lock(&loop_ctl_mutex)

Add a new Lo_deleting state to remove the need for clearing ->private_data and thus holding loop_ctl_mutex in the ioctl LOOP_CTL_REMOVE path.

Based on an analysis and earlier patch from
Ming Lei <[email protected]>.

Fixes c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk")
---
 drivers/block/loop.c | 25 +++++++------------------
 drivers/block/loop.h |  1 +
 2 files changed, 8 insertions(+), 18 deletions(-)
drivers/block/loop.c +7 -18
index d58d68f3c7cd..76e12f3482a9 100644
@@ -1879,29 +1879,18 @@ static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
 
 static int lo_open(struct block_device *bdev, fmode_t mode)
 {
-	struct loop_device *lo;
+	struct loop_device *lo = bdev->bd_disk->private_data;
 	int err;
 
-	/*
-	 * take loop_ctl_mutex to protect lo pointer from race with
-	 * loop_control_ioctl(LOOP_CTL_REMOVE), however, to reduce contention
-	 * release it prior to updating lo->lo_refcnt.
-	 */
-	err = mutex_lock_killable(&loop_ctl_mutex);
-	if (err)
-		return err;
-	lo = bdev->bd_disk->private_data;
-	if (!lo) {
-		mutex_unlock(&loop_ctl_mutex);
-		return -ENXIO;
-	}
 	err = mutex_lock_killable(&lo->lo_mutex);
-	mutex_unlock(&loop_ctl_mutex);
 	if (err)
 		return err;
-	atomic_inc(&lo->lo_refcnt);
+	if (lo->lo_state == Lo_deleting)
+		err = -ENXIO;
+	else
+		atomic_inc(&lo->lo_refcnt);
 	mutex_unlock(&lo->lo_mutex);
-	return 0;
+	return err;
 }
 
 static void lo_release(struct gendisk *disk, fmode_t mode)
@@ -2285,7 +2274,7 @@ static long loop_control_ioctl(struct file *file, unsigned int cmd,
 			mutex_unlock(&lo->lo_mutex);
 			break;
 		}
-		lo->lo_disk->private_data = NULL;
+		lo->lo_state = Lo_deleting;
 		mutex_unlock(&lo->lo_mutex);
 		idr_remove(&loop_index_idr, lo->lo_number);
 		loop_remove(lo);
drivers/block/loop.h +0 -0
index a3c04f310672..5beb959b94d3 100644

drivers/block/loop.h enum { · line 28

Quoted 7 lines
drivers/block/loop.h +1 -0
@@ -22,6 +22,7 @@ enum {
 	Lo_unbound,
 	Lo_bound,
 	Lo_rundown,
+	Lo_deleting,
 };
 
 struct loop_func_table;

Tested and no longer hangs with the stress-ng loop test. Thank you for the speedy turnaround on the fix.

Colin

Reviewed-by on v1 1/1
#2 lore raw
In reply to Christoph Hellwig
Quoted 19 lines · Christoph Hellwig

Commit c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk") adds disk->part0->bd_mutex in del_gendisk(), this way causes the following AB/BA deadlock between removing loop and opening loop:

1) loop_control_ioctl(LOOP_CTL_REMOVE)

     -> mutex_lock(&loop_ctl_mutex)
     -> del_gendisk
         -> mutex_lock(&disk->part0->bd_mutex)

2) blkdev_get_by_dev

     -> mutex_lock(&disk->part0->bd_mutex)
     -> lo_open
         -> mutex_lock(&loop_ctl_mutex)

Add a new Lo_deleting state to remove the need for clearing ->private_data and thus holding loop_ctl_mutex in the ioctl LOOP_CTL_REMOVE path.

Quoted 6 lines

Based on an analysis and earlier patch from
Ming Lei <[email protected]>.

Fixes c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk")
Signature · 1 line
Ming
maintainer Applied
#3 lore raw
In reply to Christoph Hellwig
Quoted 15 lines · Christoph Hellwig

Commit c76f48eb5c08 ("block: take bd_mutex around delete_partitions in del_gendisk") adds disk->part0->bd_mutex in del_gendisk(), this way causes the following AB/BA deadlock between removing loop and opening loop:

1) loop_control_ioctl(LOOP_CTL_REMOVE)

     -> mutex_lock(&loop_ctl_mutex)
     -> del_gendisk
         -> mutex_lock(&disk->part0->bd_mutex)

2) blkdev_get_by_dev

     -> mutex_lock(&disk->part0->bd_mutex)
     -> lo_open
         -> mutex_lock(&loop_ctl_mutex)
Quoted 6 lines

Add a new Lo_deleting state to remove the need for clearing ->private_data and thus holding loop_ctl_mutex in the ioctl LOOP_CTL_REMOVE path.

Based on an analysis and earlier patch from
Ming Lei <[email protected]>.

Applied, thanks.

Signature · 1 line
Jens Axboe