Skip to contentSkip to conversation

virtio-blk: handle block_device_operations callbacks after hot unplug

PATCH Superseded Stefan Hajnoczi 2 participants 4 messages Apr 28, 2020

Subsystems (3)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Stefan Hajnoczi

author reviewer
#0 lore raw

PATCH v2 1/1 virtio-blk: handle block_device_operations callbacks after hot unplug

A userspace process holding a file descriptor to a virtio_blk device can still invoke block_device_operations after hot unplug. For example, a program that has /dev/vdb open can call ioctl(HDIO_GETGEO) after hot unplug to invoke virtblk_getgeo().

Introduce a reference count in struct virtio_blk so that its lifetime covers both virtio_driver probe/remove and block_device_operations open/release users. This ensures that block_device_operations functions like virtblk_getgeo() can safely access struct virtio_blk.

Add remove_mutex to prevent block_device_operations functions from accessing vblk->vdev during virtblk_remove() and let the safely check for !vblk->vdev after virtblk_remove() returns.

Switching to a reference count also solves the vd_index_ida leak where vda, vdb, vdc, etc indices were lost when the device was hot unplugged while the block device was still open.

---
If someone has a simpler solution please let me know. I looked at various approaches including reusing device_lock(&vblk->vdev.dev) but they were more complex and extending the lifetime of virtio_device after remove() has been called seems questionable.

---
 drivers/block/virtio_blk.c | 85 ++++++++++++++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 8 deletions(-)
drivers/block/virtio_blk.c +77 -8
index 93468b7c6701..3dd53b445cc1 100644
@@ -44,6 +44,13 @@ struct virtio_blk {
 	/* Process context for config space updates */
 	struct work_struct config_work;
 
+	/*
+	 * Tracks references from block_device_operations open/release and
+	 * virtio_driver probe/remove so this object can be freed once no
+	 * longer in use.
+	 */
+	refcount_t refs;
+
 	/* What host tells us, plus 2 for header & tailer. */
 	unsigned int sg_elems;
 
@@ -53,6 +60,9 @@ struct virtio_blk {
 	/* num of vqs */
 	int num_vqs;
 	struct virtio_blk_vq *vqs;
+
+	/* Provides mutual exclusion with virtblk_remove(). */
+	struct mutex remove_mutex;
 };
 
 struct virtblk_req {
@@ -295,10 +305,54 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str)
 	return err;
 }
 
+static void virtblk_get(struct virtio_blk *vblk)
+{
+	refcount_inc(&vblk->refs);
+}
+
+static void virtblk_put(struct virtio_blk *vblk)
+{
+	if (refcount_dec_and_test(&vblk->refs)) {
+		ida_simple_remove(&vd_index_ida, vblk->index);
+		mutex_destroy(&vblk->remove_mutex);
+		kfree(vblk);
+	}
+}
+
+static int virtblk_open(struct block_device *bd, fmode_t mode)
+{
+	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (vblk->vdev) {
+		virtblk_get(vblk);
+		ret = 0;
+	}
+
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
+}
+
+static void virtblk_release(struct gendisk *disk, fmode_t mode)
+{
+	struct virtio_blk *vblk = disk->private_data;
+
+	virtblk_put(vblk);
+}
+
 /* We provide getgeo only to please some old bootloader/partitioning tools */
 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 {
 	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (!vblk->vdev) {
+		goto out;
+	}
 
 	/* see if the host passed in geometry config */
 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
@@ -314,11 +368,17 @@ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 		geo->sectors = 1 << 5;
 		geo->cylinders = get_capacity(bd->bd_disk) >> 11;
 	}
-	return 0;
+
+	ret = 0;
+out:
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
 }
 
 static const struct block_device_operations virtblk_fops = {
 	.owner  = THIS_MODULE,
+	.open = virtblk_open,
+	.release = virtblk_release,
 	.getgeo = virtblk_getgeo,
 };
 
@@ -655,6 +715,10 @@ static int virtblk_probe(struct virtio_device *vdev)
 		goto out_free_index;
 	}
 
+	/* This reference is dropped in virtblk_remove(). */
+	refcount_set(&vblk->refs, 1);
+	mutex_init(&vblk->remove_mutex);
+
 	vblk->vdev = vdev;
 	vblk->sg_elems = sg_elems;
 
@@ -820,8 +884,12 @@ static int virtblk_probe(struct virtio_device *vdev)
 static void virtblk_remove(struct virtio_device *vdev)
 {
 	struct virtio_blk *vblk = vdev->priv;
-	int index = vblk->index;
-	int refc;
+
+	/*
+	 * Virtqueue processing is stopped safely here but mutual exclusion is
+	 * needed for block_device_operations.
+	 */
+	mutex_lock(&vblk->remove_mutex);
 
 	/* Make sure no work handler is accessing the device. */
 	flush_work(&vblk->config_work);
@@ -834,15 +902,16 @@ static void virtblk_remove(struct virtio_device *vdev)
 	/* Stop all the virtqueues. */
 	vdev->config->reset(vdev);
 
-	refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref);
+	/* Virtqueue are stopped, nothing can use vblk->vdev anymore. */
+	vblk->vdev = NULL;
+
 	put_disk(vblk->disk);
 	vdev->config->del_vqs(vdev);
 	kfree(vblk->vqs);
-	kfree(vblk);
 
-	/* Only free device id if we don't have any users */
-	if (refc == 1)
-		ida_simple_remove(&vd_index_ida, index);
+	mutex_unlock(&vblk->remove_mutex);
+
+	virtblk_put(vblk);
 }
 
 #ifdef CONFIG_PM_SLEEP
Signature · 1 line
2.25.3
#1 lore raw
Quoted 4 lines · Stefan Hajnoczi

A userspace process holding a file descriptor to a virtio_blk device can still invoke block_device_operations after hot unplug. For example, a program that has /dev/vdb open can call ioctl(HDIO_GETGEO) after hot unplug to invoke virtblk_getgeo().

which causes what? a use after free?

Quoted 8 lines

Introduce a reference count in struct virtio_blk so that its lifetime covers both virtio_driver probe/remove and block_device_operations open/release users. This ensures that block_device_operations functions like virtblk_getgeo() can safely access struct virtio_blk.

Add remove_mutex to prevent block_device_operations functions from accessing vblk->vdev during virtblk_remove() and let the safely check

let the -> let them?

Quoted 5 lines

for !vblk->vdev after virtblk_remove() returns.

Switching to a reference count also solves the vd_index_ida leak where vda, vdb, vdc, etc indices were lost when the device was hot unplugged while the block device was still open.

Can you move this statement up so we list both issues (use after free and leak) upfront, then discuss the fix?

Quoted 31 lines

---
If someone has a simpler solution please let me know. I looked at various approaches including reusing device_lock(&vblk->vdev.dev) but they were more complex and extending the lifetime of virtio_device after remove() has been called seems questionable.

---
 drivers/block/virtio_blk.c | 85 ++++++++++++++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 8 deletions(-)
drivers/block/virtio_blk.c +7 -0
index 93468b7c6701..3dd53b445cc1 100644
@@ -44,6 +44,13 @@ struct virtio_blk {
 	/* Process context for config space updates */
 	struct work_struct config_work;
 
+	/*
+	 * Tracks references from block_device_operations open/release and
+	 * virtio_driver probe/remove so this object can be freed once no
+	 * longer in use.
+	 */
+	refcount_t refs;
+
 	/* What host tells us, plus 2 for header & tailer. */
 	unsigned int sg_elems;
 

drivers/block/virtio_blk.c struct virtio_blk { · line 64

Quoted 5 lines
drivers/block/virtio_blk.c +2 -0
@@ -53,6 +60,9 @@ struct virtio_blk {
 	/* num of vqs */
 	int num_vqs;
 	struct virtio_blk_vq *vqs;
+
+	/* Provides mutual exclusion with virtblk_remove(). */

This is not the best way to document access rules. Which fields does this protect, exactly?
I think it's just vdev. Right?
Pls add to the comment.

Quoted 16 lines
drivers/block/virtio_blk.c +9 -0
+	struct mutex remove_mutex;
 };
 
 struct virtblk_req {
@@ -295,10 +305,54 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str)
 	return err;
 }
 
+static void virtblk_get(struct virtio_blk *vblk)
+{
+	refcount_inc(&vblk->refs);
+}
+
+static void virtblk_put(struct virtio_blk *vblk)
+{
+	if (refcount_dec_and_test(&vblk->refs)) {

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 325

Quoted 10 lines
drivers/block/virtio_blk.c +10 -0
+		ida_simple_remove(&vd_index_ida, vblk->index);
+		mutex_destroy(&vblk->remove_mutex);
+		kfree(vblk);
+	}
+}
+
+static int virtblk_open(struct block_device *bd, fmode_t mode)
+{
+	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;

It's more common to do

	int ret = 0;

and on error:

	ret = -ENXIO;

let's do this.

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 332

Quoted 7 lines
drivers/block/virtio_blk.c +7 -0
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (vblk->vdev) {
+		virtblk_get(vblk);
+		ret = 0;
+	}

I prefer
else
ret = -ENXIO

here.

Quoted 7 lines
drivers/block/virtio_blk.c +7 -0
+
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
+}
+
+static void virtblk_release(struct gendisk *disk, fmode_t mode)
+{

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 349

Quoted 10 lines
drivers/block/virtio_blk.c +6 -0
+	struct virtio_blk *vblk = disk->private_data;
+
+	virtblk_put(vblk);
+}
+
 /* We provide getgeo only to please some old bootloader/partitioning tools */
 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 {
 	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;

It's more common to do

	int ret = 0;

and on error:

	ret = -ENXIO;

let's do this.

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 355

Quoted 6 lines
drivers/block/virtio_blk.c +6 -0
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (!vblk->vdev) {
+		goto out;
+	}

single lines are not supposed to use {}.
if you add ret = -ENXIO here then it won't be a single line anymore though.

Quoted 49 lines
drivers/block/virtio_blk.c +17 -3
 	/* see if the host passed in geometry config */
 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
@@ -314,11 +368,17 @@ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 		geo->sectors = 1 << 5;
 		geo->cylinders = get_capacity(bd->bd_disk) >> 11;
 	}
-	return 0;
+
+	ret = 0;
+out:
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
 }
 
 static const struct block_device_operations virtblk_fops = {
 	.owner  = THIS_MODULE,
+	.open = virtblk_open,
+	.release = virtblk_release,
 	.getgeo = virtblk_getgeo,
 };
 
@@ -655,6 +715,10 @@ static int virtblk_probe(struct virtio_device *vdev)
 		goto out_free_index;
 	}
 
+	/* This reference is dropped in virtblk_remove(). */
+	refcount_set(&vblk->refs, 1);
+	mutex_init(&vblk->remove_mutex);
+
 	vblk->vdev = vdev;
 	vblk->sg_elems = sg_elems;
 
@@ -820,8 +884,12 @@ static int virtblk_probe(struct virtio_device *vdev)
 static void virtblk_remove(struct virtio_device *vdev)
 {
 	struct virtio_blk *vblk = vdev->priv;
-	int index = vblk->index;
-	int refc;
+
+	/*
+	 * Virtqueue processing is stopped safely here but mutual exclusion is
+	 * needed for block_device_operations.
+	 */
+	mutex_lock(&vblk->remove_mutex);
 
 	/* Make sure no work handler is accessing the device. */
 	flush_work(&vblk->config_work);

drivers/block/virtio_blk.c static void virtblk_remove(struct virtio_device *vdev) · line 905

Quoted 5 lines
drivers/block/virtio_blk.c +1 -1
@@ -834,15 +902,16 @@ static void virtblk_remove(struct virtio_device *vdev)
 	/* Stop all the virtqueues. */
 	vdev->config->reset(vdev);
 
-	refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref);
+	/* Virtqueue are stopped, nothing can use vblk->vdev anymore. */

Virtqueues?

Quoted 19 lines
drivers/block/virtio_blk.c +5 -4
+	vblk->vdev = NULL;
+
 	put_disk(vblk->disk);
 	vdev->config->del_vqs(vdev);
 	kfree(vblk->vqs);
-	kfree(vblk);
 
-	/* Only free device id if we don't have any users */
-	if (refc == 1)
-		ida_simple_remove(&vd_index_ida, index);
+	mutex_unlock(&vblk->remove_mutex);
+
+	virtblk_put(vblk);
 }
 
 #ifdef CONFIG_PM_SLEEP
Signature · 1 line
2.25.3
author reviewer
#2 lore raw
Quoted 8 lines · Michael S. Tsirkin
Quoted 4 lines · Stefan Hajnoczi

A userspace process holding a file descriptor to a virtio_blk device can still invoke block_device_operations after hot unplug. For example, a program that has /dev/vdb open can call ioctl(HDIO_GETGEO) after hot unplug to invoke virtblk_getgeo().

which causes what? a use after free?

Yes, use after free. I will include the kernel panic in the next revision.

virtio_check_driver_offered_feature() accesses vdev->dev.driver, which doesn't work after virtblk_remove() has freed vdev :).

Quoted 10 lines
Quoted 8 lines

Introduce a reference count in struct virtio_blk so that its lifetime covers both virtio_driver probe/remove and block_device_operations open/release users. This ensures that block_device_operations functions like virtblk_getgeo() can safely access struct virtio_blk.

Add remove_mutex to prevent block_device_operations functions from accessing vblk->vdev during virtblk_remove() and let the safely check

let the -> let them?

Thanks, will fix.

Quoted 9 lines
Quoted 6 lines

for !vblk->vdev after virtblk_remove() returns.

Switching to a reference count also solves the vd_index_ida leak where vda, vdb, vdc, etc indices were lost when the device was hot unplugged while the block device was still open.

Can you move this statement up so we list both issues (use after free and leak) upfront, then discuss the fix?

Thanks, will fix.

Quoted 31 lines
Quoted 31 lines

---
If someone has a simpler solution please let me know. I looked at various approaches including reusing device_lock(&vblk->vdev.dev) but they were more complex and extending the lifetime of virtio_device after remove() has been called seems questionable.

---
 drivers/block/virtio_blk.c | 85 ++++++++++++++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 8 deletions(-)
drivers/block/virtio_blk.c +7 -0
index 93468b7c6701..3dd53b445cc1 100644
@@ -44,6 +44,13 @@ struct virtio_blk {
 	/* Process context for config space updates */
 	struct work_struct config_work;
 
+	/*
+	 * Tracks references from block_device_operations open/release and
+	 * virtio_driver probe/remove so this object can be freed once no
+	 * longer in use.
+	 */
+	refcount_t refs;
+
 	/* What host tells us, plus 2 for header & tailer. */
 	unsigned int sg_elems;
 
Quoted 11 lines

drivers/block/virtio_blk.c struct virtio_blk { · line 64

Quoted 6 lines
drivers/block/virtio_blk.c +2 -0
@@ -53,6 +60,9 @@ struct virtio_blk {
 	/* num of vqs */
 	int num_vqs;
 	struct virtio_blk_vq *vqs;
+
+	/* Provides mutual exclusion with virtblk_remove(). */

This is not the best way to document access rules. Which fields does this protect, exactly?
I think it's just vdev. Right?
Pls add to the comment.

Yes, vblk->vdev cannot be dereferenced after virtblk_remove() is entered.

I'll document this mutex as protecting vdev.

Quoted 31 lines
Quoted 17 lines
drivers/block/virtio_blk.c +9 -0
+	struct mutex remove_mutex;
 };
 
 struct virtblk_req {
@@ -295,10 +305,54 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str)
 	return err;
 }
 
+static void virtblk_get(struct virtio_blk *vblk)
+{
+	refcount_inc(&vblk->refs);
+}
+
+static void virtblk_put(struct virtio_blk *vblk)
+{
+	if (refcount_dec_and_test(&vblk->refs)) {

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 325

Quoted 10 lines
drivers/block/virtio_blk.c +10 -0
+		ida_simple_remove(&vd_index_ida, vblk->index);
+		mutex_destroy(&vblk->remove_mutex);
+		kfree(vblk);
+	}
+}
+
+static int virtblk_open(struct block_device *bd, fmode_t mode)
+{
+	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;

It's more common to do

Quoted 7 lines
	int ret = 0;

and on error:

	ret = -ENXIO;

let's do this.

Will fix.

Quoted 12 lines

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 332

Quoted 7 lines
drivers/block/virtio_blk.c +7 -0
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (vblk->vdev) {
+		virtblk_get(vblk);
+		ret = 0;
+	}

I prefer
else
ret = -ENXIO

Quoted 1 line

here.

Got it.

Quoted 20 lines
Quoted 7 lines
drivers/block/virtio_blk.c +7 -0
+
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
+}
+
+static void virtblk_release(struct gendisk *disk, fmode_t mode)
+{

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 349

Quoted 10 lines
drivers/block/virtio_blk.c +6 -0
+	struct virtio_blk *vblk = disk->private_data;
+
+	virtblk_put(vblk);
+}
+
 /* We provide getgeo only to please some old bootloader/partitioning tools */
 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 {
 	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;

It's more common to do

Quoted 7 lines
	int ret = 0;

and on error:

	ret = -ENXIO;

let's do this.

Will fix.

Quoted 12 lines

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 355

Quoted 7 lines
drivers/block/virtio_blk.c +6 -0
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (!vblk->vdev) {
+		goto out;
+	}

single lines are not supposed to use {}.
if you add ret = -ENXIO here then it won't be a single line anymore though.

Oops, a QEMU coding style habit :).

Quoted 52 lines
Quoted 52 lines
drivers/block/virtio_blk.c +17 -3
 	/* see if the host passed in geometry config */
 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
@@ -314,11 +368,17 @@ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 		geo->sectors = 1 << 5;
 		geo->cylinders = get_capacity(bd->bd_disk) >> 11;
 	}
-	return 0;
+
+	ret = 0;
+out:
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
 }
 
 static const struct block_device_operations virtblk_fops = {
 	.owner  = THIS_MODULE,
+	.open = virtblk_open,
+	.release = virtblk_release,
 	.getgeo = virtblk_getgeo,
 };
 
@@ -655,6 +715,10 @@ static int virtblk_probe(struct virtio_device *vdev)
 		goto out_free_index;
 	}
 
+	/* This reference is dropped in virtblk_remove(). */
+	refcount_set(&vblk->refs, 1);
+	mutex_init(&vblk->remove_mutex);
+
 	vblk->vdev = vdev;
 	vblk->sg_elems = sg_elems;
 
@@ -820,8 +884,12 @@ static int virtblk_probe(struct virtio_device *vdev)
 static void virtblk_remove(struct virtio_device *vdev)
 {
 	struct virtio_blk *vblk = vdev->priv;
-	int index = vblk->index;
-	int refc;
+
+	/*
+	 * Virtqueue processing is stopped safely here but mutual exclusion is
+	 * needed for block_device_operations.
+	 */
+	mutex_lock(&vblk->remove_mutex);
 
 	/* Make sure no work handler is accessing the device. */
 	flush_work(&vblk->config_work);
@@ -834,15 +902,16 @@ static void virtblk_remove(struct virtio_device *vdev)
 	/* Stop all the virtqueues. */
 	vdev->config->reset(vdev);
 
Quoted 4 lines

drivers/block/virtio_blk.c static void virtblk_remove(struct virtio_device *vdev) · line 905

Quoted 2 lines
drivers/block/virtio_blk.c +1 -1
-	refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref);
+	/* Virtqueue are stopped, nothing can use vblk->vdev anymore. */

Virtqueues?

Thanks, will fix.

#3 lore raw
Quoted 32 lines · Stefan Hajnoczi
Quoted 8 lines · Michael S. Tsirkin
Quoted 4 lines · Stefan Hajnoczi

A userspace process holding a file descriptor to a virtio_blk device can still invoke block_device_operations after hot unplug. For example, a program that has /dev/vdb open can call ioctl(HDIO_GETGEO) after hot unplug to invoke virtblk_getgeo().

which causes what? a use after free?

Yes, use after free. I will include the kernel panic in the next revision.

virtio_check_driver_offered_feature() accesses vdev->dev.driver, which doesn't work after virtblk_remove() has freed vdev :).

Quoted 10 lines
Quoted 8 lines

Introduce a reference count in struct virtio_blk so that its lifetime covers both virtio_driver probe/remove and block_device_operations open/release users. This ensures that block_device_operations functions like virtblk_getgeo() can safely access struct virtio_blk.

Add remove_mutex to prevent block_device_operations functions from accessing vblk->vdev during virtblk_remove() and let the safely check

let the -> let them?

Thanks, will fix.

Quoted 3 lines
Quoted 3 lines

for !vblk->vdev after virtblk_remove() returns.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

Switching to a reference count also solves the vd_index_ida leak where vda, vdb, vdc, etc indices were lost when the device was hot unplugged while the block device was still open.

Can you move this statement up so we list both issues (use after free and leak) upfront, then discuss the fix?

Thanks, will fix.

Maybe add Fixes: if there's a specific commit that added geo support.

Quoted 208 lines
Quoted 31 lines
Quoted 31 lines

---
If someone has a simpler solution please let me know. I looked at various approaches including reusing device_lock(&vblk->vdev.dev) but they were more complex and extending the lifetime of virtio_device after remove() has been called seems questionable.

---
 drivers/block/virtio_blk.c | 85 ++++++++++++++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 8 deletions(-)
drivers/block/virtio_blk.c +7 -0
index 93468b7c6701..3dd53b445cc1 100644
@@ -44,6 +44,13 @@ struct virtio_blk {
 	/* Process context for config space updates */
 	struct work_struct config_work;
 
+	/*
+	 * Tracks references from block_device_operations open/release and
+	 * virtio_driver probe/remove so this object can be freed once no
+	 * longer in use.
+	 */
+	refcount_t refs;
+
 	/* What host tells us, plus 2 for header & tailer. */
 	unsigned int sg_elems;
 
Quoted 11 lines

drivers/block/virtio_blk.c struct virtio_blk { · line 64

Quoted 6 lines
drivers/block/virtio_blk.c +2 -0
@@ -53,6 +60,9 @@ struct virtio_blk {
 	/* num of vqs */
 	int num_vqs;
 	struct virtio_blk_vq *vqs;
+
+	/* Provides mutual exclusion with virtblk_remove(). */

This is not the best way to document access rules. Which fields does this protect, exactly?
I think it's just vdev. Right?
Pls add to the comment.

Yes, vblk->vdev cannot be dereferenced after virtblk_remove() is entered.

I'll document this mutex as protecting vdev.

Quoted 31 lines
Quoted 17 lines
drivers/block/virtio_blk.c +9 -0
+	struct mutex remove_mutex;
 };
 
 struct virtblk_req {
@@ -295,10 +305,54 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str)
 	return err;
 }
 
+static void virtblk_get(struct virtio_blk *vblk)
+{
+	refcount_inc(&vblk->refs);
+}
+
+static void virtblk_put(struct virtio_blk *vblk)
+{
+	if (refcount_dec_and_test(&vblk->refs)) {

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 325

Quoted 10 lines
drivers/block/virtio_blk.c +10 -0
+		ida_simple_remove(&vd_index_ida, vblk->index);
+		mutex_destroy(&vblk->remove_mutex);
+		kfree(vblk);
+	}
+}
+
+static int virtblk_open(struct block_device *bd, fmode_t mode)
+{
+	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;

It's more common to do

Quoted 7 lines
	int ret = 0;

and on error:

	ret = -ENXIO;

let's do this.

Will fix.

Quoted 12 lines

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 332

Quoted 7 lines
drivers/block/virtio_blk.c +7 -0
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (vblk->vdev) {
+		virtblk_get(vblk);
+		ret = 0;
+	}

I prefer
else
ret = -ENXIO

Quoted 1 line

here.

Got it.

Quoted 20 lines
Quoted 7 lines
drivers/block/virtio_blk.c +7 -0
+
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
+}
+
+static void virtblk_release(struct gendisk *disk, fmode_t mode)
+{

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 349

Quoted 10 lines
drivers/block/virtio_blk.c +6 -0
+	struct virtio_blk *vblk = disk->private_data;
+
+	virtblk_put(vblk);
+}
+
 /* We provide getgeo only to please some old bootloader/partitioning tools */
 static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 {
 	struct virtio_blk *vblk = bd->bd_disk->private_data;
+	int ret = -ENXIO;

It's more common to do

Quoted 7 lines
	int ret = 0;

and on error:

	ret = -ENXIO;

let's do this.

Will fix.

Quoted 12 lines

drivers/block/virtio_blk.c static int virtblk_get_id(struct gendisk *disk, char *id_str) · line 355

Quoted 7 lines
drivers/block/virtio_blk.c +6 -0
+
+	mutex_lock(&vblk->remove_mutex);
+
+	if (!vblk->vdev) {
+		goto out;
+	}

single lines are not supposed to use {}.
if you add ret = -ENXIO here then it won't be a single line anymore though.

Oops, a QEMU coding style habit :).

Quoted 52 lines
Quoted 52 lines
drivers/block/virtio_blk.c +17 -3
 	/* see if the host passed in geometry config */
 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
@@ -314,11 +368,17 @@ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
 		geo->sectors = 1 << 5;
 		geo->cylinders = get_capacity(bd->bd_disk) >> 11;
 	}
-	return 0;
+
+	ret = 0;
+out:
+	mutex_unlock(&vblk->remove_mutex);
+	return ret;
 }
 
 static const struct block_device_operations virtblk_fops = {
 	.owner  = THIS_MODULE,
+	.open = virtblk_open,
+	.release = virtblk_release,
 	.getgeo = virtblk_getgeo,
 };
 
@@ -655,6 +715,10 @@ static int virtblk_probe(struct virtio_device *vdev)
 		goto out_free_index;
 	}
 
+	/* This reference is dropped in virtblk_remove(). */
+	refcount_set(&vblk->refs, 1);
+	mutex_init(&vblk->remove_mutex);
+
 	vblk->vdev = vdev;
 	vblk->sg_elems = sg_elems;
 
@@ -820,8 +884,12 @@ static int virtblk_probe(struct virtio_device *vdev)
 static void virtblk_remove(struct virtio_device *vdev)
 {
 	struct virtio_blk *vblk = vdev->priv;
-	int index = vblk->index;
-	int refc;
+
+	/*
+	 * Virtqueue processing is stopped safely here but mutual exclusion is
+	 * needed for block_device_operations.
+	 */
+	mutex_lock(&vblk->remove_mutex);
 
 	/* Make sure no work handler is accessing the device. */
 	flush_work(&vblk->config_work);
@@ -834,15 +902,16 @@ static void virtblk_remove(struct virtio_device *vdev)
 	/* Stop all the virtqueues. */
 	vdev->config->reset(vdev);
 
Quoted 4 lines

drivers/block/virtio_blk.c static void virtblk_remove(struct virtio_device *vdev) · line 905

Quoted 2 lines
drivers/block/virtio_blk.c +1 -1
-	refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref);
+	/* Virtqueue are stopped, nothing can use vblk->vdev anymore. */

Virtqueues?

Thanks, will fix.