block: Fix races in bdev - gendisk handling
Subsystems (5)
Jan Kara
reviewerv2 cover block: Fix races in bdev - gendisk handling
Hello,
these patches fix races happening when devices are frequently destroyed and recreated in association of block device inode with corresponding gendisk. Generally when such race happen it results in use-after-free issues, block device page cache inconsistencies, or other problems. I have verified these patches fix use-after-free issues that could be reproduced by frequent creation and destruction of loop device. Hou Tao has verified that races reported by him in [1] related to gendisk-blkdev association were also fixed. Jens, can you please merge these patches? Thanks!
Changes since v1:
* Added tested-by tags
Honza
Jan Kara
reviewerPATCH v2 1/6 genhd: Fix leaked module reference for NVME devices
Commit 8ddcd653257c "block: introduce GENHD_FL_HIDDEN" added handling of hidden devices to get_gendisk() but forgot to drop module reference which is also acquired by get_disk(). Drop the reference as necessary.
Arguably the function naming here is misleading as put_disk() is *not* the counterpart of get_disk() but let's fix that in the follow up commit since that will be more intrusive.
--- block/genhd.c | 3 +++ 1 file changed, 3 insertions(+)
block/genhd.c +3 -0
@@ -817,7 +817,10 @@ struct gendisk *get_gendisk(dev_t devt, int *partno) }if (disk &&unlikely (disk->flags &GENHD_FL_HIDDEN )) { +struct module *owner = disk->fops->owner; +put_disk (disk); +module_put (owner); disk =NULL ; }return disk;
Signature · 1 line
2.13.6
Re: [PATCH 1/6] genhd: Fix leaked module reference for NVME devices
Quoted 7 lines · Jan Kara
Commit
8ddcd653257c"block: introduceGENHD_FL_HIDDEN" added handling of hidden devices toget_gendisk()but forgot to drop module reference which is also acquired byget_disk(). Drop the reference as necessary.Arguably the function naming here is misleading as
put_disk()is *not* the counterpart ofget_disk()but let's fix that in the follow up commit since that will be more intrusive.
Looks good,
Jan Kara
reviewerPATCH v2 2/6 genhd: Rename get_disk() to get_disk_and_module()
In reply to Jan KaraRename get_disk() to get_disk_and_module() to make sure what the function does. It's not a great name but at least it is now clear that put_disk() is not it's counterpart.
--- block/genhd.c | 10 ++++------ drivers/block/amiflop.c | 2 +- drivers/block/ataflop.c | 2 +- drivers/block/brd.c | 2 +- drivers/block/floppy.c | 2 +- drivers/block/loop.c | 2 +- drivers/block/swim.c | 2 +- drivers/block/z2ram.c | 2 +- drivers/ide/ide-probe.c | 2 +- include/linux/genhd.h | 2 +- 10 files changed, 13 insertions(+), 15 deletions(-)
block/genhd.c +4 -6
@@ -547,7 +547,7 @@ static int exact_lock(dev_t devt, void *data) {struct gendisk *p = data; -if (!get_disk (p)) +if (!get_disk_and_module (p))return -1 ;return 0 ; }
@@ -809,7 +809,7 @@ struct gendisk *get_gendisk(dev_t devt, int *partno)spin_lock_bh (&ext_devt_lock); part =idr_find (&ext_devt_idr,blk_mangle_minor (MINOR (devt))); -if (part &&get_disk (part_to_disk (part))) { +if (part &&get_disk_and_module (part_to_disk (part))) { *partno = part->partno; disk =part_to_disk (part); }
@@ -1456,7 +1456,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) }EXPORT_SYMBOL (__alloc_disk_node); -struct kobject *get_disk (struct gendisk *disk) +struct kobject *get_disk_and_module (struct gendisk *disk) {struct module *owner;struct kobject *kobj;
@@ -1474,15 +1474,13 @@ struct kobject *get_disk(struct gendisk *disk)return kobj; } - -EXPORT_SYMBOL (get_disk); +EXPORT_SYMBOL (get_disk_and_module);void put_disk (struct gendisk *disk) {if (disk)kobject_put (&disk_to_dev (disk)->kobj); } -EXPORT_SYMBOL (put_disk);static void set_disk_ro_uevent (struct gendisk *gd,int ro)
drivers/block/amiflop.c +1 -1
@@ -1758,7 +1758,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)if (unit[drive].type->code ==FD_NODRIVE )return NULL ; *part =0 ; -return get_disk (unit[drive].gendisk); +return get_disk_and_module (unit[drive].gendisk); }static int __init amiga_floppy_probe (struct platform_device *pdev)
drivers/block/ataflop.c +1 -1
@@ -1917,7 +1917,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)if (drive >=FD_MAX_UNITS || type >NUM_DISK_MINORS )return NULL ; *part =0 ; -return get_disk (unit[drive].disk); +return get_disk_and_module (unit[drive].disk); }static int __init atari_floppy_init (void )
drivers/block/brd.c +1 -1
@@ -456,7 +456,7 @@ static struct kobject *brd_probe(dev_t dev, int *part, void *data)mutex_lock (&brd_devices_mutex); brd =brd_init_one (MINOR (dev) / max_part, &new); - kobj = brd ?get_disk (brd->brd_disk) :NULL ; + kobj = brd ?get_disk_and_module (brd->brd_disk) :NULL ;mutex_unlock (&brd_devices_mutex);if (new)
drivers/block/floppy.c +1 -1
@@ -4505,7 +4505,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)if (((*part >>2 ) &0x1f ) >=ARRAY_SIZE (floppy_type))return NULL ; *part =0 ; -return get_disk (disks[drive]); +return get_disk_and_module (disks[drive]); }static int __init do_floppy_init (void )
drivers/block/loop.c +1 -1
@@ -1922,7 +1922,7 @@ static struct kobject *loop_probe(dev_t dev, int *part, void *data)if (err <0 ) kobj =NULL ;else - kobj =get_disk (lo->lo_disk); + kobj =get_disk_and_module (lo->lo_disk);mutex_unlock (&loop_index_mutex); *part =0 ;
drivers/block/swim.c +1 -1
@@ -799,7 +799,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)return NULL ; *part =0 ; -return get_disk (swd->unit[drive].disk); +return get_disk_and_module (swd->unit[drive].disk); }static int swim_add_floppy (struct swim_priv *swd,enum drive_location location)
drivers/block/z2ram.c +1 -1
@@ -332,7 +332,7 @@ static const struct block_device_operations z2_fops =static struct kobject *z2_find (dev_t dev,int *part,void *data) { *part =0 ; -return get_disk (z2ram_gendisk); +return get_disk_and_module (z2ram_gendisk); }static struct request_queue *z2_queue;
drivers/ide/ide-probe.c +1 -1
@@ -928,7 +928,7 @@ static int exact_lock(dev_t dev, void *data) {struct gendisk *p = data; -if (!get_disk (p)) +if (!get_disk_and_module (p))return -1 ;return 0 ; }
include/linux/genhd.h +1 -1
@@ -600,7 +600,7 @@ extern void delete_partition(struct gendisk *, int);extern void printk_all_partitions (void );extern struct gendisk *__alloc_disk_node (int minors,int node_id); -extern struct kobject *get_disk (struct gendisk *disk); +extern struct kobject *get_disk_and_module (struct gendisk *disk);extern void put_disk (struct gendisk *disk); extern void blk_register_region(dev_t devt, unsigned long range, struct module *module,
Signature · 1 line
2.13.6
Jan Kara
reviewerPATCH v2 3/6 genhd: Add helper put_disk_and_module()
In reply to Jan KaraAdd a proper counterpart to get_disk_and_module() - put_disk_and_module(). Currently it is opencoded in several places.
--- block/blk-cgroup.c | 11 ++--------- block/genhd.c | 20 ++++++++++++++++---- fs/block_dev.c | 19 +++++-------------- include/linux/genhd.h | 1 + 4 files changed, 24 insertions(+), 27 deletions(-)
block/blk-cgroup.c +2 -9
@@ -812,7 +812,6 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,struct gendisk *disk;struct request_queue *q;struct blkcg_gq *blkg; -struct module *owner;unsigned int major, minor;int key_len, part, ret;char *body;
@@ -904,9 +903,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,spin_unlock_irq (q->queue_lock);rcu_read_unlock (); fail: - owner = disk->fops->owner; -put_disk (disk); -module_put (owner); +put_disk_and_module (disk);/* * If queue was bypassing, we should retry. Do so after a * short msleep(). It isn't strictly necessary but queue
@@ -931,13 +928,9 @@ EXPORT_SYMBOL_GPL(blkg_conf_prep);void blkg_conf_finish (struct blkg_conf_ctx *ctx) __releases(ctx->disk->queue->queue_lock) __releases(rcu) { -struct module *owner; -spin_unlock_irq (ctx->disk->queue->queue_lock);rcu_read_unlock (); - owner = ctx->disk->fops->owner; -put_disk (ctx->disk); -module_put (owner); +put_disk_and_module (ctx->disk); }EXPORT_SYMBOL_GPL (blkg_conf_finish);
block/genhd.c +16 -4
@@ -817,10 +817,7 @@ struct gendisk *get_gendisk(dev_t devt, int *partno) }if (disk &&unlikely (disk->flags &GENHD_FL_HIDDEN )) { -struct module *owner = disk->fops->owner; - -put_disk (disk); -module_put (owner); +put_disk_and_module (disk); disk =NULL ; }return disk;
@@ -1483,6 +1480,21 @@ void put_disk(struct gendisk *disk) }EXPORT_SYMBOL (put_disk); +/* +* This is a counterpart of get_disk_and_module() and thus also of +* get_gendisk(). +*/ +void put_disk_and_module (struct gendisk *disk) +{ +if (disk) { +struct module *owner = disk->fops->owner; + +put_disk (disk); +module_put (owner); + } +} +EXPORT_SYMBOL (put_disk_and_module); +static void set_disk_ro_uevent (struct gendisk *gd,int ro) {char event[] ="DISK_RO=1" ;
fs/block_dev.c +5 -14
@@ -1111,8 +1111,7 @@ static struct block_device *bd_start_claiming(struct block_device *bdev,else whole =bdgrab (bdev); -module_put (disk->fops->owner); -put_disk (disk); +put_disk_and_module (disk);if (!whole)return ERR_PTR (-ENOMEM );
@@ -1407,7 +1406,6 @@ static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);static int __blkdev_get (struct block_device *bdev,fmode_t mode,int for_part) {struct gendisk *disk; -struct module *owner;int ret;int partno;int perm =0 ;
@@ -1433,7 +1431,6 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) disk =get_gendisk (bdev->bd_dev, &partno);if (!disk) goto out; - owner = disk->fops->owner;disk_block_events (disk);mutex_lock_nested (&bdev->bd_mutex, for_part);
@@ -1463,8 +1460,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) bdev->bd_queue =NULL ;mutex_unlock (&bdev->bd_mutex);disk_unblock_events (disk); -put_disk (disk); -module_put (owner); +put_disk_and_module (disk); goto restart; } }
@@ -1525,8 +1521,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) goto out_unlock_bdev; }/* only one opener holds refs to the module and disk */ -put_disk (disk); -module_put (owner); +put_disk_and_module (disk); } bdev->bd_openers++;if (for_part)
@@ -1546,8 +1541,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) out_unlock_bdev:mutex_unlock (&bdev->bd_mutex);disk_unblock_events (disk); -put_disk (disk); -module_put (owner); +put_disk_and_module (disk); out:bdput (bdev);
@@ -1770,8 +1764,6 @@ static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part) disk->fops->release (disk, mode); }if (!bdev->bd_openers) { -struct module *owner = disk->fops->owner; -disk_put_part (bdev->bd_part); bdev->bd_part =NULL ; bdev->bd_disk =NULL ;
@@ -1779,8 +1771,7 @@ static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part) victim = bdev->bd_contains; bdev->bd_contains =NULL ; -put_disk (disk); -module_put (owner); +put_disk_and_module (disk); }mutex_unlock (&bdev->bd_mutex);bdput (bdev);
include/linux/genhd.h +1 -0
@@ -602,6 +602,7 @@ extern void printk_all_partitions(void);extern struct gendisk *__alloc_disk_node (int minors,int node_id);extern struct kobject *get_disk_and_module (struct gendisk *disk);extern void put_disk (struct gendisk *disk); +extern void put_disk_and_module (struct gendisk *disk); extern void blk_register_region(dev_t devt, unsigned long range, struct module *module, struct kobject *(*probe)(dev_t, int *, void *),
Signature · 1 line
2.13.6
Jan Kara
reviewerPATCH v2 4/6 genhd: Fix use after free in __blkdev_get()
In reply to Jan KaraWhen two blkdev_open() calls race with device removal and recreation, __blkdev_get() can use looked up gendisk after it is freed:
CPU0 CPU1 CPU2
del_gendisk(disk); bdev_unhash_inode(inode);
blkdev_open() blkdev_open()
bdev = bd_acquire(inode);
- creates and returns new inode
bdev = bd_acquire(inode);
- returns the same inode
__blkdev_get(devt) __blkdev_get(devt)
disk = get_gendisk(devt);
- got structure of device going away
<finish device removal> <new device gets
created under the same device number>
disk = get_gendisk(devt);
- got new device structure
if (!bdev->bd_openers) {
does the first open
}
if (!bdev->bd_openers)
- false
} else {
put_disk_and_module(disk)
- remember this was old device - this was last ref and disk is now freed
}
disk_unblock_events(disk); -> oops
Fix the problem by making sure we drop reference to disk in __blkdev_get() only after we are really done with it.
--- fs/block_dev.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
fs/block_dev.c +5 -2
@@ -1409,6 +1409,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)int ret;int partno;int perm =0 ; +bool first_open = false;if (mode &FMODE_READ ) perm |=MAY_READ ;
@@ -1435,6 +1436,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)disk_block_events (disk);mutex_lock_nested (&bdev->bd_mutex, for_part);if (!bdev->bd_openers) { + first_open = true; bdev->bd_disk = disk; bdev->bd_queue = disk->queue; bdev->bd_contains = bdev;
@@ -1520,14 +1522,15 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)if (ret) goto out_unlock_bdev; } -/* only one opener holds refs to the module and disk */ -put_disk_and_module (disk); } bdev->bd_openers++;if (for_part) bdev->bd_part_count++;mutex_unlock (&bdev->bd_mutex);disk_unblock_events (disk); +/* only one opener holds refs to the module and disk */ +if (!first_open) +put_disk_and_module (disk);return 0 ; out_clear:
Signature · 1 line
2.13.6
Jan Kara
reviewerPATCH v2 5/6 genhd: Fix BUG in blkdev_open()
In reply to Jan KaraWhen two blkdev_open() calls for a partition race with device removal and recreation, we can hit BUG_ON(!bd_may_claim(bdev, whole, holder)) in blkdev_open(). The race can happen as follows:
CPU0 CPU1 CPU2
del_gendisk() bdev_unhash_inode(part1);
blkdev_open(part1, O_EXCL) blkdev_open(part1, O_EXCL)
bdev = bd_acquire() bdev = bd_acquire()
blkdev_get(bdev)
bd_start_claiming(bdev)
- finds old inode 'whole'
bd_prepare_to_claim() -> 0
bdev_unhash_inode(whole);
<device removed> <new device under same
number created>
blkdev_get(bdev);
bd_start_claiming(bdev)
- finds new inode 'whole' bd_prepare_to_claim()
- this also succeeds as we have different 'whole' here...
- bad things happen now as we have two exclusive openers of the same bdev
The problem here is that block device opens can see various intermediate states while gendisk is shutting down and then being recreated.
We fix the problem by introducing new lookup_sem in gendisk that synchronizes gendisk deletion with get_gendisk() and furthermore by making sure that get_gendisk() does not return gendisk that is being (or has been) deleted. This makes sure that once we ever manage to look up newly created bdev inode, we are also guaranteed that following get_gendisk() will either return failure (and we fail open) or it returns gendisk for the new device and following bdget_disk() will return new bdev inode (i.e., blkdev_open() follows the path as if it is completely run after new device is created).
--- block/genhd.c | 21 ++++++++++++++++++++- include/linux/genhd.h | 1 + 2 files changed, 21 insertions(+), 1 deletion(-)
block/genhd.c +20 -1
@@ -717,6 +717,11 @@ void del_gendisk(struct gendisk *disk)blk_integrity_del (disk);disk_del_events (disk); +/* +* Block lookups of the disk until all bdevs are unhashed and the +* disk is marked as dead (GENHD_FL_UP cleared). +*/ +down_write (&disk->lookup_sem);/* invalidate stuff */ disk_part_iter_init (&piter, disk,DISK_PITER_INCL_EMPTY |DISK_PITER_REVERSE );
@@ -731,6 +736,7 @@ void del_gendisk(struct gendisk *disk)bdev_unhash_inode (disk_devt (disk));set_capacity (disk,0 ); disk->flags &= ~GENHD_FL_UP ; +up_write (&disk->lookup_sem);if (!(disk->flags &GENHD_FL_HIDDEN ))sysfs_remove_link (&disk_to_dev (disk)->kobj,"bdi" );
@@ -816,9 +822,21 @@ struct gendisk *get_gendisk(dev_t devt, int *partno)spin_unlock_bh (&ext_devt_lock); } -if (disk &&unlikely (disk->flags &GENHD_FL_HIDDEN )) { +if (!disk) +return NULL ; + +/* +* Synchronize with del_gendisk() to not return disk that is being +* destroyed. +*/ +down_read (&disk->lookup_sem); +if (unlikely ((disk->flags &GENHD_FL_HIDDEN ) || + !(disk->flags &GENHD_FL_UP ))) { +up_read (&disk->lookup_sem);put_disk_and_module (disk); disk =NULL ; + }else { +up_read (&disk->lookup_sem); }return disk; }
@@ -1418,6 +1436,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id)kfree (disk);return NULL ; } +init_rwsem (&disk->lookup_sem); disk->node_id = node_id;if (disk_expand_part_tbl (disk,0 )) {free_part_stats (&disk->part0);
include/linux/genhd.h +1 -0
@@ -198,6 +198,7 @@ struct gendisk {void *private_data;int flags; +struct rw_semaphore lookup_sem;struct kobject *slave_dir;struct timer_rand_state *random;
Signature · 1 line
2.13.6
Jan Kara
reviewerPATCH v2 6/6 blockdev: Avoid two active bdev inodes for one device
In reply to Jan KaraWhen blkdev_open() races with device removal and creation it can happen that unhashed bdev inode gets associated with newly created gendisk like:
CPU0 CPU1blkdev_open()
bdev = bd_acquire()
del_gendisk() bdev_unhash_inode(bdev); remove device create new device with the same number
__blkdev_get()
disk = get_gendisk()
- gets reference to gendisk of the new device
Now another blkdev_open() will not find original 'bdev' as it got unhashed, create a new one and associate it with the same 'disk' at which point problems start as we have two independent page caches for one device.
Fix the problem by verifying that the bdev inode didn't get unhashed before we acquired gendisk reference. That way we make sure gendisk can get associated only with visible bdev inodes.
--- fs/block_dev.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-)
fs/block_dev.c +23 -2
@@ -1058,6 +1058,27 @@ static int bd_prepare_to_claim(struct block_device *bdev,return 0 ; } +static struct gendisk *bdev_get_gendisk (struct block_device *bdev,int *partno) +{ +struct gendisk *disk =get_gendisk (bdev->bd_dev, partno); + +if (!disk) +return NULL ; +/* +* Now that we hold gendisk reference we make sure bdev we looked up is +* not stale. If it is, it means device got removed and created before +* we looked up gendisk and we fail open in such case. Associating +* unhashed bdev with newly created gendisk could lead to two bdevs +* (and thus two independent caches) being associated with one device +* which is bad. +*/ +if (inode_unhashed (bdev->bd_inode)) { +put_disk_and_module (disk); +return NULL ; + } +return disk; +} +/** * bd_start_claiming - start claiming a block device * @bdev: block device of interest
@@ -1094,7 +1115,7 @@ static struct block_device *bd_start_claiming(struct block_device *bdev,* @bdev might not have been initialized properly yet, look up * and grab the outer block device the hard way. */ - disk =get_gendisk (bdev->bd_dev, &partno); + disk =bdev_get_gendisk (bdev, &partno);if (!disk)return ERR_PTR (-ENXIO );
@@ -1429,7 +1450,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) restart: ret = -ENXIO ; - disk =get_gendisk (bdev->bd_dev, &partno); + disk =bdev_get_gendisk (bdev, &partno);if (!disk) goto out;
Signature · 1 line
2.13.6
Jens Axboe
maintainerQuoted 10 lines · Jan Kara
Hello,
these patches fix races happening when devices are frequently destroyed and recreated in association of block device inode with corresponding gendisk. Generally when such race happen it results in use-after-free issues, block device page cache inconsistencies, or other problems. I have verified these patches fix use-after-free issues that could be reproduced by frequent creation and destruction of loop device. Hou Tao has verified that races reported by him in [1] related to gendisk-blkdev association were also fixed. Jens, can you please merge these patches? Thanks!
What are you thinking in terms of target? I'd like to get it into 4.16, but I'd need to know if you are comfortable with that.
Signature · 1 line
Jens Axboe
Jan Kara
reviewerQuoted 12 lines · Jens Axboe
Quoted 10 lines · Jan Kara
Hello,
these patches fix races happening when devices are frequently destroyed and recreated in association of block device inode with corresponding gendisk. Generally when such race happen it results in use-after-free issues, block device page cache inconsistencies, or other problems. I have verified these patches fix use-after-free issues that could be reproduced by frequent creation and destruction of loop device. Hou Tao has verified that races reported by him in [1] related to gendisk-blkdev association were also fixed. Jens, can you please merge these patches? Thanks!
Quoted 2 lines
What are you thinking in terms of target? I'd like to get it into 4.16, but I'd need to know if you are comfortable with that.
Yeah, 4.16 should be OK. The patches are quite conservative.
Honza
Signature · 2 lines
Jan Kara <[email protected]> SUSE Labs, CR
Jens Axboe
maintainerQuoted 13 lines · Jan Kara
Quoted 12 lines · Jens Axboe
Quoted 10 lines · Jan Kara
Hello,
these patches fix races happening when devices are frequently destroyed and recreated in association of block device inode with corresponding gendisk. Generally when such race happen it results in use-after-free issues, block device page cache inconsistencies, or other problems. I have verified these patches fix use-after-free issues that could be reproduced by frequent creation and destruction of loop device. Hou Tao has verified that races reported by him in [1] related to gendisk-blkdev association were also fixed. Jens, can you please merge these patches? Thanks!
Quoted 4 lines
Quoted 2 lines
What are you thinking in terms of target? I'd like to get it into 4.16, but I'd need to know if you are comfortable with that.
Yeah, 4.16 should be OK. The patches are quite conservative.
OK good, let's get it queued up. Thanks Jan.
Signature · 1 line
Jens Axboe