block: fix partition use-after-free and optimization
cover block: fix partition use-after-free and optimization
Hi,
The 1st patch fixes one use-after-free on cached last_lookup partition.
The other 3 patches optimizes partition uses in IO path.
Ming Lei (4):
block: fix use-after-free on cached last_lookup partition
block: only define 'nr_sects_seq' in hd_part for 32bit SMP
block: re-organize fields of 'struct hd_part'
block: don't hold part0's refcount in IO path
block/blk-core.c | 15 ++------------- block/genhd.c | 7 +++++-- block/partition-generic.c | 12 ++++++++++-- include/linux/genhd.h | 25 ++++++++++++++++++------- 4 files changed, 35 insertions(+), 24 deletions(-)
Signature · 1 line
2.20.1
PATCH 1/4 block: fix use-after-free on cached last_lookup partition
delete_partition() clears the cached last_lookup partition. However the .last_lookup cache may be overwritten by one IO path after it is cleared from delete_partition(). Then another IO path may use the cached deleting partition after __delete_partition() is called, then use-after-free is triggered on the cached partition.
Fixes the issue by the following approach:
1) always get the partition's refcount via hd_struct_try_get() before setting .last_lookup
2) move clearing .last_lookup from delete_partition() to __delete_partition() which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in __delete_partition().
It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.
[1] https://lore.kernel.org/linux-block/[email protected]/T/#t
--- block/blk-core.c | 12 ------------ block/genhd.c | 6 +++++- block/partition-generic.c | 10 +++++++++- include/linux/genhd.h | 1 + 4 files changed, 15 insertions(+), 14 deletions(-)
block/blk-core.c +0 -12
@@ -1365,18 +1365,6 @@ void blk_account_io_start(struct request *rq, bool new_io)part_stat_inc (part, merges[rw]); }else { part =disk_map_sector_rcu (rq->rq_disk,blk_rq_pos (rq)); -if (!hd_struct_try_get (part)) { -/* -* The partition is already being removed, -* the request will be accounted on the disk only -* -* We take a reference on disk->part0 although that -* partition will never be deleted, so we can treat -* it as any other partition. -*/ - part = &rq->rq_disk->part0; -hd_struct_get (part); - }part_inc_in_flight (rq->q, part, rw); rq->part = part; }
block/genhd.c +5 -1
@@ -286,17 +286,21 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector) ptbl =rcu_dereference (disk->part_tbl); part =rcu_dereference (ptbl->last_lookup); -if (part &§or_in_part (part, sector)) +if (part &§or_in_part (part, sector) &&hd_struct_try_get (part))return part;for (i =1 ; i < ptbl->len; i++) { part =rcu_dereference (ptbl->part[i]);if (part &§or_in_part (part, sector)) { +if (!hd_struct_try_get (part)) + goto exit;rcu_assign_pointer (ptbl->last_lookup, part);return part; } } + exit: +hd_struct_get (&disk->part0);return &disk->part0; }EXPORT_SYMBOL_GPL (disk_map_sector_rcu);
block/partition-generic.c +9 -1
@@ -262,6 +262,12 @@ static void delete_partition_work_fn(struct work_struct *work)void __delete_partition (struct percpu_ref *ref) { struct hd_struct *part = container_of(ref, struct hd_struct, ref); +struct disk_part_tbl *ptbl = +rcu_dereference_protected (part->disk->part_tbl,1 ); + +rcu_assign_pointer (ptbl->last_lookup,NULL ); +put_device (disk_to_dev (part->disk)); +INIT_RCU_WORK (&part->rcu_work, delete_partition_work_fn);queue_rcu_work (system_wq, &part->rcu_work); }
@@ -283,8 +289,9 @@ void delete_partition(struct gendisk *disk, int partno)if (!part)return ; +get_device (disk_to_dev (disk));rcu_assign_pointer (ptbl->part[partno],NULL ); -rcu_assign_pointer (ptbl->last_lookup,NULL ); +kobject_put (part->holder_dir);device_del (part_to_dev (part));
@@ -349,6 +356,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno, p->nr_sects = len; p->partno = partno; p->policy =get_disk_ro (disk); + p->disk = disk;if (info) {struct partition_meta_info *pinfo =alloc_part_info (disk);
include/linux/genhd.h +1 -0
@@ -130,6 +130,7 @@ struct hd_struct {struct disk_stats dkstats;#endif struct percpu_ref ref; +struct gendisk *disk;struct rcu_work rcu_work; };
Signature · 1 line
2.20.1
Re: [PATCH 1/4] block: fix use-after-free on cached last_lookup partition
Hi Guys,
Did this patch have been forgotten? So, ping...
Thanks,
Yufen
Quoted 133 lines · Ming Lei
delete_partition()clears the cachedlast_lookuppartition. However the .last_lookup cache may be overwritten by one IO path after it is cleared fromdelete_partition(). Then another IO path may use the cached deleting partition after__delete_partition()is called, then use-after-free is triggered on the cached partition.Fixes the issue by the following approach:
1) always get the partition's refcount via
hd_struct_try_get()before setting .last_lookup2) move clearing .last_lookup from
delete_partition()to__delete_partition()which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in__delete_partition().It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.
[1] https://lore.kernel.org/linux-block/[email protected]/T/#t
Reported-by Yufen Yu <[email protected]>Cc Christoph Hellwig <[email protected]>Cc Hou Tao <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-core.c | 12 ------------ block/genhd.c | 6 +++++- block/partition-generic.c | 10 +++++++++- include/linux/genhd.h | 1 + 4 files changed, 15 insertions(+), 14 deletions(-)block/blk-core.c +0
-12@@ -1365,18 +1365,6 @@ void blk_account_io_start(struct request *rq, bool new_io)part_stat_inc (part, merges[rw]); }else { part =disk_map_sector_rcu (rq->rq_disk,blk_rq_pos (rq)); -if (!hd_struct_try_get (part)) { -/* -* The partition is already being removed, -* the request will be accounted on the disk only -* -* We take a reference on disk->part0 although that -* partition will never be deleted, so we can treat -* it as any other partition. -*/ - part = &rq->rq_disk->part0; -hd_struct_get (part); - }part_inc_in_flight (rq->q, part, rw); rq->part = part; }block/genhd.c +5
-1@@ -286,17 +286,21 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector) ptbl =rcu_dereference (disk->part_tbl); part =rcu_dereference (ptbl->last_lookup); -if (part &§or_in_part (part, sector)) +if (part &§or_in_part (part, sector) &&hd_struct_try_get (part))return part;for (i =1 ; i < ptbl->len; i++) { part =rcu_dereference (ptbl->part[i]);if (part &§or_in_part (part, sector)) { +if (!hd_struct_try_get (part)) + goto exit;rcu_assign_pointer (ptbl->last_lookup, part);return part; } } + exit: +hd_struct_get (&disk->part0);return &disk->part0; }EXPORT_SYMBOL_GPL (disk_map_sector_rcu);block/partition-generic.c +9
-1@@ -262,6 +262,12 @@ static void delete_partition_work_fn(struct work_struct *work)void __delete_partition (struct percpu_ref *ref) { struct hd_struct *part = container_of(ref, struct hd_struct, ref); +struct disk_part_tbl *ptbl = +rcu_dereference_protected (part->disk->part_tbl,1 ); + +rcu_assign_pointer (ptbl->last_lookup,NULL ); +put_device (disk_to_dev (part->disk)); +INIT_RCU_WORK (&part->rcu_work, delete_partition_work_fn);queue_rcu_work (system_wq, &part->rcu_work); }@@ -283,8 +289,9 @@ void delete_partition(struct gendisk *disk, int partno)if (!part)return ; +get_device (disk_to_dev (disk));rcu_assign_pointer (ptbl->part[partno],NULL ); -rcu_assign_pointer (ptbl->last_lookup,NULL ); +kobject_put (part->holder_dir);device_del (part_to_dev (part));@@ -349,6 +356,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno, p->nr_sects = len; p->partno = partno; p->policy =get_disk_ro (disk); + p->disk = disk;if (info) {struct partition_meta_info *pinfo =alloc_part_info (disk);include/linux/genhd.h +1
-0@@ -130,6 +130,7 @@ struct hd_struct {struct disk_stats dkstats;#endif struct percpu_ref ref; +struct gendisk *disk;struct rcu_work rcu_work; };
Re: [PATCH 1/4] block: fix use-after-free on cached last_lookup partition
In reply to Ming LeiQuoted 16 lines · Ming Lei
delete_partition()clears the cachedlast_lookuppartition. However the .last_lookup cache may be overwritten by one IO path after it is cleared fromdelete_partition(). Then another IO path may use the cached deleting partition after__delete_partition()is called, then use-after-free is triggered on the cached partition.Fixes the issue by the following approach:
1) always get the partition's refcount via
hd_struct_try_get()before setting .last_lookup2) move clearing .last_lookup from
delete_partition()to__delete_partition()which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in__delete_partition().
Quoted 6 lines
It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.
[1] https://lore.kernel.org/linux-block/[email protected]/T/#t
Hi Ming,
disk_map_sector_rcu() is called from the I/O path only and hence with q->q_usage_counter > 0. Has it been considered to freeze disk->queue from delete_partition() before deleting a partition and unfreezing disk->queue after partition deletion has finished? Would that approach allow to eliminate partition reference counting and thereby improve the performance of the hot path?
Thanks,
Bart.
Re: [PATCH 1/4] block: fix use-after-free on cached last_lookup partition
Quoted 24 lines · Bart Van Assche
Quoted 16 lines · Ming Lei
delete_partition()clears the cachedlast_lookuppartition. However the .last_lookup cache may be overwritten by one IO path after it is cleared fromdelete_partition(). Then another IO path may use the cached deleting partition after__delete_partition()is called, then use-after-free is triggered on the cached partition.Fixes the issue by the following approach:
1) always get the partition's refcount via
hd_struct_try_get()before setting .last_lookup2) move clearing .last_lookup from
delete_partition()to__delete_partition()which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in__delete_partition().Quoted 6 lines
It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.
[1] https://lore.kernel.org/linux-block/[email protected]/T/#t
Quoted 8 lines
Hi Ming,
disk_map_sector_rcu()is called from the I/O path only and hence withq->q_usage_counter> 0. Has it been considered to freezedisk->queuefromdelete_partition()before deleting a partition and unfreezingdisk->queueafter partition deletion has finished? Would that approach allow to eliminate partition reference counting and thereby improve the performance of the hot path?
Hi Bart,
I did consider that approach, but this way may cause performance regression, given deleting any partition drops IO performance a lot on other un-related partitions.
Thanks,
Ming
PATCH 2/4 block: only define 'nr_sects_seq' in hd_part for 32bit SMP
In reply to Ming LeiThe seqcount of 'nr_sects_seq' is only needed in case of 32bit SMP, so define it just for 32bit SMP.
--- block/genhd.c | 2 +- block/partition-generic.c | 2 +- include/linux/genhd.h | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-)
block/genhd.c +1 -1
@@ -1506,7 +1506,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id)* TODO: Ideally set_capacity() and get_capacity() should be * converted to make use of bd_mutex and sequence counters. */ -seqcount_init (&disk->part0.nr_sects_seq); +hd_sects_seq_init (&disk->part0);if (hd_ref_init (&disk->part0)) {hd_free_part (&disk->part0);kfree (disk);
block/partition-generic.c +1 -1
@@ -345,7 +345,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno, goto out_free; } -seqcount_init (&p->nr_sects_seq); +hd_sects_seq_init (p); pdev =part_to_dev (p); p->start_sect = start;
include/linux/genhd.h +9 -0
@@ -113,7 +113,9 @@ struct hd_struct {* can be non-atomic on 32bit machines with 64bit sector_t. */ sector_t nr_sects; +#if BITS_PER_LONG ==32 && defined(CONFIG_SMP )seqcount_t nr_sects_seq; +#endif sector_t alignment_offset;unsigned int discard_alignment;struct device __dev;
@@ -713,6 +715,13 @@ static inline void hd_free_part(struct hd_struct *part)percpu_ref_exit (&part->ref); } +static inline void hd_sects_seq_init (struct hd_struct *p) +{ +#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) +seqcount_init (&p->nr_sects_seq); +#endif +} +/* * Any access of part->nr_sects which is not protected by partition * bd_mutex or gendisk bdev bd_mutex, should be done using this
Signature · 1 line
2.20.1
PATCH 3/4 block: re-organize fields of 'struct hd_part'
In reply to Ming LeiPut all fields accessed in IO path together at the beginning of the struct, so that all can be fetched in single cacheline.
--- include/linux/genhd.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-)
include/linux/genhd.h +8 -7
@@ -116,6 +116,14 @@ struct hd_struct {#if BITS_PER_LONG ==32 && defined(CONFIG_SMP )seqcount_t nr_sects_seq;#endif +unsigned long stamp; +#ifdef CONFIG_SMP +struct disk_stats __percpu *dkstats; +#else +struct disk_stats dkstats; +#endif +struct percpu_ref ref; +sector_t alignment_offset;unsigned int discard_alignment;struct device __dev;
@@ -125,13 +133,6 @@ struct hd_struct {#ifdef CONFIG_FAIL_MAKE_REQUEST int make_it_fail;#endif -unsigned long stamp; -#ifdef CONFIG_SMP -struct disk_stats __percpu *dkstats; -#else -struct disk_stats dkstats; -#endif -struct percpu_ref ref;struct gendisk *disk;struct rcu_work rcu_work; };
Signature · 1 line
2.20.1
PATCH 4/4 block: don't hold part0's refcount in IO path
In reply to Ming Leigendisk can't be gone when there is IO activity, so not hold part0's refcount in IO path.
--- block/blk-core.c | 3 ++- block/genhd.c | 1 - 2 files changed, 2 insertions(+), 2 deletions(-)
block/blk-core.c +2 -1
@@ -1345,7 +1345,8 @@ void blk_account_io_done(struct request *req, u64 now)part_stat_add (part, time_in_queue,nsecs_to_jiffies64 (now - req->start_time_ns));part_dec_in_flight (req->q, part,rq_data_dir (req)); -hd_struct_put (part); +if (part->partno) +hd_struct_put (part);part_stat_unlock (); } }
block/genhd.c +0 -1
@@ -300,7 +300,6 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector) } } exit: -hd_struct_get (&disk->part0);return &disk->part0; }EXPORT_SYMBOL_GPL (disk_map_sector_rcu);
Signature · 1 line
2.20.1