Skip to contentSkip to conversation

block: fix partition use-after-free and optimization

PATCH In review Ming Lei 3 participants 8 messages Jan 9, 2020 – Feb 21, 2020 Block layer

lore
Share & tools
lore

Ming Lei

author
#0 lore raw

cover block: fix partition use-after-free and optimization

Hi,

The 1st patch fixes one use-after-free on cached last_lookup partition.

The other 3 patches optimizes partition uses in IO path.

Ming Lei (4):
block: fix use-after-free on cached last_lookup partition
block: only define 'nr_sects_seq' in hd_part for 32bit SMP
block: re-organize fields of 'struct hd_part'
block: don't hold part0's refcount in IO path

 block/blk-core.c          | 15 ++-------------
 block/genhd.c             |  7 +++++--
 block/partition-generic.c | 12 ++++++++++--
 include/linux/genhd.h     | 25 ++++++++++++++++++-------
 4 files changed, 35 insertions(+), 24 deletions(-)
Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
Signature · 1 line
2.20.1
author
#1 lore raw

PATCH 1/4 block: fix use-after-free on cached last_lookup partition

delete_partition() clears the cached last_lookup partition. However the .last_lookup cache may be overwritten by one IO path after it is cleared from delete_partition(). Then another IO path may use the cached deleting partition after __delete_partition() is called, then use-after-free is triggered on the cached partition.

Fixes the issue by the following approach:

1) always get the partition's refcount via hd_struct_try_get() before setting .last_lookup

2) move clearing .last_lookup from delete_partition() to __delete_partition() which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in __delete_partition().

It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.

[1] https://lore.kernel.org/linux-block/[email protected]/T/#t

---
 block/blk-core.c          | 12 ------------
 block/genhd.c             |  6 +++++-
 block/partition-generic.c | 10 +++++++++-
 include/linux/genhd.h     |  1 +
 4 files changed, 15 insertions(+), 14 deletions(-)
block/blk-core.c +0 -12
index 089e890ab208..79599f5fd5b7 100644
@@ -1365,18 +1365,6 @@ void blk_account_io_start(struct request *rq, bool new_io)
 		part_stat_inc(part, merges[rw]);
 	} else {
 		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
-		if (!hd_struct_try_get(part)) {
-			/*
-			 * The partition is already being removed,
-			 * the request will be accounted on the disk only
-			 *
-			 * We take a reference on disk->part0 although that
-			 * partition will never be deleted, so we can treat
-			 * it as any other partition.
-			 */
-			part = &rq->rq_disk->part0;
-			hd_struct_get(part);
-		}
 		part_inc_in_flight(rq->q, part, rw);
 		rq->part = part;
 	}
block/genhd.c +5 -1
index ff6268970ddc..6029c94510f0 100644
@@ -286,17 +286,21 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 	ptbl = rcu_dereference(disk->part_tbl);
 
 	part = rcu_dereference(ptbl->last_lookup);
-	if (part && sector_in_part(part, sector))
+	if (part && sector_in_part(part, sector) && hd_struct_try_get(part))
 		return part;
 
 	for (i = 1; i < ptbl->len; i++) {
 		part = rcu_dereference(ptbl->part[i]);
 
 		if (part && sector_in_part(part, sector)) {
+			if (!hd_struct_try_get(part))
+				goto exit;
 			rcu_assign_pointer(ptbl->last_lookup, part);
 			return part;
 		}
 	}
+ exit:
+	hd_struct_get(&disk->part0);
 	return &disk->part0;
 }
 EXPORT_SYMBOL_GPL(disk_map_sector_rcu);
block/partition-generic.c +9 -1
index 1d20c9cf213f..1739f750dbf2 100644
@@ -262,6 +262,12 @@ static void delete_partition_work_fn(struct work_struct *work)
 void __delete_partition(struct percpu_ref *ref)
 {
 	struct hd_struct *part = container_of(ref, struct hd_struct, ref);
+	struct disk_part_tbl *ptbl =
+		rcu_dereference_protected(part->disk->part_tbl, 1);
+
+	rcu_assign_pointer(ptbl->last_lookup, NULL);
+	put_device(disk_to_dev(part->disk));
+
 	INIT_RCU_WORK(&part->rcu_work, delete_partition_work_fn);
 	queue_rcu_work(system_wq, &part->rcu_work);
 }
@@ -283,8 +289,9 @@ void delete_partition(struct gendisk *disk, int partno)
 	if (!part)
 		return;
 
+	get_device(disk_to_dev(disk));
 	rcu_assign_pointer(ptbl->part[partno], NULL);
-	rcu_assign_pointer(ptbl->last_lookup, NULL);
+
 	kobject_put(part->holder_dir);
 	device_del(part_to_dev(part));
 
@@ -349,6 +356,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno,
 	p->nr_sects = len;
 	p->partno = partno;
 	p->policy = get_disk_ro(disk);
+	p->disk = disk;
 
 	if (info) {
 		struct partition_meta_info *pinfo = alloc_part_info(disk);
include/linux/genhd.h +1 -0
index 8bb63027e4d6..1b09cfe00aa3 100644
@@ -130,6 +130,7 @@ struct hd_struct {
 	struct disk_stats dkstats;
 #endif
 	struct percpu_ref ref;
+	struct gendisk *disk;
 	struct rcu_work rcu_work;
 };
 
Signature · 1 line
2.20.1
#2 lore raw

Re: [PATCH 1/4] block: fix use-after-free on cached last_lookup partition

Hi Guys,

Did this patch have been forgotten? So, ping...

Thanks,
Yufen

Quoted 133 lines · Ming Lei

delete_partition() clears the cached last_lookup partition. However the .last_lookup cache may be overwritten by one IO path after it is cleared from delete_partition(). Then another IO path may use the cached deleting partition after __delete_partition() is called, then use-after-free is triggered on the cached partition.

Fixes the issue by the following approach:

1) always get the partition's refcount via hd_struct_try_get() before setting .last_lookup

2) move clearing .last_lookup from delete_partition() to __delete_partition() which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in __delete_partition().

It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.

[1] https://lore.kernel.org/linux-block/[email protected]/T/#t

---
  block/blk-core.c          | 12 ------------
  block/genhd.c             |  6 +++++-
  block/partition-generic.c | 10 +++++++++-
  include/linux/genhd.h     |  1 +
  4 files changed, 15 insertions(+), 14 deletions(-)
block/blk-core.c +0 -12
index 089e890ab208..79599f5fd5b7 100644
@@ -1365,18 +1365,6 @@ void blk_account_io_start(struct request *rq, bool new_io)
  		part_stat_inc(part, merges[rw]);
  	} else {
  		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
-		if (!hd_struct_try_get(part)) {
-			/*
-			 * The partition is already being removed,
-			 * the request will be accounted on the disk only
-			 *
-			 * We take a reference on disk->part0 although that
-			 * partition will never be deleted, so we can treat
-			 * it as any other partition.
-			 */
-			part = &rq->rq_disk->part0;
-			hd_struct_get(part);
-		}
  		part_inc_in_flight(rq->q, part, rw);
  		rq->part = part;
  	}
block/genhd.c +5 -1
index ff6268970ddc..6029c94510f0 100644
@@ -286,17 +286,21 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
  	ptbl = rcu_dereference(disk->part_tbl);
 
  	part = rcu_dereference(ptbl->last_lookup);
-	if (part && sector_in_part(part, sector))
+	if (part && sector_in_part(part, sector) && hd_struct_try_get(part))
  		return part;
 
  	for (i = 1; i < ptbl->len; i++) {
  		part = rcu_dereference(ptbl->part[i]);
 
  		if (part && sector_in_part(part, sector)) {
+			if (!hd_struct_try_get(part))
+				goto exit;
  			rcu_assign_pointer(ptbl->last_lookup, part);
  			return part;
  		}
  	}
+ exit:
+	hd_struct_get(&disk->part0);
  	return &disk->part0;
  }
  EXPORT_SYMBOL_GPL(disk_map_sector_rcu);
block/partition-generic.c +9 -1
index 1d20c9cf213f..1739f750dbf2 100644
@@ -262,6 +262,12 @@ static void delete_partition_work_fn(struct work_struct *work)
  void __delete_partition(struct percpu_ref *ref)
  {
  	struct hd_struct *part = container_of(ref, struct hd_struct, ref);
+	struct disk_part_tbl *ptbl =
+		rcu_dereference_protected(part->disk->part_tbl, 1);
+
+	rcu_assign_pointer(ptbl->last_lookup, NULL);
+	put_device(disk_to_dev(part->disk));
+
  	INIT_RCU_WORK(&part->rcu_work, delete_partition_work_fn);
  	queue_rcu_work(system_wq, &part->rcu_work);
  }
@@ -283,8 +289,9 @@ void delete_partition(struct gendisk *disk, int partno)
  	if (!part)
  		return;
 
+	get_device(disk_to_dev(disk));
  	rcu_assign_pointer(ptbl->part[partno], NULL);
-	rcu_assign_pointer(ptbl->last_lookup, NULL);
+
  	kobject_put(part->holder_dir);
  	device_del(part_to_dev(part));
 
@@ -349,6 +356,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno,
  	p->nr_sects = len;
  	p->partno = partno;
  	p->policy = get_disk_ro(disk);
+	p->disk = disk;
 
  	if (info) {
  		struct partition_meta_info *pinfo = alloc_part_info(disk);
include/linux/genhd.h +1 -0
index 8bb63027e4d6..1b09cfe00aa3 100644
@@ -130,6 +130,7 @@ struct hd_struct {
  	struct disk_stats dkstats;
  #endif
  	struct percpu_ref ref;
+	struct gendisk *disk;
  	struct rcu_work rcu_work;
  };
 
#3 lore raw

Re: [PATCH 1/4] block: fix use-after-free on cached last_lookup partition

In reply to Ming Lei
Quoted 16 lines · Ming Lei

delete_partition() clears the cached last_lookup partition. However the .last_lookup cache may be overwritten by one IO path after it is cleared from delete_partition(). Then another IO path may use the cached deleting partition after __delete_partition() is called, then use-after-free is triggered on the cached partition.

Fixes the issue by the following approach:

1) always get the partition's refcount via hd_struct_try_get() before setting .last_lookup

2) move clearing .last_lookup from delete_partition() to __delete_partition() which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in __delete_partition().

Quoted 6 lines

It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.

[1] https://lore.kernel.org/linux-block/[email protected]/T/#t

Hi Ming,

disk_map_sector_rcu() is called from the I/O path only and hence with q->q_usage_counter > 0. Has it been considered to freeze disk->queue from delete_partition() before deleting a partition and unfreezing disk->queue after partition deletion has finished? Would that approach allow to eliminate partition reference counting and thereby improve the performance of the hot path?

Thanks,

Bart.

author
#4 lore raw

Re: [PATCH 1/4] block: fix use-after-free on cached last_lookup partition

Quoted 24 lines · Bart Van Assche
Quoted 16 lines · Ming Lei

delete_partition() clears the cached last_lookup partition. However the .last_lookup cache may be overwritten by one IO path after it is cleared from delete_partition(). Then another IO path may use the cached deleting partition after __delete_partition() is called, then use-after-free is triggered on the cached partition.

Fixes the issue by the following approach:

1) always get the partition's refcount via hd_struct_try_get() before setting .last_lookup

2) move clearing .last_lookup from delete_partition() to __delete_partition() which is release handle of the partition's percpu-refcount, so that no IO path can overwrite .last_lookup after it is cleared in __delete_partition().

Quoted 6 lines

It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.

[1] https://lore.kernel.org/linux-block/[email protected]/T/#t

Quoted 8 lines

Hi Ming,

disk_map_sector_rcu() is called from the I/O path only and hence with q->q_usage_counter > 0. Has it been considered to freeze disk->queue from delete_partition() before deleting a partition and unfreezing disk->queue after partition deletion has finished? Would that approach allow to eliminate partition reference counting and thereby improve the performance of the hot path?

Hi Bart,

I did consider that approach, but this way may cause performance regression, given deleting any partition drops IO performance a lot on other un-related partitions.

Thanks,
Ming

author
#5 lore raw

PATCH 2/4 block: only define 'nr_sects_seq' in hd_part for 32bit SMP

In reply to Ming Lei

The seqcount of 'nr_sects_seq' is only needed in case of 32bit SMP, so define it just for 32bit SMP.

Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
---
 block/genhd.c             | 2 +-
 block/partition-generic.c | 2 +-
 include/linux/genhd.h     | 9 +++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)
block/genhd.c +1 -1
index 6029c94510f0..bfc4148ec341 100644
@@ -1506,7 +1506,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id)
 		 * TODO: Ideally set_capacity() and get_capacity() should be
 		 * converted to make use of bd_mutex and sequence counters.
 		 */
-		seqcount_init(&disk->part0.nr_sects_seq);
+		hd_sects_seq_init(&disk->part0);
 		if (hd_ref_init(&disk->part0)) {
 			hd_free_part(&disk->part0);
 			kfree(disk);
block/partition-generic.c +1 -1
index 1739f750dbf2..eb606c2c3d6c 100644
@@ -345,7 +345,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno,
 		goto out_free;
 	}
 
-	seqcount_init(&p->nr_sects_seq);
+	hd_sects_seq_init(p);
 	pdev = part_to_dev(p);
 
 	p->start_sect = start;
include/linux/genhd.h +9 -0
index 1b09cfe00aa3..5f5718ce5e86 100644
@@ -113,7 +113,9 @@ struct hd_struct {
 	 * can be non-atomic on 32bit machines with 64bit sector_t.
 	 */
 	sector_t nr_sects;
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	seqcount_t nr_sects_seq;
+#endif
 	sector_t alignment_offset;
 	unsigned int discard_alignment;
 	struct device __dev;
@@ -713,6 +715,13 @@ static inline void hd_free_part(struct hd_struct *part)
 	percpu_ref_exit(&part->ref);
 }
 
+static inline void hd_sects_seq_init(struct hd_struct *p)
+{
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	seqcount_init(&p->nr_sects_seq);
+#endif
+}
+
 /*
  * Any access of part->nr_sects which is not protected by partition
  * bd_mutex or gendisk bdev bd_mutex, should be done using this
Signature · 1 line
2.20.1
author
#6 lore raw

PATCH 3/4 block: re-organize fields of 'struct hd_part'

In reply to Ming Lei

Put all fields accessed in IO path together at the beginning of the struct, so that all can be fetched in single cacheline.

Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
---
 include/linux/genhd.h | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)
include/linux/genhd.h +8 -7
index 5f5718ce5e86..4c6a998a0c56 100644
@@ -116,6 +116,14 @@ struct hd_struct {
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	seqcount_t nr_sects_seq;
 #endif
+	unsigned long stamp;
+#ifdef	CONFIG_SMP
+	struct disk_stats __percpu *dkstats;
+#else
+	struct disk_stats dkstats;
+#endif
+	struct percpu_ref ref;
+
 	sector_t alignment_offset;
 	unsigned int discard_alignment;
 	struct device __dev;
@@ -125,13 +133,6 @@ struct hd_struct {
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 	int make_it_fail;
 #endif
-	unsigned long stamp;
-#ifdef	CONFIG_SMP
-	struct disk_stats __percpu *dkstats;
-#else
-	struct disk_stats dkstats;
-#endif
-	struct percpu_ref ref;
 	struct gendisk *disk;
 	struct rcu_work rcu_work;
 };
Signature · 1 line
2.20.1
author
#7 lore raw

PATCH 4/4 block: don't hold part0's refcount in IO path

In reply to Ming Lei

gendisk can't be gone when there is IO activity, so not hold part0's refcount in IO path.

Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
---
 block/blk-core.c | 3 ++-
 block/genhd.c    | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
block/blk-core.c +2 -1
index 79599f5fd5b7..a1184a59819a 100644
@@ -1345,7 +1345,8 @@ void blk_account_io_done(struct request *req, u64 now)
 		part_stat_add(part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns));
 		part_dec_in_flight(req->q, part, rq_data_dir(req));
 
-		hd_struct_put(part);
+		if (part->partno)
+			hd_struct_put(part);
 		part_stat_unlock();
 	}
 }
block/genhd.c +0 -1
index bfc4148ec341..8c7c32b5dcc0 100644
@@ -300,7 +300,6 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 		}
 	}
  exit:
-	hd_struct_get(&disk->part0);
 	return &disk->part0;
 }
 EXPORT_SYMBOL_GPL(disk_map_sector_rcu);
Signature · 1 line
2.20.1