Skip to contentSkip to conversation

block: fix partition use-after-free and optimization

PATCH Superseded Ming Lei 2 participants 9 messages May 8, 2020 Block layer

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Ming Lei

author
#0 lore raw

v2 cover block: fix partition use-after-free and optimization

Hi,

The 1st patch fixes one use-after-free on cached last_lookup partition.

The other 3 patches optimizes partition uses in IO path.

V2:
- add comment, use part_to_disk() to retrieve disk instead of adding one field to hd_struct
- don't put part in blk_account_io_merge

Ming Lei (4):
block: fix use-after-free on cached last_lookup partition
block: only define 'nr_sects_seq' in hd_part for 32bit SMP
block: re-organize fields of 'struct hd_part'
block: don't hold part0's refcount in IO path

 block/blk-core.c        | 15 ++-------------
 block/blk-merge.c       |  3 ++-
 block/genhd.c           | 17 +++++++++++++----
 block/partitions/core.c | 14 ++++++++++++--
 include/linux/genhd.h   | 24 +++++++++++++++++-------
 5 files changed, 46 insertions(+), 27 deletions(-)
Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
Signature · 1 line
2.25.2
author
#1 lore raw

PATCH v2 1/4 block: fix use-after-free on cached last_lookup partition

delete_partition() clears the cached last_lookup partition. However the .last_lookup cache may be overwritten by one IO path after it is cleared from delete_partition(). Then another IO path may use the cached deleting partition after hd_struct_free() is called, then use-after-free is triggered on the cached partition.

Fixes the issue by the following approach:

1) always get the partition's refcount via hd_struct_try_get() before setting .last_lookup

2) move clearing .last_lookup from delete_partition() to hd_struct_free() which is the release handle of the partition's percpu-refcount, so that no IO path can cache deleteing partition via .last_lookup.

It is one candidate approach of Yufen's patch[1] which adds overhead in fast path by indirect lookup which may introduce one extra cacheline in IO path. Also this patch relies on percpu-refcount's protection, and it is easier to understand and verify.

[1] https://lore.kernel.org/linux-block/[email protected]/T/#t

---
 block/blk-core.c        | 12 ------------
 block/genhd.c           | 15 ++++++++++++---
 block/partitions/core.c | 12 +++++++++++-
 3 files changed, 23 insertions(+), 16 deletions(-)
block/blk-core.c +0 -12
index ec50d7e6be21..826a8980997d 100644
@@ -1363,18 +1363,6 @@ void blk_account_io_start(struct request *rq, bool new_io)
 		part_stat_inc(part, merges[rw]);
 	} else {
 		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
-		if (!hd_struct_try_get(part)) {
-			/*
-			 * The partition is already being removed,
-			 * the request will be accounted on the disk only
-			 *
-			 * We take a reference on disk->part0 although that
-			 * partition will never be deleted, so we can treat
-			 * it as any other partition.
-			 */
-			part = &rq->rq_disk->part0;
-			hd_struct_get(part);
-		}
 		part_inc_in_flight(rq->q, part, rw);
 		rq->part = part;
 	}
block/genhd.c +12 -3
index c05d509877fa..ec57d5d7a64d 100644
@@ -344,11 +344,12 @@ static inline int sector_in_part(struct hd_struct *part, sector_t sector)
  * primarily used for stats accounting.
  *
  * CONTEXT:
- * RCU read locked.  The returned partition pointer is valid only
- * while preemption is disabled.
+ * RCU read locked.  The returned partition pointer is always valid
+ * because its refcount is grabbed.
  *
  * RETURNS:
  * Found partition on success, part0 is returned if no partition matches
+ * or the matched partition is being deleted.
  */
 struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 {
@@ -359,17 +360,25 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 	ptbl = rcu_dereference(disk->part_tbl);
 
 	part = rcu_dereference(ptbl->last_lookup);
-	if (part && sector_in_part(part, sector))
+	if (part && sector_in_part(part, sector) && hd_struct_try_get(part))
 		return part;
 
 	for (i = 1; i < ptbl->len; i++) {
 		part = rcu_dereference(ptbl->part[i]);
 
 		if (part && sector_in_part(part, sector)) {
+			/*
+			 * only live partition can be cached for lookup,
+			 * so use-after-free on cached & deleting partition
+			 * can be avoided
+			 */
+			if (!hd_struct_try_get(part))
+				break;
 			rcu_assign_pointer(ptbl->last_lookup, part);
 			return part;
 		}
 	}
+	hd_struct_get(&disk->part0);
 	return &disk->part0;
 }
 
block/partitions/core.c +11 -1
index c085bf85509b..f4000dac23ef 100644
@@ -288,6 +288,12 @@ static void hd_struct_free_work(struct work_struct *work)
 static void hd_struct_free(struct percpu_ref *ref)
 {
 	struct hd_struct *part = container_of(ref, struct hd_struct, ref);
+	struct gendisk *disk = part_to_disk(part);
+	struct disk_part_tbl *ptbl =
+		rcu_dereference_protected(disk->part_tbl, 1);
+
+	rcu_assign_pointer(ptbl->last_lookup, NULL);
+	put_device(disk_to_dev(disk));
 
 	INIT_RCU_WORK(&part->rcu_work, hd_struct_free_work);
 	queue_rcu_work(system_wq, &part->rcu_work);
@@ -309,8 +315,12 @@ void delete_partition(struct gendisk *disk, struct hd_struct *part)
 	struct disk_part_tbl *ptbl =
 		rcu_dereference_protected(disk->part_tbl, 1);
 
+	/*
+	 * ->part_tbl is referenced in this part's release handler, so
+	 *  we have to hold the disk device
+	 */
+	get_device(disk_to_dev(part_to_disk(part)));
 	rcu_assign_pointer(ptbl->part[part->partno], NULL);
-	rcu_assign_pointer(ptbl->last_lookup, NULL);
 	kobject_put(part->holder_dir);
 	device_del(part_to_dev(part));
 
Signature · 1 line
2.25.2
author
#3 lore raw

PATCH v2 2/4 block: only define 'nr_sects_seq' in hd_part for 32bit SMP

In reply to Ming Lei

The seqcount of 'nr_sects_seq' is only needed in case of 32bit SMP, so define it just for 32bit SMP.

---
 block/genhd.c           | 2 +-
 block/partitions/core.c | 2 +-
 include/linux/genhd.h   | 9 +++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)
block/genhd.c +1 -1
index ec57d5d7a64d..bf8cbb033d64 100644
@@ -1729,7 +1729,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id)
 		 * TODO: Ideally set_capacity() and get_capacity() should be
 		 * converted to make use of bd_mutex and sequence counters.
 		 */
-		seqcount_init(&disk->part0.nr_sects_seq);
+		hd_sects_seq_init(&disk->part0);
 		if (hd_ref_init(&disk->part0)) {
 			hd_free_part(&disk->part0);
 			kfree(disk);
block/partitions/core.c +1 -1
index f4000dac23ef..ec81986b358e 100644
@@ -392,7 +392,7 @@ static struct hd_struct *add_partition(struct gendisk *disk, int partno,
 		goto out_free;
 	}
 
-	seqcount_init(&p->nr_sects_seq);
+	hd_sects_seq_init(p);
 	pdev = part_to_dev(p);
 
 	p->start_sect = start;
include/linux/genhd.h +9 -0
index f9c226f9546a..b4744035ae58 100644
@@ -68,7 +68,9 @@ struct hd_struct {
 	 * can be non-atomic on 32bit machines with 64bit sector_t.
 	 */
 	sector_t nr_sects;
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	seqcount_t nr_sects_seq;
+#endif
 	sector_t alignment_offset;
 	unsigned int discard_alignment;
 	struct device __dev;
@@ -274,6 +276,13 @@ static inline void disk_put_part(struct hd_struct *part)
 		put_device(part_to_dev(part));
 }
 
+static inline void hd_sects_seq_init(struct hd_struct *p)
+{
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	seqcount_init(&p->nr_sects_seq);
+#endif
+}
+
 /*
  * Smarter partition iterator without context limits.
  */
Signature · 1 line
2.25.2
author
#4 lore raw

PATCH v2 3/4 block: re-organize fields of 'struct hd_part'

In reply to Ming Lei

Put all fields accessed in IO path together at the beginning of the struct, so that all can be fetched in single cacheline.

---
 include/linux/genhd.h | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)
include/linux/genhd.h +8 -7
index b4744035ae58..a9384449465a 100644
@@ -71,6 +71,14 @@ struct hd_struct {
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	seqcount_t nr_sects_seq;
 #endif
+	unsigned long stamp;
+#ifdef	CONFIG_SMP
+	struct disk_stats __percpu *dkstats;
+#else
+	struct disk_stats dkstats;
+#endif
+	struct percpu_ref ref;
+
 	sector_t alignment_offset;
 	unsigned int discard_alignment;
 	struct device __dev;
@@ -80,13 +88,6 @@ struct hd_struct {
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 	int make_it_fail;
 #endif
-	unsigned long stamp;
-#ifdef	CONFIG_SMP
-	struct disk_stats __percpu *dkstats;
-#else
-	struct disk_stats dkstats;
-#endif
-	struct percpu_ref ref;
 	struct rcu_work rcu_work;
 };
 
Signature · 1 line
2.25.2
author
#5 lore raw

PATCH v2 4/4 block: don't hold part0's refcount in IO path

In reply to Ming Lei

gendisk can't be gone when there is IO activity, so not hold part0's refcount in IO path.

Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
---
 block/blk-core.c  | 3 ++-
 block/blk-merge.c | 3 ++-
 block/genhd.c     | 4 ++--
 3 files changed, 6 insertions(+), 4 deletions(-)
block/blk-core.c +2 -1
index 826a8980997d..56cc61354671 100644
@@ -1343,7 +1343,8 @@ void blk_account_io_done(struct request *req, u64 now)
 		part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);
 		part_dec_in_flight(req->q, part, rq_data_dir(req));
 
-		hd_struct_put(part);
+		if (part->partno)
+			hd_struct_put(part);
 		part_stat_unlock();
 	}
 }
block/blk-merge.c +2 -1
index a04e991b5ded..b44caa28ec04 100644
@@ -672,7 +672,8 @@ static void blk_account_io_merge(struct request *req)
 
 		part_dec_in_flight(req->q, part, rq_data_dir(req));
 
-		hd_struct_put(part);
+		if (part->partno)
+			hd_struct_put(part);
 		part_stat_unlock();
 	}
 }
block/genhd.c +2 -2
index bf8cbb033d64..d97b95d1a2fd 100644
@@ -345,7 +345,8 @@ static inline int sector_in_part(struct hd_struct *part, sector_t sector)
  *
  * CONTEXT:
  * RCU read locked.  The returned partition pointer is always valid
- * because its refcount is grabbed.
+ * because its refcount is grabbed except for part0, which lifetime
+ * is same with the disk.
  *
  * RETURNS:
  * Found partition on success, part0 is returned if no partition matches
@@ -378,7 +379,6 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 			return part;
 		}
 	}
-	hd_struct_get(&disk->part0);
 	return &disk->part0;
 }
 
Signature · 1 line
2.25.2
#6 lore raw

Re: [PATCH V2 4/4] block: don't hold part0's refcount in IO path

Quoted 7 lines · Ming Lei

gendisk can't be gone when there is IO activity, so not hold part0's refcount in IO path.

Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>

This looks correct, although I'd still prefer to centralize the partno checks in the helpers. Also hd_struct_get is unused with this patch isn't it?

author
#7 lore raw

Re: [PATCH V2 4/4] block: don't hold part0's refcount in IO path

Quoted 4 lines · Christoph Hellwig
Quoted 3 lines · Ming Lei

gendisk can't be gone when there is IO activity, so not hold part0's refcount in IO path.

Quoted 8 lines
Quoted 4 lines
Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>

This looks correct, although I'd still prefer to centralize the partno checks in the helpers. Also hd_struct_get is unused with this patch isn't it?

OK, are you fine with the following patch?

block/blk.h +6 -7
index 133fb0b99759..8efd1ca5c975 100644
@@ -376,19 +376,18 @@ int bdev_resize_partition(struct block_device *bdev, int partno,
 int disk_expand_part_tbl(struct gendisk *disk, int target);
 int hd_ref_init(struct hd_struct *part);
 
-static inline void hd_struct_get(struct hd_struct *part)
-{
-	percpu_ref_get(&part->ref);
-}
-
+/* no need to get/put refcount of part0 */
 static inline int hd_struct_try_get(struct hd_struct *part)
 {
-	return percpu_ref_tryget_live(&part->ref);
+	if (part->partno)
+		return percpu_ref_tryget_live(&part->ref);
+	return 1;
 }
 
 static inline void hd_struct_put(struct hd_struct *part)
 {
-	percpu_ref_put(&part->ref);
+	if (part->partno)
+		percpu_ref_put(&part->ref);
 }
 
 static inline void hd_free_part(struct hd_struct *part)
block/genhd.c +2 -2
index bf8cbb033d64..d97b95d1a2fd 100644
@@ -345,7 +345,8 @@ static inline int sector_in_part(struct hd_struct *part, sector_t sector)
  *
  * CONTEXT:
  * RCU read locked.  The returned partition pointer is always valid
- * because its refcount is grabbed.
+ * because its refcount is grabbed except for part0, which lifetime
+ * is same with the disk.
  *
  * RETURNS:
  * Found partition on success, part0 is returned if no partition matches
@@ -378,7 +379,6 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector)
 			return part;
 		}
 	}
-	hd_struct_get(&disk->part0);
 	return &disk->part0;
 }
 

Thanks,
Ming

Reviewed-by
#8 lore raw

Re: [PATCH V2 4/4] block: don't hold part0's refcount in IO path

Quoted 10 lines · Ming Lei
Quoted 9 lines · Christoph Hellwig
Quoted 7 lines · Ming Lei

gendisk can't be gone when there is IO activity, so not hold part0's refcount in IO path.

Cc Yufen Yu <[email protected]>
Cc Christoph Hellwig <[email protected]>
Cc Hou Tao <[email protected]>
Quoted 5 lines
Quoted 3 lines

This looks correct, although I'd still prefer to centralize the partno checks in the helpers. Also hd_struct_get is unused with this patch isn't it?

OK, are you fine with the following patch?

Yes, this looks good: