Skip to contentSkip to conversation

block: fix bio-allocation from per-cpu cache

PATCH In review Kanchan Joshi 3 participants 9 messages Oct 27, 2022 1 message not in the archive Block layer

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): CGME20221027101534epcas5p3b8335c05a1003531f1a4488dc27f27ee@epcas5p3.samsung.com

#1 lore raw

PATCH 1/1 block: fix bio-allocation from per-cpu cache

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
 <TASK>
 dump_stack_lvl+0x34/0x48
 print_report+0x490/0x4a1
 ? __virt_addr_valid+0x28/0x140
 ? bio_alloc_bioset.cold+0x2a/0x16a
 kasan_report+0xb3/0x130
 ? bio_alloc_bioset.cold+0x2a/0x16a
 bio_alloc_bioset.cold+0x2a/0x16a
 ? bvec_alloc+0xf0/0xf0
 ? iov_iter_is_aligned+0x130/0x2c0
 blkdev_direct_IO.part.0+0x16a/0x8d0
 block/bio.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
block/bio.c +6 -5
index 8f624ffaf3d0..66f088bb3736 100644
@@ -439,13 +439,14 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
 
 	cache = per_cpu_ptr(bs->cache, get_cpu());
 	if (!cache->free_list &&
-	    READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) {
+	    READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
 		bio_alloc_irq_cache_splice(cache);
-		if (!cache->free_list) {
-			put_cpu();
-			return NULL;
-		}
+
+	if (!cache->free_list) {
+		put_cpu();
+		return NULL;
 	}
+
 	bio = cache->free_list;
 	cache->free_list = bio->bi_next;
 	cache->nr--;
Signature · 1 line
2.25.1
#2 lore raw
Quoted 2 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Damn, it was done right in v2

https://lore.kernel.org/all/9fd04486d972c1f3ef273fa26b4b6bf51a5e4270.1666122465.git.asml.silence@gmail.com/

Perhaps I based v3 on a wrong version. Thanks

Quoted 34 lines
Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
  <TASK>
  dump_stack_lvl+0x34/0x48
  print_report+0x490/0x4a1
  ? __virt_addr_valid+0x28/0x140
  ? bio_alloc_bioset.cold+0x2a/0x16a
  kasan_report+0xb3/0x130
  ? bio_alloc_bioset.cold+0x2a/0x16a
  bio_alloc_bioset.cold+0x2a/0x16a
  ? bvec_alloc+0xf0/0xf0
  ? iov_iter_is_aligned+0x130/0x2c0
  blkdev_direct_IO.part.0+0x16a/0x8d0
  block/bio.c | 11 ++++++-----
  1 file changed, 6 insertions(+), 5 deletions(-)
block/bio.c +0 -0
index 8f624ffaf3d0..66f088bb3736 100644
@@ -439,13 +439,14 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
 
  	cache = per_cpu_ptr(bs->cache, get_cpu());

block/bio.c static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, · line 445

Quoted 10 lines
block/bio.c +3 -5
  	if (!cache->free_list &&
-	    READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) {
+	    READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
  		bio_alloc_irq_cache_splice(cache);
-		if (!cache->free_list) {
-			put_cpu();
-			return NULL;
-		}
+
+	if (!cache->free_list) {

Let's nest it under the other "if (!cache->free_list)"

Quoted 7 lines
block/bio.c +3 -0
+		put_cpu();
+		return NULL;
  	}
+
  	bio = cache->free_list;
  	cache->free_list = bio->bi_next;
  	cache->nr--;
Signature · 1 line
Pavel Begunkov
#3 lore raw
Quoted 48 lines · Pavel Begunkov
Quoted 2 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Damn, it was done right in v2

https://lore.kernel.org/all/9fd04486d972c1f3ef273fa26b4b6bf51a5e4270.1666122465.git.asml.silence@gmail.com/

Perhaps I based v3 on a wrong version. Thanks

Quoted 37 lines
Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
 <TASK>
 dump_stack_lvl+0x34/0x48
 print_report+0x490/0x4a1
 ? __virt_addr_valid+0x28/0x140
 ? bio_alloc_bioset.cold+0x2a/0x16a
 kasan_report+0xb3/0x130
 ? bio_alloc_bioset.cold+0x2a/0x16a
 bio_alloc_bioset.cold+0x2a/0x16a
 ? bvec_alloc+0xf0/0xf0
 ? iov_iter_is_aligned+0x130/0x2c0
 blkdev_direct_IO.part.0+0x16a/0x8d0
 block/bio.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
block/bio.c +1 -1
index 8f624ffaf3d0..66f088bb3736 100644
@@ -439,13 +439,14 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
 	cache = per_cpu_ptr(bs->cache, get_cpu());
 	if (!cache->free_list &&
-	    READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) {
+	    READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
 		bio_alloc_irq_cache_splice(cache);
Quoted 8 lines

block/bio.c static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, · line 445

Quoted 6 lines
block/bio.c +2 -4
-		if (!cache->free_list) {
-			put_cpu();
-			return NULL;
-		}
+
+	if (!cache->free_list) {

Let's nest it under the other "if (!cache->free_list)"

Not sure if I got you. It was under that if condition earlier, and that part causes trouble.
What you wrote in v2 is another way, but there also we have two checks on cache->free_list.

#4 lore raw
Quoted 58 lines · Kanchan Joshi
Quoted 48 lines · Pavel Begunkov
Quoted 2 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Damn, it was done right in v2

https://lore.kernel.org/all/9fd04486d972c1f3ef273fa26b4b6bf51a5e4270.1666122465.git.asml.silence@gmail.com/

Perhaps I based v3 on a wrong version. Thanks

Quoted 37 lines
Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
 <TASK>
 dump_stack_lvl+0x34/0x48
 print_report+0x490/0x4a1
 ? __virt_addr_valid+0x28/0x140
 ? bio_alloc_bioset.cold+0x2a/0x16a
 kasan_report+0xb3/0x130
 ? bio_alloc_bioset.cold+0x2a/0x16a
 bio_alloc_bioset.cold+0x2a/0x16a
 ? bvec_alloc+0xf0/0xf0
 ? iov_iter_is_aligned+0x130/0x2c0
 blkdev_direct_IO.part.0+0x16a/0x8d0

 block/bio.c | 11 ++++++-----

 1 file changed, 6 insertions(+), 5 deletions(-)
block/bio.c +0 -0
index 8f624ffaf3d0..66f088bb3736 100644
@@ -439,13 +439,14 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
      cache = per_cpu_ptr(bs->cache, get_cpu());

     if (!cache->free_list &&
-        READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) { +        READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) bio_alloc_irq_cache_splice(cache);

Quoted 8 lines
Quoted 6 lines

-        if (!cache->free_list) {
-            put_cpu();
-            return NULL;
-        }
+
+    if (!cache->free_list) {

Let's nest it under the other "if (!cache->free_list)"

Quoted 2 lines

Not sure if I got you. It was under that if condition earlier, and that part causes trouble.

Under the free_list check specifically, the threshold would also go in a separate if,

Quoted 2 lines

What you wrote in v2 is another way, but there also we have two checks on cache->free_list.

Your version:

if (cache_empty())

	if (check_threshold())
		try_replenish_cache(); // splice
if (cache_empty()) // still empty
	return NULL;

vs v2:

if (cache_empty()) {
	if (check_threshold())
		try_replenish_cache(); // splice
	if (cache_empty()) // still empty
		return NULL;
}

But on the other hand compilers should be smart enough to optimise repeated checks when the cache already have requests, so there should be no real difference.

Signature · 1 line
Pavel Begunkov
maintainer
#5 lore raw
In reply to Kanchan Joshi
Quoted 18 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
 <TASK>
 dump_stack_lvl+0x34/0x48
 print_report+0x490/0x4a1
Quoted 8 lines
 ? __virt_addr_valid+0x28/0x140
 ? bio_alloc_bioset.cold+0x2a/0x16a
 kasan_report+0xb3/0x130
 ? bio_alloc_bioset.cold+0x2a/0x16a
 bio_alloc_bioset.cold+0x2a/0x16a
 ? bvec_alloc+0xf0/0xf0
 ? iov_iter_is_aligned+0x130/0x2c0
 blkdev_direct_IO.part.0+0x16a/0x8d0

Was going to apply this, but after running some testing, it does fix the initial crash but I still get weird corruption crashes with the series it's fixing.

Pavel, I'm going to drop this series for now.

Signature · 1 line
Jens Axboe
#6 lore raw
Quoted 28 lines · Jens Axboe
Quoted 18 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
  <TASK>
  dump_stack_lvl+0x34/0x48
  print_report+0x490/0x4a1
Quoted 8 lines
  ? __virt_addr_valid+0x28/0x140
  ? bio_alloc_bioset.cold+0x2a/0x16a
  kasan_report+0xb3/0x130
  ? bio_alloc_bioset.cold+0x2a/0x16a
  bio_alloc_bioset.cold+0x2a/0x16a
  ? bvec_alloc+0xf0/0xf0
  ? iov_iter_is_aligned+0x130/0x2c0
  blkdev_direct_IO.part.0+0x16a/0x8d0
Quoted 5 lines

Was going to apply this, but after running some testing, it does fix the initial crash but I still get weird corruption crashes with the series it's fixing.

Pavel, I'm going to drop this series for now.

I found one yesterday. Is the issue reproducible?

Signature · 1 line
Pavel Begunkov
#7 lore raw
Quoted 29 lines · Pavel Begunkov
Quoted 28 lines · Jens Axboe
Quoted 26 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
  <TASK>
  dump_stack_lvl+0x34/0x48
  print_report+0x490/0x4a1
  ? __virt_addr_valid+0x28/0x140
  ? bio_alloc_bioset.cold+0x2a/0x16a
  kasan_report+0xb3/0x130
  ? bio_alloc_bioset.cold+0x2a/0x16a
  bio_alloc_bioset.cold+0x2a/0x16a
  ? bvec_alloc+0xf0/0xf0
  ? iov_iter_is_aligned+0x130/0x2c0
  blkdev_direct_IO.part.0+0x16a/0x8d0
Quoted 7 lines
Quoted 5 lines

Was going to apply this, but after running some testing, it does fix the initial crash but I still get weird corruption crashes with the series it's fixing.

Pavel, I'm going to drop this series for now.

I found one yesterday. Is the issue reproducible?

found one bug *

Signature · 1 line
Pavel Begunkov
maintainer
#8 lore raw
In reply to Pavel Begunkov
Quoted 29 lines · Pavel Begunkov
Quoted 28 lines · Jens Axboe
Quoted 26 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
  <TASK>
  dump_stack_lvl+0x34/0x48
  print_report+0x490/0x4a1
  ? __virt_addr_valid+0x28/0x140
  ? bio_alloc_bioset.cold+0x2a/0x16a
  kasan_report+0xb3/0x130
  ? bio_alloc_bioset.cold+0x2a/0x16a
  bio_alloc_bioset.cold+0x2a/0x16a
  ? bvec_alloc+0xf0/0xf0
  ? iov_iter_is_aligned+0x130/0x2c0
  blkdev_direct_IO.part.0+0x16a/0x8d0
Quoted 7 lines
Quoted 5 lines

Was going to apply this, but after running some testing, it does fix the initial crash but I still get weird corruption crashes with the series it's fixing.

Pavel, I'm going to drop this series for now.

I found one yesterday. Is the issue reproducible?

Oh yeah, triggers in < 1 second for me when running my usual irq peak bench:

t/io_uring -p0 -d128 -b512 -s32 -c32 -F1 -B1 -R0 -X1 -n24 -P1 /dev/nvme0n1 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1 /dev/nvme5n1 /dev/nvme6n1 /dev/nvme7n1 /dev/nvme8n1 /dev/nvme9n1 /dev/nvme10n1 /dev/nvme11n1 /dev/nvme12n1 /dev/nvme13n1 /dev/nvme14n1 /dev/nvme15n1 /dev/nvme16n1 /dev/nvme17n1 /dev/nvme18n1 /dev/nvme19n1 /dev/nvme20n1 /dev/nvme21n1 /dev/nvme22n1 /dev/nvme23n1

Interestingly, doesn't trigger in qemu with just a single device.

Signature · 1 line
Jens Axboe
#9 lore raw
Quoted 36 lines · Jens Axboe
Quoted 35 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 18 lines · Kanchan Joshi

If cache does not have any entry, make sure to detect that and return failure. Otherwise this leads to null pointer dereference.

Fixes 13a184e26965 ("block/bio: add pcpu caching for non-polling bio_put")

---
Can be reproduced by:
fio -direct=1 -iodepth=1 -rw=randread -ioengine=io_uring -bs=4k -numjobs=1 -size=4k -filename=/dev/nvme0n1 -hipri=1 -name=block

BUG: KASAN: null-ptr-deref in bio_alloc_bioset.cold+0x2a/0x16a
Read of size 8 at addr 0000000000000000 by task fio/1835

CPU: 5 PID: 1835 Comm: fio Not tainted 6.1.0-rc2+ #226
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.14.0-0-g
Call Trace:
   <TASK>
   dump_stack_lvl+0x34/0x48
   print_report+0x490/0x4a1
Quoted 8 lines
   ? __virt_addr_valid+0x28/0x140
   ? bio_alloc_bioset.cold+0x2a/0x16a
   kasan_report+0xb3/0x130
   ? bio_alloc_bioset.cold+0x2a/0x16a
   bio_alloc_bioset.cold+0x2a/0x16a
   ? bvec_alloc+0xf0/0xf0
   ? iov_iter_is_aligned+0x130/0x2c0
   blkdev_direct_IO.part.0+0x16a/0x8d0

Was going to apply this, but after running some testing, it does fix the initial crash but I still get weird corruption crashes with the series it's fixing.

Pavel, I'm going to drop this series for now.

Quoted 8 lines
Quoted 1 line

I found one yesterday. Is the issue reproducible?

Oh yeah, triggers in < 1 second for me when running my usual irq peak bench:

t/io_uring -p0 -d128 -b512 -s32 -c32 -F1 -B1 -R0 -X1 -n24 -P1 /dev/nvme0n1 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1 /dev/nvme5n1 /dev/nvme6n1 /dev/nvme7n1 /dev/nvme8n1 /dev/nvme9n1 /dev/nvme10n1 /dev/nvme11n1 /dev/nvme12n1 /dev/nvme13n1 /dev/nvme14n1 /dev/nvme15n1 /dev/nvme16n1 /dev/nvme17n1 /dev/nvme18n1 /dev/nvme19n1 /dev/nvme20n1 /dev/nvme21n1 /dev/nvme22n1 /dev/nvme23n1

Interestingly, doesn't trigger in qemu with just a single device.

The bug I mentioned is splicing from in-IRQ put, which modifies the non-irq list. We need to hit that ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK in the cache to trigger it, so makes sense you see it only with very high qd tests, matches the profile.

I'll resend the patch set with a few changes, but would be great if you can say if sth like below works for you

block/bio.c +6 -4
index 0686a3774157..af715aee239b 100644
@@ -764,6 +764,12 @@ static inline void bio_put_percpu_cache(struct bio *bio)
 	struct bio_alloc_cache *cache;
 
 	cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
+	if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX) {
+		put_cpu();
+		bio_free(bio);
+		return;
+	}
+
 	bio_uninit(bio);
 
 	if ((bio->bi_opf & REQ_POLLED) && !WARN_ON_ONCE(in_interrupt())) {
@@ -779,10 +785,6 @@ static inline void bio_put_percpu_cache(struct bio *bio)
 		cache->nr_irq++;
 		local_irq_restore(flags);
 	}
-
-	if (READ_ONCE(cache->nr_irq) + cache->nr >
-	    ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK)
-		bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK);
 	put_cpu();
 }