btrfs: scrub: fix several races which can lead to UAF
Subsystems (2)
cover btrfs: scrub: fix several races which can lead to UAF
This is inspired by the fix in raid56:
https://lore.kernel.org/linux-btrfs/[email protected]/
And there are two similar cases inside scrub.c:
- scrub_stripe::repair_wait
- scrub_stripe::io_wait
The only difference is, the window is much smaller compared to RAID56, as most scrub_stripe structure have a very long lifespan, as long as the scrub/replace ioctl. Only the RAID56 data stripes has a shorter lifespan.
But still it's possible in theory, so here comes the same fix for scrub.
For repair_wait it's pretty straight foward.
For io_wait we have several locations that we submit a batch of bios in one go, and pending_io may drop to zero during IOs.
Follow the existing behavior by grabbing one extra ref to ensure the pending_io won't reach zero during IOs.
Qu Wenruo (2):
btrfs: scrub: fix use-after-free when waking up the repair waiter
btrfs: scrub: fix use-after-free when waking up the stripe I/O waiter
fs/btrfs/scrub.c | 116 +++++++++++++++++++++++++++++++---------------- 1 file changed, 77 insertions(+), 39 deletions(-)
Signature · 1 line
2.56.0
PATCH 1/2 btrfs: scrub: fix use-after-free when waking up the repair waiter
The read-repair worker of a scrub_stripe can race with stripe freeing:
CPU0 (repair worker) CPU1 (scrub) scrub_stripe_read_repair_worker() scrub_raid56_parity_stripe() | |-wait_event() | | ^^^ checks the condition without | | taking repair_wait.lock |-set_bit(REPAIR_DONE) | | ^^^ the waiter can see it now | ^^^ sees the bit, doesn't sleep | | | \-scrub_simple_mirror() | \-... | scrub_stripe() | |-release_scrub_stripe() | \-kfree(sctx->raid56_data_stripes) | ^^^ stripe freed |
\-wake_up()
\-_raw_spin_lock_irqsave()
^^^ repair_wait.lock is in freed memory
Too late.
The same applies to flush_scrub_stripes(), where the stripes are freed by scrub_free_ctx() at the end of the scrub.
Fix it by waiting with a completion instead. complete_all() wakes the waiter under the completion's lock and does not touch it afterwards, and wait_for_completion() returns only after taking that lock, so the stripe can be freed as soon as the wait returns.
The completion is re-armed by scrub_reset_stripe(), which is called before every refill of a stripe, i.e. after the previous waiter has returned. The SCRUB_STRIPE_FLAG_REPAIR_DONE flag has no user left, remove it.
This is the same problem fixed for the RAID56 rbio in commit 26eb4d70f96d ("btrfs: raid56: fix use-after-free when waking up the rbio I/O waiter").
--- fs/btrfs/scrub.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-)
fs/btrfs/scrub.c +8 -11
@@ -80,9 +80,6 @@ enum scrub_stripe_flags {/* Set when @mirror_num, @dev, @physical and @logical are set. */ SCRUB_STRIPE_FLAG_INITIALIZED, -/* Set when the read-repair is finished. */ - SCRUB_STRIPE_FLAG_REPAIR_DONE, -/* * Set for data stripes if it's triggered from P/Q stripe. * During such scrub, we should not report errors in data stripes, nor
@@ -153,7 +150,9 @@ struct scrub_stripe {/* From btrfs_chunk_map::stripe_len. Should not exceed BTRFS_STRIPE_LEN. */ u32 stripe_len;wait_queue_head_t io_wait; -wait_queue_head_t repair_wait; + +/* Completed when the read-repair is finished. */ +struct completion repair_done;/* * Indicate the states of the stripe. Bits are defined in
@@ -359,7 +358,7 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, stripe->state =0 ;init_waitqueue_head (&stripe->io_wait); -init_waitqueue_head (&stripe->repair_wait); +init_completion (&stripe->repair_done);atomic_set (&stripe->pending_io,0 );spin_lock_init (&stripe->write_error_lock);
@@ -1234,8 +1233,7 @@ static void scrub_stripe_read_repair_worker(struct work_struct *work) }scrub_stripe_report_errors (sctx, stripe, &errors); -set_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state); -wake_up (&stripe->repair_wait); +complete_all (&stripe->repair_done); }static void scrub_read_endio (struct btrfs_bio *bbio)
@@ -1762,6 +1760,7 @@ static void scrub_reset_stripe(struct scrub_stripe *stripe) stripe->nr_meta_extents =0 ; stripe->nr_data_extents =0 ; stripe->state =0 ; +reinit_completion (&stripe->repair_done);for (int i =0 ; i < stripe->nr_sectors; i++) { stripe->sectors[i].csum =NULL ;
@@ -1960,8 +1959,7 @@ static int flush_scrub_stripes(struct scrub_ctx *sctx)for (int i =0 ; i < nr_stripes; i++) { stripe = &sctx->stripes[i]; -wait_event (stripe->repair_wait, -test_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state)); +wait_for_completion (&stripe->repair_done); }/* Submit for dev-replace. */
@@ -2236,8 +2234,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,for (int i =0 ; i < data_stripes; i++) { stripe = &sctx->raid56_data_stripes[i]; -wait_event (stripe->repair_wait, -test_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state)); +wait_for_completion (&stripe->repair_done); }/* For now, no zoned support for RAID56. */ ASSERT (!btrfs_is_zoned (sctx->fs_info));
Signature · 1 line
2.56.0
Re: [PATCH 1/2] btrfs: scrub: fix use-after-free when waking up the repair waiter
Quoted 18 lines · Qu Wenruo
The read-repair worker of a
scrub_stripecan race with stripe freeing:CPU0 (repair worker) CPU1 (scrub) scrub_stripe_read_repair_worker() scrub_raid56_parity_stripe() | |-wait_event() | | ^^^ checks the condition without | | taking repair_wait.lock |-set_bit(REPAIR_DONE) | | ^^^ the waiter can see it now | ^^^ sees the bit, doesn't sleep | | | \-scrub_simple_mirror() | \-... | scrub_stripe() | |-release_scrub_stripe() | \-kfree(sctx->raid56_data_stripes) | ^^^ stripe freed |
Quoted 4 lines
\-wake_up()
\-_raw_spin_lock_irqsave()
^^^ repair_wait.lock is in freed memory
Too late.
There's something odd and very confusing with the diagram. The way it's structured it gives the idea that scrub_raid56_parity_stripe() calls scrub_simple_mirror() for example. But it's scrub_stripe() that calls scrub_raid56_parity_stripe() and scrub_simple_mirror().
I also find it confusing the mention of scrub_simple_mirror(), since it doesn't add any value.
Something like this instead (the indentation may be screwed up by gmail):
scrub_stripe()
scrub_raid56_parity_stripe()
wait_event()
^^ ...
release_scrub_stripe()
(...)This way, the flow is clearer.
Also, the notation you are using is confusing.
On the left side set_bit() and wake_up() are called by scrub_stripe_read_repair_worker(), but one if prefixed with |- while the other with \-.
On the right side,
Quoted 9 lines
The same applies to
flush_scrub_stripes(), where the stripes are freed byscrub_free_ctx()at the end of the scrub.Fix it by waiting with a completion instead.
complete_all()wakes the waiter under the completion's lock and does not touch it afterwards, andwait_for_completion()returns only after taking that lock, so the stripe can be freed as soon as the wait returns.
Quoted 8 lines
The completion is re-armed by
scrub_reset_stripe(), which is called before every refill of a stripe, i.e. after the previous waiter has returned. TheSCRUB_STRIPE_FLAG_REPAIR_DONEflag has no user left, remove it.This is the same problem fixed for the RAID56 rbio in commit
26eb4d70f96d("btrfs: raid56: fix use-after-free when waking up the rbio I/O waiter").
We can't place the commit ID here, it's not stable yet since it's in github's for-next only and will change as it's rebased and so on.
The code itself looks good.
Thanks.
Quoted 84 lines
Fixes 9ecb5ef543d3 ("btrfs: scrub: introduce the main read repair worker for scrub_stripe")Cc [email protected] # 6.4+Signed-off-by Qu Wenruo <[email protected]>--- fs/btrfs/scrub.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-)fs/btrfs/scrub.c +8
-12@@ -80,9 +80,6 @@ enum scrub_stripe_flags {/* Set when @mirror_num, @dev, @physical and @logical are set. */ SCRUB_STRIPE_FLAG_INITIALIZED, -/* Set when the read-repair is finished. */ - SCRUB_STRIPE_FLAG_REPAIR_DONE, -/* * Set for data stripes if it's triggered from P/Q stripe. * During such scrub, we should not report errors in data stripes, nor @@ -153,7 +150,9 @@ struct scrub_stripe {/* From btrfs_chunk_map::stripe_len. Should not exceed BTRFS_STRIPE_LEN. */ u32 stripe_len;wait_queue_head_t io_wait; -wait_queue_head_t repair_wait; + +/* Completed when the read-repair is finished. */ +struct completion repair_done;/* * Indicate the states of the stripe. Bits are defined in @@ -359,7 +358,7 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, stripe->state =0 ;init_waitqueue_head (&stripe->io_wait); -init_waitqueue_head (&stripe->repair_wait); +init_completion (&stripe->repair_done);atomic_set (&stripe->pending_io,0 );spin_lock_init (&stripe->write_error_lock);@@ -1234,8 +1233,7 @@ static void scrub_stripe_read_repair_worker(struct work_struct *work) }scrub_stripe_report_errors (sctx, stripe, &errors); -set_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state); -wake_up (&stripe->repair_wait); +complete_all (&stripe->repair_done); }static void scrub_read_endio (struct btrfs_bio *bbio)@@ -1762,6 +1760,7 @@ static void scrub_reset_stripe(struct scrub_stripe *stripe) stripe->nr_meta_extents =0 ; stripe->nr_data_extents =0 ; stripe->state =0 ; +reinit_completion (&stripe->repair_done);for (int i =0 ; i < stripe->nr_sectors; i++) { stripe->sectors[i].csum =NULL ;@@ -1960,8 +1959,7 @@ static int flush_scrub_stripes(struct scrub_ctx *sctx)for (int i =0 ; i < nr_stripes; i++) { stripe = &sctx->stripes[i]; -wait_event (stripe->repair_wait, -test_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state)); +wait_for_completion (&stripe->repair_done); }/* Submit for dev-replace. */ @@ -2236,8 +2234,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,for (int i =0 ; i < data_stripes; i++) { stripe = &sctx->raid56_data_stripes[i]; -wait_event (stripe->repair_wait, -test_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state)); +wait_for_completion (&stripe->repair_done); }/* For now, no zoned support for RAID56. */ ASSERT (!btrfs_is_zoned (sctx->fs_info)); --2.56.0
Re: [PATCH 1/2] btrfs: scrub: fix use-after-free when waking up the repair waiter
Quoted 42 lines · Filipe Manana
Quoted 18 lines · Qu Wenruo
The read-repair worker of a
scrub_stripecan race with stripe freeing:CPU0 (repair worker) CPU1 (scrub) scrub_stripe_read_repair_worker() scrub_raid56_parity_stripe() | |-wait_event() | | ^^^ checks the condition without | | taking repair_wait.lock |-set_bit(REPAIR_DONE) | | ^^^ the waiter can see it now | ^^^ sees the bit, doesn't sleep | | | \-scrub_simple_mirror() | \-... | scrub_stripe() | |-release_scrub_stripe() | \-kfree(sctx->raid56_data_stripes) | ^^^ stripe freed |Quoted 4 lines
\-wake_up()
\-_raw_spin_lock_irqsave()
^^^ repair_wait.lock is in freed memory
Too late.There's something odd and very confusing with the diagram. The way it's structured it gives the idea that
scrub_raid56_parity_stripe()callsscrub_simple_mirror()for example. But it'sscrub_stripe()that callsscrub_raid56_parity_stripe()andscrub_simple_mirror().I also find it confusing the mention of
scrub_simple_mirror(), since it doesn't add any value.Something like this instead (the indentation may be screwed up by gmail):
scrub_stripe()scrub_raid56_parity_stripe() wait_event() ^^ ... release_scrub_stripe() (...)
Quoted 6 lines
This way, the flow is clearer.
Also, the notation you are using is confusing.
On the left sideset_bit()andwake_up()are called byscrub_stripe_read_repair_worker(), but one if prefixed with |- while the other with \-.
|- is for cases that there are other sibling calls. \- is for the last sibling.
That's kept consistent.
I'll update the race chart to make it more clear though.
Thanks,
Qu
Quoted 112 lines
On the right side,
Quoted 9 lines
The same applies to
flush_scrub_stripes(), where the stripes are freed byscrub_free_ctx()at the end of the scrub.Fix it by waiting with a completion instead.
complete_all()wakes the waiter under the completion's lock and does not touch it afterwards, andwait_for_completion()returns only after taking that lock, so the stripe can be freed as soon as the wait returns.Quoted 8 lines
The completion is re-armed by
scrub_reset_stripe(), which is called before every refill of a stripe, i.e. after the previous waiter has returned. TheSCRUB_STRIPE_FLAG_REPAIR_DONEflag has no user left, remove it.This is the same problem fixed for the RAID56 rbio in commit
26eb4d70f96d("btrfs: raid56: fix use-after-free when waking up the rbio I/O waiter").We can't place the commit ID here, it's not stable yet since it's in github's for-next only and will change as it's rebased and so on.
The code itself looks good.
Thanks.
Quoted 84 lines
Fixes 9ecb5ef543d3 ("btrfs: scrub: introduce the main read repair worker for scrub_stripe")Cc [email protected] # 6.4+Signed-off-by Qu Wenruo <[email protected]>--- fs/btrfs/scrub.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-)fs/btrfs/scrub.c +8
-12@@ -80,9 +80,6 @@ enum scrub_stripe_flags {/* Set when @mirror_num, @dev, @physical and @logical are set. */ SCRUB_STRIPE_FLAG_INITIALIZED, -/* Set when the read-repair is finished. */ - SCRUB_STRIPE_FLAG_REPAIR_DONE, -/* * Set for data stripes if it's triggered from P/Q stripe. * During such scrub, we should not report errors in data stripes, nor @@ -153,7 +150,9 @@ struct scrub_stripe {/* From btrfs_chunk_map::stripe_len. Should not exceed BTRFS_STRIPE_LEN. */ u32 stripe_len;wait_queue_head_t io_wait; -wait_queue_head_t repair_wait; + +/* Completed when the read-repair is finished. */ +struct completion repair_done;/* * Indicate the states of the stripe. Bits are defined in @@ -359,7 +358,7 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, stripe->state =0 ;init_waitqueue_head (&stripe->io_wait); -init_waitqueue_head (&stripe->repair_wait); +init_completion (&stripe->repair_done);atomic_set (&stripe->pending_io,0 );spin_lock_init (&stripe->write_error_lock);@@ -1234,8 +1233,7 @@ static void scrub_stripe_read_repair_worker(struct work_struct *work) }scrub_stripe_report_errors (sctx, stripe, &errors); -set_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state); -wake_up (&stripe->repair_wait); +complete_all (&stripe->repair_done); }static void scrub_read_endio (struct btrfs_bio *bbio)@@ -1762,6 +1760,7 @@ static void scrub_reset_stripe(struct scrub_stripe *stripe) stripe->nr_meta_extents =0 ; stripe->nr_data_extents =0 ; stripe->state =0 ; +reinit_completion (&stripe->repair_done);for (int i =0 ; i < stripe->nr_sectors; i++) { stripe->sectors[i].csum =NULL ;@@ -1960,8 +1959,7 @@ static int flush_scrub_stripes(struct scrub_ctx *sctx)for (int i =0 ; i < nr_stripes; i++) { stripe = &sctx->stripes[i]; -wait_event (stripe->repair_wait, -test_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state)); +wait_for_completion (&stripe->repair_done); }/* Submit for dev-replace. */ @@ -2236,8 +2234,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,for (int i =0 ; i < data_stripes; i++) { stripe = &sctx->raid56_data_stripes[i]; -wait_event (stripe->repair_wait, -test_bit (SCRUB_STRIPE_FLAG_REPAIR_DONE , &stripe->state)); +wait_for_completion (&stripe->repair_done); }/* For now, no zoned support for RAID56. */ ASSERT (!btrfs_is_zoned (sctx->fs_info)); --2.56.0
PATCH 2/2 btrfs: scrub: fix use-after-free when waking up the stripe I/O waiter
In reply to Qu WenruoThe scrub_write_endio() can race with scrub finishing, which can lead to use-after-free:
CPU0 (write endio) CPU1 (scrub) scrub_write_endio() flush_scrub_stripes() | |-wait_scrub_stripe_io() | | \-wait_event() | | ^^^ checks the condition without | | taking io_wait.lock |-atomic_dec_and_test() | | ^^^ pending_io hits 0 | | | ^^^ sees 0, doesn't sleep | | | btrfs_scrub_dev() | \-scrub_put_ctx() | \-scrub_free_ctx() | ^^^ stripe freed |
\-wake_up()
\-_raw_spin_lock_irqsave()
^^^ io_wait.lock is in freed memory
Too late.
Fix it by waiting with a completion instead. complete_all() wakes the waiter under the completion's lock and does not touch it afterwards, and wait_for_completion_io() returns only after taking that lock, so the stripe can be freed as soon as the wait returns.
There are locations that we submit multiple bios in one go, with extra care the pending_io can reach 0 during the submission, to address it:
- Re-arm the completion when the first pending_io ref is taken, in scrub_stripe_get_io(). Any previous round of IO is waited for before the next one starts.
- Hold one extra ref for batched submission
This includes:
* scrub_write_sectors()
* scrub_stripe_submit_repair_read() for non-zoned cases
This is the same pattern used in other scrub path.
The completion is initialized as completed, so waiting on a stripe without any pending IO returns immediately as before.
This is the same problem fixed for the RAID56 rbio in commit 26eb4d70f96d ("btrfs: raid56: fix use-after-free when waking up the rbio I/O waiter").
--- fs/btrfs/scrub.c | 97 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 28 deletions(-)
fs/btrfs/scrub.c +69 -28
@@ -149,7 +149,9 @@ struct scrub_stripe {atomic_t pending_io;/* From btrfs_chunk_map::stripe_len. Should not exceed BTRFS_STRIPE_LEN. */ u32 stripe_len; -wait_queue_head_t io_wait; + +/* Completed by the last endio when pending_io drops to zero. */ +struct completion io_done;/* Completed when the read-repair is finished. */ struct completion repair_done;
@@ -357,9 +359,11 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, stripe->stripe_len =BTRFS_STRIPE_LEN ; stripe->state =0 ; -init_waitqueue_head (&stripe->io_wait);init_completion (&stripe->repair_done); +init_completion (&stripe->io_done);atomic_set (&stripe->pending_io,0 ); +/* No pending IO by default, thus nothing to wait. */ +complete_all (&stripe->io_done);spin_lock_init (&stripe->write_error_lock); stripe->buffer =kvmalloc (BTRFS_STRIPE_LEN ,GFP_KERNEL );
@@ -381,9 +385,25 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info,return -ENOMEM ; } +static void scrub_stripe_get_io (struct scrub_stripe *stripe) +{ +/* The first pending IO re-arms the completion. */ +if (atomic_inc_return (&stripe->pending_io) ==1 ) +reinit_completion (&stripe->io_done); +} + +/* Drop one pending IO, return true if it was the last one. */ +static bool scrub_stripe_put_io (struct scrub_stripe *stripe) +{ +if (!atomic_dec_and_test (&stripe->pending_io)) +return false; +complete_all (&stripe->io_done); +return true; +} +static void wait_scrub_stripe_io (struct scrub_stripe *stripe) { -wait_event (stripe->io_wait,atomic_read (&stripe->pending_io) ==0 ); +wait_for_completion_io (&stripe->io_done); }static void scrub_put_ctx (struct scrub_ctx *sctx);
@@ -891,8 +911,7 @@ static void scrub_repair_read_endio(struct btrfs_bio *bbio)scrub_read_endio_common (bbio); -if (atomic_dec_and_test (&stripe->pending_io)) -wake_up (&stripe->io_wait); +scrub_stripe_put_io (stripe); }static int calc_next_mirror (int mirror,int num_copies)
@@ -941,6 +960,18 @@ static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info,return bbio; } +static void scrub_submit_repair_read_bio (struct scrub_stripe *stripe, +struct btrfs_bio *bbio,int mirror, +bool wait) +{ +ASSERT (bbio->bio.bi_iter.bi_size); + +scrub_stripe_get_io (stripe); +btrfs_submit_bbio (bbio, mirror); +if (wait) +wait_scrub_stripe_io (stripe); +} +static void scrub_stripe_submit_repair_read (struct scrub_stripe *stripe,int mirror,int blocksize,bool wait) {
@@ -953,15 +984,19 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,ASSERT (atomic_read (&stripe->pending_io) ==0 ,"atomic_read(&stripe->pending_io)=%d" ,atomic_read (&stripe->pending_io)); +/* +* Hold an extra ref during submission, so the IO can not be reported as +* finished before all bios are submitted. +* If we wait for every bio, no extra ref as it would block the wait. +*/ +if (!wait) +scrub_stripe_get_io (stripe); +for_each_set_bit (i, &old_error_bitmap, stripe->nr_sectors) {/* The current sector cannot be merged, submit the bio. */ if (bbio && ((i >0 && !test_bit (i -1 , &old_error_bitmap)) || bbio->bio.bi_iter.bi_size >= blocksize)) { -ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); -btrfs_submit_bbio (bbio, mirror); -if (wait) -wait_scrub_stripe_io (stripe); +scrub_submit_repair_read_bio (stripe, bbio, mirror, wait); bbio =NULL ; }
@@ -973,13 +1008,10 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,scrub_bio_add_sector (bbio, stripe, i); } -if (bbio) { -ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); -btrfs_submit_bbio (bbio, mirror); -if (wait) -wait_scrub_stripe_io (stripe); - } +if (bbio) +scrub_submit_repair_read_bio (stripe, bbio, mirror, wait); +if (!wait) +scrub_stripe_put_io (stripe); } static void scrub_stripe_report_errors(struct scrub_ctx *sctx,
@@ -1242,8 +1274,7 @@ static void scrub_read_endio(struct btrfs_bio *bbio)scrub_read_endio_common (bbio); -if (atomic_dec_and_test (&stripe->pending_io)) { -wake_up (&stripe->io_wait); +if (scrub_stripe_put_io (stripe)) {INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);queue_work (stripe->bg->fs_info->scrub_workers, &stripe->work); }
@@ -1269,8 +1300,7 @@ static void scrub_write_endio(struct btrfs_bio *bbio) }bio_put (&bbio->bio); -if (atomic_dec_and_test (&stripe->pending_io)) -wake_up (&stripe->io_wait); +scrub_stripe_put_io (stripe); } static void scrub_submit_write_bio(struct scrub_ctx *sctx,
@@ -1283,7 +1313,7 @@ static void scrub_submit_write_bio(struct scrub_ctx *sctx, stripe->logical;fill_writer_pointer_gap (sctx, stripe->physical + bio_off); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_repair_write (bbio, stripe->mirror_num, dev_replace);if (!btrfs_is_zoned (fs_info))return ;
@@ -1322,6 +1352,15 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *strstruct btrfs_bio *bbio =NULL ;int sector_nr; +/* +* Hold an extra ref during submission, so the IO can not be reported as +* finished before all bios are submitted. +* For zoned, we wait for every bio, no extra ref as it would block the +* wait. +*/ +if (!btrfs_is_zoned (fs_info)) +scrub_stripe_get_io (stripe); +for_each_set_bit (sector_nr, &write_bitmap, stripe->nr_sectors) {/* We should only writeback sectors covered by an extent. */ ASSERT (scrub_bitmap_test_bit_has_extent (stripe, sector_nr));
@@ -1340,6 +1379,8 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *str }if (bbio)scrub_submit_write_bio (sctx, stripe, bbio, dev_replace); +if (!btrfs_is_zoned (fs_info)) +scrub_stripe_put_io (stripe); }/*
@@ -1786,7 +1827,8 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)int mirror = stripe->mirror_num;int i; -atomic_inc (&stripe->pending_io); +/* The extra ref is dropped after all bios are submitted. */ +scrub_stripe_get_io (stripe);for_each_set_bit (i, &has_extent, stripe->nr_sectors) {/* We're beyond the chunk boundary, no need to read anymore. */
@@ -1798,7 +1840,7 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) ((i > 0 && !test_bit(i - 1, &has_extent)) || bbio->bio.bi_iter.bi_size >= stripe_len)) {ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_bbio (bbio, mirror); bbio =NULL ; }
@@ -1844,12 +1886,11 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)if (bbio) {ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_bbio (bbio, mirror); } -if (atomic_dec_and_test (&stripe->pending_io)) { -wake_up (&stripe->io_wait); +if (scrub_stripe_put_io (stripe)) {INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);queue_work (stripe->bg->fs_info->scrub_workers, &stripe->work); }
@@ -1877,7 +1918,7 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx,/* Read the whole range inside the chunk boundary. */ for (unsigned int cur =0 ; cur < nr_sectors; cur++)scrub_bio_add_sector (bbio, stripe, cur); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);/* * For dev-replace, either user asks to avoid the source dev, or
Signature · 1 line
2.56.0
Re: [PATCH 2/2] btrfs: scrub: fix use-after-free when waking up the stripe I/O waiter
Quoted 19 lines · Qu Wenruo
The
scrub_write_endio()can race with scrub finishing, which can lead to use-after-free:CPU0 (write endio) CPU1 (scrub) scrub_write_endio() flush_scrub_stripes() | |-wait_scrub_stripe_io() | | \-wait_event() | | ^^^ checks the condition without | | taking io_wait.lock |-atomic_dec_and_test() | | ^^^ pending_io hits 0 | | | ^^^ sees 0, doesn't sleep | | | btrfs_scrub_dev() | \-scrub_put_ctx() | \-scrub_free_ctx() | ^^^ stripe freed |
Quoted 4 lines
\-wake_up()
\-_raw_spin_lock_irqsave()
^^^ io_wait.lock is in freed memory
Too late.
I have the same comment here regarding the diagram.
It confuses me, it gives the wrong idea that flush_scrub_stripes() either calls btrfs_scrub_dev() or that they are called from the same function.
This can be clearer by using indentation and not skipping functions in the call chain:
btrfs_scrub_dev()
scrub_enumerate_chunks()
scrub_chunk()
scrub_stripe()
flush_scrub_stripes()
wait_scrub_stripe_io()
wait_event()
(...) scrub_put_ctx()
scrub_free_ctx()
Quoted 18 lines
Fix it by waiting with a completion instead.
complete_all()wakes the waiter under the completion's lock and does not touch it afterwards, andwait_for_completion_io()returns only after taking that lock, so the stripe can be freed as soon as the wait returns.There are locations that we submit multiple bios in one go, with extra care the
pending_iocan reach 0 during the submission, to address it:- Re-arm the completion when the first
pending_ioref is taken, inscrub_stripe_get_io(). Any previous round of IO is waited for before the next one starts.- Hold one extra ref for batched submission
This includes:
*scrub_write_sectors()
*scrub_stripe_submit_repair_read()for non-zoned cases
Quoted 8 lines
This is the same pattern used in other scrub path.
The completion is initialized as completed, so waiting on a stripe without any pending IO returns immediately as before.
This is the same problem fixed for the RAID56 rbio in commit
26eb4d70f96d("btrfs: raid56: fix use-after-free when waking up the rbio I/O waiter").
Just like for the previous patch, I have the same comment regarding the commit ID not being stable here; it can change after rebasing and so on.
The code looks good.
Thanks.
Quoted 237 lines
Fixes 9ecb5ef543d3 ("btrfs: scrub: introduce the main read repair worker for scrub_stripe")Fixes 058e09e6fe7c ("btrfs: scrub: introduce a writeback helper for scrub_stripe")Cc [email protected] # 6.4+Signed-off-by Qu Wenruo <[email protected]>--- fs/btrfs/scrub.c | 97 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 28 deletions(-)fs/btrfs/scrub.c +69
-29@@ -149,7 +149,9 @@ struct scrub_stripe {atomic_t pending_io;/* From btrfs_chunk_map::stripe_len. Should not exceed BTRFS_STRIPE_LEN. */ u32 stripe_len; -wait_queue_head_t io_wait; + +/* Completed by the last endio when pending_io drops to zero. */ +struct completion io_done;/* Completed when the read-repair is finished. */ struct completion repair_done;@@ -357,9 +359,11 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, stripe->stripe_len =BTRFS_STRIPE_LEN ; stripe->state =0 ; -init_waitqueue_head (&stripe->io_wait);init_completion (&stripe->repair_done); +init_completion (&stripe->io_done);atomic_set (&stripe->pending_io,0 ); +/* No pending IO by default, thus nothing to wait. */ +complete_all (&stripe->io_done);spin_lock_init (&stripe->write_error_lock); stripe->buffer =kvmalloc (BTRFS_STRIPE_LEN ,GFP_KERNEL );@@ -381,9 +385,25 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info,return -ENOMEM ; } +static void scrub_stripe_get_io (struct scrub_stripe *stripe) +{ +/* The first pending IO re-arms the completion. */ +if (atomic_inc_return (&stripe->pending_io) ==1 ) +reinit_completion (&stripe->io_done); +} + +/* Drop one pending IO, return true if it was the last one. */ +static bool scrub_stripe_put_io (struct scrub_stripe *stripe) +{ +if (!atomic_dec_and_test (&stripe->pending_io)) +return false; +complete_all (&stripe->io_done); +return true; +} +static void wait_scrub_stripe_io (struct scrub_stripe *stripe) { -wait_event (stripe->io_wait,atomic_read (&stripe->pending_io) ==0 ); +wait_for_completion_io (&stripe->io_done); }static void scrub_put_ctx (struct scrub_ctx *sctx);@@ -891,8 +911,7 @@ static void scrub_repair_read_endio(struct btrfs_bio *bbio)scrub_read_endio_common (bbio); -if (atomic_dec_and_test (&stripe->pending_io)) -wake_up (&stripe->io_wait); +scrub_stripe_put_io (stripe); }static int calc_next_mirror (int mirror,int num_copies)@@ -941,6 +960,18 @@ static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info,return bbio; } +static void scrub_submit_repair_read_bio (struct scrub_stripe *stripe, +struct btrfs_bio *bbio,int mirror, +bool wait) +{ +ASSERT (bbio->bio.bi_iter.bi_size); + +scrub_stripe_get_io (stripe); +btrfs_submit_bbio (bbio, mirror); +if (wait) +wait_scrub_stripe_io (stripe); +} +static void scrub_stripe_submit_repair_read (struct scrub_stripe *stripe,int mirror,int blocksize,bool wait) {@@ -953,15 +984,19 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,ASSERT (atomic_read (&stripe->pending_io) ==0 ,"atomic_read(&stripe->pending_io)=%d" ,atomic_read (&stripe->pending_io)); +/* +* Hold an extra ref during submission, so the IO can not be reported as +* finished before all bios are submitted. +* If we wait for every bio, no extra ref as it would block the wait. +*/ +if (!wait) +scrub_stripe_get_io (stripe); +for_each_set_bit (i, &old_error_bitmap, stripe->nr_sectors) {/* The current sector cannot be merged, submit the bio. */ if (bbio && ((i >0 && !test_bit (i -1 , &old_error_bitmap)) || bbio->bio.bi_iter.bi_size >= blocksize)) { -ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); -btrfs_submit_bbio (bbio, mirror); -if (wait) -wait_scrub_stripe_io (stripe); +scrub_submit_repair_read_bio (stripe, bbio, mirror, wait); bbio =NULL ; }@@ -973,13 +1008,10 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,scrub_bio_add_sector (bbio, stripe, i); } -if (bbio) { -ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); -btrfs_submit_bbio (bbio, mirror); -if (wait) -wait_scrub_stripe_io (stripe); - } +if (bbio) +scrub_submit_repair_read_bio (stripe, bbio, mirror, wait); +if (!wait) +scrub_stripe_put_io (stripe); } static void scrub_stripe_report_errors(struct scrub_ctx *sctx,@@ -1242,8 +1274,7 @@ static void scrub_read_endio(struct btrfs_bio *bbio)scrub_read_endio_common (bbio); -if (atomic_dec_and_test (&stripe->pending_io)) { -wake_up (&stripe->io_wait); +if (scrub_stripe_put_io (stripe)) {INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);queue_work (stripe->bg->fs_info->scrub_workers, &stripe->work); }@@ -1269,8 +1300,7 @@ static void scrub_write_endio(struct btrfs_bio *bbio) }bio_put (&bbio->bio); -if (atomic_dec_and_test (&stripe->pending_io)) -wake_up (&stripe->io_wait); +scrub_stripe_put_io (stripe); } static void scrub_submit_write_bio(struct scrub_ctx *sctx,@@ -1283,7 +1313,7 @@ static void scrub_submit_write_bio(struct scrub_ctx *sctx, stripe->logical;fill_writer_pointer_gap (sctx, stripe->physical + bio_off); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_repair_write (bbio, stripe->mirror_num, dev_replace);if (!btrfs_is_zoned (fs_info))return ;@@ -1322,6 +1352,15 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *strstruct btrfs_bio *bbio =NULL ;int sector_nr; +/* +* Hold an extra ref during submission, so the IO can not be reported as +* finished before all bios are submitted. +* For zoned, we wait for every bio, no extra ref as it would block the +* wait. +*/ +if (!btrfs_is_zoned (fs_info)) +scrub_stripe_get_io (stripe); +for_each_set_bit (sector_nr, &write_bitmap, stripe->nr_sectors) {/* We should only writeback sectors covered by an extent. */ ASSERT (scrub_bitmap_test_bit_has_extent (stripe, sector_nr));@@ -1340,6 +1379,8 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *str }if (bbio)scrub_submit_write_bio (sctx, stripe, bbio, dev_replace); +if (!btrfs_is_zoned (fs_info)) +scrub_stripe_put_io (stripe); }/* @@ -1786,7 +1827,8 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)int mirror = stripe->mirror_num;int i; -atomic_inc (&stripe->pending_io); +/* The extra ref is dropped after all bios are submitted. */ +scrub_stripe_get_io (stripe);for_each_set_bit (i, &has_extent, stripe->nr_sectors) {/* We're beyond the chunk boundary, no need to read anymore. */ @@ -1798,7 +1840,7 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) ((i > 0 && !test_bit(i - 1, &has_extent)) || bbio->bio.bi_iter.bi_size >= stripe_len)) {ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_bbio (bbio, mirror); bbio =NULL ; }@@ -1844,12 +1886,11 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)if (bbio) {ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_bbio (bbio, mirror); } -if (atomic_dec_and_test (&stripe->pending_io)) { -wake_up (&stripe->io_wait); +if (scrub_stripe_put_io (stripe)) {INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);queue_work (stripe->bg->fs_info->scrub_workers, &stripe->work); }@@ -1877,7 +1918,7 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx,/* Read the whole range inside the chunk boundary. */ for (unsigned int cur =0 ; cur < nr_sectors; cur++)scrub_bio_add_sector (bbio, stripe, cur); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);/* * For dev-replace, either user asks to avoid the source dev, or -- 2.56.0
Re: [PATCH 2/2] btrfs: scrub: fix use-after-free when waking up the stripe I/O waiter
Quoted 33 lines · Filipe Manana
Quoted 19 lines · Qu Wenruo
The
scrub_write_endio()can race with scrub finishing, which can lead to use-after-free:CPU0 (write endio) CPU1 (scrub) scrub_write_endio() flush_scrub_stripes() | |-wait_scrub_stripe_io() | | \-wait_event() | | ^^^ checks the condition without | | taking io_wait.lock |-atomic_dec_and_test() | | ^^^ pending_io hits 0 | | | ^^^ sees 0, doesn't sleep | | | btrfs_scrub_dev() | \-scrub_put_ctx() | \-scrub_free_ctx() | ^^^ stripe freed |Quoted 4 lines
\-wake_up()
\-_raw_spin_lock_irqsave()
^^^ io_wait.lock is in freed memory
Too late.I have the same comment here regarding the diagram.
It confuses me, it gives the wrong idea thatflush_scrub_stripes()either callsbtrfs_scrub_dev()or that they are called from the same function.This can be clearer by using indentation and not skipping functions in the call chain:
Quoted 8 lines
btrfs_scrub_dev()scrub_enumerate_chunks() scrub_chunk() scrub_stripe() flush_scrub_stripes() wait_scrub_stripe_io() wait_event() (...)
The problem is, this takes up almost all space of a colum. And the indent is not that space efficient either.
One space is not enough to distinguish them, and if you go two spaces, it's no different than using |- or \-, they also take up two spaces.
To keep thing clears and leave enough space for comments, I'd prefer not to include everything.
Thus I'd prefer to skip the scrub_enumerate_chunks() and scrub_chunk() parts, only showing btrfs_scrub_dev() -> scrub_stripe() directly.
But in this case, I'd say the btrfs_scrub_dev() part is not placed where it should be.
Quoted 277 lines
scrub_put_ctx() scrub_free_ctx()Quoted 18 lines
Fix it by waiting with a completion instead.
complete_all()wakes the waiter under the completion's lock and does not touch it afterwards, andwait_for_completion_io()returns only after taking that lock, so the stripe can be freed as soon as the wait returns.There are locations that we submit multiple bios in one go, with extra care the
pending_iocan reach 0 during the submission, to address it:- Re-arm the completion when the first
pending_ioref is taken, inscrub_stripe_get_io(). Any previous round of IO is waited for before the next one starts.- Hold one extra ref for batched submission
This includes:
*scrub_write_sectors()
*scrub_stripe_submit_repair_read()for non-zoned casesQuoted 8 lines
This is the same pattern used in other scrub path.
The completion is initialized as completed, so waiting on a stripe without any pending IO returns immediately as before.
This is the same problem fixed for the RAID56 rbio in commit
26eb4d70f96d("btrfs: raid56: fix use-after-free when waking up the rbio I/O waiter").Just like for the previous patch, I have the same comment regarding the commit ID not being stable here; it can change after rebasing and so on.
The code looks good.
Thanks.Quoted 237 lines
Fixes 9ecb5ef543d3 ("btrfs: scrub: introduce the main read repair worker for scrub_stripe")Fixes 058e09e6fe7c ("btrfs: scrub: introduce a writeback helper for scrub_stripe")Cc [email protected] # 6.4+Signed-off-by Qu Wenruo <[email protected]>--- fs/btrfs/scrub.c | 97 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 28 deletions(-)fs/btrfs/scrub.c +69
-29@@ -149,7 +149,9 @@ struct scrub_stripe {atomic_t pending_io;/* From btrfs_chunk_map::stripe_len. Should not exceed BTRFS_STRIPE_LEN. */ u32 stripe_len; -wait_queue_head_t io_wait; + +/* Completed by the last endio when pending_io drops to zero. */ +struct completion io_done;/* Completed when the read-repair is finished. */ struct completion repair_done;@@ -357,9 +359,11 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info, stripe->stripe_len =BTRFS_STRIPE_LEN ; stripe->state =0 ; -init_waitqueue_head (&stripe->io_wait);init_completion (&stripe->repair_done); +init_completion (&stripe->io_done);atomic_set (&stripe->pending_io,0 ); +/* No pending IO by default, thus nothing to wait. */ +complete_all (&stripe->io_done);spin_lock_init (&stripe->write_error_lock); stripe->buffer =kvmalloc (BTRFS_STRIPE_LEN ,GFP_KERNEL );@@ -381,9 +385,25 @@ static int init_scrub_stripe(struct btrfs_fs_info *fs_info,return -ENOMEM ; } +static void scrub_stripe_get_io (struct scrub_stripe *stripe) +{ +/* The first pending IO re-arms the completion. */ +if (atomic_inc_return (&stripe->pending_io) ==1 ) +reinit_completion (&stripe->io_done); +} + +/* Drop one pending IO, return true if it was the last one. */ +static bool scrub_stripe_put_io (struct scrub_stripe *stripe) +{ +if (!atomic_dec_and_test (&stripe->pending_io)) +return false; +complete_all (&stripe->io_done); +return true; +} +static void wait_scrub_stripe_io (struct scrub_stripe *stripe) { -wait_event (stripe->io_wait,atomic_read (&stripe->pending_io) ==0 ); +wait_for_completion_io (&stripe->io_done); }static void scrub_put_ctx (struct scrub_ctx *sctx);@@ -891,8 +911,7 @@ static void scrub_repair_read_endio(struct btrfs_bio *bbio)scrub_read_endio_common (bbio); -if (atomic_dec_and_test (&stripe->pending_io)) -wake_up (&stripe->io_wait); +scrub_stripe_put_io (stripe); }static int calc_next_mirror (int mirror,int num_copies)@@ -941,6 +960,18 @@ static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info,return bbio; } +static void scrub_submit_repair_read_bio (struct scrub_stripe *stripe, +struct btrfs_bio *bbio,int mirror, +bool wait) +{ +ASSERT (bbio->bio.bi_iter.bi_size); + +scrub_stripe_get_io (stripe); +btrfs_submit_bbio (bbio, mirror); +if (wait) +wait_scrub_stripe_io (stripe); +} +static void scrub_stripe_submit_repair_read (struct scrub_stripe *stripe,int mirror,int blocksize,bool wait) {@@ -953,15 +984,19 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,ASSERT (atomic_read (&stripe->pending_io) ==0 ,"atomic_read(&stripe->pending_io)=%d" ,atomic_read (&stripe->pending_io)); +/* +* Hold an extra ref during submission, so the IO can not be reported as +* finished before all bios are submitted. +* If we wait for every bio, no extra ref as it would block the wait. +*/ +if (!wait) +scrub_stripe_get_io (stripe); +for_each_set_bit (i, &old_error_bitmap, stripe->nr_sectors) {/* The current sector cannot be merged, submit the bio. */ if (bbio && ((i >0 && !test_bit (i -1 , &old_error_bitmap)) || bbio->bio.bi_iter.bi_size >= blocksize)) { -ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); -btrfs_submit_bbio (bbio, mirror); -if (wait) -wait_scrub_stripe_io (stripe); +scrub_submit_repair_read_bio (stripe, bbio, mirror, wait); bbio =NULL ; }@@ -973,13 +1008,10 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,scrub_bio_add_sector (bbio, stripe, i); } -if (bbio) { -ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); -btrfs_submit_bbio (bbio, mirror); -if (wait) -wait_scrub_stripe_io (stripe); - } +if (bbio) +scrub_submit_repair_read_bio (stripe, bbio, mirror, wait); +if (!wait) +scrub_stripe_put_io (stripe); } static void scrub_stripe_report_errors(struct scrub_ctx *sctx,@@ -1242,8 +1274,7 @@ static void scrub_read_endio(struct btrfs_bio *bbio)scrub_read_endio_common (bbio); -if (atomic_dec_and_test (&stripe->pending_io)) { -wake_up (&stripe->io_wait); +if (scrub_stripe_put_io (stripe)) {INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);queue_work (stripe->bg->fs_info->scrub_workers, &stripe->work); }@@ -1269,8 +1300,7 @@ static void scrub_write_endio(struct btrfs_bio *bbio) }bio_put (&bbio->bio); -if (atomic_dec_and_test (&stripe->pending_io)) -wake_up (&stripe->io_wait); +scrub_stripe_put_io (stripe); } static void scrub_submit_write_bio(struct scrub_ctx *sctx,@@ -1283,7 +1313,7 @@ static void scrub_submit_write_bio(struct scrub_ctx *sctx, stripe->logical;fill_writer_pointer_gap (sctx, stripe->physical + bio_off); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_repair_write (bbio, stripe->mirror_num, dev_replace);if (!btrfs_is_zoned (fs_info))return ;@@ -1322,6 +1352,15 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *strstruct btrfs_bio *bbio =NULL ;int sector_nr; +/* +* Hold an extra ref during submission, so the IO can not be reported as +* finished before all bios are submitted. +* For zoned, we wait for every bio, no extra ref as it would block the +* wait. +*/ +if (!btrfs_is_zoned (fs_info)) +scrub_stripe_get_io (stripe); +for_each_set_bit (sector_nr, &write_bitmap, stripe->nr_sectors) {/* We should only writeback sectors covered by an extent. */ ASSERT (scrub_bitmap_test_bit_has_extent (stripe, sector_nr));@@ -1340,6 +1379,8 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *str }if (bbio)scrub_submit_write_bio (sctx, stripe, bbio, dev_replace); +if (!btrfs_is_zoned (fs_info)) +scrub_stripe_put_io (stripe); }/* @@ -1786,7 +1827,8 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)int mirror = stripe->mirror_num;int i; -atomic_inc (&stripe->pending_io); +/* The extra ref is dropped after all bios are submitted. */ +scrub_stripe_get_io (stripe);for_each_set_bit (i, &has_extent, stripe->nr_sectors) {/* We're beyond the chunk boundary, no need to read anymore. */ @@ -1798,7 +1840,7 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) ((i > 0 && !test_bit(i - 1, &has_extent)) || bbio->bio.bi_iter.bi_size >= stripe_len)) {ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_bbio (bbio, mirror); bbio =NULL ; }@@ -1844,12 +1886,11 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)if (bbio) {ASSERT (bbio->bio.bi_iter.bi_size); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);btrfs_submit_bbio (bbio, mirror); } -if (atomic_dec_and_test (&stripe->pending_io)) { -wake_up (&stripe->io_wait); +if (scrub_stripe_put_io (stripe)) {INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);queue_work (stripe->bg->fs_info->scrub_workers, &stripe->work); }@@ -1877,7 +1918,7 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx,/* Read the whole range inside the chunk boundary. */ for (unsigned int cur =0 ; cur < nr_sectors; cur++)scrub_bio_add_sector (bbio, stripe, cur); -atomic_inc (&stripe->pending_io); +scrub_stripe_get_io (stripe);/* * For dev-replace, either user asks to avoid the source dev, or -- 2.56.0