btrfs: simplify the stripe buffer handling
v4 cover btrfs: simplify the stripe buffer handling
[CHANGELOG]
v4:
- Add a patch to extract the common handling of scrub read endios This is to reduce duplication and allow a single invalidate_kernel_vmap_range() call to be added with proper comments.
- Add a comment on why we need to call invalidate_kernel_vmap_range() for vmallocated memory
It's mostly copied from dma-api doc.
v3:
- Add a patch to remove a unused macro
- Add new ASSERT()s to make sure the kvmallocated memory is page aligned The size is 64K, which is page aligned and power of 2, thus the resulted memory should at least be page aligned.
Furthermore, we have several call sites using (BTRFS_STRIPE_LEN / PAGE_SIZE) or (BTRFS_STRIPE_LEN >> PAGE_SHIFT), add an build time assert to make sure our PAGE_SIZE is not larger than BTRFS_STRIPE_LEN.
v2:
- Fix a potential double-free during release_scrub_stripe() Reported by sashiko.
- Enlarge the scrub bbio to be able to contain the full buffer Since the buffer is allocated by kvmalloc(), it's possible that not all pages are physically contiguous.
In that case we have to enlarge the bbio to contain (BTRFS_STRIPE_LEN >> PAGE_SHIFT) bvecs.
This is to use kvmalloc() to allocate a buffer for scrub, so that we do not need to bother page-switching, and simplify the scrub progress.
Qu Wenruo (3):
btrfs: remove SCRUB_MAX_SECTORS_PER_BLOCK
btrfs: extract the common scrub read endio into a helper
btrfs: use kvmalloc() for stripe buffer of scrub_stripe
fs/btrfs/raid56.c | 18 +--- fs/btrfs/raid56.h | 2 +- fs/btrfs/scrub.c | 235 +++++++++++++++++++++------------------------- 3 files changed, 111 insertions(+), 144 deletions(-)
Signature · 1 line
2.54.0
PATCH v4 1/3 btrfs: remove SCRUB_MAX_SECTORS_PER_BLOCK
The last user of this macro is removed in commit 001e3fc263ce ("btrfs:
scrub: remove scrub_block and scrub_sector structures").
Now that macro is only utilized in an ASSERT(), which no longer makes much sense.
Just remove it completely.
--- fs/btrfs/scrub.c | 14 -------------- 1 file changed, 14 deletions(-)
fs/btrfs/scrub.c +0 -14
@@ -57,12 +57,6 @@ struct scrub_ctx;#define SCRUB_TOTAL_STRIPES (SCRUB_GROUPS_PER_SCTX * SCRUB_STRIPES_PER_GROUP) -/* -* The following value times PAGE_SIZE needs to be large enough to match the -* largest node/leaf/sector size that shall be supported. -*/ -#define SCRUB_MAX_SECTORS_PER_BLOCK (BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K) -/* Represent one sector and its needed info to verify the content. */ struct scrub_sector_verification { union {
@@ -3091,14 +3085,6 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,/* At mount time we have ensured nodesize is in the range of [4K, 64K]. */ ASSERT (fs_info->nodesize <=BTRFS_STRIPE_LEN ); -/* -* SCRUB_MAX_SECTORS_PER_BLOCK is calculated using the largest possible -* value (max nodesize / min sectorsize), thus nodesize should always -* be fine. -*/ -ASSERT (fs_info->nodesize <= -SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits); -/* Allocate outside of device_list_mutex */ sctx =scrub_setup_ctx (fs_info, is_dev_replace);if (IS_ERR (sctx))
Signature · 1 line
2.54.0
PATCH v4 2/3 btrfs: extract the common scrub read endio into a helper
In reply to Qu WenruoFor both scrub_repair_read_endio() and scrub_read_endio(), they share the same bitmap update and bio put.
Extract the common code into a helper to reduce duplication.
--- fs/btrfs/scrub.c | 52 ++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-)
fs/btrfs/scrub.c +26 -26
@@ -876,6 +876,30 @@ static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *firstreturn i; } +/* +* The common handling of read endio. +* +* The bbio will be released, so no more access to @bbio after this function. +*/ +static void scrub_read_endio_common (struct btrfs_bio *bbio) +{ +struct scrub_stripe *stripe = bbio->private; +struct btrfs_fs_info *fs_info = stripe->bg->fs_info; +int sector_nr =calc_sector_number (stripe,bio_first_bvec_all (&bbio->bio)); +const u32 bio_size =bio_get_size (&bbio->bio); +const u32 sectors = bio_size >> fs_info->sectorsize_bits; + +ASSERT (sector_nr < stripe->nr_sectors); + +if (bbio->bio.bi_status) { +scrub_bitmap_set_io_error (stripe, sector_nr, sectors); +scrub_bitmap_set_error (stripe, sector_nr, sectors); + }else { +scrub_bitmap_clear_io_error (stripe, sector_nr, sectors); + } +bio_put (&bbio->bio); +} +/* * Repair read is different to the regular read: *
@@ -885,22 +909,9 @@ static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *firststatic void scrub_repair_read_endio (struct btrfs_bio *bbio) {struct scrub_stripe *stripe = bbio->private; -struct btrfs_fs_info *fs_info = stripe->bg->fs_info; -int sector_nr =calc_sector_number (stripe,bio_first_bvec_all (&bbio->bio)); -const u32 bio_size =bio_get_size (&bbio->bio); -ASSERT (sector_nr < stripe->nr_sectors); +scrub_read_endio_common (bbio); -if (bbio->bio.bi_status) { -scrub_bitmap_set_io_error (stripe, sector_nr, - bio_size >> fs_info->sectorsize_bits); -scrub_bitmap_set_error (stripe, sector_nr, - bio_size >> fs_info->sectorsize_bits); - }else { -scrub_bitmap_clear_io_error (stripe, sector_nr, - bio_size >> fs_info->sectorsize_bits); - } -bio_put (&bbio->bio);if (atomic_dec_and_test (&stripe->pending_io))wake_up (&stripe->io_wait); }
@@ -1239,20 +1250,9 @@ static void scrub_stripe_read_repair_worker(struct work_struct *work)static void scrub_read_endio (struct btrfs_bio *bbio) {struct scrub_stripe *stripe = bbio->private; -int sector_nr =calc_sector_number (stripe,bio_first_bvec_all (&bbio->bio)); -int num_sectors; -const u32 bio_size =bio_get_size (&bbio->bio); -ASSERT (sector_nr < stripe->nr_sectors); - num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits; +scrub_read_endio_common (bbio); -if (bbio->bio.bi_status) { -scrub_bitmap_set_io_error (stripe, sector_nr, num_sectors); -scrub_bitmap_set_error (stripe, sector_nr, num_sectors); - }else { -scrub_bitmap_clear_io_error (stripe, sector_nr, num_sectors); - } -bio_put (&bbio->bio);if (atomic_dec_and_test (&stripe->pending_io)) {wake_up (&stripe->io_wait);INIT_WORK (&stripe->work, scrub_stripe_read_repair_worker);
Signature · 1 line
2.54.0
PATCH v4 3/3 btrfs: use kvmalloc() for stripe buffer of scrub_stripe
In reply to Qu WenruoCurrently we're using scrub_stripe::folios[] to store all contents of a stripe.
This means we need all the extra work to handle things like sub-page cases, and also require larger folios to handle bs > ps cases.
On the other hand, it's not hard to allocate a 64K large folio to cover the full stripe, getting rid of the cross-page handling.
Furthermore, even if that large folio allocation failed, we can still use vmalloc() to allocate a virtually contiguous space and still get rid of cross-page handling.
This patch will go with kvmalloc() to allocate 64K of memory for the stripe buffer, thus getting rid of all the complex cross-page handling.
The following aspects can be greatly simplified:
- Checksum verification for both data and metadata No more per-page iteration, all in one go.
- RAID56 data caching
Just copy the buffer into the RAID56 pages.
- No more kaddr/paddr grabbing
For most cases the virtual address is enough for csum calculation and io submission.
There is only one location left with a kernel address requirement, that's calc_sector_number, which needs to compare against the bvec virtual address.
Just open-code a simple vaddr->kaddr converter for that only caller.
- Bio assembly
There is already the helper bio_add_vmalloc() to queue vmallocated memory into a bio.
Although it means we have something else to be concerned about:
- Bio assembly
If the memory is vmallocated, we need to use bio_add_vmalloc() Otherwise use the existing bio_add_page().
- Read endio
For vmallocated memory, we need to call
invalidate_kernel_vmap_range().
- Scrub bbio bvec size
Since scrub_stripe::buffer is kvmallocated, we also need to enlarge the scrub bbio, to be able to handle the worst case, where all 64KiB is allocated by discontiguous 4K physical pages.
--- fs/btrfs/raid56.c | 18 ++--- fs/btrfs/raid56.h | 2 +- fs/btrfs/scrub.c | 169 ++++++++++++++++++++++------------------------ 3 files changed, 85 insertions(+), 104 deletions(-)
fs/btrfs/raid56.c +4 -14
@@ -2996,13 +2996,11 @@ void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio)* This is due to the fact rbio has its own page management for its cache. */ void raid56_parity_cache_data_folios (struct btrfs_raid_bio *rbio, -struct folio **data_folios,u64 data_logical) +void *vaddr,u64 data_logical) {struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;const u64 offset_in_full_stripe = data_logical - rbio->bioc->full_stripe_logical; -unsigned int findex =0 ; -unsigned int foffset =0 ;int ret;/*
@@ -3025,18 +3023,10 @@ void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, cur_off < offset_in_full_stripe +BTRFS_STRIPE_LEN ; cur_off += PAGE_SIZE) {const unsigned int pindex = cur_off >>PAGE_SHIFT ; -void *kaddr; - kaddr =kmap_local_page (rbio->stripe_pages[pindex]); -memcpy_from_folio (kaddr, data_folios[findex], foffset,PAGE_SIZE ); -kunmap_local (kaddr); - - foffset +=PAGE_SIZE ; -ASSERT (foffset <=folio_size (data_folios[findex])); -if (foffset ==folio_size (data_folios[findex])) { - findex++; - foffset =0 ; - } +ASSERT (cur_off - offset_in_full_stripe +PAGE_SIZE <=BTRFS_STRIPE_LEN ); +memcpy_to_page (rbio->stripe_pages[pindex],0 , + vaddr + cur_off - offset_in_full_stripe,PAGE_SIZE ); } bitmap_set(rbio->stripe_uptodate_bitmap, offset_in_full_stripe >> fs_info->sectorsize_bits,
fs/btrfs/raid56.h +1 -1
@@ -283,7 +283,7 @@ struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio,void raid56_parity_submit_scrub_rbio (struct btrfs_raid_bio *rbio);void raid56_parity_cache_data_folios (struct btrfs_raid_bio *rbio, -struct folio **data_folios,u64 data_logical); +void *vaddr,u64 data_logical);int btrfs_alloc_stripe_hash_table (struct btrfs_fs_info *info);void btrfs_free_stripe_hash_table (struct btrfs_fs_info *info);
fs/btrfs/scrub.c +80 -89
319 lines not on this page · raw message
Signature · 1 line
2.54.0