bio iter improvements
Subsystems (15)
Kent Overstreet
maintainerv2 cover bio iter improvements
previous lore link:
https://lore.kernel.org/linux-block/ZCxaMIP%[email protected]/T/#t
changes since v1:
- fixed squashfs issues
- renamed foliovec to folio_seg, thanks Matthew for the suggestion
- moved the WARN_ON() to an out of line function to reduce data size, per Christoph
- fixed kernel doc comment on bio_for_each_folio_all(), added one for bio_for_each_folio()
Kent Overstreet (2):
block: Rework bio_for_each_segment_all()
block: Rework bio_for_each_folio_all()
block/bio.c | 43 ++++++----- block/blk-map.c | 38 +++++----- block/bounce.c | 12 ++-- drivers/md/bcache/btree.c | 8 +-- drivers/md/dm-crypt.c | 10 +-- drivers/md/raid1.c | 4 +- fs/btrfs/disk-io.c | 10 +-- fs/btrfs/extent_io.c | 52 +++++++------- fs/btrfs/inode.c | 8 +-- fs/btrfs/raid56.c | 18 ++--- fs/crypto/bio.c | 8 +-- fs/erofs/zdata.c | 4 +- fs/ext4/page-io.c | 8 +-- fs/ext4/readpage.c | 4 +- fs/f2fs/data.c | 20 +++--- fs/gfs2/lops.c | 10 +-- fs/gfs2/meta_io.c | 8 +-- fs/iomap/buffered-io.c | 14 ++-- fs/mpage.c | 4 +- fs/squashfs/block.c | 59 +++++++-------- fs/squashfs/lz4_wrapper.c | 17 +++-- fs/squashfs/lzo_wrapper.c | 17 +++-- fs/squashfs/xz_wrapper.c | 19 ++--- fs/squashfs/zlib_wrapper.c | 18 ++--- fs/squashfs/zstd_wrapper.c | 19 ++--- fs/verity/verify.c | 4 +- include/linux/bio.h | 142 +++++++++++++++++++++++-------------- include/linux/bvec.h | 70 +++++++++++------- 28 files changed, 358 insertions(+), 290 deletions(-)
Signature · 1 line
2.40.0
Kent Overstreet
maintainerPATCH v2 1/2 block: Rework bio_for_each_segment_all()
This patch reworks bio_for_each_segment_all() to be more inline with how the other bio iterators work:
- bio_iter_all_peek() now returns a synthesized bio_vec; we don't stash one in the iterator and pass a pointer to it - bad. This way makes it clearer what's a constructed value vs. a reference to something pre-existing, and it also will help with cleaning up and consolidating code with bio_for_each_folio_all().
- We now provide bio_for_each_segment_all_continue(), for squashfs:
this makes their code clearer.
--- block/bio.c | 43 +++++++++++++++------------ block/blk-map.c | 38 ++++++++++++------------ block/bounce.c | 12 ++++---- drivers/md/bcache/btree.c | 8 ++--- drivers/md/dm-crypt.c | 10 +++---- drivers/md/raid1.c | 4 +-- fs/btrfs/disk-io.c | 10 +++---- fs/btrfs/extent_io.c | 52 ++++++++++++++++---------------- fs/btrfs/inode.c | 8 ++--- fs/btrfs/raid56.c | 18 +++++------ fs/crypto/bio.c | 8 ++--- fs/erofs/zdata.c | 4 +-- fs/ext4/page-io.c | 8 ++--- fs/ext4/readpage.c | 4 +-- fs/f2fs/data.c | 20 ++++++------- fs/gfs2/lops.c | 10 +++---- fs/gfs2/meta_io.c | 8 ++--- fs/mpage.c | 4 +-- fs/squashfs/block.c | 59 ++++++++++++++++++------------------ fs/squashfs/lz4_wrapper.c | 17 ++++++----- fs/squashfs/lzo_wrapper.c | 17 ++++++----- fs/squashfs/xz_wrapper.c | 19 ++++++------ fs/squashfs/zlib_wrapper.c | 18 ++++++----- fs/squashfs/zstd_wrapper.c | 19 ++++++------ fs/verity/verify.c | 4 +-- include/linux/bio.h | 37 ++++++++++++++++++----- include/linux/bvec.h | 61 ++++++++++++++++++++++---------------- 27 files changed, 283 insertions(+), 237 deletions(-)
block/bio.c +24 -19
@@ -1170,13 +1170,13 @@ bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len,void __bio_release_pages (struct bio *bio,bool mark_dirty) { -struct bvec_iter_all iter_all; -struct bio_vec *bvec; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (mark_dirty && !PageCompound (bvec->bv_page)) -set_page_dirty_lock (bvec->bv_page); -put_page (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) { +if (mark_dirty && !PageCompound (bvec.bv_page)) +set_page_dirty_lock (bvec.bv_page); +put_page (bvec.bv_page); } }EXPORT_SYMBOL_GPL (__bio_release_pages);
@@ -1401,6 +1401,11 @@ void __bio_advance(struct bio *bio, unsigned bytes) }EXPORT_SYMBOL (__bio_advance); +void bio_iter_all_advance_warning (void ) +{ +WARN (true,"advanced past end of bio" ); +} +void bio_copy_data_iter (struct bio *dst,struct bvec_iter *dst_iter,struct bio *src,struct bvec_iter *src_iter) {
@@ -1441,11 +1446,11 @@ EXPORT_SYMBOL(bio_copy_data);void bio_free_pages (struct bio *bio) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) -__free_page (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) +__free_page (bvec.bv_page); }EXPORT_SYMBOL (bio_free_pages);
@@ -1480,12 +1485,12 @@ EXPORT_SYMBOL(bio_free_pages);*/ void bio_set_pages_dirty (struct bio *bio) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (!PageCompound (bvec->bv_page)) -set_page_dirty_lock (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) { +if (!PageCompound (bvec.bv_page)) +set_page_dirty_lock (bvec.bv_page); } }
@@ -1528,12 +1533,12 @@ static void bio_dirty_fn(struct work_struct *work)void bio_check_pages_dirty (struct bio *bio) { -struct bio_vec *bvec; +struct bvec_iter_all iter; +struct bio_vec bvec;unsigned long flags; -struct bvec_iter_all iter_all; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (!PageDirty (bvec->bv_page) && !PageCompound (bvec->bv_page)) +bio_for_each_segment_all (bvec, bio, iter) { +if (!PageDirty (bvec.bv_page) && !PageCompound (bvec.bv_page)) goto defer; }
block/blk-map.c +19 -19
@@ -45,21 +45,21 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,*/ static int bio_copy_from_iter (struct bio *bio,struct iov_iter *iter) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all bv_iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, bv_iter) {ssize_t ret; - ret =copy_page_from_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, + ret =copy_page_from_iter (bvec.bv_page, + bvec.bv_offset, + bvec.bv_len, iter);if (!iov_iter_count (iter))break ; -if (ret < bvec->bv_len) +if (ret < bvec.bv_len)return -EFAULT ; }
@@ -76,21 +76,21 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)*/ static int bio_copy_to_iter (struct bio *bio,struct iov_iter iter) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all bv_iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, bv_iter) {ssize_t ret; - ret =copy_page_to_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, + ret =copy_page_to_iter (bvec.bv_page, + bvec.bv_offset, + bvec.bv_len, &iter);if (!iov_iter_count (&iter))break ; -if (ret < bvec->bv_len) +if (ret < bvec.bv_len)return -EFAULT ; }
@@ -443,12 +443,12 @@ static void bio_copy_kern_endio(struct bio *bio)static void bio_copy_kern_endio_read (struct bio *bio) {char *p = bio->bi_private; -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -memcpy_from_bvec (p, bvec); - p += bvec->bv_len; +bio_for_each_segment_all (bvec, bio, iter) { +memcpy_from_bvec (p, &bvec); + p += bvec.bv_len; }bio_copy_kern_endio (bio);
block/bounce.c +6 -6
@@ -102,18 +102,18 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)static void bounce_end_io (struct bio *bio) {struct bio *bio_orig = bio->bi_private; -struct bio_vec *bvec, orig_vec; +struct bio_vec bvec, orig_vec;struct bvec_iter orig_iter = bio_orig->bi_iter; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter;/* * free up bounce indirect pages used */ -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, iter) { orig_vec =bio_iter_iovec (bio_orig, orig_iter); -if (bvec->bv_page != orig_vec.bv_page) { -dec_zone_page_state (bvec->bv_page,NR_BOUNCE ); -mempool_free (bvec->bv_page, &page_pool); +if (bvec.bv_page != orig_vec.bv_page) { +dec_zone_page_state (bvec.bv_page,NR_BOUNCE ); +mempool_free (bvec.bv_page, &page_pool); }bio_advance_iter (bio_orig, &orig_iter, orig_vec.bv_len); }
drivers/md/bcache/btree.c +4 -4
@@ -373,12 +373,12 @@ static void do_btree_node_write(struct btree *b) bset_sector_offset(&b->keys, i));if (!bch_bio_alloc_pages (b->bio, __GFP_NOWARN|GFP_NOWAIT )) { -struct bio_vec *bv; +struct bio_vec bv;void *addr = (void *) ((unsigned long ) i & ~(PAGE_SIZE -1 )); -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; -bio_for_each_segment_all (bv, b->bio, iter_all) { -memcpy (page_address (bv->bv_page), addr,PAGE_SIZE ); +bio_for_each_segment_all (bv, b->bio, iter) { +memcpy (page_address (bv.bv_page), addr,PAGE_SIZE ); addr +=PAGE_SIZE ; }
drivers/md/dm-crypt.c +5 -5
@@ -1709,12 +1709,12 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)static void crypt_free_buffer_pages (struct crypt_config *cc,struct bio *clone) { -struct bio_vec *bv; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bv; -bio_for_each_segment_all (bv, clone, iter_all) { -BUG_ON (!bv->bv_page); -mempool_free (bv->bv_page, &cc->page_pool); +bio_for_each_segment_all (bv, clone, iter) { +BUG_ON (!bv.bv_page); +mempool_free (bv.bv_page, &cc->page_pool); } }
drivers/md/raid1.c +2 -2
@@ -2188,7 +2188,7 @@ static void process_checks(struct r1bio *r1_bio)blk_status_t status = sbio->bi_status;struct page **ppages =get_resync_pages (pbio)->pages;struct page **spages =get_resync_pages (sbio)->pages; -struct bio_vec *bi; +struct bio_vec bi;int page_len[RESYNC_PAGES ] = {0 };struct bvec_iter_all iter_all;
@@ -2198,7 +2198,7 @@ static void process_checks(struct r1bio *r1_bio) sbio->bi_status =0 ;bio_for_each_segment_all (bi, sbio, iter_all) - page_len[j++] = bi->bv_len; + page_len[j++] = bi.bv_len;if (!status) {for (j = vcnt; j-- ; ) {
fs/btrfs/disk-io.c +5 -5
@@ -798,15 +798,15 @@ bool btrfs_wq_submit_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_static blk_status_t btree_csum_one_bio (struct bio *bio) { -struct bio_vec *bvec; +struct bio_vec bvec;struct btrfs_root *root;int ret =0 ;struct bvec_iter_all iter_all;ASSERT (!bio_flagged (bio,BIO_CLONED ));bio_for_each_segment_all (bvec, bio, iter_all) { - root =BTRFS_I (bvec->bv_page->mapping->host)->root; - ret =csum_dirty_buffer (root->fs_info, bvec); + root =BTRFS_I (bvec.bv_page->mapping->host)->root; + ret =csum_dirty_buffer (root->fs_info, &bvec);if (ret)break ; }
@@ -3970,12 +3970,12 @@ ALLOW_ERROR_INJECTION(open_ctree, ERRNO);static void btrfs_end_super_write (struct bio *bio) {struct btrfs_device *device = bio->bi_private; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;struct page *page;bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (bio->bi_status) { btrfs_warn_rl_in_rcu(device->fs_info,
fs/btrfs/extent_io.c +26 -26
136 lines not on this page · raw message
fs/btrfs/inode.c +4 -4
@@ -10317,7 +10317,7 @@ static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio)struct btrfs_inode *inode = priv->inode;struct btrfs_fs_info *fs_info = inode->root->fs_info;u32 sectorsize = fs_info->sectorsize; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;u32 bio_offset =0 ;
@@ -10327,12 +10327,12 @@ static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio)bio_for_each_segment_all (bvec, &bbio->bio, iter_all) {unsigned int i, nr_sectors, pgoff; - nr_sectors =BTRFS_BYTES_TO_BLKS (fs_info, bvec->bv_len); - pgoff = bvec->bv_offset; + nr_sectors =BTRFS_BYTES_TO_BLKS (fs_info, bvec.bv_len); + pgoff = bvec.bv_offset;for (i =0 ; i < nr_sectors; i++) {ASSERT (pgoff <PAGE_SIZE );if (btrfs_check_data_csum (inode, bbio, bio_offset, - bvec->bv_page, pgoff)) + bvec.bv_page, pgoff))return BLK_STS_IOERR ; bio_offset += sectorsize; pgoff += sectorsize;
fs/btrfs/raid56.c +9 -9
@@ -1382,7 +1382,7 @@ static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio,static void set_bio_pages_uptodate (struct btrfs_raid_bio *rbio,struct bio *bio) {const u32 sectorsize = rbio->bioc->fs_info->sectorsize; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;ASSERT (!bio_flagged (bio,BIO_CLONED ));
@@ -1391,9 +1391,9 @@ static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio)struct sector_ptr *sector;int pgoff; -for (pgoff = bvec->bv_offset; pgoff - bvec->bv_offset < bvec->bv_len; +for (pgoff = bvec.bv_offset; pgoff - bvec.bv_offset < bvec.bv_len; pgoff += sectorsize) { - sector =find_stripe_sector (rbio, bvec->bv_page, pgoff); + sector =find_stripe_sector (rbio, bvec.bv_page, pgoff);ASSERT (sector);if (sector) sector->uptodate =1 ;
@@ -1424,12 +1424,12 @@ static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bi {int total_sector_nr =get_bio_sector_nr (rbio, bio);u32 bio_size =0 ; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;int i;bio_for_each_segment_all (bvec, bio, iter_all) - bio_size += bvec->bv_len; + bio_size += bvec.bv_len;/* * Since we can have multiple bios touching the error_bitmap, we cannot
@@ -1448,7 +1448,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio, {struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;int total_sector_nr =get_bio_sector_nr (rbio, bio); -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;/* No data csum for the whole stripe, no need to verify. */
@@ -1462,8 +1462,8 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,bio_for_each_segment_all (bvec, bio, iter_all) {int bv_offset; -for (bv_offset = bvec->bv_offset; - bv_offset < bvec->bv_offset + bvec->bv_len; +for (bv_offset = bvec.bv_offset; + bv_offset < bvec.bv_offset + bvec.bv_len; bv_offset += fs_info->sectorsize, total_sector_nr++) {u8 csum_buf[BTRFS_CSUM_SIZE ]; u8 *expected_csum = rbio->csum_buf +
@@ -1474,7 +1474,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,if (!test_bit (total_sector_nr, rbio->csum_bitmap))continue ; - ret =btrfs_check_sector_csum (fs_info, bvec->bv_page, + ret =btrfs_check_sector_csum (fs_info, bvec.bv_page, bv_offset, csum_buf, expected_csum);if (ret <0 )set_bit (total_sector_nr, rbio->error_bitmap);
fs/crypto/bio.c +4 -4
@@ -30,13 +30,13 @@*/ bool fscrypt_decrypt_bio (struct bio *bio) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; -int err =fscrypt_decrypt_pagecache_blocks (page, bv->bv_len, - bv->bv_offset); +struct page *page = bv.bv_page; +int err =fscrypt_decrypt_pagecache_blocks (page, bv.bv_len, + bv.bv_offset);if (err) { bio->bi_status =errno_to_blk_status (err);
fs/erofs/zdata.c +2 -2
@@ -1364,11 +1364,11 @@ static void z_erofs_decompressqueue_endio(struct bio *bio)tagptr1_t t =tagptr_init (tagptr1_t, bio->bi_private);struct z_erofs_decompressqueue *q =tagptr_unfold_ptr (t);blk_status_t err = bio->bi_status; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;DBG_BUGON (PageUptodate (page));DBG_BUGON (z_erofs_page_is_invalidated (page));
fs/ext4/page-io.c +4 -4
@@ -99,15 +99,15 @@ static void buffer_io_error(struct buffer_head *bh)static void ext4_finish_bio (struct bio *bio) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;struct page *bounce_page =NULL ;struct buffer_head *bh, *head; -unsigned bio_start = bvec->bv_offset; -unsigned bio_end = bio_start + bvec->bv_len; +unsigned bio_start = bvec.bv_offset; +unsigned bio_end = bio_start + bvec.bv_len;unsigned under_io =0 ;unsigned long flags;
fs/ext4/readpage.c +2 -2
@@ -69,11 +69,11 @@ struct bio_post_read_ctx {static void __read_end_io (struct bio *bio) {struct page *page; -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bio_for_each_segment_all (bv, bio, iter_all) { - page = bv->bv_page; + page = bv.bv_page;if (bio->bi_status)ClearPageUptodate (page);
fs/f2fs/data.c +10 -10
@@ -139,12 +139,12 @@ struct bio_post_read_ctx {*/ static void f2fs_finish_read_bio (struct bio *bio,bool in_task) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;struct bio_post_read_ctx *ctx = bio->bi_private;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (f2fs_is_compressed_page (page)) {if (ctx && !ctx->decompression_attempted)
@@ -189,11 +189,11 @@ static void f2fs_verify_bio(struct work_struct *work)* as those were handled separately by f2fs_end_read_compressed_page(). */ if (may_have_compressed_pages) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (!f2fs_is_compressed_page (page) && !fsverity_verify_page (page)) {
@@ -241,13 +241,13 @@ static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)static void f2fs_handle_step_decompress (struct bio_post_read_ctx *ctx,bool in_task) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bool all_compressed = true;block_t blkaddr = ctx->fs_blkaddr;bio_for_each_segment_all (bv, ctx->bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (f2fs_is_compressed_page (page)) f2fs_end_read_compressed_page(page, false, blkaddr,
@@ -329,7 +329,7 @@ static void f2fs_read_end_io(struct bio *bio)static void f2fs_write_end_io (struct bio *bio) {struct f2fs_sb_info *sbi; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;iostat_update_and_unbind_ctx (bio,1 );
@@ -341,7 +341,7 @@ static void f2fs_write_end_io(struct bio *bio) }bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;enum count_type type =WB_DATA_TYPE (page);if (page_private_dummy (page)) {
@@ -585,7 +585,7 @@ static void __submit_merged_bio(struct f2fs_bio_info *io)static bool __has_merged_page (struct bio *bio,struct inode *inode,struct page *page,nid_t ino) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;if (!bio)
@@ -595,7 +595,7 @@ static bool __has_merged_page(struct bio *bio, struct inode *inode,return true;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *target = bvec->bv_page; +struct page *target = bvec.bv_page;if (fscrypt_is_bounce_page (target)) { target =fscrypt_pagecache_page (target);
fs/gfs2/lops.c +5 -5
@@ -202,7 +202,7 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,static void gfs2_end_log_write (struct bio *bio) {struct gfs2_sbd *sdp = bio->bi_private; -struct bio_vec *bvec; +struct bio_vec bvec;struct page *page;struct bvec_iter_all iter_all;
@@ -217,9 +217,9 @@ static void gfs2_end_log_write(struct bio *bio) }bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (page_has_buffers (page)) -gfs2_end_log_write_bh (sdp, bvec, bio->bi_status); +gfs2_end_log_write_bh (sdp, &bvec, bio->bi_status);else mempool_free (page, gfs2_page_pool); }
@@ -395,11 +395,11 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)static void gfs2_end_log_read (struct bio *bio) {struct page *page; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (bio->bi_status) {int err =blk_status_to_errno (bio->bi_status);
fs/gfs2/meta_io.c +4 -4
@@ -193,15 +193,15 @@ struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno)static void gfs2_meta_read_endio (struct bio *bio) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;struct buffer_head *bh =page_buffers (page); -unsigned int len = bvec->bv_len; +unsigned int len = bvec.bv_len; -while (bh_offset (bh) < bvec->bv_offset) +while (bh_offset (bh) < bvec.bv_offset) bh = bh->b_this_page; do { struct buffer_head *next = bh->b_this_page;
fs/mpage.c +2 -2
@@ -45,11 +45,11 @@*/ static void mpage_end_io (struct bio *bio) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;page_endio (page,bio_op (bio),blk_status_to_errno (bio->bi_status)); }
fs/squashfs/block.c +30 -29
102 lines not on this page · raw message
fs/squashfs/lz4_wrapper.c +10 -7
@@ -92,20 +92,23 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter; +struct bio_vec bvec;struct squashfs_lz4 *stream = strm;void *buff = stream->input, *data;int bytes = length, res; -while (bio_next_segment (bio, &iter_all)) { -int avail =min (bytes, ((int )bvec->bv_len) - offset); +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); - data =bvec_virt (bvec); -memcpy (buff, data + offset, avail); +bio_for_each_segment_all_continue (bvec, bio, iter) { +unsigned avail =min_t (unsigned, bytes, bvec.bv_len); + +memcpy (buff,bvec_virt (&bvec), avail); buff += avail; bytes -= avail; - offset =0 ; +if (!bytes) +break ; } res = LZ4_decompress_safe(stream->input, stream->output,
fs/squashfs/lzo_wrapper.c +10 -7
@@ -66,21 +66,24 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter; +struct bio_vec bvec;struct squashfs_lzo *stream = strm;void *buff = stream->input, *data;int bytes = length, res;size_t out_len = output->length; -while (bio_next_segment (bio, &iter_all)) { -int avail =min (bytes, ((int )bvec->bv_len) - offset); +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); - data =bvec_virt (bvec); -memcpy (buff, data + offset, avail); +bio_for_each_segment_all_continue (bvec, bio, iter) { +unsigned avail =min_t (unsigned, bytes, bvec.bv_len); + +memcpy (buff,bvec_virt (&bvec), avail); buff += avail; bytes -= avail; - offset =0 ; +if (!bytes) +break ; } res = lzo1x_decompress_safe(stream->input, (size_t)length,
fs/squashfs/xz_wrapper.c +10 -9
@@ -120,8 +120,7 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter;int total =0 , error =0 ;struct squashfs_xz *stream = strm;
@@ -136,26 +135,28 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {enum xz_ret xz_err;if (stream->buf.in_pos == stream->buf.in_size) { -const void *data; -int avail; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter); +unsigned avail =min_t (unsigned, length, bvec.bv_len); -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {/* XZ_STREAM_END must be reached. */ error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); length -= avail; - stream->buf.in = data + offset; + stream->buf.in =bvec_virt (&bvec); stream->buf.in_size = avail; stream->buf.in_pos =0 ; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (stream->buf.out_pos == stream->buf.out_size) {
fs/squashfs/zlib_wrapper.c +10 -8
@@ -53,8 +53,7 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter;int zlib_init =0 , error =0 ;z_stream *stream = strm;
@@ -67,25 +66,28 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {int zlib_err;if (stream->avail_in ==0 ) { -const void *data; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter);int avail; -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {/* Z_STREAM_END must be reached. */ error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); + avail =min_t (unsigned, length, bvec.bv_len); length -= avail; - stream->next_in = data + offset; + stream->next_in =bvec_virt (&bvec); stream->avail_in = avail; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (stream->avail_out ==0 ) {
fs/squashfs/zstd_wrapper.c +10 -9
@@ -68,8 +68,7 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,int error =0 ;zstd_in_buffer in_buf = {NULL ,0 ,0 };zstd_out_buffer out_buf = {NULL ,0 ,0 }; -struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); +struct bvec_iter_all iter; stream =zstd_init_dstream (wksp->window_size, wksp->mem, wksp->mem_size);
@@ -85,25 +84,27 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {size_t zstd_err;if (in_buf.pos == in_buf.size) { -const void *data; -int avail; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter); +unsigned avail =min_t (unsigned, length, bvec.bv_len); -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) { error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); length -= avail; - in_buf.src = data + offset; + in_buf.src =bvec_virt (&bvec); in_buf.size = avail; in_buf.pos =0 ; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (out_buf.pos == out_buf.size) {
fs/verity/verify.c +2 -2
@@ -215,7 +215,7 @@ void fsverity_verify_bio(struct bio *bio)const struct fsverity_info *vi = inode->i_verity_info;const struct merkle_tree_params *params = &vi->tree_params;struct ahash_request *req; -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;unsigned long max_ra_pages =0 ;
@@ -238,7 +238,7 @@ void fsverity_verify_bio(struct bio *bio) }bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;unsigned long level0_index = page->index >> params->log_arity;unsigned long level0_ra_pages =min (max_ra_pages, params->level0_blocks - level0_index);
include/linux/bio.h +29 -8
@@ -76,22 +76,43 @@ static inline void *bio_data(struct bio *bio)return NULL ; } -static inline bool bio_next_segment (const struct bio *bio, -struct bvec_iter_all *iter) +static inline struct bio_vec bio_iter_all_peek (const struct bio *bio, +struct bvec_iter_all *iter) { -if (iter->idx >= bio->bi_vcnt) -return false; +if (WARN_ON (iter->idx >= bio->bi_vcnt)) +return (struct bio_vec ) {NULL }; -bvec_advance (&bio->bi_io_vec[iter->idx], iter); -return true; +return bvec_iter_all_peek (bio->bi_io_vec, iter); } +void bio_iter_all_advance_warning (void ); + +static inline void bio_iter_all_advance (const struct bio *bio, +struct bvec_iter_all *iter, +unsigned bytes) +{ +bvec_iter_all_advance (bio->bi_io_vec, iter, bytes); + +if (iter->idx > bio->bi_vcnt || + (iter->idx == bio->bi_vcnt && iter->done)) +bio_iter_all_advance_warning (); +} + +#define bio_for_each_segment_all_continue (bvl, bio, iter) \ + for (; \ + iter.idx < bio->bi_vcnt && \ + ((bvl = bio_iter_all_peek(bio, &iter)), true); \ + bio_iter_all_advance((bio), &iter, bvl.bv_len)) +/* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */ -#define bio_for_each_segment_all (bvl, bio, iter) \ - for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); ) +#define bio_for_each_segment_all (bvl, bio, iter) \ + for (bvec_iter_all_init(&iter); \ + iter.idx < (bio)->bi_vcnt && \ + ((bvl = bio_iter_all_peek((bio), &iter)), true); \ + bio_iter_all_advance((bio), &iter, bvl.bv_len))static inline void bio_advance_iter (const struct bio *bio,struct bvec_iter *iter,unsigned int bytes)
include/linux/bvec.h +35 -26
87 lines not on this page · raw message
Signature · 1 line
2.40.0
Kent Overstreet
maintainerPATCH v2 2/2 block: Rework bio_for_each_folio_all()
In reply to Kent OverstreetThis reimplements bio_for_each_folio_all() on top of the newly-reworked bvec_iter_all, and since it's now trivial we also provide bio_for_each_folio.
--- fs/iomap/buffered-io.c | 14 +++--- include/linux/bio.h | 105 +++++++++++++++++++++++------------------ include/linux/bvec.h | 15 ++++-- 3 files changed, 78 insertions(+), 56 deletions(-)
fs/iomap/buffered-io.c +8 -6
@@ -187,10 +187,11 @@ static void iomap_finish_folio_read(struct folio *folio, size_t offset,static void iomap_read_end_io (struct bio *bio) {int error =blk_status_to_errno (bio->bi_status); -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fv; -bio_for_each_folio_all (fi, bio) -iomap_finish_folio_read (fi.folio, fi.offset, fi.length, error); +bio_for_each_folio_all (fv, bio, iter) +iomap_finish_folio_read (fv.fv_folio, fv.fv_offset, fv.fv_len, error);bio_put (bio); }
@@ -1299,7 +1300,8 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error)u32 folio_count =0 ;for (bio = &ioend->io_inline_bio; bio; bio = next) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fv;/* * For the last bio, bi_private points to the ioend, so we
@@ -1311,8 +1313,8 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error) next = bio->bi_private;/* walk all folios in bio, ending page IO on them */ -bio_for_each_folio_all (fi, bio) { -iomap_finish_folio_write (inode, fi.folio, fi.length, +bio_for_each_folio_all (fv, bio, iter) { +iomap_finish_folio_write (inode, fv.fv_folio, fv.fv_len, error); folio_count++; }
include/linux/bio.h +58 -47
123 lines not on this page · raw message
include/linux/bvec.h +12 -3
@@ -166,18 +166,27 @@ static inline void bvec_iter_all_init(struct bvec_iter_all *iter_all) iter_all->idx =0 ; } -static inline struct bio_vec bvec_iter_all_peek (const struct bio_vec *bvec, -struct bvec_iter_all *iter) +static inline struct bio_vec __bvec_iter_all_peek (const struct bio_vec *bvec, +const struct bvec_iter_all *iter) {struct bio_vec bv = bvec[iter->idx]; +BUG_ON (iter->done >= bv.bv_len); + bv.bv_offset += iter->done; bv.bv_len -= iter->done; bv.bv_page += bv.bv_offset >>PAGE_SHIFT ; bv.bv_offset &= ~PAGE_MASK ; - bv.bv_len =min_t (unsigned,PAGE_SIZE - bv.bv_offset, bv.bv_len); +return bv; +} + +static inline struct bio_vec bvec_iter_all_peek (const struct bio_vec *bvec, +const struct bvec_iter_all *iter) +{ +struct bio_vec bv =__bvec_iter_all_peek (bvec, iter); + bv.bv_len =min_t (unsigned,PAGE_SIZE - bv.bv_offset, bv.bv_len);return bv; }
Signature · 1 line
2.40.0