block: Rework bio_for_each_segment_all()
Subsystems (20)
Kent Overstreet
maintainerPATCH v2 1/5 block: Rework bio_for_each_segment_all()
This patch reworks bio_for_each_segment_all() to be more inline with how the other bio iterators work:
- bio_iter_all_peek() now returns a synthesized bio_vec; we don't stash one in the iterator and pass a pointer to it - bad. This way makes it clearer what's a constructed value vs. a reference to something pre-existing, and it also will help with cleaning up and consolidating code with bio_for_each_folio_all().
- We now provide bio_for_each_segment_all_continue(), for squashfs:
this makes their code clearer.
--- block/bio.c | 38 ++++++++++++------------ block/blk-map.c | 38 ++++++++++++------------ block/bounce.c | 12 ++++---- drivers/md/bcache/btree.c | 8 ++--- drivers/md/dm-crypt.c | 10 +++---- drivers/md/raid1.c | 4 +-- fs/btrfs/disk-io.c | 4 +-- fs/btrfs/extent_io.c | 50 +++++++++++++++---------------- fs/btrfs/raid56.c | 14 ++++----- fs/erofs/zdata.c | 4 +-- fs/f2fs/data.c | 20 ++++++------- fs/gfs2/lops.c | 10 +++---- fs/gfs2/meta_io.c | 8 ++--- fs/squashfs/block.c | 48 +++++++++++++++++------------- fs/squashfs/lz4_wrapper.c | 17 ++++++----- fs/squashfs/lzo_wrapper.c | 17 ++++++----- fs/squashfs/xz_wrapper.c | 19 ++++++------ fs/squashfs/zlib_wrapper.c | 18 ++++++----- fs/squashfs/zstd_wrapper.c | 19 ++++++------ include/linux/bio.h | 28 ++++++++++++----- include/linux/bvec.h | 61 ++++++++++++++++++++++---------------- 21 files changed, 242 insertions(+), 205 deletions(-)
block/bio.c +19 -19
@@ -1163,13 +1163,13 @@ EXPORT_SYMBOL(bio_add_folio);void __bio_release_pages (struct bio *bio,bool mark_dirty) { -struct bvec_iter_all iter_all; -struct bio_vec *bvec; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (mark_dirty && !PageCompound (bvec->bv_page)) -set_page_dirty_lock (bvec->bv_page); -bio_release_page (bio, bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) { +if (mark_dirty && !PageCompound (bvec.bv_page)) +set_page_dirty_lock (bvec.bv_page); +bio_release_page (bio, bvec.bv_page); } }EXPORT_SYMBOL_GPL (__bio_release_pages);
@@ -1435,11 +1435,11 @@ EXPORT_SYMBOL(bio_copy_data);void bio_free_pages (struct bio *bio) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) -__free_page (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) +__free_page (bvec.bv_page); }EXPORT_SYMBOL (bio_free_pages);
@@ -1474,12 +1474,12 @@ EXPORT_SYMBOL(bio_free_pages);*/ void bio_set_pages_dirty (struct bio *bio) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (!PageCompound (bvec->bv_page)) -set_page_dirty_lock (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) { +if (!PageCompound (bvec.bv_page)) +set_page_dirty_lock (bvec.bv_page); } }
@@ -1522,12 +1522,12 @@ static void bio_dirty_fn(struct work_struct *work)void bio_check_pages_dirty (struct bio *bio) { -struct bio_vec *bvec; +struct bvec_iter_all iter; +struct bio_vec bvec;unsigned long flags; -struct bvec_iter_all iter_all; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (!PageDirty (bvec->bv_page) && !PageCompound (bvec->bv_page)) +bio_for_each_segment_all (bvec, bio, iter) { +if (!PageDirty (bvec.bv_page) && !PageCompound (bvec.bv_page)) goto defer; }
block/blk-map.c +19 -19
@@ -47,21 +47,21 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,*/ static int bio_copy_from_iter (struct bio *bio,struct iov_iter *iter) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all bv_iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, bv_iter) {ssize_t ret; - ret =copy_page_from_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, + ret =copy_page_from_iter (bvec.bv_page, + bvec.bv_offset, + bvec.bv_len, iter);if (!iov_iter_count (iter))break ; -if (ret < bvec->bv_len) +if (ret < bvec.bv_len)return -EFAULT ; }
@@ -78,21 +78,21 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)*/ static int bio_copy_to_iter (struct bio *bio,struct iov_iter iter) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all bv_iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, bv_iter) {ssize_t ret; - ret =copy_page_to_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, + ret =copy_page_to_iter (bvec.bv_page, + bvec.bv_offset, + bvec.bv_len, &iter);if (!iov_iter_count (&iter))break ; -if (ret < bvec->bv_len) +if (ret < bvec.bv_len)return -EFAULT ; }
@@ -443,12 +443,12 @@ static void bio_copy_kern_endio(struct bio *bio)static void bio_copy_kern_endio_read (struct bio *bio) {char *p = bio->bi_private; -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -memcpy_from_bvec (p, bvec); - p += bvec->bv_len; +bio_for_each_segment_all (bvec, bio, iter) { +memcpy_from_bvec (p, &bvec); + p += bvec.bv_len; }bio_copy_kern_endio (bio);
block/bounce.c +6 -6
@@ -102,18 +102,18 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)static void bounce_end_io (struct bio *bio) {struct bio *bio_orig = bio->bi_private; -struct bio_vec *bvec, orig_vec; +struct bio_vec bvec, orig_vec;struct bvec_iter orig_iter = bio_orig->bi_iter; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter;/* * free up bounce indirect pages used */ -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, iter) { orig_vec =bio_iter_iovec (bio_orig, orig_iter); -if (bvec->bv_page != orig_vec.bv_page) { -dec_zone_page_state (bvec->bv_page,NR_BOUNCE ); -mempool_free (bvec->bv_page, &page_pool); +if (bvec.bv_page != orig_vec.bv_page) { +dec_zone_page_state (bvec.bv_page,NR_BOUNCE ); +mempool_free (bvec.bv_page, &page_pool); }bio_advance_iter (bio_orig, &orig_iter, orig_vec.bv_len); }
drivers/md/bcache/btree.c +4 -4
@@ -373,12 +373,12 @@ static void do_btree_node_write(struct btree *b) bset_sector_offset(&b->keys, i));if (!bch_bio_alloc_pages (b->bio, __GFP_NOWARN|GFP_NOWAIT )) { -struct bio_vec *bv; +struct bio_vec bv;void *addr = (void *) ((unsigned long ) i & ~(PAGE_SIZE -1 )); -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; -bio_for_each_segment_all (bv, b->bio, iter_all) { -memcpy (page_address (bv->bv_page), addr,PAGE_SIZE ); +bio_for_each_segment_all (bv, b->bio, iter) { +memcpy (page_address (bv.bv_page), addr,PAGE_SIZE ); addr +=PAGE_SIZE ; }
drivers/md/dm-crypt.c +5 -5
@@ -1713,12 +1713,12 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)static void crypt_free_buffer_pages (struct crypt_config *cc,struct bio *clone) { -struct bio_vec *bv; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bv; -bio_for_each_segment_all (bv, clone, iter_all) { -BUG_ON (!bv->bv_page); -mempool_free (bv->bv_page, &cc->page_pool); +bio_for_each_segment_all (bv, clone, iter) { +BUG_ON (!bv.bv_page); +mempool_free (bv.bv_page, &cc->page_pool); } }
drivers/md/raid1.c +2 -2
@@ -2188,7 +2188,7 @@ static void process_checks(struct r1bio *r1_bio)blk_status_t status = sbio->bi_status;struct page **ppages =get_resync_pages (pbio)->pages;struct page **spages =get_resync_pages (sbio)->pages; -struct bio_vec *bi; +struct bio_vec bi;int page_len[RESYNC_PAGES ] = {0 };struct bvec_iter_all iter_all;
@@ -2198,7 +2198,7 @@ static void process_checks(struct r1bio *r1_bio) sbio->bi_status =0 ;bio_for_each_segment_all (bi, sbio, iter_all) - page_len[j++] = bi->bv_len; + page_len[j++] = bi.bv_len;if (!status) {for (j = vcnt; j-- ; ) {
fs/btrfs/disk-io.c +2 -2
@@ -3807,12 +3807,12 @@ ALLOW_ERROR_INJECTION(open_ctree, ERRNO);static void btrfs_end_super_write (struct bio *bio) {struct btrfs_device *device = bio->bi_private; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;struct page *page;bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (bio->bi_status) { btrfs_warn_rl_in_rcu(device->fs_info,
fs/btrfs/extent_io.c +25 -25
125 lines not on this page · raw message
fs/btrfs/raid56.c +7 -7
@@ -1393,7 +1393,7 @@ static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio,static void set_bio_pages_uptodate (struct btrfs_raid_bio *rbio,struct bio *bio) {const u32 sectorsize = rbio->bioc->fs_info->sectorsize; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;ASSERT (!bio_flagged (bio,BIO_CLONED ));
@@ -1402,9 +1402,9 @@ static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio)struct sector_ptr *sector;int pgoff; -for (pgoff = bvec->bv_offset; pgoff - bvec->bv_offset < bvec->bv_len; +for (pgoff = bvec.bv_offset; pgoff - bvec.bv_offset < bvec.bv_len; pgoff += sectorsize) { - sector =find_stripe_sector (rbio, bvec->bv_page, pgoff); + sector =find_stripe_sector (rbio, bvec.bv_page, pgoff);ASSERT (sector);if (sector) sector->uptodate =1 ;
@@ -1458,7 +1458,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio, {struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;int total_sector_nr =get_bio_sector_nr (rbio, bio); -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;/* No data csum for the whole stripe, no need to verify. */
@@ -1472,8 +1472,8 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,bio_for_each_segment_all (bvec, bio, iter_all) {int bv_offset; -for (bv_offset = bvec->bv_offset; - bv_offset < bvec->bv_offset + bvec->bv_len; +for (bv_offset = bvec.bv_offset; + bv_offset < bvec.bv_offset + bvec.bv_len; bv_offset += fs_info->sectorsize, total_sector_nr++) {u8 csum_buf[BTRFS_CSUM_SIZE ]; u8 *expected_csum = rbio->csum_buf +
@@ -1484,7 +1484,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,if (!test_bit (total_sector_nr, rbio->csum_bitmap))continue ; - ret =btrfs_check_sector_csum (fs_info, bvec->bv_page, + ret =btrfs_check_sector_csum (fs_info, bvec.bv_page, bv_offset, csum_buf, expected_csum);if (ret <0 )set_bit (total_sector_nr, rbio->error_bitmap);
fs/erofs/zdata.c +2 -2
@@ -1648,11 +1648,11 @@ static void z_erofs_decompressqueue_endio(struct bio *bio) {struct z_erofs_decompressqueue *q = bio->bi_private;blk_status_t err = bio->bi_status; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;DBG_BUGON (PageUptodate (page));DBG_BUGON (z_erofs_page_is_invalidated (page));
fs/f2fs/data.c +10 -10
@@ -139,12 +139,12 @@ struct bio_post_read_ctx {*/ static void f2fs_finish_read_bio (struct bio *bio,bool in_task) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;struct bio_post_read_ctx *ctx = bio->bi_private;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (f2fs_is_compressed_page (page)) {if (ctx && !ctx->decompression_attempted)
@@ -189,11 +189,11 @@ static void f2fs_verify_bio(struct work_struct *work)* as those were handled separately by f2fs_end_read_compressed_page(). */ if (may_have_compressed_pages) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (!f2fs_is_compressed_page (page) && !fsverity_verify_page (page)) {
@@ -241,13 +241,13 @@ static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)static void f2fs_handle_step_decompress (struct bio_post_read_ctx *ctx,bool in_task) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bool all_compressed = true;block_t blkaddr = ctx->fs_blkaddr;bio_for_each_segment_all (bv, ctx->bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (f2fs_is_compressed_page (page)) f2fs_end_read_compressed_page(page, false, blkaddr,
@@ -327,7 +327,7 @@ static void f2fs_read_end_io(struct bio *bio)static void f2fs_write_end_io (struct bio *bio) {struct f2fs_sb_info *sbi; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;iostat_update_and_unbind_ctx (bio);
@@ -337,7 +337,7 @@ static void f2fs_write_end_io(struct bio *bio) bio->bi_status =BLK_STS_IOERR ;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;enum count_type type =WB_DATA_TYPE (page);if (page_private_dummy (page)) {
@@ -583,7 +583,7 @@ static void __submit_merged_bio(struct f2fs_bio_info *io)static bool __has_merged_page (struct bio *bio,struct inode *inode,struct page *page,nid_t ino) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;if (!bio)
@@ -593,7 +593,7 @@ static bool __has_merged_page(struct bio *bio, struct inode *inode,return true;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *target = bvec->bv_page; +struct page *target = bvec.bv_page;if (fscrypt_is_bounce_page (target)) { target =fscrypt_pagecache_page (target);
fs/gfs2/lops.c +5 -5
@@ -202,7 +202,7 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,static void gfs2_end_log_write (struct bio *bio) {struct gfs2_sbd *sdp = bio->bi_private; -struct bio_vec *bvec; +struct bio_vec bvec;struct page *page;struct bvec_iter_all iter_all;
@@ -217,9 +217,9 @@ static void gfs2_end_log_write(struct bio *bio) }bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (page_has_buffers (page)) -gfs2_end_log_write_bh (sdp, bvec, bio->bi_status); +gfs2_end_log_write_bh (sdp, &bvec, bio->bi_status);else mempool_free (page, gfs2_page_pool); }
@@ -395,11 +395,11 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)static void gfs2_end_log_read (struct bio *bio) {struct page *page; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (bio->bi_status) {int err =blk_status_to_errno (bio->bi_status);
fs/gfs2/meta_io.c +4 -4
@@ -193,15 +193,15 @@ struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno)static void gfs2_meta_read_endio (struct bio *bio) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;struct buffer_head *bh =page_buffers (page); -unsigned int len = bvec->bv_len; +unsigned int len = bvec.bv_len; -while (bh_offset (bh) < bvec->bv_offset) +while (bh_offset (bh) < bvec.bv_offset) bh = bh->b_this_page; do { struct buffer_head *next = bh->b_this_page;
fs/squashfs/block.c +27 -21
97 lines not on this page · raw message
fs/squashfs/lz4_wrapper.c +10 -7
@@ -92,20 +92,23 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter; +struct bio_vec bvec;struct squashfs_lz4 *stream = strm;void *buff = stream->input, *data;int bytes = length, res; -while (bio_next_segment (bio, &iter_all)) { -int avail =min (bytes, ((int )bvec->bv_len) - offset); +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); - data =bvec_virt (bvec); -memcpy (buff, data + offset, avail); +bio_for_each_segment_all_continue (bvec, bio, iter) { +unsigned avail =min_t (unsigned, bytes, bvec.bv_len); + +memcpy (buff,bvec_virt (&bvec), avail); buff += avail; bytes -= avail; - offset =0 ; +if (!bytes) +break ; } res = LZ4_decompress_safe(stream->input, stream->output,
fs/squashfs/lzo_wrapper.c +10 -7
@@ -66,21 +66,24 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter; +struct bio_vec bvec;struct squashfs_lzo *stream = strm;void *buff = stream->input, *data;int bytes = length, res;size_t out_len = output->length; -while (bio_next_segment (bio, &iter_all)) { -int avail =min (bytes, ((int )bvec->bv_len) - offset); +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); - data =bvec_virt (bvec); -memcpy (buff, data + offset, avail); +bio_for_each_segment_all_continue (bvec, bio, iter) { +unsigned avail =min_t (unsigned, bytes, bvec.bv_len); + +memcpy (buff,bvec_virt (&bvec), avail); buff += avail; bytes -= avail; - offset =0 ; +if (!bytes) +break ; } res = lzo1x_decompress_safe(stream->input, (size_t)length,
fs/squashfs/xz_wrapper.c +10 -9
@@ -120,8 +120,7 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter;int total =0 , error =0 ;struct squashfs_xz *stream = strm;
@@ -136,26 +135,28 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {enum xz_ret xz_err;if (stream->buf.in_pos == stream->buf.in_size) { -const void *data; -int avail; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter); +unsigned avail =min_t (unsigned, length, bvec.bv_len); -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {/* XZ_STREAM_END must be reached. */ error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); length -= avail; - stream->buf.in = data + offset; + stream->buf.in =bvec_virt (&bvec); stream->buf.in_size = avail; stream->buf.in_pos =0 ; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (stream->buf.out_pos == stream->buf.out_size) {
fs/squashfs/zlib_wrapper.c +10 -8
@@ -53,8 +53,7 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter;int zlib_init =0 , error =0 ;z_stream *stream = strm;
@@ -67,25 +66,28 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {int zlib_err;if (stream->avail_in ==0 ) { -const void *data; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter);int avail; -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {/* Z_STREAM_END must be reached. */ error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); + avail =min_t (unsigned, length, bvec.bv_len); length -= avail; - stream->next_in = data + offset; + stream->next_in =bvec_virt (&bvec); stream->avail_in = avail; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (stream->avail_out ==0 ) {
fs/squashfs/zstd_wrapper.c +10 -9
@@ -68,8 +68,7 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,int error =0 ;zstd_in_buffer in_buf = {NULL ,0 ,0 };zstd_out_buffer out_buf = {NULL ,0 ,0 }; -struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); +struct bvec_iter_all iter; stream =zstd_init_dstream (wksp->window_size, wksp->mem, wksp->mem_size);
@@ -85,25 +84,27 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {size_t zstd_err;if (in_buf.pos == in_buf.size) { -const void *data; -int avail; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter); +unsigned avail =min_t (unsigned, length, bvec.bv_len); -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) { error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); length -= avail; - in_buf.src = data + offset; + in_buf.src =bvec_virt (&bvec); in_buf.size = avail; in_buf.pos =0 ; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (out_buf.pos == out_buf.size) {
include/linux/bio.h +20 -8
@@ -78,22 +78,34 @@ static inline void *bio_data(struct bio *bio)return NULL ; } -static inline bool bio_next_segment (const struct bio *bio, -struct bvec_iter_all *iter) +static inline struct bio_vec bio_iter_all_peek (const struct bio *bio, +struct bvec_iter_all *iter) { -if (iter->idx >= bio->bi_vcnt) -return false; +return bvec_iter_all_peek (bio->bi_io_vec, iter); +} -bvec_advance (&bio->bi_io_vec[iter->idx], iter); -return true; +static inline void bio_iter_all_advance (const struct bio *bio, +struct bvec_iter_all *iter, +unsigned bytes) +{ +bvec_iter_all_advance (bio->bi_io_vec, iter, bytes); } +#define bio_for_each_segment_all_continue (bvl, bio, iter) \ + for (; \ + iter.idx < bio->bi_vcnt && \ + ((bvl = bio_iter_all_peek(bio, &iter)), true); \ + bio_iter_all_advance((bio), &iter, bvl.bv_len)) +/* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */ -#define bio_for_each_segment_all (bvl, bio, iter) \ - for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); ) +#define bio_for_each_segment_all (bvl, bio, iter) \ + for (bvec_iter_all_init(&iter); \ + iter.idx < (bio)->bi_vcnt && \ + ((bvl = bio_iter_all_peek((bio), &iter)), true); \ + bio_iter_all_advance((bio), &iter, bvl.bv_len))static inline void bio_advance_iter (const struct bio *bio,struct bvec_iter *iter,unsigned int bytes)
include/linux/bvec.h +35 -26
87 lines not on this page · raw message
Signature · 1 line
2.40.1
Kent Overstreet
maintainerPATCH v2 2/5 block: Rework bio_for_each_folio_all()
This reimplements bio_for_each_folio_all() on top of the newly-reworked bvec_iter_all, and since it's now trivial we also provide bio_for_each_folio.
--- fs/crypto/bio.c | 9 ++-- fs/ext4/page-io.c | 11 ++--- fs/ext4/readpage.c | 7 ++-- fs/iomap/buffered-io.c | 14 ++++--- fs/mpage.c | 22 +++++----- fs/verity/verify.c | 9 ++-- include/linux/bio.h | 95 ++++++++++++++++++++++-------------------- include/linux/bvec.h | 13 ++++-- 8 files changed, 99 insertions(+), 81 deletions(-)
fs/crypto/bio.c +5 -4
@@ -30,11 +30,12 @@*/ bool fscrypt_decrypt_bio (struct bio *bio) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs; -bio_for_each_folio_all (fi, bio) { -int err =fscrypt_decrypt_pagecache_blocks (fi.folio, fi.length, - fi.offset); +bio_for_each_folio_all (fs, bio, iter) { +int err =fscrypt_decrypt_pagecache_blocks (fs.fs_folio, fs.fs_len, + fs.fs_offset);if (err) { bio->bi_status =errno_to_blk_status (err);
fs/ext4/page-io.c +6 -5
@@ -99,14 +99,15 @@ static void buffer_io_error(struct buffer_head *bh)static void ext4_finish_bio (struct bio *bio) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs; -bio_for_each_folio_all (fi, bio) { -struct folio *folio = fi.folio; +bio_for_each_folio_all (fs, bio, iter) { +struct folio *folio = fs.fs_folio;struct folio *io_folio =NULL ;struct buffer_head *bh, *head; -size_t bio_start = fi.offset; -size_t bio_end = bio_start + fi.length; +size_t bio_start = fs.fs_offset; +size_t bio_end = bio_start + fs.fs_len;unsigned under_io =0 ;unsigned long flags;
fs/ext4/readpage.c +4 -3
@@ -68,10 +68,11 @@ struct bio_post_read_ctx {static void __read_end_io (struct bio *bio) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs; -bio_for_each_folio_all (fi, bio) { -struct folio *folio = fi.folio; +bio_for_each_folio_all (fs, bio, iter) { +struct folio *folio = fs.fs_folio;if (bio->bi_status)folio_clear_uptodate (folio);
fs/iomap/buffered-io.c +8 -6
@@ -187,10 +187,11 @@ static void iomap_finish_folio_read(struct folio *folio, size_t offset,static void iomap_read_end_io (struct bio *bio) {int error =blk_status_to_errno (bio->bi_status); -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs; -bio_for_each_folio_all (fi, bio) -iomap_finish_folio_read (fi.folio, fi.offset, fi.length, error); +bio_for_each_folio_all (fs, bio, iter) +iomap_finish_folio_read (fs.fs_folio, fs.fs_offset, fs.fs_len, error);bio_put (bio); }
@@ -1321,7 +1322,8 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error)u32 folio_count =0 ;for (bio = &ioend->io_inline_bio; bio; bio = next) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs;/* * For the last bio, bi_private points to the ioend, so we
@@ -1333,8 +1335,8 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error) next = bio->bi_private;/* walk all folios in bio, ending page IO on them */ -bio_for_each_folio_all (fi, bio) { -iomap_finish_folio_write (inode, fi.folio, fi.length, +bio_for_each_folio_all (fs, bio, iter) { +iomap_finish_folio_write (inode, fs.fs_folio, fs.fs_len, error); folio_count++; }
fs/mpage.c +12 -10
@@ -45,15 +45,16 @@*/ static void mpage_read_end_io (struct bio *bio) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs;int err =blk_status_to_errno (bio->bi_status); -bio_for_each_folio_all (fi, bio) { +bio_for_each_folio_all (fs, bio, iter) {if (err) -folio_set_error (fi.folio); +folio_set_error (fs.fs_folio);else -folio_mark_uptodate (fi.folio); -folio_unlock (fi.folio); +folio_mark_uptodate (fs.fs_folio); +folio_unlock (fs.fs_folio); }bio_put (bio);
@@ -61,15 +62,16 @@ static void mpage_read_end_io(struct bio *bio)static void mpage_write_end_io (struct bio *bio) { -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs;int err =blk_status_to_errno (bio->bi_status); -bio_for_each_folio_all (fi, bio) { +bio_for_each_folio_all (fs, bio, iter) {if (err) { -folio_set_error (fi.folio); -mapping_set_error (fi.folio->mapping, err); +folio_set_error (fs.fs_folio); +mapping_set_error (fs.fs_folio->mapping, err); } -folio_end_writeback (fi.folio); +folio_end_writeback (fs.fs_folio); }bio_put (bio);
fs/verity/verify.c +5 -4
@@ -340,7 +340,8 @@ void fsverity_verify_bio(struct bio *bio)struct inode *inode =bio_first_page_all (bio)->mapping->host;struct fsverity_info *vi = inode->i_verity_info;struct ahash_request *req; -struct folio_iter fi; +struct bvec_iter_all iter; +struct folio_seg fs;unsigned long max_ra_pages =0 ;/* This allocation never fails, since it's mempool-backed. */
@@ -359,9 +360,9 @@ void fsverity_verify_bio(struct bio *bio) max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT +2 ); } -bio_for_each_folio_all (fi, bio) { -if (!verify_data_blocks (inode, vi, req, fi.folio, fi.length, - fi.offset, max_ra_pages)) { +bio_for_each_folio_all (fs, bio, iter) { +if (!verify_data_blocks (inode, vi, req, fs.fs_folio, fs.fs_len, + fs.fs_offset, max_ra_pages)) { bio->bi_status =BLK_STS_IOERR ;break ; }
include/linux/bio.h +49 -46
114 lines not on this page · raw message
include/linux/bvec.h +10 -3
@@ -205,8 +205,8 @@ static inline void bvec_iter_all_init(struct bvec_iter_all *iter_all) iter_all->idx =0 ; } -static inline struct bio_vec bvec_iter_all_peek (const struct bio_vec *bvec, -struct bvec_iter_all *iter) +static inline struct bio_vec __bvec_iter_all_peek (const struct bio_vec *bvec, +const struct bvec_iter_all *iter) {struct bio_vec bv = bvec[iter->idx];
@@ -215,8 +215,15 @@ static inline struct bio_vec bvec_iter_all_peek(const struct bio_vec *bvec, bv.bv_page += bv.bv_offset >>PAGE_SHIFT ; bv.bv_offset &= ~PAGE_MASK ; - bv.bv_len =min_t (unsigned,PAGE_SIZE - bv.bv_offset, bv.bv_len); +return bv; +} + +static inline struct bio_vec bvec_iter_all_peek (const struct bio_vec *bvec, +const struct bvec_iter_all *iter) +{ +struct bio_vec bv =__bvec_iter_all_peek (bvec, iter); + bv.bv_len =min_t (unsigned,PAGE_SIZE - bv.bv_offset, bv.bv_len);return bv; }
Signature · 1 line
2.40.1
Kent Overstreet
maintainerPATCH v2 3/5 block: Add documentation for bio iterator macros
In reply to Kent OverstreetWe've now got 3x2 interfaces for iterating over bios: by page, by bvec, or by folio, and variants that iterate over what bi_iter points to, or the entire bio as created by the filesystem/originator.
This adds more detailed kerneldoc comments for each variant.
--- include/linux/bio.h | 54 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-)
include/linux/bio.h +48 -6
@@ -97,9 +97,14 @@ static inline void bio_iter_all_advance(const struct bio *bio, ((bvl =bio_iter_all_peek (bio, &iter)), true); \ bio_iter_all_advance((bio), &iter, bvl.bv_len)) -/* -* drivers should _never_ use the all version - the bio may have been split -* before it got to the driver and the driver won't own all of it +/** +* bio_for_each_segment_all - iterate over single pages in a bio +* +* Like other _all versions, this is for the filesystem, or the owner/creator of +* a bio; it iterates over the original contents of a bio. +* +* Drivers that are working with bios that were submitted to them should not use +* the _all version. */ #define bio_for_each_segment_all (bvl, bio, iter) \ for (bvec_iter_all_init(&iter); \
@@ -160,6 +165,13 @@ static inline void bio_advance(struct bio *bio, unsigned int nbytes) ((bvl =bio_iter_iovec ((bio), (iter))),1 ); \ bio_advance_iter_single((bio), &(iter), (bvl).bv_len)) +/** +* bio_for_each_segment - iterate over single pages in a bio +* +* Like other non-_all versions, this iterates over what bio->bi_iter currently +* points to. This version is for drivers, where the bio may have previously +* been split or cloned. +*/ #define bio_for_each_segment (bvl, bio, iter) \ __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
@@ -200,6 +212,13 @@ static inline struct folio_seg bio_iter_iovec_folio(struct bio *bio, ((fs =bio_iter_iovec_folio ((bio), (iter))),1 ); \ bio_advance_iter_single((bio), &(iter), (fs).fs_len)) +/** +* bio_for_each_folio - iterate over folios within a bio +* +* Like other non-_all versions, this iterates over what bio->bi_iter currently +* points to. This version is for drivers, where the bio may have previously +* been split or cloned. +*/ #define bio_for_each_folio (fs, bio, iter) \ __bio_for_each_folio(fs, bio, iter, (bio)->bi_iter)
@@ -209,13 +228,30 @@ static inline struct folio_seg bio_iter_iovec_folio(struct bio *bio, ((bvl =mp_bvec_iter_bvec ((bio)->bi_io_vec, (iter))),1 ); \ bio_advance_iter_single((bio), &(iter), (bvl).bv_len)) -/* iterate over multi-page bvec */ +/** +* bio_for_each_bvec - iterate over bvecs within a bio +* +* This version iterates over entire bio_vecs, which will be a range of +* contiguous pages. +* +* Like other non-_all versions, this iterates over what bio->bi_iter currently +* points to. This version is for drivers, where the bio may have previously +* been split or cloned. +*/ #define bio_for_each_bvec (bvl, bio, iter) \ __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)/* -* Iterate over all multi-page bvecs. Drivers shouldn't use this version for the -* same reasons as bio_for_each_segment_all(). +* bio_for_each_bvec_all - iterate over bvecs within a bio +* +* This version iterates over entire bio_vecs, which will be a range of +* contiguous pages. +* +* Like other _all versions, this is for the filesystem, or the owner/creator of +* a bio; it iterates over the original contents of a bio. +* +* Drivers that are working with bios that were submitted to them should not use +* the _all version. */ #define bio_for_each_bvec_all (bvl, bio, i) \ for (i = 0, bvl = bio_first_bvec_all(bio); \
@@ -321,6 +357,12 @@ static inline struct folio_seg bio_folio_iter_all_peek(const struct bio *bio,* bio_for_each_folio_all - Iterate over each folio in a bio. * @fi: struct bio_folio_iter_all which is updated for each folio. * @bio: struct bio to iterate over. +* +* Like other _all versions, this is for the filesystem, or the owner/creator of +* a bio; it iterates over the original contents of a bio. +* +* Drivers that are working with bios that were submitted to them should not use +* the _all version. */ #define bio_for_each_folio_all (fs, bio, iter) \ for (bvec_iter_all_init(&iter); \
Signature · 1 line
2.40.1
Kent Overstreet
maintainerPATCH v2 4/5 block: Change memcpy_(to|from)_bvec to pass bvec by value
In reply to Kent OverstreetThis is a small code size decrease; memcpy_from_bvec() is inlined so there's no benefit to passing the bvec by referene, but taking a pointer to it may inhibit optimization in the caller.
--- block/blk-map.c | 2 +- block/bounce.c | 4 ++-- drivers/block/floppy.c | 4 ++-- drivers/block/ps3disk.c | 2 +- drivers/block/zram/zram_drv.c | 4 ++-- drivers/md/dm-log-writes.c | 2 +- drivers/scsi/aha1542.c | 2 +- include/linux/bvec.h | 8 ++++---- 8 files changed, 14 insertions(+), 14 deletions(-)
block/blk-map.c +1 -1
@@ -447,7 +447,7 @@ static void bio_copy_kern_endio_read(struct bio *bio)struct bio_vec bvec;bio_for_each_segment_all (bvec, bio, iter) { -memcpy_from_bvec (p, &bvec); +memcpy_from_bvec (p, bvec); p += bvec.bv_len; }
block/bounce.c +2 -2
@@ -92,7 +92,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)* been modified by the block layer, so use the original * copy, bounce_copy_vec already uses tovec->bv_len */ -memcpy_to_bvec (&tovec,page_address (fromvec.bv_page) + +memcpy_to_bvec (tovec,page_address (fromvec.bv_page) + tovec.bv_offset); }bio_advance_iter (from, &from_iter, tovec.bv_len);
@@ -249,7 +249,7 @@ struct bio *__blk_queue_bounce(struct bio *bio_orig, struct request_queue *q)if (rw ==WRITE ) {flush_dcache_page (to->bv_page); -memcpy_from_bvec (page_address (bounce_page), to); +memcpy_from_bvec (page_address (bounce_page), *to); } to->bv_page = bounce_page; }
drivers/block/floppy.c +2 -2
@@ -2483,9 +2483,9 @@ static void copy_buffer(int ssize, int max_sector, int max_sector_2) }if (CT (raw_cmd->cmd[COMMAND ]) ==FD_READ ) -memcpy_to_bvec (&bv, dma_buffer); +memcpy_to_bvec (bv, dma_buffer);else -memcpy_from_bvec (dma_buffer, &bv); +memcpy_from_bvec (dma_buffer, bv); remaining -= size; dma_buffer += size;
drivers/block/ps3disk.c +1 -1
@@ -86,7 +86,7 @@ static void ps3disk_scatter_gather(struct ps3_storage_device *dev,rq_for_each_segment (bvec, req, iter) {if (gather) -memcpy_from_bvec (dev->bounce_buf + offset, &bvec); +memcpy_from_bvec (dev->bounce_buf + offset, bvec);else memcpy_to_bvec (&bvec, dev->bounce_buf + offset); }
drivers/block/zram/zram_drv.c +2 -2
@@ -1396,7 +1396,7 @@ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec,return -ENOMEM ; ret =zram_read_page (zram, page, index,NULL );if (likely (!ret)) -memcpy_to_bvec (bvec,page_address (page) + offset); +memcpy_to_bvec (*bvec,page_address (page) + offset);__free_page (page);return ret; }
@@ -1548,7 +1548,7 @@ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, ret =zram_read_page (zram, page, index, bio);if (!ret) { -memcpy_from_bvec (page_address (page) + offset, bvec); +memcpy_from_bvec (page_address (page) + offset, *bvec); ret =zram_write_page (zram, page, index); }__free_page (page);
drivers/md/dm-log-writes.c +1 -1
@@ -744,7 +744,7 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio) } dst =kmap_local_page (page); -memcpy_from_bvec (dst, &bv); +memcpy_from_bvec (dst, bv);kunmap_local (dst); block->vecs[i].bv_page = page; block->vecs[i].bv_len = bv.bv_len;
drivers/scsi/aha1542.c +1 -1
@@ -450,7 +450,7 @@ static int aha1542_queuecommand(struct Scsi_Host *sh, struct scsi_cmnd *cmd)struct bio_vec bv;rq_for_each_segment (bv, rq, iter) { -memcpy_from_bvec (buf, &bv); +memcpy_from_bvec (buf, bv); buf += bv.bv_len; } }
include/linux/bvec.h +4 -4
@@ -257,9 +257,9 @@ static inline void *bvec_kmap_local(struct bio_vec *bvec)* * Must be called on single-page bvecs only. */ -static inline void memcpy_from_bvec (char *to,struct bio_vec *bvec) +static inline void memcpy_from_bvec (char *to,struct bio_vec bvec) { -memcpy_from_page (to, bvec->bv_page, bvec->bv_offset, bvec->bv_len); +memcpy_from_page (to, bvec.bv_page, bvec.bv_offset, bvec.bv_len); }/**
@@ -268,9 +268,9 @@ static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec)* * Must be called on single-page bvecs only. */ -static inline void memcpy_to_bvec (struct bio_vec *bvec,const char *from) +static inline void memcpy_to_bvec (struct bio_vec bvec,const char *from) { -memcpy_to_page (bvec->bv_page, bvec->bv_offset, from, bvec->bv_len); +memcpy_to_page (bvec.bv_page, bvec.bv_offset, from, bvec.bv_len); }/**
Signature · 1 line
2.40.1
kernel test robot
build botRe: [PATCH v2 4/5] block: Change memcpy_(to|from)_bvec to pass bvec by value
kernel test robot noticed the following build errors:
Hi Kent,
kernel test robot noticed the following build errors:
[auto build test ERROR on axboe-block/for-next]
[cannot apply to kdave/for-next device-mapper-dm/for-next gfs2/for-next tytso-ext4/dev linus/master v6.4-rc5 next-20230606] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Kent-Overstreet/block-Rework-bio_for_each_folio_all/20230606-052850 base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next patch link: https://lore.kernel.org/r/20230605212717.2570570-4-kent.overstreet%40linux.dev patch subject: [PATCH v2 4/5] block: Change memcpy_(to|from)_bvec to pass bvec by value config: alpha-allyesconfig (https://download.01.org/0day-ci/archive/20230606/[email protected]/config) compiler: alpha-linux-gcc (GCC) 12.3.0
reproduce (this is a W=1 build):
mkdir -p ~/bin
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
git remote add axboe-block https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git
git fetch axboe-block for-next
git checkout axboe-block/for-next
b4 shazam https://lore.kernel.org/r/[email protected]
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.3.0 ~/bin/make.cross W=1 O=build_dir ARCH=alpha olddefconfig COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.3.0 ~/bin/make.cross W=1 O=build_dir ARCH=alpha SHELL=/bin/bash drivers/scsi/
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
drivers/scsi/aha1542.c: In function 'aha1542_free_cmd':
>> drivers/scsi/aha1542.c:270:40: error: incompatible type for argument 1 of 'memcpy_to_bvec'
270 | memcpy_to_bvec(&bv, buf);
| ^~~
| |
| struct bio_vec *
In file included from include/linux/blk_types.h:10,
from include/linux/blkdev.h:9,
from include/scsi/scsi_cmnd.h:6,
from drivers/scsi/aha1542.c:23: include/linux/bvec.h:271:50: note: expected 'struct bio_vec' but argument is of type 'struct bio_vec *'
271 | static inline void memcpy_to_bvec(struct bio_vec bvec, const char *from)
| ~~~~~~~~~~~~~~~^~~~
vim +/memcpy_to_bvec +270 drivers/scsi/aha1542.c
^1da177e4c3f41 Linus Torvalds 2005-04-16 258
1794ef2b150dd5 Christoph Hellwig 2018-11-10 259 static void aha1542_free_cmd(struct scsi_cmnd *cmd)
1794ef2b150dd5 Christoph Hellwig 2018-11-10 260 {
1794ef2b150dd5 Christoph Hellwig 2018-11-10 261 struct aha1542_cmd *acmd = scsi_cmd_priv(cmd);
1794ef2b150dd5 Christoph Hellwig 2018-11-10 262
2f2fef022c3e7a Christoph Hellwig 2021-03-31 263 if (cmd->sc_data_direction == DMA_FROM_DEVICE) {
11bf4ec580737c Bart Van Assche 2021-08-09 264 struct request *rq = scsi_cmd_to_rq(cmd);
2f2fef022c3e7a Christoph Hellwig 2021-03-31 265 void *buf = acmd->data_buffer;
2f2fef022c3e7a Christoph Hellwig 2021-03-31 266 struct req_iterator iter;
2f2fef022c3e7a Christoph Hellwig 2021-03-31 267 struct bio_vec bv;
2f2fef022c3e7a Christoph Hellwig 2021-03-31 268
11bf4ec580737c Bart Van Assche 2021-08-09 269 rq_for_each_segment(bv, rq, iter) {
e6ab6113526aa4 Christoph Hellwig 2021-10-18 @270 memcpy_to_bvec(&bv, buf);
2f2fef022c3e7a Christoph Hellwig 2021-03-31 271 buf += bv.bv_len;
2f2fef022c3e7a Christoph Hellwig 2021-03-31 272 }
1794ef2b150dd5 Christoph Hellwig 2018-11-10 273 }
1794ef2b150dd5 Christoph Hellwig 2018-11-10 274
1794ef2b150dd5 Christoph Hellwig 2018-11-10 275 scsi_dma_unmap(cmd);
1794ef2b150dd5 Christoph Hellwig 2018-11-10 276 }
1794ef2b150dd5 Christoph Hellwig 2018-11-10 277 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Kent Overstreet
maintainerPATCH v2 5/5 block: CONFIG_BLK_DEBUG
In reply to Kent OverstreetThis adds a new kconfig option to enable some extra debug mode assertions. Currently this is just used for some bio iterator assertions, but hopefully more will be added in the future.
--- block/Kconfig | 7 +++++++ include/linux/bio.h | 5 +++++ include/linux/bvec.h | 8 ++++++++ 3 files changed, 20 insertions(+)
block/Kconfig +7 -0
@@ -165,6 +165,13 @@ config BLK_CGROUP_IOPRIO scheduler and block devices process requests. Only some I/O schedulers and some block devices support I/O priorities. +config BLK_DEBUG + bool "Extra block layer assertions" + depends on DEBUG_KERNEL + help + Enable extra assertions in the block layer. Currently this is just the + bio iterator code, but may be expanded. + config BLK_DEBUG_FS bool "Block layer debugging information in debugfs" default y
include/linux/bio.h +5 -0
@@ -81,6 +81,8 @@ static inline void *bio_data(struct bio *bio)static inline struct bio_vec bio_iter_all_peek (const struct bio *bio,struct bvec_iter_all *iter) { +BLK_BUG_ON (iter->idx >= bio->bi_vcnt); +return bvec_iter_all_peek (bio->bi_io_vec, iter); }
@@ -89,6 +91,9 @@ static inline void bio_iter_all_advance(const struct bio *bio, unsigned bytes) {bvec_iter_all_advance (bio->bi_io_vec, iter, bytes); + +BLK_BUG_ON (iter->idx > bio->bi_vcnt || + (iter->idx == bio->bi_vcnt && iter->done)); }#define bio_for_each_segment_all_continue (bvl, bio, iter) \
include/linux/bvec.h +8 -0
@@ -16,6 +16,12 @@struct page ; +#ifdef CONFIG_BLK_DEBUG +#define BLK_BUG_ON (cond) BUG_ON(cond) +#else +#define BLK_BUG_ON (cond) BUILD_BUG_ON_INVALID(cond) +#endif +/** * struct bio_vec - a contiguous range of physical memory addresses * @bv_page: First page associated with the address range.
@@ -210,6 +216,8 @@ static inline struct bio_vec __bvec_iter_all_peek(const struct bio_vec *bvec, {struct bio_vec bv = bvec[iter->idx]; +BLK_BUG_ON (iter->done >= bv.bv_len); + bv.bv_offset += iter->done; bv.bv_len -= iter->done;
Signature · 1 line
2.40.1
Re: [PATCH v2 5/5] block: CONFIG_BLK_DEBUG
Hi,
Quoted 5 lines · Kent Overstreet
block/Kconfig +0
-0
block/Kconfig config BLK_CGROUP_IOPRIO · line 173
Quoted 9 lines
block/Kconfig +6
-0@@ -165,6 +165,13 @@ config BLK_CGROUP_IOPRIO scheduler and block devices process requests. Only some I/O schedulers and some block devices support I/O priorities. +config BLK_DEBUG + bool "Extra block layer assertions" + depends on DEBUG_KERNEL + help + Enable extra assertions in the block layer. Currently this is just the + bio iterator code, but may be expanded.
I see that block/Kconfig isn't consistent about indenting help text, but Documentation/process/coding-style.rst says:
10) Kconfig configuration files
-------------------------------
For all of the Kconfig* configuration files throughout the source tree, the indentation is somewhat different. Lines under a ``config`` definition are indented with one tab, while help text is indented an additional two spaces.
so please add 2 spaces after the tab at the beginning of the help text lines (but not the "help" line itself).
Signature · 1 line
~Randy
Quoted 8 lines · Kent Overstreet
This patch reworks
bio_for_each_segment_all()to be more inline with how the other bio iterators work:-
bio_iter_all_peek()now returns a synthesizedbio_vec; we don't stash one in the iterator and pass a pointer to it - bad. This way makes it clearer what's a constructed value vs. a reference to something pre-existing, and it also will help with cleaning up and consolidating code withbio_for_each_folio_all().
by-value is more readable & clean, the current by-reference is just for aligning with original interface.
Quoted 3 lines
- We now provide
bio_for_each_segment_all_continue(), for squashfs:
this makes their code clearer.
Probably it is better to move the new helper into new patch, but I guess it could be hard for this patch.
Quoted 901 lines
Signed-off-by Kent Overstreet <[email protected]>Cc Jens Axboe <[email protected]>Cc Ming Lei <[email protected]>Cc Phillip Lougher <[email protected]>--- block/bio.c | 38 ++++++++++++------------ block/blk-map.c | 38 ++++++++++++------------ block/bounce.c | 12 ++++---- drivers/md/bcache/btree.c | 8 ++--- drivers/md/dm-crypt.c | 10 +++---- drivers/md/raid1.c | 4 +-- fs/btrfs/disk-io.c | 4 +-- fs/btrfs/extent_io.c | 50 +++++++++++++++---------------- fs/btrfs/raid56.c | 14 ++++----- fs/erofs/zdata.c | 4 +-- fs/f2fs/data.c | 20 ++++++------- fs/gfs2/lops.c | 10 +++---- fs/gfs2/meta_io.c | 8 ++--- fs/squashfs/block.c | 48 +++++++++++++++++------------- fs/squashfs/lz4_wrapper.c | 17 ++++++----- fs/squashfs/lzo_wrapper.c | 17 ++++++----- fs/squashfs/xz_wrapper.c | 19 ++++++------ fs/squashfs/zlib_wrapper.c | 18 ++++++----- fs/squashfs/zstd_wrapper.c | 19 ++++++------ include/linux/bio.h | 28 ++++++++++++----- include/linux/bvec.h | 61 ++++++++++++++++++++++---------------- 21 files changed, 242 insertions(+), 205 deletions(-)block/bio.c +19
-19@@ -1163,13 +1163,13 @@ EXPORT_SYMBOL(bio_add_folio);void __bio_release_pages (struct bio *bio,bool mark_dirty) { -struct bvec_iter_all iter_all; -struct bio_vec *bvec; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (mark_dirty && !PageCompound (bvec->bv_page)) -set_page_dirty_lock (bvec->bv_page); -bio_release_page (bio, bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) { +if (mark_dirty && !PageCompound (bvec.bv_page)) +set_page_dirty_lock (bvec.bv_page); +bio_release_page (bio, bvec.bv_page); } }EXPORT_SYMBOL_GPL (__bio_release_pages);@@ -1435,11 +1435,11 @@ EXPORT_SYMBOL(bio_copy_data);void bio_free_pages (struct bio *bio) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) -__free_page (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) +__free_page (bvec.bv_page); }EXPORT_SYMBOL (bio_free_pages);@@ -1474,12 +1474,12 @@ EXPORT_SYMBOL(bio_free_pages);*/ void bio_set_pages_dirty (struct bio *bio) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (!PageCompound (bvec->bv_page)) -set_page_dirty_lock (bvec->bv_page); +bio_for_each_segment_all (bvec, bio, iter) { +if (!PageCompound (bvec.bv_page)) +set_page_dirty_lock (bvec.bv_page); } }@@ -1522,12 +1522,12 @@ static void bio_dirty_fn(struct work_struct *work)void bio_check_pages_dirty (struct bio *bio) { -struct bio_vec *bvec; +struct bvec_iter_all iter; +struct bio_vec bvec;unsigned long flags; -struct bvec_iter_all iter_all; -bio_for_each_segment_all (bvec, bio, iter_all) { -if (!PageDirty (bvec->bv_page) && !PageCompound (bvec->bv_page)) +bio_for_each_segment_all (bvec, bio, iter) { +if (!PageDirty (bvec.bv_page) && !PageCompound (bvec.bv_page)) goto defer; }block/blk-map.c +19
-19@@ -47,21 +47,21 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,*/ static int bio_copy_from_iter (struct bio *bio,struct iov_iter *iter) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all bv_iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, bv_iter) {ssize_t ret; - ret =copy_page_from_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, + ret =copy_page_from_iter (bvec.bv_page, + bvec.bv_offset, + bvec.bv_len, iter);if (!iov_iter_count (iter))break ; -if (ret < bvec->bv_len) +if (ret < bvec.bv_len)return -EFAULT ; }@@ -78,21 +78,21 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)*/ static int bio_copy_to_iter (struct bio *bio,struct iov_iter iter) { -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all bv_iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, bv_iter) {ssize_t ret; - ret =copy_page_to_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, + ret =copy_page_to_iter (bvec.bv_page, + bvec.bv_offset, + bvec.bv_len, &iter);if (!iov_iter_count (&iter))break ; -if (ret < bvec->bv_len) +if (ret < bvec.bv_len)return -EFAULT ; }@@ -443,12 +443,12 @@ static void bio_copy_kern_endio(struct bio *bio)static void bio_copy_kern_endio_read (struct bio *bio) {char *p = bio->bi_private; -struct bio_vec *bvec; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bvec; -bio_for_each_segment_all (bvec, bio, iter_all) { -memcpy_from_bvec (p, bvec); - p += bvec->bv_len; +bio_for_each_segment_all (bvec, bio, iter) { +memcpy_from_bvec (p, &bvec); + p += bvec.bv_len; }bio_copy_kern_endio (bio);block/bounce.c +6
-6@@ -102,18 +102,18 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)static void bounce_end_io (struct bio *bio) {struct bio *bio_orig = bio->bi_private; -struct bio_vec *bvec, orig_vec; +struct bio_vec bvec, orig_vec;struct bvec_iter orig_iter = bio_orig->bi_iter; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter;/* * free up bounce indirect pages used */ -bio_for_each_segment_all (bvec, bio, iter_all) { +bio_for_each_segment_all (bvec, bio, iter) { orig_vec =bio_iter_iovec (bio_orig, orig_iter); -if (bvec->bv_page != orig_vec.bv_page) { -dec_zone_page_state (bvec->bv_page,NR_BOUNCE ); -mempool_free (bvec->bv_page, &page_pool); +if (bvec.bv_page != orig_vec.bv_page) { +dec_zone_page_state (bvec.bv_page,NR_BOUNCE ); +mempool_free (bvec.bv_page, &page_pool); }bio_advance_iter (bio_orig, &orig_iter, orig_vec.bv_len); }drivers/md/bcache/btree.c +4
-4@@ -373,12 +373,12 @@ static void do_btree_node_write(struct btree *b) bset_sector_offset(&b->keys, i));if (!bch_bio_alloc_pages (b->bio, __GFP_NOWARN|GFP_NOWAIT )) { -struct bio_vec *bv; +struct bio_vec bv;void *addr = (void *) ((unsigned long ) i & ~(PAGE_SIZE -1 )); -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; -bio_for_each_segment_all (bv, b->bio, iter_all) { -memcpy (page_address (bv->bv_page), addr,PAGE_SIZE ); +bio_for_each_segment_all (bv, b->bio, iter) { +memcpy (page_address (bv.bv_page), addr,PAGE_SIZE ); addr +=PAGE_SIZE ; }drivers/md/dm-crypt.c +5
-5@@ -1713,12 +1713,12 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)static void crypt_free_buffer_pages (struct crypt_config *cc,struct bio *clone) { -struct bio_vec *bv; -struct bvec_iter_all iter_all; +struct bvec_iter_all iter; +struct bio_vec bv; -bio_for_each_segment_all (bv, clone, iter_all) { -BUG_ON (!bv->bv_page); -mempool_free (bv->bv_page, &cc->page_pool); +bio_for_each_segment_all (bv, clone, iter) { +BUG_ON (!bv.bv_page); +mempool_free (bv.bv_page, &cc->page_pool); } }drivers/md/raid1.c +2
-2@@ -2188,7 +2188,7 @@ static void process_checks(struct r1bio *r1_bio)blk_status_t status = sbio->bi_status;struct page **ppages =get_resync_pages (pbio)->pages;struct page **spages =get_resync_pages (sbio)->pages; -struct bio_vec *bi; +struct bio_vec bi;int page_len[RESYNC_PAGES ] = {0 };struct bvec_iter_all iter_all;@@ -2198,7 +2198,7 @@ static void process_checks(struct r1bio *r1_bio) sbio->bi_status =0 ;bio_for_each_segment_all (bi, sbio, iter_all) - page_len[j++] = bi->bv_len; + page_len[j++] = bi.bv_len;if (!status) {for (j = vcnt; j-- ; ) {fs/btrfs/disk-io.c +2
-2@@ -3807,12 +3807,12 @@ ALLOW_ERROR_INJECTION(open_ctree, ERRNO);static void btrfs_end_super_write (struct bio *bio) {struct btrfs_device *device = bio->bi_private; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;struct page *page;bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (bio->bi_status) { btrfs_warn_rl_in_rcu(device->fs_info,fs/btrfs/extent_io.c +25
-25@@ -573,34 +573,34 @@ static void end_bio_extent_writepage(struct btrfs_bio *bbio) {struct bio *bio = &bbio->bio;int error =blk_status_to_errno (bio->bi_status); -struct bio_vec *bvec; +struct bio_vec bvec;u64 start;u64 end;struct bvec_iter_all iter_all;ASSERT (!bio_flagged (bio,BIO_CLONED ));bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;struct inode *inode = page->mapping->host;struct btrfs_fs_info *fs_info =btrfs_sb (inode->i_sb);const u32 sectorsize = fs_info->sectorsize;/* Our read/write should always be sector aligned. */ -if (!IS_ALIGNED (bvec->bv_offset, sectorsize)) +if (!IS_ALIGNED (bvec.bv_offset, sectorsize))btrfs_err (fs_info,"partial page write in btrfs with offset %u and length %u" , - bvec->bv_offset, bvec->bv_len); -else if (!IS_ALIGNED (bvec->bv_len, sectorsize)) + bvec.bv_offset, bvec.bv_len); +else if (!IS_ALIGNED (bvec.bv_len, sectorsize))btrfs_info (fs_info,"incomplete page write with offset %u and length %u" , - bvec->bv_offset, bvec->bv_len); + bvec.bv_offset, bvec.bv_len); - start =page_offset (page) + bvec->bv_offset; - end = start + bvec->bv_len -1 ; + start =page_offset (page) + bvec.bv_offset; + end = start + bvec.bv_len -1 ;end_extent_writepage (page, error, start, end); -btrfs_page_clear_writeback (fs_info, page, start, bvec->bv_len); +btrfs_page_clear_writeback (fs_info, page, start, bvec.bv_len); }bio_put (bio);@@ -728,7 +728,7 @@ static struct extent_buffer *find_extent_buffer_readpage(static void end_bio_extent_readpage (struct btrfs_bio *bbio) {struct bio *bio = &bbio->bio; -struct bio_vec *bvec; +struct bio_vec bvec;struct processed_extent processed = {0 };/* * The offset to the beginning of a bio, since one bio can never be @@ -741,7 +741,7 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio)ASSERT (!bio_flagged (bio,BIO_CLONED ));bio_for_each_segment_all (bvec, bio, iter_all) {bool uptodate = !bio->bi_status; -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;struct inode *inode = page->mapping->host;struct btrfs_fs_info *fs_info =btrfs_sb (inode->i_sb);const u32 sectorsize = fs_info->sectorsize;@@ -761,19 +761,19 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio)* for unaligned offsets, and an error if they don't add up to * a full sector. */ -if (!IS_ALIGNED (bvec->bv_offset, sectorsize)) +if (!IS_ALIGNED (bvec.bv_offset, sectorsize))btrfs_err (fs_info,"partial page read in btrfs with offset %u and length %u" , - bvec->bv_offset, bvec->bv_len); -else if (!IS_ALIGNED (bvec->bv_offset + bvec->bv_len, + bvec.bv_offset, bvec.bv_len); +else if (!IS_ALIGNED (bvec.bv_offset + bvec.bv_len, sectorsize))btrfs_info (fs_info,"incomplete page read with offset %u and length %u" , - bvec->bv_offset, bvec->bv_len); + bvec.bv_offset, bvec.bv_len); - start =page_offset (page) + bvec->bv_offset; - end = start + bvec->bv_len -1 ; - len = bvec->bv_len; + start =page_offset (page) + bvec.bv_offset; + end = start + bvec.bv_len -1 ; + len = bvec.bv_len; mirror = bbio->mirror_num; if (uptodate && !is_data_inode(inode) &&@@ -1879,7 +1879,7 @@ static void end_bio_subpage_eb_writepage(struct btrfs_bio *bbio) {struct bio *bio = &bbio->bio;struct btrfs_fs_info *fs_info; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all; fs_info =btrfs_sb (bio_first_page_all (bio)->mapping->host->i_sb);@@ -1887,12 +1887,12 @@ static void end_bio_subpage_eb_writepage(struct btrfs_bio *bbio)ASSERT (!bio_flagged (bio,BIO_CLONED ));bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; -u64 bvec_start =page_offset (page) + bvec->bv_offset; -u64 bvec_end = bvec_start + bvec->bv_len -1 ; +struct page *page = bvec.bv_page; +u64 bvec_start =page_offset (page) + bvec.bv_offset; +u64 bvec_end = bvec_start + bvec.bv_len -1 ;u64 cur_bytenr = bvec_start; -ASSERT (IS_ALIGNED (bvec->bv_len, fs_info->nodesize)); +ASSERT (IS_ALIGNED (bvec.bv_len, fs_info->nodesize));/* Iterate through all extent buffers in the range */ while (cur_bytenr <= bvec_end) {@@ -1936,14 +1936,14 @@ static void end_bio_subpage_eb_writepage(struct btrfs_bio *bbio)static void end_bio_extent_buffer_writepage (struct btrfs_bio *bbio) {struct bio *bio = &bbio->bio; -struct bio_vec *bvec; +struct bio_vec bvec;struct extent_buffer *eb;int done;struct bvec_iter_all iter_all;ASSERT (!bio_flagged (bio,BIO_CLONED ));bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page; eb = (struct extent_buffer *)page->private;BUG_ON (!eb);fs/btrfs/raid56.c +7
-7@@ -1393,7 +1393,7 @@ static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio,static void set_bio_pages_uptodate (struct btrfs_raid_bio *rbio,struct bio *bio) {const u32 sectorsize = rbio->bioc->fs_info->sectorsize; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;ASSERT (!bio_flagged (bio,BIO_CLONED ));@@ -1402,9 +1402,9 @@ static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio)struct sector_ptr *sector;int pgoff; -for (pgoff = bvec->bv_offset; pgoff - bvec->bv_offset < bvec->bv_len; +for (pgoff = bvec.bv_offset; pgoff - bvec.bv_offset < bvec.bv_len; pgoff += sectorsize) { - sector =find_stripe_sector (rbio, bvec->bv_page, pgoff); + sector =find_stripe_sector (rbio, bvec.bv_page, pgoff);ASSERT (sector);if (sector) sector->uptodate =1 ;@@ -1458,7 +1458,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio, {struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;int total_sector_nr =get_bio_sector_nr (rbio, bio); -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;/* No data csum for the whole stripe, no need to verify. */ @@ -1472,8 +1472,8 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,bio_for_each_segment_all (bvec, bio, iter_all) {int bv_offset; -for (bv_offset = bvec->bv_offset; - bv_offset < bvec->bv_offset + bvec->bv_len; +for (bv_offset = bvec.bv_offset; + bv_offset < bvec.bv_offset + bvec.bv_len; bv_offset += fs_info->sectorsize, total_sector_nr++) {u8 csum_buf[BTRFS_CSUM_SIZE ]; u8 *expected_csum = rbio->csum_buf +@@ -1484,7 +1484,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,if (!test_bit (total_sector_nr, rbio->csum_bitmap))continue ; - ret =btrfs_check_sector_csum (fs_info, bvec->bv_page, + ret =btrfs_check_sector_csum (fs_info, bvec.bv_page, bv_offset, csum_buf, expected_csum);if (ret <0 )set_bit (total_sector_nr, rbio->error_bitmap);fs/erofs/zdata.c +2
-2@@ -1648,11 +1648,11 @@ static void z_erofs_decompressqueue_endio(struct bio *bio) {struct z_erofs_decompressqueue *q = bio->bi_private;blk_status_t err = bio->bi_status; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;DBG_BUGON (PageUptodate (page));DBG_BUGON (z_erofs_page_is_invalidated (page));fs/f2fs/data.c +10
-10@@ -139,12 +139,12 @@ struct bio_post_read_ctx {*/ static void f2fs_finish_read_bio (struct bio *bio,bool in_task) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;struct bio_post_read_ctx *ctx = bio->bi_private;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (f2fs_is_compressed_page (page)) {if (ctx && !ctx->decompression_attempted)@@ -189,11 +189,11 @@ static void f2fs_verify_bio(struct work_struct *work)* as those were handled separately by f2fs_end_read_compressed_page(). */ if (may_have_compressed_pages) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bio_for_each_segment_all (bv, bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (!f2fs_is_compressed_page (page) && !fsverity_verify_page (page)) {@@ -241,13 +241,13 @@ static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)static void f2fs_handle_step_decompress (struct bio_post_read_ctx *ctx,bool in_task) { -struct bio_vec *bv; +struct bio_vec bv;struct bvec_iter_all iter_all;bool all_compressed = true;block_t blkaddr = ctx->fs_blkaddr;bio_for_each_segment_all (bv, ctx->bio, iter_all) { -struct page *page = bv->bv_page; +struct page *page = bv.bv_page;if (f2fs_is_compressed_page (page)) f2fs_end_read_compressed_page(page, false, blkaddr,@@ -327,7 +327,7 @@ static void f2fs_read_end_io(struct bio *bio)static void f2fs_write_end_io (struct bio *bio) {struct f2fs_sb_info *sbi; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;iostat_update_and_unbind_ctx (bio);@@ -337,7 +337,7 @@ static void f2fs_write_end_io(struct bio *bio) bio->bi_status =BLK_STS_IOERR ;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;enum count_type type =WB_DATA_TYPE (page);if (page_private_dummy (page)) {@@ -583,7 +583,7 @@ static void __submit_merged_bio(struct f2fs_bio_info *io)static bool __has_merged_page (struct bio *bio,struct inode *inode,struct page *page,nid_t ino) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;if (!bio)@@ -593,7 +593,7 @@ static bool __has_merged_page(struct bio *bio, struct inode *inode,return true;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *target = bvec->bv_page; +struct page *target = bvec.bv_page;if (fscrypt_is_bounce_page (target)) { target =fscrypt_pagecache_page (target);fs/gfs2/lops.c +5
-5@@ -202,7 +202,7 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,static void gfs2_end_log_write (struct bio *bio) {struct gfs2_sbd *sdp = bio->bi_private; -struct bio_vec *bvec; +struct bio_vec bvec;struct page *page;struct bvec_iter_all iter_all;@@ -217,9 +217,9 @@ static void gfs2_end_log_write(struct bio *bio) }bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (page_has_buffers (page)) -gfs2_end_log_write_bh (sdp, bvec, bio->bi_status); +gfs2_end_log_write_bh (sdp, &bvec, bio->bi_status);else mempool_free (page, gfs2_page_pool); }@@ -395,11 +395,11 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)static void gfs2_end_log_read (struct bio *bio) {struct page *page; -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { - page = bvec->bv_page; + page = bvec.bv_page;if (bio->bi_status) {int err =blk_status_to_errno (bio->bi_status);fs/gfs2/meta_io.c +4
-4@@ -193,15 +193,15 @@ struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno)static void gfs2_meta_read_endio (struct bio *bio) { -struct bio_vec *bvec; +struct bio_vec bvec;struct bvec_iter_all iter_all;bio_for_each_segment_all (bvec, bio, iter_all) { -struct page *page = bvec->bv_page; +struct page *page = bvec.bv_page;struct buffer_head *bh =page_buffers (page); -unsigned int len = bvec->bv_len; +unsigned int len = bvec.bv_len; -while (bh_offset (bh) < bvec->bv_offset) +while (bh_offset (bh) < bvec.bv_offset) bh = bh->b_this_page; do { struct buffer_head *next = bh->b_this_page;fs/squashfs/block.c +27
-21@@ -35,30 +35,33 @@ static int copy_bio_to_actor(struct bio *bio, int offset, int req_length) { void *actor_addr; -struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); +struct bvec_iter_all iter; +struct bio_vec bvec;int copied_bytes =0 ;int actor_offset =0 ; +int bytes_to_copy;squashfs_actor_nobuff (actor); actor_addr =squashfs_first_page (actor); -if (WARN_ON_ONCE (!bio_next_segment (bio, &iter_all))) -return 0 ; +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); -while (copied_bytes < req_length) { -int bytes_to_copy =min_t (int, bvec->bv_len - offset, +while (copied_bytes < req_length && + iter.idx < bio->bi_vcnt) { + bvec =bio_iter_all_peek (bio, &iter); + + bytes_to_copy =min_t (int, bvec.bv_len,PAGE_SIZE - actor_offset); bytes_to_copy =min_t (int, bytes_to_copy, req_length - copied_bytes);if (!IS_ERR (actor_addr)) -memcpy (actor_addr + actor_offset,bvec_virt (bvec) + - offset, bytes_to_copy); +memcpy (actor_addr + actor_offset,bvec_virt (&bvec), + bytes_to_copy); actor_offset += bytes_to_copy; copied_bytes += bytes_to_copy; - offset += bytes_to_copy;if (actor_offset >=PAGE_SIZE ) { actor_addr =squashfs_next_page (actor);@@ -66,11 +69,8 @@ static int copy_bio_to_actor(struct bio *bio,break ; actor_offset =0 ; } -if (offset >= bvec->bv_len) { -if (!bio_next_segment (bio, &iter_all)) -break ; - offset =0 ; - } + +bio_iter_all_advance (bio, &iter, bytes_to_copy); }squashfs_finish_page (actor);return copied_bytes;@@ -159,8 +159,10 @@ int squashfs_read_data(struct super_block *sb, u64 index, int length,* Metadata block. */ const u8 *data; -struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); +struct bvec_iter_all iter; +struct bio_vec bvec; + +bvec_iter_all_init (&iter);if (index +2 > msblk->bytes_used) { res = -EIO ;@@ -170,21 +172,25 @@ int squashfs_read_data(struct super_block *sb, u64 index, int length,if (res) goto out; -if (WARN_ON_ONCE (!bio_next_segment (bio, &iter_all))) { + bvec =bio_iter_all_peek (bio, &iter); + +if (WARN_ON_ONCE (!bvec.bv_len)) { res = -EIO ; goto out_free_bio; }/* Extract the length of the metadata block */ - data =bvec_virt (bvec); + data =bvec_virt (&bvec); length = data[offset]; -if (offset < bvec->bv_len -1 ) { +if (offset < bvec.bv_len -1 ) { length |= data[offset +1 ] <<8 ; }else { -if (WARN_ON_ONCE (!bio_next_segment (bio, &iter_all))) { +bio_iter_all_advance (bio, &iter, bvec.bv_len); + +if (WARN_ON_ONCE (!bvec.bv_len)) { res = -EIO ; goto out_free_bio; } - data =bvec_virt (bvec); + data =bvec_virt (&bvec); length |= data[0 ] <<8 ; }bio_free_pages (bio);fs/squashfs/lz4_wrapper.c +10
-7@@ -92,20 +92,23 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter; +struct bio_vec bvec;struct squashfs_lz4 *stream = strm;void *buff = stream->input, *data;int bytes = length, res; -while (bio_next_segment (bio, &iter_all)) { -int avail =min (bytes, ((int )bvec->bv_len) - offset); +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); - data =bvec_virt (bvec); -memcpy (buff, data + offset, avail); +bio_for_each_segment_all_continue (bvec, bio, iter) { +unsigned avail =min_t (unsigned, bytes, bvec.bv_len); + +memcpy (buff,bvec_virt (&bvec), avail); buff += avail; bytes -= avail; - offset =0 ; +if (!bytes) +break ; } res = LZ4_decompress_safe(stream->input, stream->output,fs/squashfs/lzo_wrapper.c +10
-7@@ -66,21 +66,24 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter; +struct bio_vec bvec;struct squashfs_lzo *stream = strm;void *buff = stream->input, *data;int bytes = length, res;size_t out_len = output->length; -while (bio_next_segment (bio, &iter_all)) { -int avail =min (bytes, ((int )bvec->bv_len) - offset); +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); - data =bvec_virt (bvec); -memcpy (buff, data + offset, avail); +bio_for_each_segment_all_continue (bvec, bio, iter) { +unsigned avail =min_t (unsigned, bytes, bvec.bv_len); + +memcpy (buff,bvec_virt (&bvec), avail); buff += avail; bytes -= avail; - offset =0 ; +if (!bytes) +break ; } res = lzo1x_decompress_safe(stream->input, (size_t)length,fs/squashfs/xz_wrapper.c +10
-9@@ -120,8 +120,7 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter;int total =0 , error =0 ;struct squashfs_xz *stream = strm;@@ -136,26 +135,28 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {enum xz_ret xz_err;if (stream->buf.in_pos == stream->buf.in_size) { -const void *data; -int avail; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter); +unsigned avail =min_t (unsigned, length, bvec.bv_len); -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {/* XZ_STREAM_END must be reached. */ error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); length -= avail; - stream->buf.in = data + offset; + stream->buf.in =bvec_virt (&bvec); stream->buf.in_size = avail; stream->buf.in_pos =0 ; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (stream->buf.out_pos == stream->buf.out_size) {fs/squashfs/zlib_wrapper.c +4
-2@@ -53,8 +53,7 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, struct bio *bio, int offset, int length, struct squashfs_page_actor *output) { - struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); + struct bvec_iter_all iter;int zlib_init =0 , error =0 ;z_stream *stream = strm;@@ -67,25 +66,28 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +
fs/squashfs/zlib_wrapper.c static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, · line 79
Quoted 10 lines
fs/squashfs/zlib_wrapper.c +2
-2for (;;) {int zlib_err;if (stream->avail_in ==0 ) { -const void *data; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter);int avail; -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {
It is bad to expose iterator/bio internal here, probably you can return bool from bio_iter_all_peek() by moveing the check into helper, meantime passing bvec reference to bio_iter_all_peek().
Quoted 40 lines
fs/squashfs/zlib_wrapper.c +4
-4/* Z_STREAM_END must be reached. */ error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); + avail =min_t (unsigned, length, bvec.bv_len); length -= avail; - stream->next_in = data + offset; + stream->next_in =bvec_virt (&bvec); stream->avail_in = avail; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (stream->avail_out ==0 ) {fs/squashfs/zstd_wrapper.c +4
-2@@ -68,8 +68,7 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,int error =0 ;zstd_in_buffer in_buf = {NULL ,0 ,0 };zstd_out_buffer out_buf = {NULL ,0 ,0 }; -struct bvec_iter_all iter_all = {}; -struct bio_vec *bvec =bvec_init_iter_all (&iter_all); +struct bvec_iter_all iter; stream =zstd_init_dstream (wksp->window_size, wksp->mem, wksp->mem_size);@@ -85,25 +84,27 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, goto finish; } +bvec_iter_all_init (&iter); +bio_iter_all_advance (bio, &iter, offset); +for (;;) {
fs/squashfs/zstd_wrapper.c static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, · line 97
Quoted 10 lines
fs/squashfs/zstd_wrapper.c +3
-3size_t zstd_err;if (in_buf.pos == in_buf.size) { -const void *data; -int avail; +struct bio_vec bvec =bio_iter_all_peek (bio, &iter); +unsigned avail =min_t (unsigned, length, bvec.bv_len); -if (!bio_next_segment (bio, &iter_all)) { +if (iter.idx >= bio->bi_vcnt) {
Same with above.
Quoted 50 lines
fs/squashfs/zstd_wrapper.c +3
-4error = -EIO ;break ; } - avail =min (length, ((int )bvec->bv_len) - offset); - data =bvec_virt (bvec); length -= avail; - in_buf.src = data + offset; + in_buf.src =bvec_virt (&bvec); in_buf.size = avail; in_buf.pos =0 ; - offset =0 ; + +bio_iter_all_advance (bio, &iter, avail); }if (out_buf.pos == out_buf.size) {include/linux/bio.h +15
-6@@ -78,22 +78,34 @@ static inline void *bio_data(struct bio *bio)return NULL ; } -static inline bool bio_next_segment (const struct bio *bio, -struct bvec_iter_all *iter) +static inline struct bio_vec bio_iter_all_peek (const struct bio *bio, +struct bvec_iter_all *iter) { -if (iter->idx >= bio->bi_vcnt) -return false; +return bvec_iter_all_peek (bio->bi_io_vec, iter); +} -bvec_advance (&bio->bi_io_vec[iter->idx], iter); -return true; +static inline void bio_iter_all_advance (const struct bio *bio, +struct bvec_iter_all *iter, +unsigned bytes) +{ +bvec_iter_all_advance (bio->bi_io_vec, iter, bytes); } +#define bio_for_each_segment_all_continue (bvl, bio, iter) \ + for (; \ + iter.idx < bio->bi_vcnt && \ + ((bvl = bio_iter_all_peek(bio, &iter)), true); \ + bio_iter_all_advance((bio), &iter, bvl.bv_len)) +
include/linux/bio.h static inline void *bio_data(struct bio *bio) · line 107
Quoted 10 lines
include/linux/bio.h +4
-2/* * drivers should _never_ use the all version - the bio may have been split * before it got to the driver and the driver won't own all of it */ -#define bio_for_each_segment_all (bvl, bio, iter) \ - for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); ) +#define bio_for_each_segment_all (bvl, bio, iter) \ + for (bvec_iter_all_init(&iter); \ + iter.idx < (bio)->bi_vcnt && \ + ((bvl = bio_iter_all_peek((bio), &iter)), true); \
The check of 'iter.idx < (bio)->bi_vcnt' could be moved to bio_iter_all_peek(), meantime passing &bvl to bio_iter_all_peek().
Quoted 58 lines
include/linux/bio.h +1
-0+ bio_iter_all_advance((bio), &iter, bvl.bv_len))static inline void bio_advance_iter (const struct bio *bio,struct bvec_iter *iter,unsigned int bytes)include/linux/bvec.h +19
-9@@ -85,12 +85,6 @@ struct bvec_iter {current bvec */ } __packed; -struct bvec_iter_all { -struct bio_vec bv; -int idx; -unsigned done; -}; -/* * various member access, note that bio_data should of course not be used * on highmem page vectors @@ -184,7 +178,10 @@ static inline void bvec_iter_advance_single(const struct bio_vec *bv, ((bvl =bvec_iter_bvec ((bio_vec), (iter))),1 ); \ bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) -/* for iterating one bio from start to end */ +/* +* bvec_iter_all: for advancing over a bio as it was originally created, but +* with the usual bio_for_each_segment interface - nonstandard, do not use: +*/ #define BVEC_ITER_ALL_INIT (struct bvec_iter) \ { \ .bi_sector = 0, \@@ -193,33 +190,45 @@ static inline void bvec_iter_advance_single(const struct bio_vec *bv, .bi_bvec_done = 0, \ } -static inline struct bio_vec *bvec_init_iter_all (struct bvec_iter_all *iter_all) +/* +* bvec_iter_all: for advancing over individual pages in a bio, as it was when +* it was first created: +*/ +struct bvec_iter_all { +int idx; +unsigned done; +}; + +static inline void bvec_iter_all_init (struct bvec_iter_all *iter_all) { iter_all->done =0 ; iter_all->idx =0 ; +} -return &iter_all->bv; +static inline struct bio_vec bvec_iter_all_peek (const struct bio_vec *bvec, +struct bvec_iter_all *iter) +{ +struct bio_vec bv = bvec[iter->idx];
include/linux/bvec.h static inline void bvec_iter_advance_single(const struct bio_vec *bv, · line 221
Quoted 10 lines
include/linux/bvec.h +9
-0+ + bv.bv_offset += iter->done; + bv.bv_len -= iter->done; + + bv.bv_page += bv.bv_offset >>PAGE_SHIFT ; + bv.bv_offset &= ~PAGE_MASK ; + bv.bv_len =min_t (unsigned,PAGE_SIZE - bv.bv_offset, bv.bv_len); + +return bv; }
Here I think compiler optimizes the last bvec copy by using external bvec directly, and I run fio/t/io_uring and not find IOPS degrade.
As mentioned above, it can be done explicitly by passing &bvl to bvec_iter_all_peek(), then user of bvec_iter_all_peek() can know if one real bvec is peeked, and check of 'if (iter.idx >= bio->bi_vcnt)' can be avoided in external callers.
Otherwise, this patch looks fine for me.
thanks,
Ming
Kent Overstreet
maintainerQuoted 4 lines · Ming Lei
Quoted 3 lines · Kent Overstreet
This patch reworks
bio_for_each_segment_all()to be more inline with how the other bio iterators work:
Quoted 8 lines
Quoted 5 lines
-
bio_iter_all_peek()now returns a synthesizedbio_vec; we don't stash one in the iterator and pass a pointer to it - bad. This way makes it clearer what's a constructed value vs. a reference to something pre-existing, and it also will help with cleaning up and consolidating code withbio_for_each_folio_all().by-value is more readable & clean, the current by-reference is just for aligning with original interface.
This is the fourth or fifth time this patch set has been out on the list, and I specifically asked you to review it awhile back, and you're just now looking at it?
Kent Overstreet
maintainerJens - I've updated the branch at
https://evilpiepirate.org/git/bcachefs.git block-for-bcachefs
which now has the patches you'll want to take for now (bio iterator) first, only changes being bio_iter_all_end() based on Ming's feedback.