Skip to contentSkip to conversation

Multipage biovecs

PATCH New Kent Overstreet 2 participants 10 messages Sep 25, 2013

Subsystems (6)
x86 VFS SCSI Block layer +2 more subsystems
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Multipage biovecs

This patch series is most definitely not done yet - I wanted to get it out so other people could look at where I'm going with it.

It implements multipage biovecs - a single biovec can point to an arbitrary amount of physically contiguous memory. For code that still needs to work on individual pages, we add bio_for_each_page() which does magic to allow iteration over single page bvecs.

This then changes how segment merging works: instead of blk-merge.c trying to figure out which segments it can merge, we start out by merging everything that could possibly be merged, and we add blk_max_segment() which various code can use to find the maximum segment size, and when building up sglists we split as needed. The end result is a lot cleaner, IMO.

This patch then deletes bi_seg_front_size and bi_seg_back_size - they're not needed anymore since the first and last segment size are the lengths of the biovecs themselves. The patch series does drop segment merging across bios - my belief is that with the upper layers always building up the largest bios they can (which is part of what all the immutable biovec work is intended to help with) that functionality won't be needed so much anymore - if it still is needed, we can readd it without bringing back bi_seg_front_size and bi_seg_back_size.

bi_phys_segments can also be deleted now - the only reason this series doesn't delete it is because the md code is abusing it for other purposes (I think it's only using it for counting bio splits, which it could use bi_remaining for instead).

The series is on top of immutable biovecs, and some other stuff not included here - it's also in my git repository (there's a bunch of other random crap in that branch though)
git://evilpiepirate.org/~kent/linux-bcache.git block_stuff.

#1 lore raw

PATCH 1/6 block: Introduce bio_for_each_page()

Prep work for multipage bvecs: various code will still need to iterate over individual pages, so we add primitives to do so

---
 drivers/scsi/sd_dif.c |  4 +--
 fs/bio.c              | 20 +++++++-----
 fs/mpage.c            |  8 ++---
 include/linux/bio.h   | 85 ++++++++++++++++++++++++++++++++++-----------------
 4 files changed, 76 insertions(+), 41 deletions(-)
drivers/scsi/sd_dif.c +2 -2
index a7a691d..ea92592 100644
@@ -385,7 +385,7 @@ void sd_dif_prepare(struct request *rq, sector_t hw_sector,
 
 		virt = bio->bi_integrity->bip_iter.bi_sector & 0xffffffff;
 
-		bip_for_each_vec(iv, bio->bi_integrity, iter) {
+		bip_for_each_page(iv, bio->bi_integrity, iter) {
 			sdt = kmap_atomic(iv.bv_page)
 				+ iv.bv_offset;
 
@@ -436,7 +436,7 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes)
 
 		virt = bio->bi_integrity->bip_iter.bi_sector & 0xffffffff;
 
-		bip_for_each_vec(iv, bio->bi_integrity, iter) {
+		bip_for_each_page(iv, bio->bi_integrity, iter) {
 			sdt = kmap_atomic(iv.bv_page)
 				+ iv.bv_offset;
 
fs/bio.c +13 -7
index e9d1c05..da8aa81 100644
@@ -496,16 +496,22 @@ EXPORT_SYMBOL(bio_alloc_bioset);
 
 void zero_fill_bio(struct bio *bio)
 {
-	unsigned long flags;
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	bio_for_each_segment(bv, bio, iter) {
+#if defined(CONFIG_HIGHMEM) || defined(ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)
+	bio_for_each_page(bv, bio, iter) {
+		unsigned long flags;
 		char *data = bvec_kmap_irq(&bv, &flags);
 		memset(data, 0, bv.bv_len);
 		flush_dcache_page(bv.bv_page);
 		bvec_kunmap_irq(data, &flags);
 	}
+#else
+	bio_for_each_segment(bv, bio, iter)
+		memset(page_address(bv.bv_page) + bv.bv_offset,
+		       0, bv.bv_len);
+#endif
 }
 EXPORT_SYMBOL(zero_fill_bio);
 
@@ -1474,11 +1480,11 @@ EXPORT_SYMBOL(bio_copy_kern);
  */
 void bio_set_pages_dirty(struct bio *bio)
 {
-	struct bio_vec *bvec;
-	int i;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 
-	bio_for_each_segment_all(bvec, bio, i) {
-		struct page *page = bvec->bv_page;
+	bio_for_each_page_all(bvec, bio, iter) {
+		struct page *page = bvec.bv_page;
 
 		if (page && !PageCompound(page))
 			set_page_dirty_lock(page);
@@ -1574,7 +1580,7 @@ void bio_flush_dcache_pages(struct bio *bi)
 	struct bio_vec *bvec;
 	struct bvec_iter iter;
 
-	bio_for_each_segment(bvec, bi, iter)
+	bio_for_each_page(bvec, bi, iter)
 		flush_dcache_page(bvec->bv_page);
 }
 EXPORT_SYMBOL(bio_flush_dcache_pages);
fs/mpage.c +4 -4
index 8e0a471..7507811 100644
@@ -43,11 +43,11 @@
  */
 static void mpage_end_io(struct bio *bio, int err)
 {
-	struct bio_vec *bv;
-	int i;
+	struct bio_vec bv;
+	struct bvec_iter iter;
 
-	bio_for_each_segment_all(bv, bio, i) {
-		struct page *page = bv->bv_page;
+	bio_for_each_page_all(bv, bio, iter) {
+		struct page *page = bv.bv_page;
 
 		if (bio_data_dir(bio) == READ) {
 			if (!err) {
include/linux/bio.h +57 -28

136 lines not on this page · raw message

Signature · 1 line
1.8.4.rc3
#2 lore raw

Re: [PATCH 1/6] block: Introduce bio_for_each_page()

Quoted 10 lines

void zero_fill_bio(struct bio *bio)
{

fs/bio.c +3 -2
-	unsigned long flags;
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	bio_for_each_segment(bv, bio, iter) {
+#if defined(CONFIG_HIGHMEM) || defined(ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)
+	bio_for_each_page(bv, bio, iter) {
+		unsigned long flags;

fs/bio.c EXPORT_SYMBOL(bio_alloc_bioset); · line 514

Quoted 10 lines
fs/bio.c +5 -0
 		char *data = bvec_kmap_irq(&bv, &flags);
 		memset(data, 0, bv.bv_len);
 		flush_dcache_page(bv.bv_page);
 		bvec_kunmap_irq(data, &flags);
 	}
+#else
+	bio_for_each_segment(bv, bio, iter)
+		memset(page_address(bv.bv_page) + bv.bv_offset,
+		       0, bv.bv_len);
+#endif

This looks pretty sketchy. I'd expect this to be doable with one loop and that bvec_kmap_irq() and flush_dcache_page() would fall back to page_address() and nops when they're not needed.

Where did this come from?

- z

#3 lore raw

Re: [PATCH 1/6] block: Introduce bio_for_each_page()

Quoted 25 lines · Zach Brown
Quoted 10 lines

void zero_fill_bio(struct bio *bio)
{

fs/bio.c +3 -2
-	unsigned long flags;
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	bio_for_each_segment(bv, bio, iter) {
+#if defined(CONFIG_HIGHMEM) || defined(ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)
+	bio_for_each_page(bv, bio, iter) {
+		unsigned long flags;

fs/bio.c EXPORT_SYMBOL(bio_alloc_bioset); · line 514

Quoted 10 lines
fs/bio.c +5 -0
 		char *data = bvec_kmap_irq(&bv, &flags);
 		memset(data, 0, bv.bv_len);
 		flush_dcache_page(bv.bv_page);
 		bvec_kunmap_irq(data, &flags);
 	}
+#else
+	bio_for_each_segment(bv, bio, iter)
+		memset(page_address(bv.bv_page) + bv.bv_offset,
+		       0, bv.bv_len);
+#endif

This looks pretty sketchy. I'd expect this to be doable with one loop and that bvec_kmap_irq() and flush_dcache_page() would fall back to page_address() and nops when they're not needed.

Quoted 1 line

Where did this come from?

It's just that if we need the kmap or the flush_dcache_page we have to process the bio one 4k page at a time - if not, we can process 64k (or whatever) bvecs all at once. That doesn't just save us memcpy calls, we can also avoid all the machinery in bio_for_each_page() for chunking up large bvecs into single page bvecs.

I can definitely think of better ways to do this, but I figured I'd wait and see if other code ends up wanting to switch between bio_for_each_page() and bio_for_each_segment() and why.

#4 lore raw

Re: [PATCH 1/6] block: Introduce bio_for_each_page()

Quoted 26 lines · Kent Overstreet
Quoted 25 lines · Zach Brown
Quoted 10 lines

void zero_fill_bio(struct bio *bio)
{

fs/bio.c +3 -2
-	unsigned long flags;
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	bio_for_each_segment(bv, bio, iter) {
+#if defined(CONFIG_HIGHMEM) || defined(ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE)
+	bio_for_each_page(bv, bio, iter) {
+		unsigned long flags;

fs/bio.c EXPORT_SYMBOL(bio_alloc_bioset); · line 514

Quoted 10 lines
fs/bio.c +5 -0
 		char *data = bvec_kmap_irq(&bv, &flags);
 		memset(data, 0, bv.bv_len);
 		flush_dcache_page(bv.bv_page);
 		bvec_kunmap_irq(data, &flags);
 	}
+#else
+	bio_for_each_segment(bv, bio, iter)
+		memset(page_address(bv.bv_page) + bv.bv_offset,
+		       0, bv.bv_len);
+#endif

This looks pretty sketchy. I'd expect this to be doable with one loop and that bvec_kmap_irq() and flush_dcache_page() would fall back to page_address() and nops when they're not needed.

Quoted 7 lines
Quoted 1 line

Where did this come from?

It's just that if we need the kmap or the flush_dcache_page we have to process the bio one 4k page at a time - if not, we can process 64k (or whatever) bvecs all at once. That doesn't just save us memcpy calls, we can also avoid all the machinery in bio_for_each_page() for chunking up large bvecs into single page bvecs.

Understood. A comment would probably be wise as that ifdefery is going to raise all the eyebrows.

- z

#5 lore raw

PATCH 2/6 block: Convert various code to bio_for_each_page()

In reply to Kent Overstreet

From: Kent Overstreet <[email protected]>

---
 fs/btrfs/compression.c |  8 ++++----
 fs/btrfs/disk-io.c     | 11 ++++++-----
 fs/btrfs/extent_io.c   | 48 +++++++++++++++++++++++++-----------------------
 fs/btrfs/file-item.c   | 42 +++++++++++++++++++-----------------------
 fs/btrfs/inode.c       | 35 +++++++++++++++++++----------------
 fs/logfs/dev_bdev.c    | 10 +++++-----
 6 files changed, 78 insertions(+), 76 deletions(-)
fs/btrfs/compression.c +4 -4
index b4dc421..51e5cc5 100644
@@ -200,15 +200,15 @@ csum_failed:
 	if (cb->errors) {
 		bio_io_error(cb->orig_bio);
 	} else {
-		int i;
-		struct bio_vec *bvec;
+		struct bio_vec bvec;
+		struct bvec_iter iter;
 
 		/*
 		 * we have verified the checksum already, set page
 		 * checked so the end_io handlers know about it
 		 */
-		bio_for_each_segment_all(bvec, cb->orig_bio, i)
-			SetPageChecked(bvec->bv_page);
+		bio_for_each_page_all(bvec, cb->orig_bio, iter)
+			SetPageChecked(bvec.bv_page);
 
 		bio_endio(cb->orig_bio, 0);
 	}
fs/btrfs/disk-io.c +6 -5
index 6f84032..3ad7b5c 100644
@@ -850,13 +850,14 @@ int btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct inode *inode,
 
 static int btree_csum_one_bio(struct bio *bio)
 {
-	struct bio_vec *bvec;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 	struct btrfs_root *root;
-	int i, ret = 0;
+	int ret = 0;
 
-	bio_for_each_segment_all(bvec, bio, i) {
-		root = BTRFS_I(bvec->bv_page->mapping->host)->root;
-		ret = csum_dirty_buffer(root, bvec->bv_page);
+	bio_for_each_page_all(bvec, bio, iter) {
+		root = BTRFS_I(bvec.bv_page->mapping->host)->root;
+		ret = csum_dirty_buffer(root, bvec.bv_page);
 		if (ret)
 			break;
 	}
fs/btrfs/extent_io.c +25 -23

113 lines not on this page · raw message

fs/btrfs/file-item.c +19 -23

109 lines not on this page · raw message

fs/btrfs/inode.c +19 -16
index 5978a18..98de70c 100644
@@ -6765,31 +6765,30 @@ unlock_err:
 static void btrfs_endio_direct_read(struct bio *bio, int err)
 {
 	struct btrfs_dio_private *dip = bio->bi_private;
-	struct bio_vec *bvec;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 	struct inode *inode = dip->inode;
 	struct btrfs_root *root = BTRFS_I(inode)->root;
 	struct bio *dio_bio;
 	u32 *csums = (u32 *)dip->csum;
 	u64 start;
-	int i;
 
 	start = dip->logical_offset;
-	bio_for_each_segment_all(bvec, bio, i) {
+	bio_for_each_page_all(bvec, bio, iter) {
 		if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
-			struct page *page = bvec->bv_page;
 			char *kaddr;
 			u32 csum = ~(u32)0;
 			unsigned long flags;
 
 			local_irq_save(flags);
-			kaddr = kmap_atomic(page);
-			csum = btrfs_csum_data(kaddr + bvec->bv_offset,
-					       csum, bvec->bv_len);
+			kaddr = kmap_atomic(bvec.bv_page);
+			csum = btrfs_csum_data(kaddr + bvec.bv_offset,
+					       csum, bvec.bv_len);
 			btrfs_csum_final(csum, (char *)&csum);
 			kunmap_atomic(kaddr);
 			local_irq_restore(flags);
 
-			flush_dcache_page(bvec->bv_page);
+			flush_dcache_page(bvec.bv_page);
 			if (csum != csums[i]) {
 				btrfs_err(root->fs_info, "csum failed ino %llu off %llu csum %u expected csum %u",
 					  btrfs_ino(inode), start, csum,
@@ -6798,7 +6797,7 @@ static void btrfs_endio_direct_read(struct bio *bio, int err)
 			}
 		}
 
-		start += bvec->bv_len;
+		start += bvec.bv_len;
 	}
 
 	unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
@@ -6964,7 +6963,8 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 	struct btrfs_root *root = BTRFS_I(inode)->root;
 	struct bio *bio;
 	struct bio *orig_bio = dip->orig_bio;
-	struct bio_vec *bvec = orig_bio->bi_io_vec;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 	u64 start_sector = orig_bio->bi_iter.bi_sector;
 	u64 file_offset = dip->logical_offset;
 	u64 submit_len = 0;
@@ -7001,10 +7001,13 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 	bio->bi_end_io = btrfs_end_dio_bio;
 	atomic_inc(&dip->pending_bios);
 
-	while (bvec <= (orig_bio->bi_io_vec + orig_bio->bi_vcnt - 1)) {
-		if (unlikely(map_length < submit_len + bvec->bv_len ||
-		    bio_add_page(bio, bvec->bv_page, bvec->bv_len,
-				 bvec->bv_offset) < bvec->bv_len)) {
+	iter = orig_bio->bi_iter;
+	while (iter.bi_size) {
+		bvec = bio_iovec_iter(orig_bio, iter);
+
+		if (unlikely(map_length < submit_len + bvec.bv_len ||
+		    bio_add_page(bio, bvec.bv_page, bvec.bv_len,
+				 bvec.bv_offset) < bvec.bv_len)) {
 			/*
 			 * inc the count before we submit the bio so
 			 * we know the end IO handler won't happen before
@@ -7043,9 +7046,9 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 				goto out_err;
 			}
 		} else {
-			submit_len += bvec->bv_len;
+			submit_len += bvec.bv_len;
 			nr_pages ++;
-			bvec++;
+			bio_advance_iter(orig_bio, &iter, bvec.bv_len);
 		}
 	}
 
fs/logfs/dev_bdev.c +5 -5
index 685ae02..c3c6361 100644
@@ -61,17 +61,17 @@ static DECLARE_WAIT_QUEUE_HEAD(wq);
 static void writeseg_end_io(struct bio *bio, int err)
 {
 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
-	struct bio_vec *bvec;
-	int i;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 	struct super_block *sb = bio->bi_private;
 	struct logfs_super *super = logfs_super(sb);
 
 	BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */
 	BUG_ON(err);
 
-	bio_for_each_segment_all(bvec, bio, i) {
-		end_page_writeback(bvec->bv_page);
-		page_cache_release(bvec->bv_page);
+	bio_for_each_page_all(bvec, bio, iter) {
+		end_page_writeback(bvec.bv_page);
+		page_cache_release(bvec.bv_page);
 	}
 	bio_put(bio);
 	if (atomic_dec_and_test(&super->s_pending_writes))
Signature · 1 line
1.8.4.rc3
#6 lore raw

PATCH 3/6 Introduce blk_max_segment()

In reply to Kent Overstreet

From: Kent Overstreet <[email protected]>

---
 arch/x86/include/asm/io.h |  3 +++
 block/blk-merge.c         | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 drivers/xen/biomerge.c    |  8 ++++++++
 include/linux/blkdev.h    |  1 +
 4 files changed, 63 insertions(+)
arch/x86/include/asm/io.h +3 -0
index 34f69cb..4799845 100644
@@ -335,6 +335,9 @@ extern bool is_early_ioremap_ptep(pte_t *ptep);
 #include <xen/xen.h>
 struct bio_vec;
 
+extern bool xen_page_phys_mergeable(const struct page *p1,
+				    const struct page *p2);
+
 extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
 				      const struct bio_vec *vec2);
 
block/blk-merge.c +51 -0

57 lines not on this page · raw message

drivers/xen/biomerge.c +8 -0
index 0edb91c..d9fb598 100644
@@ -3,6 +3,14 @@
 #include <linux/export.h>
 #include <xen/page.h>
 
+bool xen_page_phys_mergeable(const struct page *p1, const struct page *p2)
+{
+	unsigned long mfn1 = pfn_to_mfn(page_to_pfn(p1));
+	unsigned long mfn2 = pfn_to_mfn(page_to_pfn(p2));
+
+	return mfn1 + 1 == mfn2;
+}
+
 bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
 			       const struct bio_vec *vec2)
 {
include/linux/blkdev.h +1 -0
index 1afe1bb..4a45e35 100644
@@ -746,6 +746,7 @@ extern void blk_rq_unprep_clone(struct request *rq);
 extern int blk_insert_cloned_request(struct request_queue *q,
 				     struct request *rq);
 extern void blk_delay_queue(struct request_queue *, unsigned long);
+extern unsigned int blk_max_segment(struct request_queue *, struct bio_vec *);
 extern void blk_recount_segments(struct request_queue *, struct bio *);
 extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
 extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
Signature · 1 line
1.8.4.rc3
#7 lore raw

PATCH 4/6 convert nvme to blk_max_segment()

In reply to Kent Overstreet

From: Kent Overstreet <[email protected]>

---
 drivers/block/nvme-core.c | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)
drivers/block/nvme-core.c +16 -15
index ddcb405..e4a9502 100644
@@ -465,6 +465,7 @@ static int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
 static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
 		struct bio *bio, enum dma_data_direction dma_dir, int psegs)
 {
+	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 	struct bio_vec bvec, bvprv;
 	struct bvec_iter iter;
 	struct scatterlist *sg = NULL;
@@ -477,26 +478,26 @@ static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
 			 (nvmeq->dev->stripe_size - 1));
 
 	sg_init_table(iod->sg, psegs);
-	bio_for_each_segment(bvec, bio, iter) {
-		if (!first && BIOVEC_PHYS_MERGEABLE(&bvprv, &bvec)) {
-			sg->length += bvec.bv_len;
-		} else {
-			if (!first && BIOVEC_NOT_VIRT_MERGEABLE(&bvprv, &bvec))
-				return nvme_split_and_submit(bio, nvmeq,
-							     length);
-
-			sg = sg ? sg + 1 : iod->sg;
-			sg_set_page(sg, bvec.bv_page,
-				    bvec.bv_len, bvec.bv_offset);
-			nsegs++;
-		}
 
-		if (split_len - length < bvec.bv_len)
-			return nvme_split_and_submit(bio, nvmeq, split_len);
+	iter = bio->bi_iter;
+	while (iter.bi_size) {
+		bvec = bio_iovec_iter(bio, iter);
+		bvec.bv_len = blk_max_segment(q, &bvec);
+
+		if (!first && BIOVEC_NOT_VIRT_MERGEABLE(&bvprv, &bvec))
+			return nvme_split_and_submit(bio, nvmeq, length);
+
+		sg = sg ? sg + 1 : iod->sg;
+		sg_set_page(sg, bvec.bv_page, bvec.bv_len, bvec.bv_offset);
+		nsegs++;
+
 		length += bvec.bv_len;
 		bvprv = bvec;
 		first = 0;
+
+		bio_advance_iter(bio, &iter, bvec.bv_len);
 	}
+
 	iod->nents = nsegs;
 	sg_mark_end(sg);
 	if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0)
Signature · 1 line
1.8.4.rc3
#8 lore raw

PATCH 5/6 convert integrity to new merging

In reply to Kent Overstreet

From: Kent Overstreet <[email protected]>

---
 block/blk-integrity.c | 54 +++++++++++++--------------------------------------
 fs/bio-integrity.c    | 10 ++++++++++
 2 files changed, 24 insertions(+), 40 deletions(-)
block/blk-integrity.c +14 -40
index 861fcae..22d8e9c 100644
@@ -43,33 +43,19 @@ static const char *bi_unsupported_name = "unsupported";
  */
 int blk_rq_count_integrity_sg(struct request_queue *q, struct bio *bio)
 {
-	struct bio_vec iv, ivprv;
 	unsigned int segments = 0;
-	unsigned int seg_size = 0;
-	struct bvec_iter iter;
-	int prev = 0;
 
-	bio_for_each_integrity_vec(iv, bio, iter) {
+	for_each_bio(bio) {
+		struct bio_integrity_payload *bip = bio->bi_integrity;
+		struct bvec_iter iter = bip->bip_iter;
 
-		if (prev) {
-			if (!BIOVEC_PHYS_MERGEABLE(&ivprv, &iv))
-				goto new_segment;
+		while (iter.bi_size) {
+			struct bio_vec iv = bvec_iter_bvec(bip->bip_vec, iter);
+			unsigned nbytes = blk_max_segment(q, &iv);
 
-			if (!BIOVEC_SEG_BOUNDARY(q, &ivprv, &iv))
-				goto new_segment;
-
-			if (seg_size + iv.bv_len > queue_max_segment_size(q))
-				goto new_segment;
-
-			seg_size += iv.bv_len;
-		} else {
-new_segment:
 			segments++;
-			seg_size = iv.bv_len;
+			bio_advance_iter(bio, &iter, nbytes);
 		}
-
-		prev = 1;
-		ivprv = iv;
 	}
 
 	return segments;
@@ -89,27 +75,17 @@ EXPORT_SYMBOL(blk_rq_count_integrity_sg);
 int blk_rq_map_integrity_sg(struct request_queue *q, struct bio *bio,
 			    struct scatterlist *sglist)
 {
-	struct bio_vec iv, ivprv;
 	struct scatterlist *sg = NULL;
 	unsigned int segments = 0;
-	struct bvec_iter iter;
-	int prev = 0;
 
-	bio_for_each_integrity_vec(iv, bio, iter) {
+	for_each_bio(bio) {
+		struct bio_integrity_payload *bip = bio->bi_integrity;
+		struct bvec_iter iter = bip->bip_iter;
 
-		if (prev) {
-			if (!BIOVEC_PHYS_MERGEABLE(&ivprv, &iv))
-				goto new_segment;
+		while (iter.bi_size) {
+			struct bio_vec iv= bvec_iter_bvec(bip->bip_vec, iter);
+			unsigned nbytes = blk_max_segment(q, &iv);
 
-			if (!BIOVEC_SEG_BOUNDARY(q, &ivprv, &iv))
-				goto new_segment;
-
-			if (sg->length + iv.bv_len > queue_max_segment_size(q))
-				goto new_segment;
-
-			sg->length += iv.bv_len;
-		} else {
-new_segment:
 			if (!sg)
 				sg = sglist;
 			else {
@@ -119,10 +95,8 @@ new_segment:
 
 			sg_set_page(sg, iv.bv_page, iv.bv_len, iv.bv_offset);
 			segments++;
+			bio_advance_iter(bio, &iter, nbytes);
 		}
-
-		prev = 1;
-		ivprv = iv;
 	}
 
 	if (sg)
fs/bio-integrity.c +10 -0
index 31f2d5a..2578c84 100644
@@ -129,6 +129,16 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
 	struct bio_integrity_payload *bip = bio->bi_integrity;
 	struct bio_vec *iv;
 
+	if (bip->bip_vcnt) {
+		iv = bip->bip_vec + bip->bip_vcnt - 1;
+
+		if (bvec_to_phys(iv) + iv->bv_len ==
+		    page_to_phys(page) + offset) {
+			iv->bv_len += len;
+			return len;
+		}
+	}
+
 	if (bip->bip_vcnt >= bvec_nr_vecs(bip->bip_slab)) {
 		printk(KERN_ERR "%s: bip_vec full\n", __func__);
 		return 0;
Signature · 1 line
1.8.4.rc3
#9 lore raw

PATCH 6/6 Multipage bvecs

In reply to Kent Overstreet

Convert merging to bio_add_page()/blk_max_segment()

---
 arch/x86/include/asm/io.h |   7 --
 block/blk-merge.c         | 227 ++++++++++++----------------------------------
 drivers/xen/biomerge.c    |  11 ---
 fs/bio.c                  |  14 ++-
 include/linux/bio.h       |  25 -----
 include/linux/blk_types.h |   7 --
 6 files changed, 65 insertions(+), 226 deletions(-)
arch/x86/include/asm/io.h +0 -7
index 4799845..9060a25 100644
@@ -333,17 +333,10 @@ extern bool is_early_ioremap_ptep(pte_t *ptep);
 
 #ifdef CONFIG_XEN
 #include <xen/xen.h>
-struct bio_vec;
 
 extern bool xen_page_phys_mergeable(const struct page *p1,
 				    const struct page *p2);
 
-extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
-				      const struct bio_vec *vec2);
-
-#define BIOVEC_PHYS_MERGEABLE(vec1, vec2)				\
-	(__BIOVEC_PHYS_MERGEABLE(vec1, vec2) &&				\
-	 (!xen_domain() || xen_biovec_phys_mergeable(vec1, vec2)))
 #endif	/* CONFIG_XEN */
 
 #define IO_SPACE_LIMIT 0xffff
block/blk-merge.c +59 -168

307 lines not on this page · raw message

drivers/xen/biomerge.c +0 -11
index d9fb598..9934266 100644
@@ -10,14 +10,3 @@ bool xen_page_phys_mergeable(const struct page *p1, const struct page *p2)
 
 	return mfn1 + 1 == mfn2;
 }
-
-bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
-			       const struct bio_vec *vec2)
-{
-	unsigned long mfn1 = pfn_to_mfn(page_to_pfn(vec1->bv_page));
-	unsigned long mfn2 = pfn_to_mfn(page_to_pfn(vec2->bv_page));
-
-	return __BIOVEC_PHYS_MERGEABLE(vec1, vec2) &&
-		((mfn1 == mfn2) || ((mfn1+1) == mfn2));
-}
-EXPORT_SYMBOL(xen_biovec_phys_mergeable);
fs/bio.c +6 -8
index da8aa81..93aee5c 100644
@@ -678,8 +678,8 @@ int bio_add_page(struct bio *bio, struct page *page,
 	if (bio->bi_vcnt > 0) {
 		bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
 
-		if (page == bv->bv_page &&
-		    offset == bv->bv_offset + bv->bv_len) {
+		if (bvec_to_phys(bv) + bv->bv_len ==
+		    page_to_phys(page) + offset) {
 			bv->bv_len += len;
 			goto done;
 		}
@@ -688,12 +688,10 @@ int bio_add_page(struct bio *bio, struct page *page,
 	if (bio->bi_vcnt >= bio->bi_max_vecs)
 		return 0;
 
-	bv		= &bio->bi_io_vec[bio->bi_vcnt];
-	bv->bv_page	= page;
-	bv->bv_len	= len;
-	bv->bv_offset	= offset;
-
-	bio->bi_vcnt++;
+	bv = &bio->bi_io_vec[bio->bi_vcnt++];
+	bv->bv_page = page;
+	bv->bv_len = len;
+	bv->bv_offset = offset;
 done:
 	bio->bi_iter.bi_size += len;
 	return len;
include/linux/bio.h +0 -25
index f6f0e99..3d9473d 100644
@@ -175,27 +175,6 @@ static inline void *bio_data(struct bio *bio)
 
 #define __bio_kunmap_atomic(addr)	kunmap_atomic(addr)
 
-/*
- * merge helpers etc
- */
-
-/* Default implementation of BIOVEC_PHYS_MERGEABLE */
-#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2)	\
-	((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2)))
-
-/*
- * allow arch override, for eg virtualized architectures (put in asm/io.h)
- */
-#ifndef BIOVEC_PHYS_MERGEABLE
-#define BIOVEC_PHYS_MERGEABLE(vec1, vec2)	\
-	__BIOVEC_PHYS_MERGEABLE(vec1, vec2)
-#endif
-
-#define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \
-	(((addr1) | (mask)) == (((addr2) - 1) | (mask)))
-#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \
-	__BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
-
 #define bio_io_error(bio) bio_endio((bio), -EIO)
 
 static inline void bvec_iter_advance(struct bio_vec *bv, struct bvec_iter *iter,
@@ -652,10 +631,6 @@ struct biovec_slab {
 #define bip_for_each_page(bvl, bip, iter)				\
 	__bip_for_each(bvl, bip, iter, bvec_iter_page_bytes)
 
-#define bio_for_each_integrity_vec(_bvl, _bio, _iter)			\
-	for_each_bio(_bio)						\
-		bip_for_each_segment(_bvl, _bio->bi_integrity, _iter)
-
 #define bio_integrity(bio) (bio->bi_integrity != NULL)
 
 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
include/linux/blk_types.h +0 -7
index 8fca6e3..4aebb73 100644
@@ -57,13 +57,6 @@ struct bio {
 	 */
 	unsigned int		bi_phys_segments;
 
-	/*
-	 * To keep track of the max segment size, we account for the
-	 * sizes of the first and last mergeable segments in this bio.
-	 */
-	unsigned int		bi_seg_front_size;
-	unsigned int		bi_seg_back_size;
-
 	atomic_t		bi_remaining;
 
 	bio_end_io_t		*bi_end_io;
Signature · 1 line
1.8.4.rc3