direct io dma alignment
Subsystems (2)
v3 cover direct io dma alignment
From: Keith Busch <[email protected]>
Here is the 3rd version enabling non-block aligned user space addresses for O_DIRECT.
Changes since v2:
Folded in improvements cleanin gup zone append pages (Christoph)
Added documentation the exported attribute (Bart)
Split bdev_dma_alignment() into its own patch (Christoph)
Removed fs/ from implementing support for these address offsets for now
Fixed up a couple places assuming SECTOR_SIZE multiple bv_len
On that last point, I searched through much of the code and found a few other places that also assumed this bv_len size, but they didn't apply to this change since they don't set 'dma_alignment'. This got me thinking, though, should this be a new attribute, 'dio_alignment', instead of using the pre-existing 'dma_alignment' for this purpose? Or is it clear enough that drivers setting the existing attribute ought to know what they're getting into?
Keith Busch (6):
block/bio: remove duplicate append pages code
block: export dma_alignment attribute
block: introduce bdev_dma_alignment helper
block/merge: count bytes instead of sectors
block/bounce: count bytes instead of sectors
block: relax direct io memory alignment
Documentation/ABI/stable/sysfs-block | 9 ++ block/bio.c | 118 +++++++++++++-------------- block/blk-merge.c | 35 ++++---- block/blk-sysfs.c | 7 ++ block/bounce.c | 5 +- block/fops.c | 41 +++++++--- include/linux/blkdev.h | 5 ++ 7 files changed, 127 insertions(+), 93 deletions(-)
Signature · 1 line
2.30.2
PATCH v3 1/6 block/bio: remove duplicate append pages code
From: Keith Busch <[email protected]>
The getting pages setup for zone append and normal IO are identical. Use common code for each.
---v2->v3:
Folded in improvement suggestions from Christoph
block/bio.c | 105 +++++++++++++++++++++------------------------------- 1 file changed, 42 insertions(+), 63 deletions(-)
block/bio.c +42 -63
@@ -1158,6 +1158,37 @@ static void bio_put_pages(struct page **pages, size_t size, size_t off)put_page (pages[i]); } +static int bio_iov_add_page (struct bio *bio,struct page *page, +unsigned int len,unsigned int offset) +{ +bool same_page = false; + +if (!__bio_try_merge_page (bio, page, len, offset, &same_page)) { +if (WARN_ON_ONCE (bio_full (bio, len))) +return -EINVAL ; +__bio_add_page (bio, page, len, offset); +return 0 ; + } + +if (same_page) +put_page (page); +return 0 ; +} + +static int bio_iov_add_zone_append_page (struct bio *bio,struct page *page, +unsigned int len,unsigned int offset) +{ +struct request_queue *q =bdev_get_queue (bio->bi_bdev); +bool same_page = false; + +if (bio_add_hw_page (q, bio, page, len, offset, +queue_max_zone_append_sectors (q), &same_page) != len) +return -EINVAL ; +if (same_page) +put_page (page); +return 0 ; +} +#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))/**
@@ -1176,7 +1207,6 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;struct page **pages = (struct page **)bv; -bool same_page = false;ssize_t size, left;unsigned len, i;size_t offset;
@@ -1185,7 +1215,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)* Move page array up in the allocated memory for the bio vecs as far as * possible so that we can start filling biovecs from the beginning * without overwriting the temporary page array. -*/ +*/ BUILD_BUG_ON (PAGE_PTRS_PER_BVEC <2 ); pages += entries_left * (PAGE_PTRS_PER_BVEC -1 );
@@ -1195,18 +1225,18 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)for (left = size, i =0 ; left >0 ; left -= len, i++) {struct page *page = pages[i]; +int ret; len =min_t (size_t,PAGE_SIZE - offset, left); +if (bio_op (bio) ==REQ_OP_ZONE_APPEND ) + ret =bio_iov_add_zone_append_page (bio, page, len, + offset); +else + ret =bio_iov_add_page (bio, page, len, offset); -if (__bio_try_merge_page (bio, page, len, offset, &same_page)) { -if (same_page) -put_page (page); - }else { -if (WARN_ON_ONCE (bio_full (bio, len))) { -bio_put_pages (pages + i, left, offset); -return -EINVAL ; - } -__bio_add_page (bio, page, len, offset); +if (ret) { +bio_put_pages (pages + i, left, offset); +return ret; } offset =0 ; }
@@ -1215,54 +1245,6 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)return 0 ; } -static int __bio_iov_append_get_pages (struct bio *bio,struct iov_iter *iter) -{ -unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; -unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; -struct request_queue *q =bdev_get_queue (bio->bi_bdev); -unsigned int max_append_sectors =queue_max_zone_append_sectors (q); -struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; -struct page **pages = (struct page **)bv; -ssize_t size, left; -unsigned len, i; -size_t offset; -int ret =0 ; - -if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ; - -/* -* Move page array up in the allocated memory for the bio vecs as far as -* possible so that we can start filling biovecs from the beginning -* without overwriting the temporary page array. -*/ -BUILD_BUG_ON (PAGE_PTRS_PER_BVEC <2 ); - pages += entries_left * (PAGE_PTRS_PER_BVEC -1 ); - - size =iov_iter_get_pages (iter, pages,LONG_MAX , nr_pages, &offset); -if (unlikely (size <=0 )) -return size ? size : -EFAULT ; - -for (left = size, i =0 ; left >0 ; left -= len, i++) { -struct page *page = pages[i]; -bool same_page = false; - - len =min_t (size_t,PAGE_SIZE - offset, left); -if (bio_add_hw_page (q, bio, page, len, offset, - max_append_sectors, &same_page) != len) { -bio_put_pages (pages + i, left, offset); - ret = -EINVAL ; -break ; - } -if (same_page) -put_page (page); - offset =0 ; - } - -iov_iter_advance (iter, size - left); -return ret; -} -/** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to
@@ -1297,10 +1279,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) }do { -if (bio_op (bio) ==REQ_OP_ZONE_APPEND ) - ret =__bio_iov_append_get_pages (bio, iter); -else - ret =__bio_iov_iter_get_pages (bio, iter); + ret =__bio_iov_iter_get_pages (bio, iter); }while (!ret &&iov_iter_count (iter) && !bio_full (bio,0 ));/* don't account direct I/O as memory stall */
Signature · 1 line
2.30.2
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
Looks good:
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
In reply to Keith BuschLooks good,
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
In reply to Keith BuschQuoted 28 lines · Keith Busch
From: Keith Busch <[email protected]>
The getting pages setup for zone append and normal IO are identical. Use common code for each.
Signed-off-by Keith Busch <[email protected]>block/bio.c +15
-6+ +static int bio_iov_add_zone_append_page (struct bio *bio,struct page *page, +unsigned int len,unsigned int offset) +{ +struct request_queue *q =bdev_get_queue (bio->bi_bdev); +bool same_page = false; + +if (bio_add_hw_page (q, bio, page, len, offset, +queue_max_zone_append_sectors (q), &same_page) != len) +return -EINVAL ; +if (same_page) +put_page (page); +return 0 ; +} + -static int __bio_iov_append_get_pages (struct bio *bio,struct iov_iter *iter) -{ -unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; -unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; -struct request_queue *q =bdev_get_queue (bio->bi_bdev); -unsigned int max_append_sectors =queue_max_zone_append_sectors (q);
block/bio.c static void bio_put_pages(struct page **pages, size_t size, size_t off)
Quoted 10 lines
block/bio.c +0
-9-struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; -struct page **pages = (struct page **)bv; -ssize_t size, left; -unsigned len, i; -size_t offset; -int ret =0 ; - -if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ;
I don't see this check in the append path. Should it be added in bio_iov_add_zone_append_page() function?
Quoted 11 lines
block/bio.c +0
-11- -/* -* Move page array up in the allocated memory for the bio vecs as far as -* possible so that we can start filling biovecs from the beginning -* without overwriting the temporary page array. -*/ -BUILD_BUG_ON (PAGE_PTRS_PER_BVEC <2 ); - pages += entries_left * (PAGE_PTRS_PER_BVEC -1 ); - - size =iov_iter_get_pages (iter, pages,LONG_MAX , nr_pages, &offset); -if (unlikely (size <=0 ))
Signature · 1 line
Pankaj Raghav
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
Quoted 5 lines · Pankaj Raghav
block/bio.c static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · line 1232
Quoted 2 lines · Keith Busch
block/bio.c +0
-2-if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ;I don't see this check in the append path. Should it be added in
bio_iov_add_zone_append_page()function?
I'm not sure this check makes a lot of sense. If it just returns 0 here, then won't that get bio_iov_iter_get_pages() stuck in an infinite loop? The bio isn't filling, the iov isn't advancing, and 0 indicates keep-going.
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
Quoted 10 lines · Keith Busch
Quoted 5 lines · Pankaj Raghav
block/bio.c static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · line 1232
Quoted 2 lines · Keith Busch
block/bio.c +0
-2-if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ;I don't see this check in the append path. Should it be added in
bio_iov_add_zone_append_page()function?I'm not sure this check makes a lot of sense. If it just returns 0 here, then won't that get
bio_iov_iter_get_pages()stuck in an infinite loop? The bio isn't filling, the iov isn't advancing, and 0 indicates keep-going.
Yeah but if max_append_sectors is zero, then bio_add_hw_page() also returns 0 as follows:
....
if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) return 0;
....
With WARN_ON_ONCE, we at least get a warning message if it gets stuck in an infinite loop because of max_append_sectors being zero right?
Signature · 1 line
Pankaj Raghav
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
Quoted 11 lines · Pankaj Raghav
Quoted 10 lines · Keith Busch
Quoted 5 lines · Pankaj Raghav
block/bio.c static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · line 1232
Quoted 2 lines · Keith Busch
block/bio.c +0
-2-if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ;I don't see this check in the append path. Should it be added in
bio_iov_add_zone_append_page()function?I'm not sure this check makes a lot of sense. If it just returns 0 here, then won't that get
bio_iov_iter_get_pages()stuck in an infinite loop? The bio isn't filling, the iov isn't advancing, and 0 indicates keep-going.
Quoted 9 lines
Yeah but if
max_append_sectorsis zero, thenbio_add_hw_page()also returns 0 as follows:
....if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) return 0;....
WithWARN_ON_ONCE, we at least get a warning message if it gets stuck in an infinite loop because ofmax_append_sectorsbeing zero right?
Warning about an infinite loop that can be recovered from only by rebooting the machine is not very useful...
If max_append_sectors is zero and bio_iov_add_zone_append_page() is called, this is an error (stupid user) and everything should be failed with -ENOSUPP or -EIO.
Signature · 2 lines
Damien Le Moal Western Digital Research
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
In reply to Pankaj RaghavQuoted 11 lines · Pankaj Raghav
Quoted 10 lines · Keith Busch
Quoted 5 lines · Pankaj Raghav
block/bio.c static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · line 1232
Quoted 2 lines · Keith Busch
block/bio.c +0
-2-if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ;I don't see this check in the append path. Should it be added in
bio_iov_add_zone_append_page()function?I'm not sure this check makes a lot of sense. If it just returns 0 here, then won't that get
bio_iov_iter_get_pages()stuck in an infinite loop? The bio isn't filling, the iov isn't advancing, and 0 indicates keep-going.
Quoted 8 lines
Yeah but if
max_append_sectorsis zero, thenbio_add_hw_page()also returns 0 as follows:
....if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) return 0;....
WithWARN_ON_ONCE, we at least get a warning message if it gets stuck in an infinite loop because ofmax_append_sectorsbeing zero right?
The return for this function is the added length, not an indicator of success. And we already handle '0' as an error from bio_iov_add_zone_append_page():
if (bio_add_hw_page(q, bio, page, len, offset, queue_max_zone_append_sectors(q), &same_page) != len) return -EINVAL;
Re: [PATCHv3 1/6] block/bio: remove duplicate append pages code
Quoted 21 lines · Keith Busch
Quoted 11 lines · Pankaj Raghav
Quoted 10 lines · Keith Busch
Quoted 5 lines · Pankaj Raghav
block/bio.c static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · line 1232
Quoted 2 lines · Keith Busch
block/bio.c +0
-2-if (WARN_ON_ONCE (!max_append_sectors)) -return 0 ;I don't see this check in the append path. Should it be added in
bio_iov_add_zone_append_page()function?I'm not sure this check makes a lot of sense. If it just returns 0 here, then won't that get
bio_iov_iter_get_pages()stuck in an infinite loop? The bio isn't filling, the iov isn't advancing, and 0 indicates keep-going.Quoted 8 lines
Yeah but if
max_append_sectorsis zero, thenbio_add_hw_page()also returns 0 as follows:
....if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) return 0;....
WithWARN_ON_ONCE, we at least get a warning message if it gets stuck in an infinite loop because ofmax_append_sectorsbeing zero right?
Quoted 5 lines
The return for this function is the added length, not an indicator of success. And we already handle '0' as an error from
bio_iov_add_zone_append_page():if (bio_add_hw_page(q, bio, page, len, offset, queue_max_zone_append_sectors(q), &same_page) != len)
Ah. I didn't see the !=len part. Sorry for the noise and ignore this comment.
Quoted 1 line
return -EINVAL;
Signature · 1 line
Pankaj Raghav
PATCH v3 2/6 block: export dma_alignment attribute
In reply to Keith BuschFrom: Keith Busch <[email protected]>
User space may want to know how to align their buffers to avoid bouncing. Export the queue attribute.
---v2->v3:
Added Documentation (Bart)
Documentation/ABI/stable/sysfs-block | 9 +++++++++ block/blk-sysfs.c | 7 +++++++ 2 files changed, 16 insertions(+)
Documentation/ABI/stable/sysfs-block +9 -0
@@ -142,6 +142,15 @@ Description: Default value of this file is '1'(on). +What: /sys/block/<disk>/queue/dma_alignment +Date: May 2022 +Contact: [email protected] +Description: + Reports the alignment that user space addresses must have to be + used for raw block device access with O_DIRECT and other driver + specific passthrough mechanisms. + + What: /sys/block/<disk>/queue/chunk_sectors Date: September 2016 Contact: Hannes Reinecke <[email protected]>
block/blk-sysfs.c +7 -0
@@ -274,6 +274,11 @@ static ssize_t queue_virt_boundary_mask_show(struct request_queue *q, char *pagereturn queue_var_show (q->limits.virt_boundary_mask, page); } +static ssize_t queue_dma_alignment_show (struct request_queue *q,char *page) +{ +return queue_var_show (queue_dma_alignment (q), page); +} +#define QUEUE_SYSFS_BIT_FNS (name, flag, neg) \ static ssize_t \ queue_##name##_show(struct request_queue *q, char *page) \
@@ -606,6 +611,7 @@ QUEUE_RO_ENTRY(queue_dax, "dax");QUEUE_RW_ENTRY (queue_io_timeout,"io_timeout" );QUEUE_RW_ENTRY (queue_wb_lat,"wbt_lat_usec" );QUEUE_RO_ENTRY (queue_virt_boundary_mask,"virt_boundary_mask" ); +QUEUE_RO_ENTRY (queue_dma_alignment,"dma_alignment" );#ifdef CONFIG_BLK_DEV_THROTTLING_LOW QUEUE_RW_ENTRY (blk_throtl_sample_time,"throttle_sample_time" );
@@ -667,6 +673,7 @@ static struct attribute *queue_attrs[] = { &blk_throtl_sample_time_entry.attr, #endif &queue_virt_boundary_mask_entry.attr, + &queue_dma_alignment_entry.attr, NULL, };
Signature · 1 line
2.30.2
Re: [PATCHv3 2/6] block: export dma_alignment attribute
Looks good:
Re: [PATCHv3 2/6] block: export dma_alignment attribute
In reply to Keith BuschLooks good,
PATCH v3 3/6 block: introduce bdev_dma_alignment helper
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Preparing for upcoming dma_alignment users that have a block_device, but don't need the request_queue.
--- include/linux/blkdev.h | 5 +++++ 1 file changed, 5 insertions(+)
include/linux/blkdev.h +5 -0
@@ -1365,6 +1365,11 @@ static inline int queue_dma_alignment(const struct request_queue *q)return q ? q->dma_alignment :511 ; } +static inline unsigned int bdev_dma_alignment (struct block_device *bdev) +{ +return queue_dma_alignment (bdev_get_queue (bdev)); +} +static inline int blk_rq_aligned (struct request_queue *q,unsigned long addr,unsigned int len) {
Signature · 1 line
2.30.2
Re: [PATCHv3 3/6] block: introduce bdev_dma_alignment helper
Quoted 4 lines · Keith Busch
From: Keith Busch <[email protected]>
Preparing for upcoming
dma_alignmentusers that have ablock_device, but don't need therequest_queue.
Looks good:
Re: [PATCHv3 3/6] block: introduce bdev_dma_alignment helper
In reply to Keith BuschLooks good,
PATCH v3 4/6 block/merge: count bytes instead of sectors
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Individual bv_len's may not be a sector size.
--- block/blk-merge.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-)
block/blk-merge.c +18 -17
@@ -201,11 +201,11 @@ static inline unsigned get_max_segment_size(const struct request_queue *q,* @nsegs: [in,out] Number of segments in the bio being built. Incremented * by the number of segments from @bv that may be appended to that * bio without exceeding @max_segs -* @sectors: [in,out] Number of sectors in the bio being built. Incremented -* by the number of sectors from @bv that may be appended to that -* bio without exceeding @max_sectors +* @bytes: [in,out] Number of bytes in the bio being built. Incremented +* by the number of bytes from @bv that may be appended to that +* bio without exceeding @max_bytes * @max_segs: [in] upper bound for *@nsegs -* @max_sectors: [in] upper bound for *@sectors +* @max_bytes: [in] upper bound for *@bytes * * When splitting a bio, it can happen that a bvec is encountered that is too * big to fit in a single segment and hence that it has to be split in the
@@ -216,10 +216,10 @@ static inline unsigned get_max_segment_size(const struct request_queue *q,*/ static bool bvec_split_segs (const struct request_queue *q,const struct bio_vec *bv,unsigned *nsegs, -unsigned *sectors,unsigned max_segs, -unsigned max_sectors) +unsigned *bytes,unsigned max_segs, +unsigned max_bytes) { -unsigned max_len = (min (max_sectors,UINT_MAX >>9 ) - *sectors) <<9 ; +unsigned max_len =min (max_bytes,UINT_MAX ) - *bytes;unsigned len =min (bv->bv_len, max_len);unsigned total_len =0 ;unsigned seg_size =0 ;
@@ -237,7 +237,7 @@ static bool bvec_split_segs(const struct request_queue *q,break ; } - *sectors += total_len >>9 ; + *bytes += total_len;/* tell the caller to split the bvec if it is too big to fit */ return len >0 || bv->bv_len > max_len;
@@ -269,8 +269,8 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, {struct bio_vec bv, bvprv, *bvprvp =NULL ;struct bvec_iter iter; -unsigned nsegs =0 , sectors =0 ; -const unsigned max_sectors =get_max_io_size (q, bio); +unsigned nsegs =0 , bytes =0 ; +const unsigned max_bytes =get_max_io_size (q, bio) <<9 ;const unsigned max_segs =queue_max_segments (q);bio_for_each_bvec (bv, bio, iter) {
@@ -282,12 +282,12 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, goto split;if (nsegs < max_segs && - sectors + (bv.bv_len >>9 ) <= max_sectors && + bytes + bv.bv_len <= max_bytes && bv.bv_offset + bv.bv_len <=PAGE_SIZE ) { nsegs++; - sectors += bv.bv_len >>9 ; - }else if (bvec_split_segs (q, &bv, &nsegs, §ors, max_segs, - max_sectors)) { + bytes += bv.bv_len; + }else if (bvec_split_segs (q, &bv, &nsegs, &bytes, max_segs, + max_bytes)) { goto split; }
@@ -299,6 +299,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,return NULL ; split: *segs = nsegs; + bytes =ALIGN_DOWN (bytes,queue_logical_block_size (q));/* * Bio splitting may cause subtle trouble such as hang when doing sync
@@ -306,7 +307,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,* big IO can be trival, disable iopoll when split needed. */ bio_clear_polled (bio); -return bio_split (bio, sectors,GFP_NOIO , bs); +return bio_split (bio, bytes >>9 ,GFP_NOIO , bs); }/**
@@ -375,7 +376,7 @@ EXPORT_SYMBOL(blk_queue_split);unsigned int blk_recalc_rq_segments (struct request *rq) {unsigned int nr_phys_segs =0 ; -unsigned int nr_sectors =0 ; +unsigned int bytes =0 ;struct req_iterator iter;struct bio_vec bv;
@@ -398,7 +399,7 @@ unsigned int blk_recalc_rq_segments(struct request *rq) }rq_for_each_bvec (bv, rq, iter) -bvec_split_segs (rq->q, &bv, &nr_phys_segs, &nr_sectors, +bvec_split_segs (rq->q, &bv, &nr_phys_segs, &bytes,UINT_MAX ,UINT_MAX );return nr_phys_segs; }
Signature · 1 line
2.30.2
Re: [PATCHv3 4/6] block/merge: count bytes instead of sectors
block/blk-merge.c static struct bio *blk_bio_segment_split(struct request_queue *q,
Quoted 3 lines · Keith Busch
block/blk-merge.c +2
-0+const unsigned max_bytes =get_max_io_size (q, bio) <<9 ; +return bio_split (bio, bytes >>9 ,GFP_NOIO , bs);
These should use SECTOR_SHIFT.
block/blk-merge.c static struct bio *blk_bio_segment_split(struct request_queue *q, · line 302
Quoted 3 lines
split:
block/blk-merge.c +1
-0*segs = nsegs; + bytes =ALIGN_DOWN (bytes,queue_logical_block_size (q));
Please add a comment explaining the added alignment here.
PATCH v3 5/6 block/bounce: count bytes instead of sectors
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Individual bv_len's may not be a sector size.
--- block/bounce.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
block/bounce.c +3 -2
@@ -207,17 +207,18 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)struct bvec_iter iter;unsigned i =0 ;bool bounce = false; -int sectors =0 ; +int sectors =0 , bytes =0 ;bio_for_each_segment (from, *bio_orig, iter) {if (i++ <BIO_MAX_VECS ) - sectors += from.bv_len >>9 ; + bytes += from.bv_len;if (PageHighMem (from.bv_page)) bounce = true; }if (!bounce)return ; + sectors =ALIGN_DOWN (bytes,queue_logical_block_size (q)) >>9 ;if (sectors <bio_sectors (*bio_orig)) { bio =bio_split (*bio_orig, sectors,GFP_NOIO , &bounce_bio_split);bio_chain (bio, *bio_orig);
Signature · 1 line
2.30.2
Re: [PATCHv3 5/6] block/bounce: count bytes instead of sectors
Quoted 21 lines · Keith Busch
From: Keith Busch <[email protected]>
Individual bv_len's may not be a sector size.
Signed-off-by Keith Busch <[email protected]>--- block/bounce.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)block/bounce.c +1
-1@@ -207,17 +207,18 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)struct bvec_iter iter;unsigned i =0 ;bool bounce = false; -int sectors =0 ; +int sectors =0 , bytes =0 ;bio_for_each_segment (from, *bio_orig, iter) {
block/bounce.c void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · line 221
Quoted 10 lines
block/bounce.c +2
-1if (i++ <BIO_MAX_VECS ) - sectors += from.bv_len >>9 ; + bytes += from.bv_len;if (PageHighMem (from.bv_page)) bounce = true; }if (!bounce)return ; + sectors =ALIGN_DOWN (bytes,queue_logical_block_size (q)) >>9 ;
Same comment about SECTOR_SHIFT and a comment here. That being said, why do we even align here? Shouldn't bytes always be setor aligned here and this should be a WARN_ON or other sanity check? Probably the same for the previous patch.
Re: [PATCHv3 5/6] block/bounce: count bytes instead of sectors
Quoted 31 lines · Christoph Hellwig
Quoted 30 lines · Keith Busch
From: Keith Busch <[email protected]>
Individual bv_len's may not be a sector size.
Signed-off-by Keith Busch <[email protected]>--- block/bounce.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)block/bounce.c +2
-2@@ -207,17 +207,18 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)struct bvec_iter iter;unsigned i =0 ;bool bounce = false; -int sectors =0 ; +int sectors =0 , bytes =0 ;bio_for_each_segment (from, *bio_orig, iter) {if (i++ <BIO_MAX_VECS ) - sectors += from.bv_len >>9 ; + bytes += from.bv_len;if (PageHighMem (from.bv_page)) bounce = true; }if (!bounce)return ;
Quoted 6 lines
block/bounce.c void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · line 221
Quoted 1 line
block/bounce.c +1
-0+ sectors =ALIGN_DOWN (bytes,queue_logical_block_size (q)) >>9 ;Same comment about
SECTOR_SHIFTand a comment here. That being said, why do we even align here? Shouldn't bytes always be setor aligned here and this should be aWARN_ONor other sanity check? Probably the same for the previous patch.
Yes, the total bytes should be sector aligned.
I'm not exactly sure why we're counting it this way, though. We could just skip the iterative addition and get the total from bio->bi_iter.bi_size. Unless bio_orig has more segments that BIO_MAX_VECS, which doesn't seem possible.
Re: [PATCHv3 5/6] block/bounce: count bytes instead of sectors
Quoted 18 lines · Keith Busch
Quoted 10 lines · Christoph Hellwig
Quoted 10 lines
block/bounce.c +1
-1bio_for_each_segment (from, *bio_orig, iter) {if (i++ <BIO_MAX_VECS ) - sectors += from.bv_len >>9 ; + bytes += from.bv_len;if (PageHighMem (from.bv_page)) bounce = true; }if (!bounce)return ;Quoted 6 lines
block/bounce.c void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · line 221
Quoted 1 line
block/bounce.c +1
-0+ sectors =ALIGN_DOWN (bytes,queue_logical_block_size (q)) >>9 ;Same comment about
SECTOR_SHIFTand a comment here. That being said, why do we even align here? Shouldn't bytes always be setor aligned here and this should be aWARN_ONor other sanity check? Probably the same for the previous patch.
Quoted 5 lines
Yes, the total bytes should be sector aligned.
I'm not exactly sure why we're counting it this way, though. We could just skip the iterative addition and get the total from
bio->bi_iter.bi_size. Unlessbio_orighas more segments thatBIO_MAX_VECS, which doesn't seem possible.
Oh, there's a comment explaining the original can have more than BIO_MAX_VECS, so the ALIGN_DOWN is necessary to ensure a logical block sized split.
Re: [PATCHv3 5/6] block/bounce: count bytes instead of sectors
In reply to Keith BuschQuoted 14 lines · Keith Busch
From: Keith Busch <[email protected]>
Individual bv_len's may not be a sector size.
Signed-off-by Keith Busch <[email protected]>--- block/bounce.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)block/bounce.c +0
-0
block/bounce.c void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · line 214
Quoted 10 lines
block/bounce.c +2
-2@@ -207,17 +207,18 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)struct bvec_iter iter;unsigned i =0 ;bool bounce = false; -int sectors =0 ; +int sectors =0 , bytes =0 ;bio_for_each_segment (from, *bio_orig, iter) {if (i++ <BIO_MAX_VECS ) - sectors += from.bv_len >>9 ; + bytes += from.bv_len;
bv.len is unsigned int. bytes variable should also unsigned int to be on the safe side.
Quoted 13 lines
block/bounce.c +1
-0if (PageHighMem (from.bv_page)) bounce = true; }if (!bounce)return ; + sectors =ALIGN_DOWN (bytes,queue_logical_block_size (q)) >>9 ;if (sectors <bio_sectors (*bio_orig)) { bio =bio_split (*bio_orig, sectors,GFP_NOIO , &bounce_bio_split);bio_chain (bio, *bio_orig);Signature · 1 line
2.30.2
Signature · 1 line
Pankaj Raghav
PATCH v3 6/6 block: relax direct io memory alignment
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Use the address alignment requirements from the hardware for direct io instead of requiring addresses be aligned to the block size. User space can discover the alignment requirements from the dma_alignment queue attribute.
User space can specify any hardware compatible DMA offset for each segment, but every segment length is still required to be a multiple of the block size.
---v2->v3:
Removed iomap support for now
Added alignment help function instead of duplicating it (Christoph)
Added comment explaining ALIGN_DOWN
Added check for iov alignment in _async case
block/bio.c | 13 +++++++++++++ block/fops.c | 41 ++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 11 deletions(-)
block/bio.c +13 -0
@@ -1205,6 +1205,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) {unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; +struct request_queue *q =bdev_get_queue (bio->bi_bdev);struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;struct page **pages = (struct page **)bv;ssize_t size, left;
@@ -1219,7 +1220,19 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)BUILD_BUG_ON (PAGE_PTRS_PER_BVEC <2 ); pages += entries_left * (PAGE_PTRS_PER_BVEC -1 ); +/* +* Each segment in the iov is required to be a block size multiple. +* However, we may not be able to get the entire segment if it spans +* more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the +* result to ensure the bio's total size is correct. The remainder of +* the iov data will be picked up in the next bio iteration. +* +* If the result is ever 0, that indicates the iov fails the segment +* size requirement and is an error. +*/ size =iov_iter_get_pages (iter, pages,LONG_MAX , nr_pages, &offset); +if (size >0 ) + size =ALIGN_DOWN (size,queue_logical_block_size (q));if (unlikely (size <=0 ))return size ? size : -EFAULT ;
block/fops.c +30 -11
79 lines not on this page · raw message
Signature · 1 line
2.30.2
Re: [PATCHv3 6/6] block: relax direct io memory alignment
Quoted 1 line · Keith Busch
Removed iomap support for now
Do you plan to add a separate patch for it? It would be a shame to miss it especially as you said you already tested xfs.
Quoted 11 lines
block/bio.c +4
-0+struct request_queue *q =bdev_get_queue (bio->bi_bdev);struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;struct page **pages = (struct page **)bv;ssize_t size, left;@@ -1219,7 +1220,19 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)BUILD_BUG_ON (PAGE_PTRS_PER_BVEC <2 ); pages += entries_left * (PAGE_PTRS_PER_BVEC -1 ); +/* +* Each segment in the iov is required to be a block size multiple. +* However, we may not be able to get the entire segment if it spans
block/bio.c static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) · line 1235
Quoted 10 lines
block/bio.c +9
-0+* more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the +* result to ensure the bio's total size is correct. The remainder of +* the iov data will be picked up in the next bio iteration. +* +* If the result is ever 0, that indicates the iov fails the segment +* size requirement and is an error. +*/ size =iov_iter_get_pages (iter, pages,LONG_MAX , nr_pages, &offset); +if (size >0 ) + size =ALIGN_DOWN (size,queue_logical_block_size (q));
I think we can simply use bdev_logical_block_size here and remove the need for the q local variable.
Given that bio_iov_iter_get_pages is used by more than the block device direct I/O code maybe split this up further?
block/fops.c static unsigned int dio_bio_write_op(struct kiocb *iocb) · line 53
Quoted 13 lines
block/fops.c +9
-0@@ -42,6 +42,16 @@ static unsigned int dio_bio_write_op(struct kiocb *iocb)return op; } +static int blkdev_dio_aligned (struct block_device *bdev,loff_t pos, +struct iov_iter *iter) +{ +if ((pos |iov_iter_count (iter)) & (bdev_logical_block_size (bdev) -1 )) +return -EINVAL ; +if (iov_iter_alignment (iter) &bdev_dma_alignment (bdev)) +return -EINVAL ; +return 0 ; +}
I'd also split adding this helper into another prep patch to see the actual change in behavior more easily.
Re: [PATCHv3 6/6] block: relax direct io memory alignment
In reply to Keith Buschblock/fops.c static unsigned int dio_bio_write_op(struct kiocb *iocb) · line 48
Quoted 13 lines · Keith Busch
From: Keith Busch <[email protected]>
block/fops.c +4
-0@@ -42,6 +42,16 @@ static unsigned int dio_bio_write_op(struct kiocb *iocb)return op; } +static int blkdev_dio_aligned (struct block_device *bdev,loff_t pos, +struct iov_iter *iter) +{ +if ((pos |iov_iter_count (iter)) & (bdev_logical_block_size (bdev) -1 ))
Minor nit as ALIGN* macros have been used in other places, this alignment check can also be changed to using IS_ALIGNED macro:
if (!IS_ALIGNED(pos | iov_iter_count(iter), bdev_logical_block_size(bdev)))
Quoted 20 lines
block/fops.c +9
-3+return -EINVAL ; +if (iov_iter_alignment (iter) &bdev_dma_alignment (bdev)) +return -EINVAL ; +return 0 ; +} +#define DIO_INLINE_BIO_VECS 4 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,@@ -54,9 +64,9 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,struct bio bio;ssize_t ret; -if ((pos |iov_iter_alignment (iter)) & - (bdev_logical_block_size (bdev) -1 )) -return -EINVAL ; + ret =blkdev_dio_aligned (bdev, pos, iter); +if (ret) +return ret;
Signature · 1 line
Pankaj Raghav