block: advance by bvec's length for bio_for_each_bvec
PATCH 1/1 block: advance by bvec's length for bio_for_each_bvec
bio_for_each_bvec is used in fast path of bio splitting and sg mapping, and what we want to do is to iterate over multi-page bvecs, instead of pages. However, bvec_iter_advance() is invisble for this requirement, and always advance by page size.
This way isn't efficient for multipage bvec iterator, also bvec_iter_len() isn't as fast as mp_bvec_iter_len().
So advance by multi-page bvec's length instead of page size for bio_for_each_bvec().
More than 1% IOPS improvement can be observed in io_uring test on null_blk.
--- include/linux/bio.h | 13 +++++++++---- include/linux/bvec.h | 13 ++++++++++--- 2 files changed, 19 insertions(+), 7 deletions(-)
include/linux/bio.h +9 -4
@@ -134,17 +134,22 @@ static inline bool bio_full(struct bio *bio)for (i =0 , iter_all.idx =0 ; iter_all.idx < (bio)->bi_vcnt; iter_all.idx++) \ mp_bvec_for_each_segment(bvl, &((bio)->bi_io_vec[iter_all.idx]), i, iter_all) -static inline void bio_advance_iter (struct bio *bio,struct bvec_iter *iter, -unsigned bytes) +static inline void __bio_advance_iter (struct bio *bio,struct bvec_iter *iter, +unsigned bytes,bool bvec) { iter->bi_sector += bytes >>9 ;if (bio_no_advance_iter (bio)) iter->bi_size -= bytes;else -bvec_iter_advance (bio->bi_io_vec, iter, bytes); +__bvec_iter_advance (bio->bi_io_vec, iter, bytes, bvec);/* TODO: It is reasonable to complete bio with error here. */ } +static inline void bio_advance_iter (struct bio *bio,struct bvec_iter *iter, +unsigned bytes) +{ +return __bio_advance_iter (bio, iter, bytes, false); +}#define __bio_for_each_segment (bvl, bio, iter, start) \ for (iter = (start); \
@@ -159,7 +164,7 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,for (iter = (start); \ (iter).bi_size && \ ((bvl =mp_bvec_iter_bvec ((bio)->bi_io_vec, (iter))),1 ); \ -bio_advance_iter ((bio), &(iter), (bvl).bv_len)) +__bio_advance_iter ((bio), &(iter), (bvl).bv_len, true))/* iterate over multi-page bvec */ #define bio_for_each_bvec(bvl, bio, iter) \
include/linux/bvec.h +10 -3
@@ -102,8 +102,8 @@ static inline struct page *bvec_nth_page(struct page *page, int idx) .bv_offset = bvec_iter_offset((bvec), (iter)), \ }) -static inline bool bvec_iter_advance (const struct bio_vec *bv, -struct bvec_iter *iter,unsigned bytes) +static inline bool __bvec_iter_advance (const struct bio_vec *bv, +struct bvec_iter *iter,unsigned bytes,bool bvec) {if (WARN_ONCE (bytes > iter->bi_size,"Attempted to advance past end of bvec iter\n" )) {
@@ -112,7 +112,8 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, }while (bytes) { -unsigned iter_len =bvec_iter_len (bv, *iter); +unsigned iter_len = bvec ?mp_bvec_iter_len (bv, *iter) : +bvec_iter_len (bv, *iter);unsigned len =min (bytes, iter_len); bytes -= len;
@@ -127,6 +128,12 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,return true; } +static inline bool bvec_iter_advance (const struct bio_vec *bv, +struct bvec_iter *iter,unsigned bytes) +{ +return __bvec_iter_advance (bv, iter, bytes, false); +} +#define for_each_bvec (bvl, bio_vec, iter, start) \ for (iter = (start); \ (iter).bi_size && \
Signature · 1 line
2.9.5
Jens Axboe
maintainerQuoted 11 lines · Ming Lei
bio_for_each_bvecis used in fast path of bio splitting and sg mapping, and what we want to do is to iterate over multi-page bvecs, instead of pages. However,bvec_iter_advance()is invisble for this requirement, and always advance by page size.This way isn't efficient for multipage bvec iterator, also
bvec_iter_len()isn't as fast asmp_bvec_iter_len().So advance by multi-page bvec's length instead of page size for
bio_for_each_bvec().More than 1% IOPS improvement can be observed in
io_uringtest onnull_blk.
Thanks Ming, I tested this last night with good results.
Signature · 1 line
Jens Axboe
Quoted 11 lines · Ming Lei
bio_for_each_bvecis used in fast path of bio splitting and sg mapping, and what we want to do is to iterate over multi-page bvecs, instead of pages. However,bvec_iter_advance()is invisble for this requirement, and always advance by page size.This way isn't efficient for multipage bvec iterator, also
bvec_iter_len()isn't as fast asmp_bvec_iter_len().So advance by multi-page bvec's length instead of page size for
bio_for_each_bvec().More than 1% IOPS improvement can be observed in
io_uringtest onnull_blk.
We've been there before, and I still insist that there is not good reason ever to clamp the iteration to page size in bvec_iter_advance. Callers that iterate over it already do that in the callers.
So here is a resurretion and rebase of my patch from back then to just do the right thing:
include/linux/bvec.h +4 -3
@@ -112,14 +112,15 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, }while (bytes) { -unsigned iter_len =bvec_iter_len (bv, *iter); -unsigned len =min (bytes, iter_len); +const struct bio_vec *cur = bv + iter->bi_idx; +unsigned len =min3 (bytes, iter->bi_size, + cur->bv_len - iter->bi_bvec_done); bytes -= len; iter->bi_size -= len; iter->bi_bvec_done += len; -if (iter->bi_bvec_done ==__bvec_iter_bvec (bv, *iter)->bv_len) { +if (iter->bi_bvec_done == cur->bv_len) { iter->bi_bvec_done =0 ; iter->bi_idx++; }
Quoted 33 lines · Christoph Hellwig
Quoted 11 lines · Ming Lei
bio_for_each_bvecis used in fast path of bio splitting and sg mapping, and what we want to do is to iterate over multi-page bvecs, instead of pages. However,bvec_iter_advance()is invisble for this requirement, and always advance by page size.This way isn't efficient for multipage bvec iterator, also
bvec_iter_len()isn't as fast asmp_bvec_iter_len().So advance by multi-page bvec's length instead of page size for
bio_for_each_bvec().More than 1% IOPS improvement can be observed in
io_uringtest onnull_blk.We've been there before, and I still insist that there is not good reason ever to clamp the iteration to page size in
bvec_iter_advance. Callers that iterate over it already do that in the callers.So here is a resurretion and rebase of my patch from back then to just do the right thing:
include/linux/bvec.h +3
-2@@ -112,14 +112,15 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, }while (bytes) { -unsigned iter_len =bvec_iter_len (bv, *iter); -unsigned len =min (bytes, iter_len); +const struct bio_vec *cur = bv + iter->bi_idx; +unsigned len =min3 (bytes, iter->bi_size, + cur->bv_len - iter->bi_bvec_done);
include/linux/bvec.h static inline bool bvec_iter_advance(const struct bio_vec *bv, · line 126
Quoted 10 lines
include/linux/bvec.h +1
-1bytes -= len; iter->bi_size -= len; iter->bi_bvec_done += len; -if (iter->bi_bvec_done ==__bvec_iter_bvec (bv, *iter)->bv_len) { +if (iter->bi_bvec_done == cur->bv_len) { iter->bi_bvec_done =0 ; iter->bi_idx++; }
Yeah, this change is the correct thing to do, and there shouldn't be performance drop with this patch for Jens' test case, I guess.
Thanks,
Ming
Jens Axboe
maintainerQuoted 11 lines · Christoph Hellwig
Quoted 10 lines · Ming Lei
bio_for_each_bvecis used in fast path of bio splitting and sg mapping, and what we want to do is to iterate over multi-page bvecs, instead of pages. However,bvec_iter_advance()is invisble for this requirement, and always advance by page size.This way isn't efficient for multipage bvec iterator, also
bvec_iter_len()isn't as fast asmp_bvec_iter_len().So advance by multi-page bvec's length instead of page size for
bio_for_each_bvec().
Quoted 8 lines
Quoted 1 line
More than 1% IOPS improvement can be observed in
io_uringtest onnull_blk.We've been there before, and I still insist that there is not good reason ever to clamp the iteration to page size in
bvec_iter_advance. Callers that iterate over it already do that in the callers.So here is a resurretion and rebase of my patch from back then to just do the right thing:
Care to resend as a proper patch?
Signature · 1 line
Jens Axboe