io_uring: support vectored fixed kernel buffer
Subsystems (3)
cover io_uring: support vectored fixed kernel buffer
Hello Jens,
This patchset supports vectored fixed buffer for kernel bvec buffer, and use it on for ublk/stripe.
Please review.
Thanks,
Ming
Ming Lei (4):
io_uring: add validate_fixed_range() for validate fixed buffer
block: add for_each_mp_bvec()
io_uring: support vectored kernel fixed buffer
selftests: ublk: enable zero copy for stripe target
include/linux/bvec.h | 6 ++ io_uring/rsrc.c | 125 +++++++++++++++++++++++--- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/stripe.c | 69 ++++++++++---- 4 files changed, 170 insertions(+), 31 deletions(-)
Signature · 1 line
2.47.0
PATCH 1/4 io_uring: add validate_fixed_range() for validate fixed buffer
Add helper of validate_fixed_range() for validating fixed buffer range.
--- io_uring/rsrc.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-)
io_uring/rsrc.c +23 -11
@@ -1002,20 +1002,32 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, }EXPORT_SYMBOL_GPL (io_buffer_unregister_bvec); -static int io_import_fixed (int ddir,struct iov_iter *iter, -struct io_mapped_ubuf *imu, -u64 buf_addr,size_t len) +static int validate_fixed_range (u64 buf_addr,size_t len, +const struct io_mapped_ubuf *imu) {u64 buf_end; -size_t offset; -if (WARN_ON_ONCE (!imu)) -return -EFAULT ;if (unlikely (check_add_overflow (buf_addr, (u64 )len, &buf_end)))return -EFAULT ;/* not inside the mapped region */ if (unlikely (buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))return -EFAULT ; + +return 0 ; +} + +static int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len) +{ +size_t offset; +int ret; + +if (WARN_ON_ONCE (!imu)) +return -EFAULT ; + ret =validate_fixed_range (buf_addr, len, imu); +if (ret) +return ret;if (!(imu->dir & (1 << ddir)))return -EFAULT ;
@@ -1305,12 +1317,12 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter,u64 buf_addr = (u64 )(uintptr_t )iovec[iov_idx].iov_base;struct bio_vec *src_bvec;size_t offset; -u64 buf_end; +int ret; + + ret =validate_fixed_range (buf_addr, iov_len, imu); +if (unlikely (ret)) +return ret; -if (unlikely (check_add_overflow (buf_addr, (u64 )iov_len, &buf_end))) -return -EFAULT ; -if (unlikely (buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len))) -return -EFAULT ;if (unlikely (!iov_len))return -EFAULT ;if (unlikely (check_add_overflow (total_len, iov_len, &total_len)))
Signature · 1 line
2.47.0
Re: [PATCH 1/4] io_uring: add validate_fixed_range() for validate fixed buffer
Quoted 25 lines · Ming Lei
Add helper of
validate_fixed_range()for validating fixed buffer range.Signed-off-by Ming Lei <[email protected]>--- io_uring/rsrc.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-)io_uring/rsrc.c +2
-4@@ -1002,20 +1002,32 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, }EXPORT_SYMBOL_GPL (io_buffer_unregister_bvec); -static int io_import_fixed (int ddir,struct iov_iter *iter, -struct io_mapped_ubuf *imu, -u64 buf_addr,size_t len) +static int validate_fixed_range (u64 buf_addr,size_t len, +const struct io_mapped_ubuf *imu) {u64 buf_end; -size_t offset;
io_uring/rsrc.c int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, · line 1016
Quoted 10 lines
io_uring/rsrc.c +2
-2-if (WARN_ON_ONCE (!imu)) -return -EFAULT ;if (unlikely (check_add_overflow (buf_addr, (u64 )len, &buf_end)))return -EFAULT ;/* not inside the mapped region */ if (unlikely (buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))return -EFAULT ; + +return 0 ;
It's nice to avoid this code duplication. It looks like validate_fixed_range() could return a bool instead and leave the return code up to the caller, but I don't feel strongly either way.
Quoted 38 lines
io_uring/rsrc.c +19
-6+} + +static int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len) +{ +size_t offset; +int ret; + +if (WARN_ON_ONCE (!imu)) +return -EFAULT ; + ret =validate_fixed_range (buf_addr, len, imu); +if (ret) +return ret;if (!(imu->dir & (1 << ddir)))return -EFAULT ;@@ -1305,12 +1317,12 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter,u64 buf_addr = (u64 )(uintptr_t )iovec[iov_idx].iov_base;struct bio_vec *src_bvec;size_t offset; -u64 buf_end; +int ret; + + ret =validate_fixed_range (buf_addr, iov_len, imu); +if (unlikely (ret)) +return ret; -if (unlikely (check_add_overflow (buf_addr, (u64 )iov_len, &buf_end))) -return -EFAULT ; -if (unlikely (buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len))) -return -EFAULT ;if (unlikely (!iov_len))return -EFAULT ;if (unlikely (check_add_overflow (total_len, iov_len, &total_len))) --2.47.0
PATCH 2/4 block: add for_each_mp_bvec()
In reply to Ming LeiAdd helper of for_each_mp_bvec() for io_uring to import fixed kernel buffer.
--- include/linux/bvec.h | 6 ++++++ 1 file changed, 6 insertions(+)
include/linux/bvec.h +6 -0
@@ -184,6 +184,12 @@ static inline void bvec_iter_advance_single(const struct bio_vec *bv, ((bvl =bvec_iter_bvec ((bio_vec), (iter))),1 ); \ bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) +#define for_each_mp_bvec (bvl, bio_vec, iter, start) \ + for (iter = (start); \ + (iter).bi_size && \ + ((bvl = mp_bvec_iter_bvec((bio_vec), (iter))), 1); \ + bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) +/* for iterating one bio from start to end */ #define BVEC_ITER_ALL_INIT (struct bvec_iter) \ { \
Signature · 1 line
2.47.0
PATCH 3/4 io_uring: support vectored kernel fixed buffer
In reply to Ming Leiio_uring has supported fixed kernel buffer via io_buffer_register_bvec() and io_buffer_unregister_bvec().
The vectored fixed buffer has been ready, so it is natural to support fixed kernel buffer, one use case is ublk.
--- io_uring/rsrc.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-)
io_uring/rsrc.c +88 -3
@@ -1361,6 +1361,82 @@ static int io_estimate_bvec_size(struct iovec *iov, unsigned nr_iovs,return max_segs; } +static int io_vec_fill_kern_bvec (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +struct iovec *iovec,unsigned nr_iovs, +struct iou_vec *vec) +{ +const struct bio_vec *src_bvec = imu->bvec; +struct bio_vec *res_bvec = vec->bvec; +unsigned res_idx =0 ; +size_t total_len =0 ; +unsigned iov_idx; + +for (iov_idx =0 ; iov_idx < nr_iovs; iov_idx++) { +size_t offset = (size_t )(uintptr_t )iovec[iov_idx].iov_base; +size_t iov_len = iovec[iov_idx].iov_len; +struct bvec_iter bi = { + .bi_size = offset + iov_len, + }; +struct bio_vec bv; + +bvec_iter_advance (src_bvec, &bi, offset); +for_each_mp_bvec (bv, src_bvec, bi, bi) + res_bvec[res_idx++] = bv; + total_len += iov_len; + } +iov_iter_bvec (iter, ddir, res_bvec, res_idx, total_len); +return 0 ; +} + +static int iov_kern_bvec_size (const struct iovec *iov, +const struct io_mapped_ubuf *imu, +unsigned int *nr_seg) +{ +size_t offset = (size_t )(uintptr_t )iov->iov_base; +const struct bio_vec *bvec = imu->bvec; +int start =0 , i =0 ; +size_t off =0 ; +int ret; + + ret =validate_fixed_range (offset, iov->iov_len, imu); +if (unlikely (ret)) +return ret; + +for (i =0 ; off < offset + iov->iov_len && i < imu->nr_bvecs; + off += bvec[i].bv_len, i++) { +if (offset >= off && offset < off + bvec[i].bv_len) + start = i; + } + *nr_seg = i - start; +return 0 ; +} + +static int io_kern_bvec_size (struct iovec *iov,unsigned nr_iovs, +struct io_mapped_ubuf *imu,unsigned *nr_segs) +{ +unsigned max_segs =0 ; +size_t total_len =0 ; +unsigned i; +int ret; + + *nr_segs =0 ; +for (i =0 ; i < nr_iovs; i++) { +if (unlikely (!iov[i].iov_len)) +return -EFAULT ; +if (unlikely (check_add_overflow (total_len, iov[i].iov_len, + &total_len))) +return -EOVERFLOW ; + ret =iov_kern_bvec_size (&iov[i], imu, &max_segs); +if (unlikely (ret)) +return ret; + *nr_segs += max_segs; + } +if (total_len >MAX_RW_COUNT ) +return -EINVAL ; +return 0 ; +} +int io_import_reg_vec (int ddir,struct iov_iter *iter,struct io_kiocb *req,struct iou_vec *vec,unsigned nr_iovs,unsigned issue_flags)
@@ -1375,14 +1451,20 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,if (!node)return -EFAULT ; imu = node->buf; -if (imu->is_kbuf) -return -EOPNOTSUPP ;if (!(imu->dir & (1 << ddir)))return -EFAULT ; iovec_off = vec->nr - nr_iovs; iov = vec->iovec + iovec_off; - nr_segs =io_estimate_bvec_size (iov, nr_iovs, imu); + +if (imu->is_kbuf) { +int ret =io_kern_bvec_size (iov, nr_iovs, imu, &nr_segs); + +if (unlikely (ret)) +return ret; + }else { + nr_segs =io_estimate_bvec_size (iov, nr_iovs, imu); + }if (sizeof (struct bio_vec ) >sizeof (struct iovec )) {size_t bvec_bytes;
@@ -1409,6 +1491,9 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter, req->flags |=REQ_F_NEED_CLEANUP ; } +if (imu->is_kbuf) +return io_vec_fill_kern_bvec (ddir, iter, imu, iov, nr_iovs, vec); +return io_vec_fill_bvec (ddir, iter, imu, iov, nr_iovs, vec); }
Signature · 1 line
2.47.0
PATCH 4/4 selftests: ublk: enable zero copy for stripe target
In reply to Ming LeiUse io_uring vectored fixed kernel buffer for handling stripe IO.
--- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/stripe.c | 69 ++++++++++++++++++++------- 2 files changed, 53 insertions(+), 17 deletions(-)
tools/testing/selftests/ublk/Makefile +1 -0
@@ -17,6 +17,7 @@ TEST_PROGS += test_loop_05.sh TEST_PROGS += test_stripe_01.sh TEST_PROGS += test_stripe_02.sh TEST_PROGS += test_stripe_03.sh +TEST_PROGS += test_stripe_04.sh TEST_PROGS += test_stress_01.sh TEST_PROGS += test_stress_02.sh
tools/testing/selftests/ublk/stripe.c +52 -17
129 lines not on this page · raw message
Signature · 1 line
2.47.0
Re: [PATCH 4/4] selftests: ublk: enable zero copy for stripe target
Quoted 13 lines · Ming Lei
Use
io_uringvectored fixed kernel buffer for handling stripe IO.Signed-off-by Ming Lei <[email protected]>--- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/stripe.c | 69 ++++++++++++++++++++------- 2 files changed, 53 insertions(+), 17 deletions(-)tools/testing/selftests/ublk/Makefile +0
-0
tools/testing/selftests/ublk/Makefile TEST_PROGS += test_loop_05.sh · line 20
Quoted 4 lines
tools/testing/selftests/ublk/Makefile +1
-0@@ -17,6 +17,7 @@ TEST_PROGS += test_loop_05.sh TEST_PROGS += test_stripe_01.sh TEST_PROGS += test_stripe_02.sh TEST_PROGS += test_stripe_03.sh +TEST_PROGS += test_stripe_04.sh
This patch is missing the new file test_stripe_04.sh, causing ublk selftests to be broken on block/for-next. Can you fix?
Re: [PATCH 4/4] selftests: ublk: enable zero copy for stripe target
Quoted 9 lines · Uday Shankar
Quoted 8 lines · Ming Lei
Use
io_uringvectored fixed kernel buffer for handling stripe IO.Signed-off-by Ming Lei <[email protected]>--- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/stripe.c | 69 ++++++++++++++++++++------- 2 files changed, 53 insertions(+), 17 deletions(-)
Quoted 12 lines
tools/testing/selftests/ublk/Makefile TEST_PROGS += test_loop_05.sh · line 20
Quoted 9 lines
tools/testing/selftests/ublk/Makefile +1
-0@@ -17,6 +17,7 @@ TEST_PROGS += test_loop_05.sh TEST_PROGS += test_stripe_01.sh TEST_PROGS += test_stripe_02.sh TEST_PROGS += test_stripe_03.sh +TEST_PROGS += test_stripe_04.shThis patch is missing the new file
test_stripe_04.sh, causing ublk selftests to be broken onblock/for-next. Can you fix?
The fix is posted out:
https://lore.kernel.org/linux-block/[email protected]/
Thanks for the report and sorry for the trouble.
Thanks,
Ming
Quoted 11 lines · Ming Lei
Hello Jens,
This patchset supports vectored fixed buffer for kernel bvec buffer, and use it on for ublk/stripe.
Please review.
Thanks,
Ming
Quoted 5 lines
Ming Lei (4):
io_uring: addvalidate_fixed_range()for validate fixed buffer
block: addfor_each_mp_bvec()
io_uring: support vectored kernel fixed buffer
selftests: ublk: enable zero copy for stripe target
Hello,
Ping...
Thanks,
Ming
Jens Axboe
maintainerQuoted 18 lines · Ming Lei
Quoted 11 lines · Ming Lei
Hello Jens,
This patchset supports vectored fixed buffer for kernel bvec buffer, and use it on for ublk/stripe.
Please review.
Thanks,
MingQuoted 5 lines
Ming Lei (4):
io_uring: addvalidate_fixed_range()for validate fixed buffer
block: addfor_each_mp_bvec()
io_uring: support vectored kernel fixed buffer
selftests: ublk: enable zero copy for stripe target
Quoted 3 lines
Hello,
Ping...
Looks fine to me and pretty straight forward, but it was in the merge window. Anything that makes this important for 6.15? We can still include it if so. If not, let's take a look for 6.16 when the merge window closes.
Signature · 1 line
Jens Axboe
Quoted 19 lines · Jens Axboe
Quoted 18 lines · Ming Lei
Quoted 16 lines · Ming Lei
Hello Jens,
This patchset supports vectored fixed buffer for kernel bvec buffer, and use it on for ublk/stripe.
Please review.
Thanks,
MingMing Lei (4):
io_uring: addvalidate_fixed_range()for validate fixed buffer
block: addfor_each_mp_bvec()
io_uring: support vectored kernel fixed buffer
selftests: ublk: enable zero copy for stripe target
Quoted 8 lines
Quoted 3 lines
Hello,
Ping...
Looks fine to me and pretty straight forward, but it was in the merge window. Anything that makes this important for 6.15? We can still include it if so. If not, let's take a look for 6.16 when the merge window closes.
fwiw, I looked through it, looks correct, but it'd be a good to test it aside from ublk. I guess I'll just extend a kernel hack I used before.
No opinion on 6.15 vs 6.16, but the argument might be to have it in the same release with ublk zc, and have less probing for the user.
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerQuoted 29 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 18 lines · Ming Lei
Quoted 16 lines · Ming Lei
Hello Jens,
This patchset supports vectored fixed buffer for kernel bvec buffer, and use it on for ublk/stripe.
Please review.
Thanks,
MingMing Lei (4):
io_uring: addvalidate_fixed_range()for validate fixed buffer
block: addfor_each_mp_bvec()
io_uring: support vectored kernel fixed buffer
selftests: ublk: enable zero copy for stripe targetQuoted 8 lines
Quoted 3 lines
Hello,
Ping...
Looks fine to me and pretty straight forward, but it was in the merge window. Anything that makes this important for 6.15? We can still include it if so. If not, let's take a look for 6.16 when the merge window closes.
Quoted 7 lines
fwiw, I looked through it, looks correct, but it'd be a good to test it aside from ublk. I guess I'll just extend a kernel hack I used before.
No opinion on 6.15 vs 6.16, but the argument might be to have it in the same release with ublk zc, and have less probing for the user.
That's a good argument for 6.15 actually, rather than split the vectored feature over two releases... I'll queue it up.
Signature · 1 line
Jens Axboe
Quoted 9 lines · Ming Lei
This patchset supports vectored fixed buffer for kernel bvec buffer, and use it on for ublk/stripe.
Please review.
Thanks,
Ming[...]
Applied, thanks!
[1/4] io_uring: add validate_fixed_range() for validate fixed buffer
(no commit info)
[2/4] block: add for_each_mp_bvec()
(no commit info)
[3/4] io_uring: support vectored kernel fixed buffer
(no commit info)
[4/4] selftests: ublk: enable zero copy for stripe target
(no commit info)
Best regards,
Signature · 1 line
Jens Axboe