pre-mapped rw attributes
cover pre-mapped rw attributes
warning: not properly tested
Follow up on the discussion about optimising copy_from_user() for read/write attributes. The idea here is to use registered regions (see IORING_REGISTER_MEM_REGION) for that purpose pretty much in the same way registered wait arguments work.
Putting it simply, a region is a user provided chunk of memory that has been registered and pre-mapped into io_uring / kernel, but it has more modes like mmap'ing kernel memory. For attributes the user passes an offset into a region, and the kernel can read from it directly without copy_from_user().
The other alternative is to store attributes into the upper half of SQE128, but then we might run out of space in SQE for larger and/or compound attributes. It'd also require SQE128, which has a (perhaps minor) downside when other types of requests don't need it.
Pavel Begunkov (4):
io_uring: add structure for registered arguments
io_uring: add registered request arguments io_uring/rw: use READ_ONCE with rw attributes io_uring/rw: pre-mapped rw attributes
include/linux/io_uring_types.h | 11 ++++++++-- include/uapi/linux/io_uring.h | 4 +++- io_uring/io_uring.c | 23 +++------------------ io_uring/io_uring.h | 16 +++++++++++++++ io_uring/register.c | 7 +++++-- io_uring/rw.c | 37 ++++++++++++++++++++++++---------- 6 files changed, 62 insertions(+), 36 deletions(-)
Signature · 1 line
2.47.1
PATCH 1/4 io_uring: add structure for registered arguments
A preparation patch making infra for wait arguments a bit more general to use in in following patches.
--- include/linux/io_uring_types.h | 9 +++++++-- io_uring/io_uring.c | 23 +++-------------------- io_uring/io_uring.h | 16 ++++++++++++++++ io_uring/register.c | 4 ++-- 4 files changed, 28 insertions(+), 24 deletions(-)
include/linux/io_uring_types.h +7 -2
@@ -83,6 +83,11 @@ struct io_mapped_region {unsigned flags; }; +struct io_reg_args { +void *ptr; +size_t size; +}; +/* * Arbitrary limit, can be raised if need be */
@@ -332,8 +337,8 @@ struct io_ring_ctx {struct io_ev_fd __rcu *io_ev_fd;unsigned cq_extra; -void *cq_wait_arg; -size_t cq_wait_size; +struct io_reg_args wait_args; + }____cacheline_aligned_in_smp ;/*
io_uring/io_uring.c +3 -20
@@ -3178,25 +3178,6 @@ void __io_uring_cancel(bool cancel_all)io_uring_cancel_generic (cancel_all,NULL ); } -static struct io_uring_reg_wait *io_get_ext_arg_reg (struct io_ring_ctx *ctx, -const struct io_uring_getevents_arg __user *uarg) -{ -unsigned long size =sizeof (struct io_uring_reg_wait ); -unsigned long offset = (uintptr_t )uarg; -unsigned long end; - -if (unlikely (offset %sizeof (long ))) -return ERR_PTR (-EFAULT ); - -/* also protects from NULL ->cq_wait_arg as the size would be 0 */ -if (unlikely (check_add_overflow (offset, size, &end) || - end > ctx->cq_wait_size)) -return ERR_PTR (-EFAULT ); - - offset =array_index_nospec (offset, ctx->cq_wait_size - size); -return ctx->cq_wait_arg + offset; -} -static int io_validate_ext_arg (struct io_ring_ctx *ctx,unsigned flags,const void __user *argp,size_t argsz) {
@@ -3233,7 +3214,9 @@ static int io_get_ext_arg(struct io_ring_ctx *ctx, unsigned flags,if (ext_arg->argsz !=sizeof (struct io_uring_reg_wait ))return -EINVAL ; - w =io_get_ext_arg_reg (ctx, argp); + + w =io_args_get_ptr (&ctx->wait_args, (uintptr_t )argp, +sizeof (struct io_uring_reg_wait ));if (IS_ERR (w))return PTR_ERR (w);
io_uring/io_uring.h +16 -0
@@ -514,4 +514,20 @@ static inline bool io_has_work(struct io_ring_ctx *ctx)return test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq) || io_local_work_pending(ctx); } + +static inline void *io_args_get_ptr (struct io_reg_args *args, +unsigned long offset,size_t size) +{ +unsigned long end; + +if (unlikely (offset %sizeof (long ))) +return ERR_PTR (-EFAULT ); + +/* also protects from NULL as the size would be 0 */ +if (unlikely (check_add_overflow (offset, size, &end) || end > args->size)) +return ERR_PTR (-EFAULT ); + +return args->ptr +array_index_nospec (offset, args->size - size); +} +#endif
io_uring/register.c +2 -2
@@ -604,8 +604,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) }if (reg.flags &IORING_MEM_REGION_REG_WAIT_ARG ) { - ctx->cq_wait_arg =io_region_get_ptr (&ctx->param_region); - ctx->cq_wait_size = rd.size; + ctx->wait_args.ptr =io_region_get_ptr (&ctx->param_region); + ctx->wait_args.size = rd.size; }return 0 ; }
Signature · 1 line
2.47.1
PATCH 2/4 io_uring: add registered request arguments
In reply to Pavel BegunkovSimilarly to registered wait arguments we want to have a pre-mapped space for various request arguments. Use the same parameter region, however as ->wait_args has different lifetime rules, add a new instance of struct io_reg_args.
--- include/linux/io_uring_types.h | 2 ++ io_uring/register.c | 3 +++ 2 files changed, 5 insertions(+)
include/linux/io_uring_types.h +2 -0
@@ -299,6 +299,8 @@ struct io_ring_ctx {struct io_submit_state submit_state; +struct io_reg_args sqe_args; +/* * Modifications are protected by ->uring_lock and ->mmap_lock. * The flags, buf_pages and buf_nr_pages fields should be stable
io_uring/register.c +3 -0
@@ -607,6 +607,9 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) ctx->wait_args.ptr =io_region_get_ptr (&ctx->param_region); ctx->wait_args.size = rd.size; } + + ctx->sqe_args.ptr =io_region_get_ptr (&ctx->param_region); + ctx->sqe_args.size = rd.size;return 0 ; }
Signature · 1 line
2.47.1
RE: [PATCH 2/4] io_uring: add registered request arguments
Hi,
Quoted 38 lines
-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, December 30, 2024 9:30 PM
To: [email protected]
Cc: [email protected]; Anuj Gupta <[email protected]>; Kanchan Joshi <[email protected]>
Subject: [PATCH 2/4]io_uring: add registered request argumentsSimilarly to registered wait arguments we want to have a pre-mapped space for various request arguments. Use the same parameter region, however as -
Quoted 1 line
wait_argshas different lifetime rules, add a new instance of struct
io_reg_args.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 2 ++ io_uring/register.c | 3 +++ 2 files changed, 5 insertions(+)include/linux/io_uring_types.h +2
-0@@ -299,6 +299,8 @@ struct io_ring_ctx {struct io_submit_state submit_state; +struct io_reg_args sqe_args; +/* * Modifications are protected by ->uring_lock and - Quoted 1 line
mmap_lock.* The flags,
buf_pagesandbuf_nr_pagesfields should be
stable diff --gita/io_uring/register.cb/io_uring/register.cindexb926eb053408..d2232b90a81d100644
io_uring/register.c static int io_register_mem_region(struct io_ring_ctx · line 613
Quoted 7 lines
io_uring/register.c +3
-0@@ -607,6 +607,9 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) ctx->wait_args.ptr =io_region_get_ptr (&ctx->param_region); ctx->wait_args.size = rd.size; } + + ctx->sqe_args.ptr =io_region_get_ptr (&ctx->param_region); + ctx->sqe_args.size = rd.size;
Why not add an enum value against sqe_args? Will mixing it with IORING_MEM_REGION_REG_WAIT_ARG cause management confusion?
Quoted 6 lines
io_uring/register.c +0
-1return 0 ; } --2.47.1
---
Li Zetao
Re: [PATCH 2/4] io_uring: add registered request arguments
Quoted 34 lines · lizetao
Hi,
Quoted 32 lines
-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, December 30, 2024 9:30 PM
To: [email protected]
Cc: [email protected]; Anuj Gupta <[email protected]>; Kanchan Joshi <[email protected]>
Subject: [PATCH 2/4]io_uring: add registered request argumentsSimilarly to registered wait arguments we want to have a pre-mapped space for various request arguments. Use the same parameter region, however as -
Quoted 1 line
wait_argshas different lifetime rules, add a new instance of struct
io_reg_args.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 2 ++ io_uring/register.c | 3 +++ 2 files changed, 5 insertions(+)include/linux/io_uring_types.h +2
-0@@ -299,6 +299,8 @@ struct io_ring_ctx {struct io_submit_state submit_state; +struct io_reg_args sqe_args; +/* * Modifications are protected by ->uring_lock and - Quoted 1 line
mmap_lock.
Quoted 15 lines
io_uring/register.c static int io_register_mem_region(struct io_ring_ctx · line 613
Quoted 13 lines
* The flags,
buf_pagesandbuf_nr_pagesfields should be
stable diff --gita/io_uring/register.cb/io_uring/register.cindexb926eb053408..d2232b90a81d100644io_uring/register.c +3
-0@@ -607,6 +607,9 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) ctx->wait_args.ptr =io_region_get_ptr (&ctx->param_region); ctx->wait_args.size = rd.size; } + + ctx->sqe_args.ptr =io_region_get_ptr (&ctx->param_region); + ctx->sqe_args.size = rd.size;Why not add an enum value against
sqe_args? Will mixing it with
Not sure what you mean
Quoted 1 line
IORING_MEM_REGION_REG_WAIT_ARGcause management confusion?
This specific region was added to be shared for multiple use cases, the rest is up to the userspace. liburing will need to have a clear API
Signature · 1 line
Pavel Begunkov
PATCH 3/4 io_uring/rw: use READ_ONCE with rw attributes
In reply to Pavel BegunkovIn preparation to pre-mapped attributes read struct io_uring_attr_pi with READ_ONCE and use an intermediate pointer.
--- io_uring/rw.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
io_uring/rw.c +14 -8
@@ -264,23 +264,29 @@ static inline void io_meta_restore(struct io_async_rw *io, struct kiocb *kiocb)static int io_prep_rw_pi (struct io_kiocb *req,struct io_rw *rw,int ddir,u64 attr_ptr,u64 attr_type_mask) { -struct io_uring_attr_pi pi_attr; +struct io_uring_attr_pi __pi_attr; +struct io_uring_attr_pi *pi_attr;struct io_async_rw *io; +void __user *pi_addr; +size_t pi_len;int ret; -if (copy_from_user (&pi_attr,u64_to_user_ptr (attr_ptr), +if (copy_from_user (&__pi_attr,u64_to_user_ptr (attr_ptr),sizeof (pi_attr)))return -EFAULT ; + pi_attr = &__pi_attr; -if (pi_attr.rsvd) +if (pi_attr->rsvd)return -EINVAL ; io = req->async_data; - io->meta.flags = pi_attr.flags; - io->meta.app_tag = pi_attr.app_tag; - io->meta.seed = pi_attr.seed; - ret =import_ubuf (ddir,u64_to_user_ptr (pi_attr.addr), - pi_attr.len, &io->meta.iter); + io->meta.flags =READ_ONCE (pi_attr->flags); + io->meta.app_tag =READ_ONCE (pi_attr->app_tag); + io->meta.seed =READ_ONCE (pi_attr->seed); + + pi_addr =u64_to_user_ptr (READ_ONCE (pi_attr->addr)); + pi_len =READ_ONCE (pi_attr->len); + ret =import_ubuf (ddir, pi_addr, pi_len, &io->meta.iter);if (unlikely (ret <0 ))return ret; req->flags |=REQ_F_HAS_METADATA ;
Signature · 1 line
2.47.1
PATCH 4/4 io_uring/rw: pre-mapped rw attributes
In reply to Pavel BegunkovInstead of copy_from_user()'ing request attributes, allow it to be grabbwd from a registered pre-registered parameter region like we do with registered wait arguments.
--- include/uapi/linux/io_uring.h | 4 +++- io_uring/rw.c | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-)
include/uapi/linux/io_uring.h +3 -1
@@ -112,7 +112,9 @@ struct io_uring_sqe { };/* sqe->attr_type_mask flags */ -#define IORING_RW_ATTR_FLAG_PI (1U << 0) +#define IORING_RW_ATTR_FLAG_PI (1UL << 0) +#define IORING_RW_ATTR_REGISTERED (1UL << 63) +/* PI attribute information */ struct io_uring_attr_pi {__u16 flags;
io_uring/rw.c +14 -5
@@ -271,10 +271,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,size_t pi_len;int ret; -if (copy_from_user (&__pi_attr,u64_to_user_ptr (attr_ptr), -sizeof (pi_attr))) -return -EFAULT ; - pi_attr = &__pi_attr; +if (attr_type_mask &IORING_RW_ATTR_REGISTERED ) { + pi_attr =io_args_get_ptr (&req->ctx->sqe_args, attr_ptr, +sizeof (pi_attr)); +if (IS_ERR (pi_attr)) +return PTR_ERR (pi_attr); + }else { +if (copy_from_user (&__pi_attr,u64_to_user_ptr (attr_ptr), +sizeof (pi_attr))) +return -EFAULT ; + pi_attr = &__pi_attr; + }if (pi_attr->rsvd)return -EINVAL ;
@@ -294,6 +301,8 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,return ret; } +#define IO_RW_ATTR_ALLOWED_MASK (IORING_RW_ATTR_FLAG_PI | IORING_RW_ATTR_REGISTERED) +static int io_prep_rw (struct io_kiocb *req,const struct io_uring_sqe *sqe,int ddir,bool do_import) {
@@ -332,7 +341,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,u64 attr_ptr;/* only PI attribute is supported currently */ -if (attr_type_mask !=IORING_RW_ATTR_FLAG_PI ) +if (attr_type_mask &IO_RW_ATTR_ALLOWED_MASK )return -EINVAL ; attr_ptr =READ_ONCE (sqe->attr_ptr);
Signature · 1 line
2.47.1
RE: [PATCH 4/4] io_uring/rw: pre-mapped rw attributes
Hi,
Quoted 24 lines
-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, December 30, 2024 9:30 PM
To: [email protected]
Cc: [email protected]; Anuj Gupta <[email protected]>; Kanchan Joshi <[email protected]>
Subject: [PATCH 4/4]io_uring/rw: pre-mapped rw attributesInstead of copy_from_user()'ing request attributes, allow it to be grabbwd from a registered pre-registered parameter region like we do with registered wait arguments.
Suggested-by Anuj Gupta <[email protected]>Signed-off-by Pavel Begunkov <[email protected]>--- include/uapi/linux/io_uring.h | 4 +++- io_uring/rw.c | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-)include/uapi/linux/io_uring.h index renamed from include/uapi/linux/io_uring.h +0
-0
38f0d6b10eaf..ec6e6fd37d1c100644
include/uapi/linux/io_uring.h struct io_uring_sqe { }; · line 115
Quoted 5 lines
include/uapi/linux/io_uring.h +2
-1@@ -112,7 +112,9 @@ struct io_uring_sqe { };/* sqe->attr_type_mask flags */ -#define IORING_RW_ATTR_FLAG_PI (1U << 0) +#define IORING_RW_ATTR_FLAG_PI (1UL << 0) +#define IORING_RW_ATTR_REGISTERED (1UL << 63)
Why use (1UL << 63) instead of (1UL << 1) here?
Quoted 10 lines
include/uapi/linux/io_uring.h +1
-0+/* PI attribute information */ struct io_uring_attr_pi {__u16 flags;io_uring/rw.c index dc1acaf95db1..b1db4595788b renamed from io_uring/rw.c +0
-0100644
io_uring/rw.c +0
-0@@ -271,10 +271,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,
io_uring/rw.c static int io_prep_rw_pi(struct io_kiocb *req, struct · line 277
Quoted 10 lines
io_uring/rw.c +3
-4size_t pi_len;int ret; -if (copy_from_user (&__pi_attr,u64_to_user_ptr (attr_ptr), -sizeof (pi_attr))) -return -EFAULT ; - pi_attr = &__pi_attr; +if (attr_type_mask &IORING_RW_ATTR_REGISTERED ) { + pi_attr =io_args_get_ptr (&req->ctx->sqe_args, attr_ptr, +sizeof (pi_attr));
Here pi_attr is just pointer, so maybe sizeof(__pi_attr) or sizeof(struct io_uring_attr_pi) will be better.
io_uring/rw.c static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, · line 281
Quoted 5 lines
io_uring/rw.c +5
-0+if (IS_ERR (pi_attr)) +return PTR_ERR (pi_attr); + }else { +if (copy_from_user (&__pi_attr,u64_to_user_ptr (attr_ptr), +sizeof (pi_attr)))
Just like above, shoule be sizeof(struct io_uring_attr_pi) here.
Quoted 18 lines
io_uring/rw.c +3
-0+return -EFAULT ; + pi_attr = &__pi_attr; + }if (pi_attr->rsvd)io_uring/rw.c +3
-0return -EINVAL ;@@ -294,6 +301,8 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, return ret; } +#define IO_RW_ATTR_ALLOWED_MASK (IORING_RW_ATTR_FLAG_PI | +IORING_RW_ATTR_REGISTERED) + static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,int ddir,bool do_import) {
io_uring/rw.c static int io_prep_rw(struct io_kiocb *req, const struct · line 336
Quoted 5 lines
io_uring/rw.c +0
-1@@ -332,7 +341,7 @@ static int io_prep_rw(struct io_kiocb *req, const structio_uring_sqe *sqe, u64 attr_ptr;/* only PI attribute is supported currently */ -if (attr_type_mask !=IORING_RW_ATTR_FLAG_PI )
The comment needs to be adjusted.
io_uring/rw.c static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, · line 344
Quoted 1 line
io_uring/rw.c +1
-0+if (attr_type_mask &IO_RW_ATTR_ALLOWED_MASK )
Here should be attr_type_mask & ~IO_RW_ATTR_ALLOWED_MASK ?
Quoted 6 lines
io_uring/rw.c +0
-0return -EINVAL ;
attr_ptr=READ_ONCE(sqe->attr_ptr);
--
2.47.1
--
Li Zetao
Re: [PATCH 4/4] io_uring/rw: pre-mapped rw attributes
On 1/6/25 11:13, lizetao wrote:
...
Quoted 5 lines
include/uapi/linux/io_uring.h struct io_uring_sqe { · line 116
Quoted 4 lines
/*
sqe->attr_type_maskflags */include/uapi/linux/io_uring.h +2
-1-#define IORING_RW_ATTR_FLAG_PI (1U << 0) +#define IORING_RW_ATTR_FLAG_PI (1UL << 0) +#define IORING_RW_ATTR_REGISTERED (1UL << 63)Why use (1UL << 63) instead of (1UL << 1) here?
They serve different purposes, think how you'd be iterating more attribute types, calculating the total size, etc.
Quoted 9 lines
Quoted 9 lines
io_uring/rw.c index dc1acaf95db1..b1db4595788b renamed from io_uring/rw.c +0
-0100644
io_uring/rw.c +0
-0@@ -271,10 +271,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, size_t pi_len;int ret;
Quoted 8 lines
io_uring/rw.c static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, · line 276
Quoted 7 lines
io_uring/rw.c +3
-4-if (copy_from_user (&__pi_attr,u64_to_user_ptr (attr_ptr), -sizeof (pi_attr))) -return -EFAULT ; - pi_attr = &__pi_attr; +if (attr_type_mask &IORING_RW_ATTR_REGISTERED ) { + pi_attr =io_args_get_ptr (&req->ctx->sqe_args, attr_ptr, +sizeof (pi_attr));Here
pi_attris just pointer, so maybesizeof(__pi_attr)orsizeof(struct io_uring_attr_pi)will be better.
Good catch, and that's an issue from 3/4. As noted it's not even tested and posted to discuss the API. I'd even prefer them to be thrown away, and for Anuj / Kanchan to take over if that's interesting.
Signature · 1 line
Pavel Begunkov