Add support for registered waits
Jens Axboe
maintainerv3 cover Add support for registered waits
Hi,
v3 of the registered wait support, avoiding needless copies for doing high frequency waits. For v1, see the posting here:
https://lore.kernel.org/io-uring/[email protected]/T/#m2d1eb2cc648b9f9c292fd75fc6bc2a8d71eadd49
As with v1, find the kernel repo here:
https://git.kernel.dk/cgit/linux/log/?h=io_uring-reg-wait
and the liburing side here:
https://git.kernel.dk/cgit/liburing/log/?h=reg-wait
include/linux/io_uring_types.h | 10 ++++ include/uapi/linux/io_uring.h | 41 +++++++++++++ io_uring/io_uring.c | 105 ++++++++++++++++++++++++++------- io_uring/register.c | 82 +++++++++++++++++++++++++ io_uring/register.h | 1 + 5 files changed, 217 insertions(+), 22 deletions(-)
Since v2:
- Wrap registration with a setup struct, to better future proof the feature. That would allow doing a registered wait variant with the sigmask embedded as well, should that be interesting in the future.
- Update liburing reg-wait branch with the new registration API
- Fixup copy for 32 vs 64-bit
- Minor cleanups and comments
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v3 1/3 io_uring: switch struct ext_arg from __kernel_timespec to timespec64
This avoids intermediate storage for turning a __kernel_timespec user pointer into an on-stack struct timespec64, only then to turn it into a ktime_t.
--- io_uring/io_uring.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
io_uring/io_uring.c +9 -10
@@ -2495,9 +2495,10 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,struct ext_arg {size_t argsz; -struct __kernel_timespec __user *ts; +struct timespec64 ts;const sigset_t __user *sig;ktime_t min_time; +bool ts_set; };/*
@@ -2535,13 +2536,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, iowq.timeout =KTIME_MAX ; start_time =io_get_time (ctx); -if (ext_arg->ts) { -struct timespec64 ts; - -if (get_timespec64 (&ts, ext_arg->ts)) -return -EFAULT ; - - iowq.timeout =timespec64_to_ktime (ts); +if (ext_arg->ts_set) { + iowq.timeout =timespec64_to_ktime (ext_arg->ts);if (!(flags &IORING_ENTER_ABS_TIMER )) iowq.timeout =ktime_add (iowq.timeout, start_time); }
@@ -3252,7 +3248,6 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,*/ if (!(flags &IORING_ENTER_EXT_ARG )) { ext_arg->sig = (const sigset_t __user *) argp; - ext_arg->ts =NULL ;return 0 ; }
@@ -3267,7 +3262,11 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, ext_arg->min_time = arg.min_wait_usec *NSEC_PER_USEC ; ext_arg->sig =u64_to_user_ptr (arg.sigmask); ext_arg->argsz = arg.sigmask_sz; - ext_arg->ts =u64_to_user_ptr (arg.ts); +if (arg.ts) { +if (get_timespec64 (&ext_arg->ts,u64_to_user_ptr (arg.ts))) +return -EFAULT ; + ext_arg->ts_set = true; + }return 0 ; }
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 2/3 io_uring: change io_get_ext_arg() to use uaccess begin + end
In reply to Jens AxboeIn scenarios where a high frequency of wait events are seen, the copy of the struct io_uring_getevents_arg is quite noticeable in the profiles in terms of time spent. It can be seen as up to 3.5-4.5%. Rewrite the copy-in logic, saving about 0.5% of the time.
--- io_uring/io_uring.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)
io_uring/io_uring.c +17 -1
@@ -3240,6 +3240,7 @@ static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t astatic int io_get_ext_arg (unsigned flags,const void __user *argp,struct ext_arg *ext_arg) { +const struct io_uring_getevents_arg __user *uarg = argp;struct io_uring_getevents_arg arg;/*
@@ -3257,8 +3258,18 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,*/ if (ext_arg->argsz !=sizeof (arg))return -EINVAL ; -if (copy_from_user (&arg, argp,sizeof (arg))) +#ifdef CONFIG_64BIT +if (!user_access_begin (uarg,sizeof (*uarg)))return -EFAULT ; +unsafe_get_user (arg.sigmask, &uarg->sigmask, uaccess_end); +unsafe_get_user (arg.sigmask_sz, &uarg->sigmask_sz, uaccess_end); +unsafe_get_user (arg.min_wait_usec, &uarg->min_wait_usec, uaccess_end); +unsafe_get_user (arg.ts, &uarg->ts, uaccess_end); +user_access_end (); +#else +if (copy_from_user (&arg, uarg,sizeof (arg))) +return -EFAULT ; +#endif ext_arg->min_time = arg.min_wait_usec *NSEC_PER_USEC ; ext_arg->sig =u64_to_user_ptr (arg.sigmask); ext_arg->argsz = arg.sigmask_sz;
@@ -3268,6 +3279,11 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, ext_arg->ts_set = true; }return 0 ; +#ifdef CONFIG_64BIT +uaccess_end: +user_access_end (); +return -EFAULT ; +#endif } SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 3/3 io_uring: add support for fixed wait regions
In reply to Jens AxboeGenerally applications have 1 or a few waits of waiting, yet they pass in a struct io_uring_getevents_arg every time. This needs to get copied and, in turn, the timeout value needs to get copied.
Rather than do this for every invocation, allow the application to register a fixed set of wait regions that can simply be indexed when asking the kernel to wait on events.
At ring setup time, the application can register a number of these wait regions and initialize region/index 0 upfront:
struct io_uring_reg_wait *reg; reg = io_uring_setup_reg_wait(ring, nr_regions, &ret); /* set timeout and mark as set, sigmask/sigmask_sz as needed */ reg->ts.tv_sec = 0; reg->ts.tv_nsec = 100000; reg->flags = IORING_REG_WAIT_TS;
where nr_regions >= 1 && nr_regions <= PAGE_SIZE / sizeof(*reg). The above initializes index 0, but 63 other regions can be initialized, if needed. Now, instead of doing:
struct __kernel_timespec timeout = { .tv_nsec = 100000, };
io_uring_submit_and_wait_timeout(ring, &cqe, nr, &t, NULL);to wait for events for each submit_and_wait, or just wait, operation, it can just reference the above region at offset 0 and do:
io_uring_submit_and_wait_reg(ring, &cqe, nr, 0);
to achieve the same goal of waiting 100usec without needing to copy both struct io_uring_getevents_arg (24b) and struct __kernel_timeout (16b) for each invocation. Struct io_uring_reg_wait looks as follows:
struct io_uring_reg_wait {
struct __kernel_timespec ts;
__u32 min_wait_usec;
__u32 flags;
__u64 sigmask;
__u32 sigmask_sz;
__u32 pad[3];
__u64 pad2[2];
};embedding the timeout itself in the region, rather than passing it as a pointer as well. Note that the signal mask is still passed as a pointer, both for compatability reasons, but also because there doesn't seem to be a lot of high frequency waits scenarios that involve setting and resetting the signal mask for each wait.
The application is free to modify any region before a wait call, or it can use keep multiple regions with different settings to avoid needing to modify the same one for wait calls. Up to a page size of regions is mapped by default, allowing PAGE_SIZE / 64 available regions for use.
The registered region must fit within a page. On a 4kb page size system, that allows for 64 wait regions if a full page is used, as the size of struct io_uring_reg_wait is 64b. The region registered must be aligned to io_uring_reg_wait in size. It's valid to register less than 64 entries.
In network performance testing with zero-copy, this reduced the time spent waiting on the TX side from 3.12% to 0.3% and the RX side from 4.4% to 0.3%.
Wait regions are fixed for the lifetime of the ring - once registered, they are persistent until the ring is torn down. The regions support minimum wait timeout as well as the regular waits.
--- include/linux/io_uring_types.h | 10 +++++ include/uapi/linux/io_uring.h | 41 +++++++++++++++++ io_uring/io_uring.c | 68 +++++++++++++++++++++++----- io_uring/register.c | 82 ++++++++++++++++++++++++++++++++++ io_uring/register.h | 1 + 5 files changed, 191 insertions(+), 11 deletions(-)
include/linux/io_uring_types.h +10 -0
@@ -327,6 +327,14 @@ struct io_ring_ctx {atomic_t cq_wait_nr;atomic_t cq_timeouts;struct wait_queue_head cq_wait; + +/* +* If registered with IORING_REGISTER_CQWAIT_REG, a single +* page holds N entries, mapped in cq_wait_arg. cq_wait_index +* is the maximum allowable index. +*/ +struct io_uring_reg_wait *cq_wait_arg; +unsigned char cq_wait_index; }____cacheline_aligned_in_smp ;/* timeouts */
@@ -430,6 +438,8 @@ struct io_ring_ctx {unsigned short n_sqe_pages;struct page **ring_pages;struct page **sqe_pages; + +struct page **cq_wait_page; };struct io_tw_state {
include/uapi/linux/io_uring.h +41 -0
59 lines not on this page · raw message
io_uring/io_uring.c +57 -11
111 lines not on this page · raw message
io_uring/register.c +82 -0
94 lines not on this page · raw message
io_uring/register.h +1 -0
@@ -5,5 +5,6 @@int io_eventfd_unregister (struct io_ring_ctx *ctx);int io_unregister_personality (struct io_ring_ctx *ctx,unsigned id);struct file *io_uring_register_get_file (unsigned int fd,bool registered); +void io_unregister_cqwait_reg (struct io_ring_ctx *ctx);#endif
Signature · 1 line
2.45.2