Skip to contentSkip to conversation

Add support for registered waits

RFC PATCH Superseded Jens Axboe 2 participants 6 messages Oct 22, 2024 – Oct 23, 2024 io_uring

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

cover Add support for registered waits

Hi,

While doing testing on zero-copy tx/rx with io_uring, I noticed that a LOT of time is being spent copying in data structures related to waiting on events. While the structures aren't big (24b and 16b), it's a user copy, and a dependent user copy on top of that - first get the main struct io_uring_getevents_arg, and then copy in the timeout from that. Details are in patch 3 on the numbers.

I then toyed around with the idea of having registered regions of wait data. This can be set by the application, and then directly seen by the kernel. On top of that, embed the timeout itself in this region, rather than it being supplied as an out-of-band pointer.

Patch 1 is just a generic cleanup, and patch 2 improves the copying a both. Both of these can stand alone. Patch 3 implements the meat of this, which adds IORING_REGISTER_CQWAIT_REG, allowing an application to register a number of struct io_uring_reg_wait regions that it can index for wait scenarios. The kernel always registers a full page, so on 4k page size archs, 64 regions are available by default. Then rather than pass in a pointer to a struct io_uring_getevents_arg, an index is passed instead, telling the kernel which registered wait region should be used for this wait.

This basically removes all of the copying seen, which was anywhere from 3.5-4.5% of the time, when doing high frequency operations where the number of wait invocations can be quite high.

Patches sit on top of the io_uring-ring-resize branch, as both end up adding register opcodes.

Kernel branch here:

https://git.kernel.dk/cgit/linux/log/?h=io_uring-reg-wait

and liburing (pretty basic right now) branch here:

https://git.kernel.dk/cgit/liburing/log/?h=reg-wait

 include/linux/io_uring_types.h |   7 +++
 include/uapi/linux/io_uring.h  |  18 ++++++
 io_uring/io_uring.c            | 102 ++++++++++++++++++++++++++-------
 io_uring/register.c            |  48 ++++++++++++++++
 4 files changed, 153 insertions(+), 22 deletions(-)
Signature · 1 line
Jens Axboe
author maintainer
#1 lore raw

PATCH 1/3 io_uring: switch struct ext_arg from __kernel_timespec to timespec64

This avoids intermediate storage for turning a __kernel_timespec user pointer into an on-stack struct timespec64, only then to turn it into a ktime_t.

---
 io_uring/io_uring.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
io_uring/io_uring.c +9 -10
index b5974bdad48b..8952453ea807 100644
@@ -2494,9 +2494,10 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 
 struct ext_arg {
 	size_t argsz;
-	struct __kernel_timespec __user *ts;
+	struct timespec64 ts;
 	const sigset_t __user *sig;
 	ktime_t min_time;
+	bool ts_set;
 };
 
 /*
@@ -2534,13 +2535,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 	iowq.timeout = KTIME_MAX;
 	start_time = io_get_time(ctx);
 
-	if (ext_arg->ts) {
-		struct timespec64 ts;
-
-		if (get_timespec64(&ts, ext_arg->ts))
-			return -EFAULT;
-
-		iowq.timeout = timespec64_to_ktime(ts);
+	if (ext_arg->ts_set) {
+		iowq.timeout = timespec64_to_ktime(ext_arg->ts);
 		if (!(flags & IORING_ENTER_ABS_TIMER))
 			iowq.timeout = ktime_add(iowq.timeout, start_time);
 	}
@@ -3251,7 +3247,6 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 	 */
 	if (!(flags & IORING_ENTER_EXT_ARG)) {
 		ext_arg->sig = (const sigset_t __user *) argp;
-		ext_arg->ts = NULL;
 		return 0;
 	}
 
@@ -3266,7 +3261,11 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 	ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC;
 	ext_arg->sig = u64_to_user_ptr(arg.sigmask);
 	ext_arg->argsz = arg.sigmask_sz;
-	ext_arg->ts = u64_to_user_ptr(arg.ts);
+	if (arg.ts) {
+		if (get_timespec64(&ext_arg->ts, u64_to_user_ptr(arg.ts)))
+			return -EFAULT;
+		ext_arg->ts_set = true;
+	}
 	return 0;
 }
 
Signature · 1 line
2.45.2
author maintainer
#2 lore raw

PATCH 2/3 io_uring: change io_get_ext_arg() to uaccess begin + end

In reply to Jens Axboe

In scenarios where a high frequency of wait events are seen, the copy of the struct io_uring_getevents_arg is quite noticeable in the profiles in terms of time spent. It can be seen as up to 3.5-4.5%. Rewrite the copy-in logic, saving about 0.5% of the time.

---
 io_uring/io_uring.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
io_uring/io_uring.c +10 -1
index 8952453ea807..612e7d66f845 100644
@@ -3239,6 +3239,7 @@ static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t a
 static int io_get_ext_arg(unsigned flags, const void __user *argp,
 			  struct ext_arg *ext_arg)
 {
+	const struct io_uring_getevents_arg __user *uarg = argp;
 	struct io_uring_getevents_arg arg;
 
 	/*
@@ -3256,8 +3257,13 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 	 */
 	if (ext_arg->argsz != sizeof(arg))
 		return -EINVAL;
-	if (copy_from_user(&arg, argp, sizeof(arg)))
+	if (!user_access_begin(uarg, sizeof(*uarg)))
 		return -EFAULT;
+	unsafe_get_user(arg.sigmask, &uarg->sigmask, uaccess_end);
+	unsafe_get_user(arg.min_wait_usec, &uarg->min_wait_usec, uaccess_end);
+	unsafe_get_user(arg.ts, &uarg->ts, uaccess_end);
+	unsafe_get_user(arg.sigmask_sz, &uarg->sigmask_sz, uaccess_end);
+	user_access_end();
 	ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC;
 	ext_arg->sig = u64_to_user_ptr(arg.sigmask);
 	ext_arg->argsz = arg.sigmask_sz;
@@ -3267,6 +3273,9 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 		ext_arg->ts_set = true;
 	}
 	return 0;
+uaccess_end:
+	user_access_end();
+	return -EFAULT;
 }
 
 SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Signature · 1 line
2.45.2
Reviewed-by on v1 2/3
#3 lore raw

Re: [PATCH 2/3] io_uring: change io_get_ext_arg() to uaccess begin + end

Quoted 4 lines · Jens Axboe

In scenarios where a high frequency of wait events are seen, the copy of the struct io_uring_getevents_arg is quite noticeable in the profiles in terms of time spent. It can be seen as up to 3.5-4.5%. Rewrite the copy-in logic, saving about 0.5% of the time.

I'm surprised it's faster to retrieve field by field instead of a straight copy. But you can't argue with the results!

Quoted 33 lines
---
 io_uring/io_uring.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
io_uring/io_uring.c +7 -1
index 8952453ea807..612e7d66f845 100644
@@ -3239,6 +3239,7 @@ static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t a
 static int io_get_ext_arg(unsigned flags, const void __user *argp,
 			  struct ext_arg *ext_arg)
 {
+	const struct io_uring_getevents_arg __user *uarg = argp;
 	struct io_uring_getevents_arg arg;
 
 	/*
@@ -3256,8 +3257,13 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 	 */
 	if (ext_arg->argsz != sizeof(arg))
 		return -EINVAL;
-	if (copy_from_user(&arg, argp, sizeof(arg)))
+	if (!user_access_begin(uarg, sizeof(*uarg)))
 		return -EFAULT;
+	unsafe_get_user(arg.sigmask, &uarg->sigmask, uaccess_end);
+	unsafe_get_user(arg.min_wait_usec, &uarg->min_wait_usec, uaccess_end);
+	unsafe_get_user(arg.ts, &uarg->ts, uaccess_end);
+	unsafe_get_user(arg.sigmask_sz, &uarg->sigmask_sz, uaccess_end);
+	user_access_end();
 	ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC;
 	ext_arg->sig = u64_to_user_ptr(arg.sigmask);
 	ext_arg->argsz = arg.sigmask_sz;

io_uring/io_uring.c static int io_get_ext_arg(unsigned flags, const void __user *argp, · line 3279

Quoted 7 lines
io_uring/io_uring.c +3 -0
@@ -3267,6 +3273,9 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 		ext_arg->ts_set = true;
 	}
 	return 0;
+uaccess_end:
+	user_access_end();
+	return -EFAULT;
 }

I was looking for the 'goto' for this label and discovered it's in the macro. Neat.

author maintainer
#4 lore raw

Re: [PATCH 2/3] io_uring: change io_get_ext_arg() to uaccess begin + end

Quoted 8 lines · Keith Busch
Quoted 4 lines · Jens Axboe

In scenarios where a high frequency of wait events are seen, the copy of the struct io_uring_getevents_arg is quite noticeable in the profiles in terms of time spent. It can be seen as up to 3.5-4.5%. Rewrite the copy-in logic, saving about 0.5% of the time.

I'm surprised it's faster to retrieve field by field instead of a straight copy. But you can't argue with the results!

It's a pretty common setup - mostly when copying separate entities. But works here too!

Quoted 4 lines

I was looking for the 'goto' for this label and discovered it's in the macro. Neat.

Thanks, will add.

Signature · 1 line
Jens Axboe
author maintainer
#5 lore raw

PATCH 3/3 io_uring: add support for fixed wait regions

In reply to Jens Axboe

Generally applications have 1 or a few waits of waiting, yet they pass in a struct io_uring_getevents_arg every time. This needs to get copied and, in turn, the timeout value needs to get copied.

Rather than do this for every invocation, allow the application to register a fixed set of wait regions that can simply be indexed when asking the kernel to wait on events.

At ring setup time, the application can register a number of these wait regions ala:

	struct io_uring_reg_wait *reg;

	posix_memalign((void **) &reg, page_size, page_size);
	memset(reg, 0, page_size);

	/* set timeout and mark it as set, sigmask/sigmask_sz as needed */
	reg->ts.tv_sec = 0;
	reg->ts.tv_nsec = 100000;
	reg->flags = IORING_REG_WAIT_TS;

	io_uring_register_cqwait_reg(ring, reg, nr_regions);

and instead of doing:

	struct __kernel_timespec timeout = { .tv_nsec = 100000, };

	io_uring_submit_and_wait_timeout(ring, &cqe, nr, &t, NULL);

for each submit_and_wait, or just wait, operation, it can just reference the above region at offset 0 and do:

	io_uring_submit_and_wait_reg(ring, &cqe, nr, 0);

to achieve the same goal of waiting 100usec without needing to copy both struct io_uring_getevents_arg (24b) and struct __kernel_timeout (16b) for each invocation. Struct io_uring_reg_wait looks as follows:

struct io_uring_reg_wait {
	struct __kernel_timespec	ts;
	__u32				min_wait_usec;
	__u32				flags;
	__u64				sigmask;
	__u32				sigmask_sz;
	__u32				pad[3];
	__u64				pad2[2];
};

embedding the timeout itself in the region, rather than passing it as a pointer as well. Note that the signal mask is still passed as a pointer, both for compatability reasons, but also because there doesn't seem to be a lot of high frequency waits scenarios that involve setting and resetting the signal mask for each wait.

The application is free to modify any region before a wait call, or it can use keep multiple regions with different settings to avoid needing to modify the same one for wait calls. Up to a page size of regions is mapped by default, allowing PAGE_SIZE / 64 available regions for use.

In network performance testing with zero-copy, this reduced the time spent waiting on the TX side from 3.12% to 0.3% and the RX side from 4.4% to 0.3%.

Wait regions are fixed for the lifetime of the ring - once registered, they are persistent until the ring is torn down.

---
 include/linux/io_uring_types.h |  7 ++++
 include/uapi/linux/io_uring.h  | 18 +++++++++
 io_uring/io_uring.c            | 72 ++++++++++++++++++++++++++++------
 io_uring/register.c            | 48 +++++++++++++++++++++++
 4 files changed, 134 insertions(+), 11 deletions(-)
include/linux/io_uring_types.h +7 -0
index 6d3ee71bd832..40dc1ec37a42 100644
@@ -327,6 +327,13 @@ struct io_ring_ctx {
 		atomic_t		cq_wait_nr;
 		atomic_t		cq_timeouts;
 		struct wait_queue_head	cq_wait;
+
+		/*
+		 * If registered with IORING_REGISTER_CQWAIT_REG, a single
+		 * page holds N entries, mapped in cq_wait_arg.
+		 */
+		struct page			**cq_wait_page;
+		struct io_uring_reg_wait	*cq_wait_arg;
 	} ____cacheline_aligned_in_smp;
 
 	/* timeouts */
include/uapi/linux/io_uring.h +18 -0
index c4737892c7cd..4ead32fa9275 100644
@@ -518,6 +518,7 @@ struct io_cqring_offsets {
 #define IORING_ENTER_EXT_ARG		(1U << 3)
 #define IORING_ENTER_REGISTERED_RING	(1U << 4)
 #define IORING_ENTER_ABS_TIMER		(1U << 5)
+#define IORING_ENTER_EXT_ARG_REG	(1U << 6)
 
 /*
  * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -618,6 +619,9 @@ enum io_uring_register_op {
 	/* resize CQ ring */
 	IORING_REGISTER_RESIZE_RINGS		= 33,
 
+	/* register fixed io_uring_reg_wait arguments */
+	IORING_REGISTER_CQWAIT_REG		= 34,
+
 	/* this goes last */
 	IORING_REGISTER_LAST,
 
@@ -801,6 +805,20 @@ enum io_uring_register_restriction_op {
 	IORING_RESTRICTION_LAST
 };
 
+enum {
+	IORING_REG_WAIT_TS		= (1U << 0),
+};
+
+struct io_uring_reg_wait {
+	struct __kernel_timespec	ts;
+	__u32				min_wait_usec;
+	__u32				flags;
+	__u64				sigmask;
+	__u32				sigmask_sz;
+	__u32				pad[3];
+	__u64				pad2[2];
+};
+
 struct io_uring_getevents_arg {
 	__u64	sigmask;
 	__u32	sigmask_sz;
io_uring/io_uring.c +61 -11

115 lines not on this page · raw message

io_uring/register.c +48 -0

60 lines not on this page · raw message

Signature · 1 line
2.45.2