Skip to contentSkip to conversation

Add support for batched min timeout

PATCH Superseded Jens Axboe 3 participants 27 messages Aug 19, 2024 – Aug 21, 2024 io_uring

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

v4 cover Add support for batched min timeout

Hi,

After somewhat of a lengthy delay from v2, here's v3 of this patchset. We've now validated this actually makes sense and helps for production workloads where there's a big gap between the peak and slow hours in terms of load. Having a min time enables the same settings to be used for peak hours (guaranteeing minimum response time), while avoiding excessive context switches during lulls.

For a full description, see the v2 posting:

https://lore.kernel.org/io-uring/[email protected]/

As before, there's a liburing branch with added test cases, it can be found here:

https://git.kernel.dk/cgit/liburing/log/?h=min-wait

The patches are on top of for-6.12/io_uring, and with David Wei's new iowait feat and enter flag added to avoid conflicts with that.

Changes since v3:
- Rebase on current 6.12 tree, without the NOWAIT toggle patch. Mostly mechanical, and FEAT flag reordering.

 include/uapi/linux/io_uring.h |   3 +-
 io_uring/io_uring.c           | 186 +++++++++++++++++++++++++---------
 io_uring/io_uring.h           |   4 +
 3 files changed, 146 insertions(+), 47 deletions(-)
Signature · 1 line
Jens Axboe
author maintainer
#1 lore raw

PATCH v4 1/5 io_uring: encapsulate extraneous wait flags into a separate struct

Rather than need to pass in 2 or 3 separate arguments, add a struct to encapsulate the timeout and sigset_t parts of waiting. In preparation for adding another argument for waiting.

---
 io_uring/io_uring.c | 45 ++++++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 21 deletions(-)
io_uring/io_uring.c +24 -21
index 20229e72b65c..37053d32c668 100644
@@ -2384,13 +2384,18 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	return ret;
 }
 
+struct ext_arg {
+	size_t argsz;
+	struct __kernel_timespec __user *ts;
+	const sigset_t __user *sig;
+};
+
 /*
  * Wait until events become available, if we don't already have some. The
  * application must reap them itself, as they reside on the shared cq ring.
  */
 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
-			  const sigset_t __user *sig, size_t sigsz,
-			  struct __kernel_timespec __user *uts)
+			  struct ext_arg *ext_arg)
 {
 	struct io_wait_queue iowq;
 	struct io_rings *rings = ctx->rings;
@@ -2415,10 +2420,10 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 	iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
 	iowq.timeout = KTIME_MAX;
 
-	if (uts) {
+	if (ext_arg->ts) {
 		struct timespec64 ts;
 
-		if (get_timespec64(&ts, uts))
+		if (get_timespec64(&ts, ext_arg->ts))
 			return -EFAULT;
 
 		iowq.timeout = timespec64_to_ktime(ts);
@@ -2426,14 +2431,14 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 			iowq.timeout = ktime_add(iowq.timeout, io_get_time(ctx));
 	}
 
-	if (sig) {
+	if (ext_arg->sig) {
 #ifdef CONFIG_COMPAT
 		if (in_compat_syscall())
-			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
-						      sigsz);
+			ret = set_compat_user_sigmask((const compat_sigset_t __user *)ext_arg->sig,
+						      ext_arg->argsz);
 		else
 #endif
-			ret = set_user_sigmask(sig, sigsz);
+			ret = set_user_sigmask(ext_arg->sig, ext_arg->argsz);
 
 		if (ret)
 			return ret;
@@ -3112,9 +3117,8 @@ static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t a
 	return 0;
 }
 
-static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
-			  struct __kernel_timespec __user **ts,
-			  const sigset_t __user **sig)
+static int io_get_ext_arg(unsigned flags, const void __user *argp,
+			  struct ext_arg *ext_arg)
 {
 	struct io_uring_getevents_arg arg;
 
@@ -3123,8 +3127,8 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz
 	 * is just a pointer to the sigset_t.
 	 */
 	if (!(flags & IORING_ENTER_EXT_ARG)) {
-		*sig = (const sigset_t __user *) argp;
-		*ts = NULL;
+		ext_arg->sig = (const sigset_t __user *) argp;
+		ext_arg->ts = NULL;
 		return 0;
 	}
 
@@ -3132,15 +3136,15 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz
 	 * EXT_ARG is set - ensure we agree on the size of it and copy in our
 	 * timespec and sigset_t pointers if good.
 	 */
-	if (*argsz != sizeof(arg))
+	if (ext_arg->argsz != sizeof(arg))
 		return -EINVAL;
 	if (copy_from_user(&arg, argp, sizeof(arg)))
 		return -EFAULT;
 	if (arg.pad)
 		return -EINVAL;
-	*sig = u64_to_user_ptr(arg.sigmask);
-	*argsz = arg.sigmask_sz;
-	*ts = u64_to_user_ptr(arg.ts);
+	ext_arg->sig = u64_to_user_ptr(arg.sigmask);
+	ext_arg->argsz = arg.sigmask_sz;
+	ext_arg->ts = u64_to_user_ptr(arg.ts);
 	return 0;
 }
 
@@ -3246,15 +3250,14 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 			}
 			mutex_unlock(&ctx->uring_lock);
 		} else {
-			const sigset_t __user *sig;
-			struct __kernel_timespec __user *ts;
+			struct ext_arg ext_arg = { .argsz = argsz };
 
-			ret2 = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
+			ret2 = io_get_ext_arg(flags, argp, &ext_arg);
 			if (likely(!ret2)) {
 				min_complete = min(min_complete,
 						   ctx->cq_entries);
 				ret2 = io_cqring_wait(ctx, min_complete, flags,
-						      sig, argsz, ts);
+						      &ext_arg);
 			}
 		}
 
Signature · 1 line
2.43.0
author maintainer
#2 lore raw

PATCH v4 2/5 io_uring: move schedule wait logic into helper

In reply to Jens Axboe

In preparation for expanding how we handle waits, move the actual schedule and schedule_timeout() handling into a helper.

---
 io_uring/io_uring.c | 37 +++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)
io_uring/io_uring.c +21 -16
index 37053d32c668..9e2b8d4c05db 100644
@@ -2350,22 +2350,10 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 
-/* when returns >0, the caller should retry */
-static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
-					  struct io_wait_queue *iowq)
+static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+				     struct io_wait_queue *iowq)
 {
-	int ret;
-
-	if (unlikely(READ_ONCE(ctx->check_cq)))
-		return 1;
-	if (unlikely(!llist_empty(&ctx->work_llist)))
-		return 1;
-	if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
-		return 1;
-	if (unlikely(task_sigpending(current)))
-		return -EINTR;
-	if (unlikely(io_should_wake(iowq)))
-		return 0;
+	int ret = 0;
 
 	/*
 	 * Mark us as being in io_wait if we have pending requests, so cpufreq
@@ -2374,7 +2362,6 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	 */
 	if (current_pending_io())
 		current->in_iowait = 1;
-	ret = 0;
 	if (iowq->timeout == KTIME_MAX)
 		schedule();
 	else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
@@ -2384,6 +2371,24 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	return ret;
 }
 
+/* If this returns > 0, the caller should retry */
+static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+					  struct io_wait_queue *iowq)
+{
+	if (unlikely(READ_ONCE(ctx->check_cq)))
+		return 1;
+	if (unlikely(!llist_empty(&ctx->work_llist)))
+		return 1;
+	if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
+		return 1;
+	if (unlikely(task_sigpending(current)))
+		return -EINTR;
+	if (unlikely(io_should_wake(iowq)))
+		return 0;
+
+	return __io_cqring_wait_schedule(ctx, iowq);
+}
+
 struct ext_arg {
 	size_t argsz;
 	struct __kernel_timespec __user *ts;
Signature · 1 line
2.43.0
author maintainer
#3 lore raw

PATCH v4 3/5 io_uring: implement our own schedule timeout handling

In reply to Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
 io_uring/io_uring.h |  2 ++
 2 files changed, 38 insertions(+), 5 deletions(-)
io_uring/io_uring.c +36 -5
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
 	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
 	 * the task, and the next invocation will do it.
 	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)
 		return autoremove_wake_function(curr, mode, wake_flags, key);
 	return -1;
 }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);
+	return HRTIMER_NORESTART;
+}
+
+static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
+				      clockid_t clock_id)
+{
+	iowq->hit_timeout = 0;
+	hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
+	iowq->t.function = io_cqring_timer_wakeup;
+	hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
+	hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
+
+	if (!READ_ONCE(iowq->hit_timeout))
+		schedule();
+
+	hrtimer_cancel(&iowq->t);
+	destroy_hrtimer_on_stack(&iowq->t);
+	__set_current_state(TASK_RUNNING);
+
+	return READ_ONCE(iowq->hit_timeout) ? -ETIME : 0;
+}
+
 static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 				     struct io_wait_queue *iowq)
 {
@@ -2362,11 +2394,10 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	 */
 	if (current_pending_io())
 		current->in_iowait = 1;
-	if (iowq->timeout == KTIME_MAX)
+	if (iowq->timeout != KTIME_MAX)
+		ret = io_cqring_schedule_timeout(iowq, ctx->clockid);
+	else
 		schedule();
-	else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
-						 HRTIMER_MODE_ABS, ctx->clockid))
-		ret = -ETIME;
 	current->in_iowait = 0;
 	return ret;
 }
io_uring/io_uring.h +2 -0
index 9935819f12b7..f95c1b080f4b 100644
@@ -40,7 +40,9 @@ struct io_wait_queue {
 	struct io_ring_ctx *ctx;
 	unsigned cq_tail;
 	unsigned nr_timeouts;
+	int hit_timeout;
 	ktime_t timeout;
+	struct hrtimer t;
 
 #ifdef CONFIG_NET_RX_BUSY_POLL
 	ktime_t napi_busy_poll_dt;
Signature · 1 line
2.43.0
#4 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 14 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
 io_uring/io_uring.h |  2 ++
 2 files changed, 38 insertions(+), 5 deletions(-)
io_uring/io_uring.c +0 -0
index 9e2b8d4c05db..ddfbe04c61ed 100644

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 5 lines
io_uring/io_uring.c +1 -1
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
 	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
 	 * the task, and the next invocation will do it.
 	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Quoted 7 lines
io_uring/io_uring.c +0 -0
 		return autoremove_wake_function(curr, mode, wake_flags, key);
 	return -1;
 }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

Quoted 54 lines
io_uring/io_uring.c +25 -4
+	return HRTIMER_NORESTART;
+}
+
+static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
+				      clockid_t clock_id)
+{
+	iowq->hit_timeout = 0;
+	hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
+	iowq->t.function = io_cqring_timer_wakeup;
+	hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
+	hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
+
+	if (!READ_ONCE(iowq->hit_timeout))
+		schedule();
+
+	hrtimer_cancel(&iowq->t);
+	destroy_hrtimer_on_stack(&iowq->t);
+	__set_current_state(TASK_RUNNING);
+
+	return READ_ONCE(iowq->hit_timeout) ? -ETIME : 0;
+}
+
 static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 				     struct io_wait_queue *iowq)
 {
@@ -2362,11 +2394,10 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	 */
 	if (current_pending_io())
 		current->in_iowait = 1;
-	if (iowq->timeout == KTIME_MAX)
+	if (iowq->timeout != KTIME_MAX)
+		ret = io_cqring_schedule_timeout(iowq, ctx->clockid);
+	else
 		schedule();
-	else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
-						 HRTIMER_MODE_ABS, ctx->clockid))
-		ret = -ETIME;
 	current->in_iowait = 0;
 	return ret;
 }
io_uring/io_uring.h +2 -0
index 9935819f12b7..f95c1b080f4b 100644
@@ -40,7 +40,9 @@ struct io_wait_queue {
 	struct io_ring_ctx *ctx;
 	unsigned cq_tail;
 	unsigned nr_timeouts;
+	int hit_timeout;
 	ktime_t timeout;
+	struct hrtimer t;
 
 #ifdef CONFIG_NET_RX_BUSY_POLL
 	ktime_t napi_busy_poll_dt;
author maintainer
#5 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
 io_uring/io_uring.h |  2 ++
 2 files changed, 38 insertions(+), 5 deletions(-)
Quoted 14 lines

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 10 lines
io_uring/io_uring.c +1 -1
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
 	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
 	 * the task, and the next invocation will do it.
 	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Yes probably not a bad idea to make it READ_ONCE().

Quoted 7 lines
Quoted 7 lines
io_uring/io_uring.c +0 -0
 		return autoremove_wake_function(curr, mode, wake_flags, key);
 	return -1;
 }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 
Quoted 13 lines

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

That's how the wakeups work - for defer taskrun, the task isn't on a waitqueue at all. Hence we need to wake the task itself. For any other setup, they will be on the waitqueue, and we just call io_cqring_wake() to wake up anyone waiting on the waitqueue. That will iterate the wake queue and call handlers for each item. Having a separate handler for that will allow to NOT wake up the task if we don't need to. taskrun, the waker

Signature · 1 line
Jens Axboe
#6 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 49 lines · Jens Axboe
Quoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
 io_uring/io_uring.h |  2 ++
 2 files changed, 38 insertions(+), 5 deletions(-)
Quoted 14 lines

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 10 lines
io_uring/io_uring.c +1 -1
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
 	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
 	 * the task, and the next invocation will do it.
 	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Yes probably not a bad idea to make it READ_ONCE().

Quoted 7 lines
Quoted 7 lines
io_uring/io_uring.c +0 -0
 		return autoremove_wake_function(curr, mode, wake_flags, key);
 	return -1;
 }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 
Quoted 13 lines

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

Quoted 7 lines

That's how the wakeups work - for defer taskrun, the task isn't on a waitqueue at all. Hence we need to wake the task itself. For any other setup, they will be on the waitqueue, and we just call io_cqring_wake() to wake up anyone waiting on the waitqueue. That will iterate the wake queue and call handlers for each item. Having a separate handler for that will allow to NOT wake up the task if we don't need to. taskrun, the waker

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

author maintainer
#7 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 58 lines · David Wei
Quoted 49 lines · Jens Axboe
Quoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
 io_uring/io_uring.h |  2 ++
 2 files changed, 38 insertions(+), 5 deletions(-)
Quoted 14 lines

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 10 lines
io_uring/io_uring.c +1 -1
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
 	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
 	 * the task, and the next invocation will do it.
 	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Yes probably not a bad idea to make it READ_ONCE().

Quoted 7 lines
Quoted 7 lines
io_uring/io_uring.c +0 -0
 		return autoremove_wake_function(curr, mode, wake_flags, key);
 	return -1;
 }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 
Quoted 13 lines

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

Quoted 7 lines

That's how the wakeups work - for defer taskrun, the task isn't on a waitqueue at all. Hence we need to wake the task itself. For any other setup, they will be on the waitqueue, and we just call io_cqring_wake() to wake up anyone waiting on the waitqueue. That will iterate the wake queue and call handlers for each item. Having a separate handler for that will allow to NOT wake up the task if we don't need to. taskrun, the waker

Quoted 3 lines

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

Signature · 1 line
Jens Axboe
#8 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 63 lines · Jens Axboe
Quoted 58 lines · David Wei
Quoted 49 lines · Jens Axboe
Quoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
  io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
  io_uring/io_uring.h |  2 ++
  2 files changed, 38 insertions(+), 5 deletions(-)
Quoted 14 lines

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 10 lines
io_uring/io_uring.c +1 -1
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
  	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
  	 * the task, and the next invocation will do it.
  	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Yes probably not a bad idea to make it READ_ONCE().

Quoted 7 lines
Quoted 7 lines
io_uring/io_uring.c +0 -0
  		return autoremove_wake_function(curr, mode, wake_flags, key);
  	return -1;
  }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
  	return percpu_counter_read_positive(&tctx->inflight);
  }
 
Quoted 13 lines

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

Quoted 7 lines

That's how the wakeups work - for defer taskrun, the task isn't on a waitqueue at all. Hence we need to wake the task itself. For any other setup, they will be on the waitqueue, and we just call io_cqring_wake() to wake up anyone waiting on the waitqueue. That will iterate the wake queue and call handlers for each item. Having a separate handler for that will allow to NOT wake up the task if we don't need to. taskrun, the waker

Quoted 3 lines

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Quoted 5 lines

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

Answering the question, for !IORING_SETUP_DEFER_TASKRUN, before it was waking only the task that started the timeout. Now, io_cqring_wake(ctx) wakes all waiters, so if there are multiple tasks waiting, a timeout of one waiter will try to wake all of them.

I believe it's unintentional, but I don't think we care either. You can save the waiter task struct and wake it specifically.

Signature · 1 line
Pavel Begunkov
#9 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

In reply to Jens Axboe
Quoted 63 lines · Jens Axboe
Quoted 58 lines · David Wei
Quoted 49 lines · Jens Axboe
Quoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
 io_uring/io_uring.h |  2 ++
 2 files changed, 38 insertions(+), 5 deletions(-)
Quoted 14 lines

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 10 lines
io_uring/io_uring.c +1 -1
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
 	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
 	 * the task, and the next invocation will do it.
 	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Yes probably not a bad idea to make it READ_ONCE().

Quoted 7 lines
Quoted 7 lines
io_uring/io_uring.c +0 -0
 		return autoremove_wake_function(curr, mode, wake_flags, key);
 	return -1;
 }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
 	return percpu_counter_read_positive(&tctx->inflight);
 }
 
Quoted 13 lines

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

Quoted 7 lines

That's how the wakeups work - for defer taskrun, the task isn't on a waitqueue at all. Hence we need to wake the task itself. For any other setup, they will be on the waitqueue, and we just call io_cqring_wake() to wake up anyone waiting on the waitqueue. That will iterate the wake queue and call handlers for each item. Having a separate handler for that will allow to NOT wake up the task if we don't need to. taskrun, the waker

Quoted 3 lines

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Quoted 6 lines

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

io_cqring_timer_wakeup() is the timer expired callback which calls wake_up_process() or io_cqring_wake() depending on DEFER_TASKRUN.

The original code calling schedule_hrtimeout_range_clock() uses hrtimer_sleeper instead, which has a default timer expired callback set to hrtimer_wakeup().

hrtimer_wakeup() only calls wake_up_process().

My question is: why this asymmetry? Why does the new custom callback require io_cqring_wake()?

#10 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 74 lines · David Wei
Quoted 63 lines · Jens Axboe
Quoted 58 lines · David Wei
Quoted 49 lines · Jens Axboe
Quoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe

In preparation for having two distinct timeouts and avoid waking the task if we don't need to.

---
  io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++-----
  io_uring/io_uring.h |  2 ++
  2 files changed, 38 insertions(+), 5 deletions(-)
Quoted 14 lines

io_uring/io_uring.c static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · line 2325

Quoted 10 lines
io_uring/io_uring.c +1 -1
index 9e2b8d4c05db..ddfbe04c61ed 100644
@@ -2322,7 +2322,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
  	 * Cannot safely flush overflowed CQEs from here, ensure we wake up
  	 * the task, and the next invocation will do it.
  	 */
-	if (io_should_wake(iowq) || io_has_work(iowq->ctx))
+	if (io_should_wake(iowq) || io_has_work(iowq->ctx) || iowq->hit_timeout)

iowq->hit_timeout may be modified in a timer softirq context, while this wait_queue_func_t (AIUI) may get called from any context e.g. net_rx_softirq for sockets. Does this need a READ_ONLY()?

Yes probably not a bad idea to make it READ_ONCE().

Quoted 7 lines
Quoted 7 lines
io_uring/io_uring.c +0 -0
  		return autoremove_wake_function(curr, mode, wake_flags, key);
  	return -1;
  }
@@ -2350,6 +2350,38 @@ static bool current_pending_io(void)
  	return percpu_counter_read_positive(&tctx->inflight);
  }
 
Quoted 13 lines

io_uring/io_uring.c static bool current_pending_io(void) · line 2362

Quoted 10 lines
io_uring/io_uring.c +10 -0
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	WRITE_ONCE(iowq->hit_timeout, 1);
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		wake_up_process(ctx->submitter_task);
+	else
+		io_cqring_wake(ctx);

This is a bit different to schedule_hrtimeout_range_clock(). Why is io_cqring_wake() needed here for non-DEFER_TASKRUN?

Quoted 7 lines

That's how the wakeups work - for defer taskrun, the task isn't on a waitqueue at all. Hence we need to wake the task itself. For any other setup, they will be on the waitqueue, and we just call io_cqring_wake() to wake up anyone waiting on the waitqueue. That will iterate the wake queue and call handlers for each item. Having a separate handler for that will allow to NOT wake up the task if we don't need to. taskrun, the waker

Quoted 3 lines

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Quoted 6 lines

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

io_cqring_timer_wakeup() is the timer expired callback which calls wake_up_process() or io_cqring_wake() depending on DEFER_TASKRUN.

Quoted 8 lines

The original code calling schedule_hrtimeout_range_clock() uses hrtimer_sleeper instead, which has a default timer expired callback set to hrtimer_wakeup().

hrtimer_wakeup() only calls wake_up_process().

My question is: why this asymmetry? Why does the new custom callback require io_cqring_wake()?

That's what I'm saying, it doesn't need and doesn't really want it. From the correctness point of view, it's ok since we wake up a (unnecessarily) larger set of tasks.

Signature · 1 line
Pavel Begunkov
#11 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 19 lines · Pavel Begunkov
Quoted 19 lines
Quoted 11 lines
Quoted 4 lines
Quoted 1 line · David Wei

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

io_cqring_timer_wakeup() is the timer expired callback which calls wake_up_process() or io_cqring_wake() depending on DEFER_TASKRUN.

The original code calling schedule_hrtimeout_range_clock() uses hrtimer_sleeper instead, which has a default timer expired callback set to hrtimer_wakeup().

Quoted 9 lines
Quoted 4 lines

hrtimer_wakeup() only calls wake_up_process().

My question is: why this asymmetry? Why does the new custom callback require io_cqring_wake()?

That's what I'm saying, it doesn't need and doesn't really want it. From the correctness point of view, it's ok since we wake up a (unnecessarily) larger set of tasks.

Yeah your explanation that came in while I was writing the email answered it, thanks Pavel.

author maintainer
#12 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 25 lines · David Wei
Quoted 24 lines · Pavel Begunkov
Quoted 23 lines
Quoted 11 lines
Quoted 4 lines
Quoted 1 line · David Wei

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

io_cqring_timer_wakeup() is the timer expired callback which calls wake_up_process() or io_cqring_wake() depending on DEFER_TASKRUN.

The original code calling schedule_hrtimeout_range_clock() uses hrtimer_sleeper instead, which has a default timer expired callback set to hrtimer_wakeup().

hrtimer_wakeup() only calls wake_up_process().

My question is: why this asymmetry? Why does the new custom callback require io_cqring_wake()?

Quoted 7 lines
Quoted 4 lines

That's what I'm saying, it doesn't need and doesn't really want it. From the correctness point of view, it's ok since we wake up a (unnecessarily) larger set of tasks.

Yeah your explanation that came in while I was writing the email answered it, thanks Pavel.

Hah and now I see what you meant - yeah we can just remove the distinction. I didn't see anything in testing, but I also didn't have multiple tasks waiting on a ring, nor would you. So it doesn't really matter, but I'll clean it up so there's no confusion.

Signature · 1 line
Jens Axboe
author maintainer
#13 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 31 lines · Jens Axboe
Quoted 30 lines · David Wei
Quoted 28 lines · Pavel Begunkov
Quoted 15 lines
Quoted 11 lines
Quoted 4 lines
Quoted 1 line · David Wei

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

io_cqring_timer_wakeup() is the timer expired callback which calls wake_up_process() or io_cqring_wake() depending on DEFER_TASKRUN.

Quoted 8 lines

The original code calling schedule_hrtimeout_range_clock() uses hrtimer_sleeper instead, which has a default timer expired callback set to hrtimer_wakeup().

hrtimer_wakeup() only calls wake_up_process().

My question is: why this asymmetry? Why does the new custom callback require io_cqring_wake()?

That's what I'm saying, it doesn't need and doesn't really want it. From the correctness point of view, it's ok since we wake up a (unnecessarily) larger set of tasks.

Quoted 7 lines
Quoted 2 lines

Yeah your explanation that came in while I was writing the email answered it, thanks Pavel.

Hah and now I see what you meant - yeah we can just remove the distinction. I didn't see anything in testing, but I also didn't have multiple tasks waiting on a ring, nor would you. So it doesn't really matter, but I'll clean it up so there's no confusion.

Actually probably better to just leave it as-is, as we'd otherwise need to store a task in io_wait_queue. Which we of course could, and would remove a branch in there...

Signature · 1 line
Jens Axboe
author maintainer
#14 lore raw

Re: [PATCH 3/5] io_uring: implement our own schedule timeout handling

Quoted 35 lines · Jens Axboe
Quoted 34 lines · Jens Axboe
Quoted 32 lines · David Wei
Quoted 19 lines · Pavel Begunkov
Quoted 19 lines
Quoted 11 lines
Quoted 4 lines
Quoted 1 line · David Wei

To rephase the question, why is the original code calling schedule_hrtimeout_range_clock() not needing to differentiate behaviour between defer taskrun and not?

Because that part is the same, the task schedules out and goes to sleep. That has always been the same regardless of how the ring is setup. Only difference is that DEFER_TASKRUN doesn't add itself to ctx->wait, and hence cannot be woken by a wake_up(ctx->wait). We have to wake the task manually.

io_cqring_timer_wakeup() is the timer expired callback which calls wake_up_process() or io_cqring_wake() depending on DEFER_TASKRUN.

The original code calling schedule_hrtimeout_range_clock() uses hrtimer_sleeper instead, which has a default timer expired callback set to hrtimer_wakeup().

Quoted 9 lines
Quoted 4 lines

hrtimer_wakeup() only calls wake_up_process().

My question is: why this asymmetry? Why does the new custom callback require io_cqring_wake()?

That's what I'm saying, it doesn't need and doesn't really want it. From the correctness point of view, it's ok since we wake up a (unnecessarily) larger set of tasks.

Yeah your explanation that came in while I was writing the email answered it, thanks Pavel.

Quoted 8 lines
Quoted 4 lines

Hah and now I see what you meant - yeah we can just remove the distinction. I didn't see anything in testing, but I also didn't have multiple tasks waiting on a ring, nor would you. So it doesn't really matter, but I'll clean it up so there's no confusion.

Actually probably better to just leave it as-is, as we'd otherwise need to store a task in io_wait_queue. Which we of course could, and would remove a branch in there...

I guess I should actually look at the code first, we have it via wq->private already. Hence:

io_uring/io_uring.c +1 -5
index ddfbe04c61ed..4ba5292137c3 100644
@@ -2353,13 +2353,9 @@ static bool current_pending_io(void)
 static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
 {
 	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
-	struct io_ring_ctx *ctx = iowq->ctx;
 
 	WRITE_ONCE(iowq->hit_timeout, 1);
-	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
-		wake_up_process(ctx->submitter_task);
-	else
-		io_cqring_wake(ctx);
+	wake_up_process(iowq->wq.private);
 	return HRTIMER_NORESTART;
 }
 
Signature · 1 line
Jens Axboe
author maintainer
#15 lore raw

PATCH v4 4/5 io_uring: add support for batch wait timeout

In reply to Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

---
 io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------
 io_uring/io_uring.h |  2 ++
 2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +65 -10
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
 	return HRTIMER_NORESTART;
 }
 
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	/* no general timeout, or shorter, we are done */
+	if (iowq->timeout == KTIME_MAX ||
+	    ktime_after(iowq->min_timeout, iowq->timeout))
+		goto out_wake;
+	/* work we may need to run, wake function will see if we need to wake */
+	if (io_has_work(ctx))
+		goto out_wake;
+	/* got events since we started waiting, min timeout is done */
+	if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+		goto out_wake;
+	/* if we have any events and min timeout expired, we're done */
+	if (io_cqring_events(ctx))
+		goto out_wake;
+
+	/*
+	 * If using deferred task_work running and application is waiting on
+	 * more than one request, ensure we reset it now where we are switching
+	 * to normal sleeps. Any request completion post min_wait should wake
+	 * the task and return.
+	 */
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		atomic_set(&ctx->cq_wait_nr, 1);
+
+	iowq->t.function = io_cqring_timer_wakeup;
+	hrtimer_set_expires(timer, iowq->timeout);
+	return HRTIMER_RESTART;
+out_wake:
+	return io_cqring_timer_wakeup(timer);
+}
+
 static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
-				      clockid_t clock_id)
+				      clockid_t clock_id, ktime_t start_time)
 {
+	ktime_t timeout;
+
 	iowq->hit_timeout = 0;
 	hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
-	iowq->t.function = io_cqring_timer_wakeup;
-	hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
+	if (iowq->min_timeout) {
+		timeout = ktime_add_ns(iowq->min_timeout, start_time);
+		iowq->t.function = io_cqring_min_timer_wakeup;
+	} else {
+		timeout = iowq->timeout;
+		iowq->t.function = io_cqring_timer_wakeup;
+	}
+
+	hrtimer_set_expires_range_ns(&iowq->t, timeout, 0);
 	hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
 
 	if (!READ_ONCE(iowq->hit_timeout))
@@ -2383,7 +2432,8 @@ static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
 }
 
 static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
-				     struct io_wait_queue *iowq)
+				     struct io_wait_queue *iowq,
+				     ktime_t start_time)
 {
 	int ret = 0;
 
@@ -2394,8 +2444,8 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	 */
 	if (current_pending_io())
 		current->in_iowait = 1;
-	if (iowq->timeout != KTIME_MAX)
-		ret = io_cqring_schedule_timeout(iowq, ctx->clockid);
+	if (iowq->timeout != KTIME_MAX || iowq->min_timeout != KTIME_MAX)
+		ret = io_cqring_schedule_timeout(iowq, ctx->clockid, start_time);
 	else
 		schedule();
 	current->in_iowait = 0;
@@ -2404,7 +2454,8 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 
 /* If this returns > 0, the caller should retry */
 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
-					  struct io_wait_queue *iowq)
+					  struct io_wait_queue *iowq,
+					  ktime_t start_time)
 {
 	if (unlikely(READ_ONCE(ctx->check_cq)))
 		return 1;
@@ -2417,7 +2468,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
 	if (unlikely(io_should_wake(iowq)))
 		return 0;
 
-	return __io_cqring_wait_schedule(ctx, iowq);
+	return __io_cqring_wait_schedule(ctx, iowq, start_time);
 }
 
 struct ext_arg {
@@ -2435,6 +2486,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 {
 	struct io_wait_queue iowq;
 	struct io_rings *rings = ctx->rings;
+	ktime_t start_time;
 	int ret;
 
 	if (!io_allowed_run_tw(ctx))
@@ -2453,8 +2505,11 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 	INIT_LIST_HEAD(&iowq.wq.entry);
 	iowq.ctx = ctx;
 	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
+	iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail);
 	iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
+	iowq.min_timeout = 0;
 	iowq.timeout = KTIME_MAX;
+	start_time = io_get_time(ctx);
 
 	if (ext_arg->ts) {
 		struct timespec64 ts;
@@ -2464,7 +2519,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 
 		iowq.timeout = timespec64_to_ktime(ts);
 		if (!(flags & IORING_ENTER_ABS_TIMER))
-			iowq.timeout = ktime_add(iowq.timeout, io_get_time(ctx));
+			iowq.timeout = ktime_add(iowq.timeout, start_time);
 	}
 
 	if (ext_arg->sig) {
@@ -2495,7 +2550,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 							TASK_INTERRUPTIBLE);
 		}
 
-		ret = io_cqring_wait_schedule(ctx, &iowq);
+		ret = io_cqring_wait_schedule(ctx, &iowq, start_time);
 		__set_current_state(TASK_RUNNING);
 		atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
 
io_uring/io_uring.h +2 -0
index f95c1b080f4b..65078e641390 100644
@@ -39,8 +39,10 @@ struct io_wait_queue {
 	struct wait_queue_entry wq;
 	struct io_ring_ctx *ctx;
 	unsigned cq_tail;
+	unsigned cq_min_tail;
 	unsigned nr_timeouts;
 	int hit_timeout;
+	ktime_t min_timeout;
 	ktime_t timeout;
 	struct hrtimer t;
 
Signature · 1 line
2.43.0
#16 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 20 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

Quoted 5 lines

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

I think the batch request count should be applied to the min_timeout, such that:

start_time min_timeout timeout

    |--------------------|--------------------|

Return to user between [start_time, min_timeout) if there are wait_nr number of completions, checked by io_req_local_work_add(), or is it io_wake_function()?

Return to user between [min_timeout, timeout) if there are at least one completion.

Return to user at timeout always.

Quoted 28 lines
---
 io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------
 io_uring/io_uring.h |  2 ++
 2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +13 -0
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
 	return HRTIMER_NORESTART;
 }
 
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	/* no general timeout, or shorter, we are done */
+	if (iowq->timeout == KTIME_MAX ||
+	    ktime_after(iowq->min_timeout, iowq->timeout))

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2388

Quoted 10 lines
io_uring/io_uring.c +10 -0
+		goto out_wake;
+	/* work we may need to run, wake function will see if we need to wake */
+	if (io_has_work(ctx))
+		goto out_wake;
+	/* got events since we started waiting, min timeout is done */
+	if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+		goto out_wake;
+	/* if we have any events and min timeout expired, we're done */
+	if (io_cqring_events(ctx))
+		goto out_wake;

How can ctx->rings->cq.tail be modified if the task is sleeping while waiting for completions? What is doing the work?

Quoted 39 lines
io_uring/io_uring.c +29 -3
+
+	/*
+	 * If using deferred task_work running and application is waiting on
+	 * more than one request, ensure we reset it now where we are switching
+	 * to normal sleeps. Any request completion post min_wait should wake
+	 * the task and return.
+	 */
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		atomic_set(&ctx->cq_wait_nr, 1);
+
+	iowq->t.function = io_cqring_timer_wakeup;
+	hrtimer_set_expires(timer, iowq->timeout);
+	return HRTIMER_RESTART;
+out_wake:
+	return io_cqring_timer_wakeup(timer);
+}
+
 static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
-				      clockid_t clock_id)
+				      clockid_t clock_id, ktime_t start_time)
 {
+	ktime_t timeout;
+
 	iowq->hit_timeout = 0;
 	hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
-	iowq->t.function = io_cqring_timer_wakeup;
-	hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
+	if (iowq->min_timeout) {
+		timeout = ktime_add_ns(iowq->min_timeout, start_time);
+		iowq->t.function = io_cqring_min_timer_wakeup;
+	} else {
+		timeout = iowq->timeout;
+		iowq->t.function = io_cqring_timer_wakeup;
+	}
+
+	hrtimer_set_expires_range_ns(&iowq->t, timeout, 0);
 	hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
 
 	if (!READ_ONCE(iowq->hit_timeout))
author maintainer
#17 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 30 lines · David Wei
Quoted 20 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

Quoted 5 lines

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

I think the batch request count should be applied to the min_timeout, such that:

Quoted 6 lines

start_time min_timeout timeout

    |--------------------|--------------------|

Return to user between [start_time, min_timeout) if there are wait_nr number of completions, checked by io_req_local_work_add(), or is it io_wake_function()?

Right, if we get the batch fulfilled, we should ALWAYS return.

If we have any events and min_timeout expires, return.

If not, sleep the full timeout.

Quoted 2 lines

Return to user between [min_timeout, timeout) if there are at least one completion.

Yes

Quoted 1 line

Return to user at timeout always.

Yes

This should be how it works, and how I described it in the commit message.

Signature · 1 line
Jens Axboe
#18 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 47 lines · Jens Axboe
Quoted 30 lines · David Wei
Quoted 20 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

Quoted 5 lines

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

I think the batch request count should be applied to the min_timeout, such that:

Quoted 6 lines

start_time min_timeout timeout

    |--------------------|--------------------|

Return to user between [start_time, min_timeout) if there are wait_nr number of completions, checked by io_req_local_work_add(), or is it io_wake_function()?

Right, if we get the batch fulfilled, we should ALWAYS return.

If we have any events and min_timeout expires, return.

If not, sleep the full timeout.

Quoted 2 lines

Return to user between [min_timeout, timeout) if there are at least one completion.

Quoted 9 lines

Yes

Quoted 1 line

Return to user at timeout always.

Yes

This should be how it works, and how I described it in the commit message.

You're right, thanks. With DEFER_TASKRUN, the wakeup either happens in the timer expired callback io_cqring_min_timer_wakeup(), or in io_req_local_work_add().

In both cases control returns to after schedule() in io_cqring_schedule_timeout() and the timer is cancelled.

Is it possible for the two to race at all?

author maintainer
#19 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

In reply to David Wei
Quoted 18 lines · David Wei
Quoted 18 lines
io_uring/io_uring.c +18 -0
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	/* no general timeout, or shorter, we are done */
+	if (iowq->timeout == KTIME_MAX ||
+	    ktime_after(iowq->min_timeout, iowq->timeout))
+		goto out_wake;
+	/* work we may need to run, wake function will see if we need to wake */
+	if (io_has_work(ctx))
+		goto out_wake;
+	/* got events since we started waiting, min timeout is done */
Quoted 8 lines

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2388

Quoted 5 lines
io_uring/io_uring.c +5 -0
+	if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+		goto out_wake;
+	/* if we have any events and min timeout expired, we're done */
+	if (io_cqring_events(ctx))
+		goto out_wake;

How can ctx->rings->cq.tail be modified if the task is sleeping while waiting for completions? What is doing the work?

Good question. If we have a min_timeout of <something> and a batch count of <something>, ideally we don't want to wake the task to process when a single completion comes in. And this is how we handle DEFER_TASKRUN, but for anything else, the task will wake and process items. So it may have woken up to process an item and posted a completion before this timeout triggers. If that's the case, and min_timeout has expired (which it has when this handler is called), then we should wake up and return.

Signature · 1 line
Jens Axboe
#20 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 28 lines · Jens Axboe
Quoted 18 lines · David Wei
Quoted 18 lines
io_uring/io_uring.c +18 -0
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	/* no general timeout, or shorter, we are done */
+	if (iowq->timeout == KTIME_MAX ||
+	    ktime_after(iowq->min_timeout, iowq->timeout))
+		goto out_wake;
+	/* work we may need to run, wake function will see if we need to wake */
+	if (io_has_work(ctx))
+		goto out_wake;
+	/* got events since we started waiting, min timeout is done */
Quoted 8 lines

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2388

Quoted 5 lines
io_uring/io_uring.c +5 -0
+	if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+		goto out_wake;
+	/* if we have any events and min timeout expired, we're done */
+	if (io_cqring_events(ctx))
+		goto out_wake;

How can ctx->rings->cq.tail be modified if the task is sleeping while waiting for completions? What is doing the work?

Quoted 7 lines

Good question. If we have a min_timeout of <something> and a batch count of <something>, ideally we don't want to wake the task to process when a single completion comes in. And this is how we handle DEFER_TASKRUN, but for anything else, the task will wake and process items. So it may have woken up to process an item and posted a completion before this timeout triggers. If that's the case, and min_timeout has expired (which it has when this handler is called), then we should wake up and return.

Also, for !DEFER_TASKRUN, it can be iowq or another user thread sharing the ring.

Signature · 1 line
Pavel Begunkov
#21 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

In reply to Jens Axboe
Quoted 62 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

---
  io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------
  io_uring/io_uring.h |  2 ++
  2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +22 -0
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
  	return HRTIMER_NORESTART;
  }
 
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+	struct io_ring_ctx *ctx = iowq->ctx;
+
+	/* no general timeout, or shorter, we are done */
+	if (iowq->timeout == KTIME_MAX ||
+	    ktime_after(iowq->min_timeout, iowq->timeout))
+		goto out_wake;
+	/* work we may need to run, wake function will see if we need to wake */
+	if (io_has_work(ctx))
+		goto out_wake;
+	/* got events since we started waiting, min timeout is done */
+	if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+		goto out_wake;
+	/* if we have any events and min timeout expired, we're done */
+	if (io_cqring_events(ctx))

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2397

Quoted 10 lines
io_uring/io_uring.c +10 -0
+		goto out_wake;
+
+	/*
+	 * If using deferred task_work running and application is waiting on
+	 * more than one request, ensure we reset it now where we are switching
+	 * to normal sleeps. Any request completion post min_wait should wake
+	 * the task and return.
+	 */
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+		atomic_set(&ctx->cq_wait_nr, 1);

racy

atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
if (llist_empty(&ctx->work_llist))

	// wake;
Quoted 8 lines
io_uring/io_uring.c +8 -0
+
+	iowq->t.function = io_cqring_timer_wakeup;
+	hrtimer_set_expires(timer, iowq->timeout);
+	return HRTIMER_RESTART;
+out_wake:
+	return io_cqring_timer_wakeup(timer);
+}
+
Signature · 1 line
Pavel Begunkov
#22 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 75 lines · Pavel Begunkov
Quoted 61 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

---
  io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------ io_uring/io_uring.h |  2 ++

  2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +22 -0
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
       return HRTIMER_NORESTART;
   }
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+    struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+    struct io_ring_ctx *ctx = iowq->ctx;
+
+    /* no general timeout, or shorter, we are done */
+    if (iowq->timeout == KTIME_MAX ||
+        ktime_after(iowq->min_timeout, iowq->timeout))
+        goto out_wake;
+    /* work we may need to run, wake function will see if we need to wake */
+    if (io_has_work(ctx))
+        goto out_wake;
+    /* got events since we started waiting, min timeout is done */
+    if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+        goto out_wake;
+    /* if we have any events and min timeout expired, we're done */
+    if (io_cqring_events(ctx))

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2396

Quoted 10 lines
io_uring/io_uring.c +10 -0
+        goto out_wake;
+
+    /*
+     * If using deferred task_work running and application is waiting on
+     * more than one request, ensure we reset it now where we are switching
+     * to normal sleeps. Any request completion post min_wait should wake
+     * the task and return.
+     */
+    if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+        atomic_set(&ctx->cq_wait_nr, 1);

racy

Quoted 4 lines

atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
if (llist_empty(&ctx->work_llist))
    // wake;

rather if _not_ empty

Quoted 8 lines
Quoted 8 lines
io_uring/io_uring.c +8 -0
+
+    iowq->t.function = io_cqring_timer_wakeup;
+    hrtimer_set_expires(timer, iowq->timeout);
+    return HRTIMER_RESTART;
+out_wake:
+    return io_cqring_timer_wakeup(timer);
+}
+
Signature · 1 line
Pavel Begunkov
author maintainer
#23 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 76 lines · Pavel Begunkov
Quoted 75 lines · Pavel Begunkov
Quoted 61 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

---
  io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------ io_uring/io_uring.h |  2 ++

  2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +22 -0
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
       return HRTIMER_NORESTART;
   }
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+    struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+    struct io_ring_ctx *ctx = iowq->ctx;
+
+    /* no general timeout, or shorter, we are done */
+    if (iowq->timeout == KTIME_MAX ||
+        ktime_after(iowq->min_timeout, iowq->timeout))
+        goto out_wake;
+    /* work we may need to run, wake function will see if we need to wake */
+    if (io_has_work(ctx))
+        goto out_wake;
+    /* got events since we started waiting, min timeout is done */
+    if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+        goto out_wake;
+    /* if we have any events and min timeout expired, we're done */
+    if (io_cqring_events(ctx))

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2396

Quoted 10 lines
io_uring/io_uring.c +10 -0
+        goto out_wake;
+
+    /*
+     * If using deferred task_work running and application is waiting on
+     * more than one request, ensure we reset it now where we are switching
+     * to normal sleeps. Any request completion post min_wait should wake
+     * the task and return.
+     */
+    if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+        atomic_set(&ctx->cq_wait_nr, 1);

racy

Quoted 6 lines
Quoted 4 lines

atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
if (llist_empty(&ctx->work_llist))
     // wake;

rather if _not_ empty

Yep that one was a given :-)

Updated it, we'll punt to out_wake at that point.

Signature · 1 line
Jens Axboe
#24 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 82 lines · Jens Axboe
Quoted 81 lines · Pavel Begunkov
Quoted 79 lines · Pavel Begunkov
Quoted 61 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

---
   io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------
   io_uring/io_uring.h |  2 ++
   2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +22 -0
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
       return HRTIMER_NORESTART;
   }
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+    struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+    struct io_ring_ctx *ctx = iowq->ctx;
+
+    /* no general timeout, or shorter, we are done */
+    if (iowq->timeout == KTIME_MAX ||
+        ktime_after(iowq->min_timeout, iowq->timeout))
+        goto out_wake;
+    /* work we may need to run, wake function will see if we need to wake */
+    if (io_has_work(ctx))
+        goto out_wake;
+    /* got events since we started waiting, min timeout is done */
+    if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+        goto out_wake;
+    /* if we have any events and min timeout expired, we're done */
+    if (io_cqring_events(ctx))

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2396

Quoted 10 lines
io_uring/io_uring.c +10 -0
+        goto out_wake;
+
+    /*
+     * If using deferred task_work running and application is waiting on
+     * more than one request, ensure we reset it now where we are switching
+     * to normal sleeps. Any request completion post min_wait should wake
+     * the task and return.
+     */
+    if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+        atomic_set(&ctx->cq_wait_nr, 1);

racy

atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
if (llist_empty(&ctx->work_llist))
     // wake;

Quoted 5 lines
Quoted 1 line

rather if _not_ empty

Yep that one was a given :-)

Updated it, we'll punt to out_wake at that point.

Another concern is racing with the task [re]setting ->cq_wait_nr in io_cqring_wait(), e.g. because of a spurious wake up.

Signature · 1 line
Pavel Begunkov
author maintainer
#25 lore raw

Re: [PATCH 4/5] io_uring: add support for batch wait timeout

Quoted 83 lines · Pavel Begunkov
Quoted 82 lines · Jens Axboe
Quoted 81 lines · Pavel Begunkov
Quoted 79 lines · Pavel Begunkov
Quoted 61 lines · Jens Axboe

Waiting for events with io_uring has two knobs that can be set:

1) The number of events to wake for
2) The timeout associated with the event

Waiting will abort when either of those conditions are met, as expected.

This adds support for a third event, which is associated with the number of events to wait for. Applications generally like to handle batches of completions, and right now they'd set a number of events to wait for and the timeout for that. If no events have been received but the timeout triggers, control is returned to the application and it can wait again. However, if the application doesn't have anything to do until events are reaped, then it's possible to make this waiting more efficient.

For example, the application may have a latency time of 50 usecs and wanting to handle a batch of 8 requests at the time. If it uses 50 usecs as the timeout, then it'll be doing 20K context switches per second even if nothing is happening.

This introduces the notion of min batch wait time. If the min batch wait time expires, then we'll return to userspace if we have any events at all. If none are available, the general wait time is applied. Any request arriving after the min batch wait time will cause waiting to stop and return control to the application.

---
   io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++++++++++++------
   io_uring/io_uring.h |  2 ++
   2 files changed, 67 insertions(+), 10 deletions(-)
io_uring/io_uring.c +22 -0
index ddfbe04c61ed..d09a7c2e1096 100644
@@ -2363,13 +2363,62 @@ static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
       return HRTIMER_NORESTART;
   }
+/*
+ * Doing min_timeout portion. If we saw any timeouts, events, or have work,
+ * wake up. If not, and we have a normal timeout, switch to that and keep
+ * sleeping.
+ */
+static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
+{
+    struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+    struct io_ring_ctx *ctx = iowq->ctx;
+
+    /* no general timeout, or shorter, we are done */
+    if (iowq->timeout == KTIME_MAX ||
+        ktime_after(iowq->min_timeout, iowq->timeout))
+        goto out_wake;
+    /* work we may need to run, wake function will see if we need to wake */
+    if (io_has_work(ctx))
+        goto out_wake;
+    /* got events since we started waiting, min timeout is done */
+    if (iowq->cq_min_tail != READ_ONCE(ctx->rings->cq.tail))
+        goto out_wake;
+    /* if we have any events and min timeout expired, we're done */
+    if (io_cqring_events(ctx))

io_uring/io_uring.c static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer) · line 2396

Quoted 10 lines
io_uring/io_uring.c +10 -0
+        goto out_wake;
+
+    /*
+     * If using deferred task_work running and application is waiting on
+     * more than one request, ensure we reset it now where we are switching
+     * to normal sleeps. Any request completion post min_wait should wake
+     * the task and return.
+     */
+    if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+        atomic_set(&ctx->cq_wait_nr, 1);

racy

atomic_set(&ctx->cq_wait_nr, 1);
smp_mb();
if (llist_empty(&ctx->work_llist))

      // wake;
Quoted 8 lines
Quoted 5 lines
Quoted 1 line

rather if _not_ empty

Yep that one was a given :-)

Updated it, we'll punt to out_wake at that point.

Another concern is racing with the task [re]setting ->cq_wait_nr in io_cqring_wait(), e.g. because of a spurious wake up.

I tried to close that up somewhat in the next iteration.

Signature · 1 line
Jens Axboe
author maintainer
#26 lore raw

PATCH v4 5/5 io_uring: wire up min batch wake timeout

In reply to Jens Axboe

Expose min_wait_usec in io_uring_getevents_arg, replacing the pad member that is currently in there. The value is in usecs, which is explained in the name as well.

Note that if min_wait_usec and a normal timeout is used in conjunction, the normal timeout is still relative to the base time. For example, if min_wait_usec is set to 100 and the normal timeout is 1000, the max total time waited is still 1000. This also means that if the normal timeout is shorter than min_wait_usec, then only the min_wait_usec will take effect.

See previous commit for an explanation of how this works.

IORING_FEAT_MIN_TIMEOUT is added as a feature flag for this, as applications doing submit_and_wait_timeout() style operations will generally not see the -EINVAL from the wait side as they return the number of IOs submitted. Only if no IOs are submitted will the -EINVAL bubble back up to the application.

---
 include/uapi/linux/io_uring.h | 3 ++-
 io_uring/io_uring.c           | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)
include/uapi/linux/io_uring.h +2 -1
index 7af716136df9..042eab793e26 100644
@@ -543,6 +543,7 @@ struct io_uring_params {
 #define IORING_FEAT_LINKED_FILE		(1U << 12)
 #define IORING_FEAT_REG_REG_RING	(1U << 13)
 #define IORING_FEAT_RECVSEND_BUNDLE	(1U << 14)
+#define IORING_FEAT_MIN_TIMEOUT		(1U << 15)
 
 /*
  * io_uring_register(2) opcodes and arguments
@@ -766,7 +767,7 @@ enum io_uring_register_restriction_op {
 struct io_uring_getevents_arg {
 	__u64	sigmask;
 	__u32	sigmask_sz;
-	__u32	pad;
+	__u32	min_wait_usec;
 	__u64	ts;
 };
 
io_uring/io_uring.c +4 -4
index d09a7c2e1096..20b67fea645d 100644
@@ -2475,6 +2475,7 @@ struct ext_arg {
 	size_t argsz;
 	struct __kernel_timespec __user *ts;
 	const sigset_t __user *sig;
+	ktime_t min_time;
 };
 
 /*
@@ -2507,7 +2508,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
 	iowq.cq_min_tail = READ_ONCE(ctx->rings->cq.tail);
 	iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
-	iowq.min_timeout = 0;
+	iowq.min_timeout = ext_arg->min_time;
 	iowq.timeout = KTIME_MAX;
 	start_time = io_get_time(ctx);
 
@@ -3231,8 +3232,7 @@ static int io_get_ext_arg(unsigned flags, const void __user *argp,
 		return -EINVAL;
 	if (copy_from_user(&arg, argp, sizeof(arg)))
 		return -EFAULT;
-	if (arg.pad)
-		return -EINVAL;
+	ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC;
 	ext_arg->sig = u64_to_user_ptr(arg.sigmask);
 	ext_arg->argsz = arg.sigmask_sz;
 	ext_arg->ts = u64_to_user_ptr(arg.ts);
@@ -3633,7 +3633,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
 			IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
 			IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP |
 			IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING |
-			IORING_FEAT_RECVSEND_BUNDLE;
+			IORING_FEAT_RECVSEND_BUNDLE | IORING_FEAT_MIN_TIMEOUT;
 
 	if (copy_to_user(params, p, sizeof(*p))) {
 		ret = -EFAULT;
Signature · 1 line
2.43.0