Skip to contentSkip to conversation

drain cleanups and extra

PATCH Superseded Pavel Begunkov 1 participant 8 messages May 8, 2025 io_uring

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover drain cleanups and extra

Patches 1-4 clean up draininng, on top I'll be trying to remove the cq_extra mess. It sacrifices some performance in the path, but shouldn't be critical, and we'll probably get rid of the allocation later on.

Patches 5 and 6 are not related but just bundled together.

Pavel Begunkov (6):
io_uring: account drain memory to cgroup
io_uring: simplify drain ret passing
io_uring: remove drain prealloc checks
io_uring: consolidate drain seq checking io_uring/net: move CONFIG_NET guards to Makefile
io_uring: add lockdep asserts to io_add_aux_cqe

 io_uring/Makefile   |  4 ++--
 io_uring/io_uring.c | 52 +++++++++++++++++++--------------------------
 io_uring/net.c      |  2 --
 3 files changed, 24 insertions(+), 34 deletions(-)
Signature · 1 line
2.49.0
#1 lore raw

PATCH 1/6 io_uring: account drain memory to cgroup

Account drain allocations against memcg. It's not a big problem as each such allocation is paired with a request, which is accounted, but it's nicer to follow the limits more closely.

---
 io_uring/io_uring.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/io_uring.c +1 -1
index 0d051476008c..23e283e65eeb 100644
@@ -1672,7 +1672,7 @@ static __cold void io_drain_req(struct io_kiocb *req)
 	spin_unlock(&ctx->completion_lock);
 
 	io_prep_async_link(req);
-	de = kmalloc(sizeof(*de), GFP_KERNEL);
+	de = kmalloc(sizeof(*de), GFP_KERNEL_ACCOUNT);
 	if (!de) {
 		ret = -ENOMEM;
 		io_req_defer_failed(req, ret);
Signature · 1 line
2.49.0
#2 lore raw

PATCH 2/6 io_uring: simplify drain ret passing

In reply to Pavel Begunkov

"ret" in io_drain_req() is only used in one place, remove it and pass -ENOMEM directly.

---
 io_uring/io_uring.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
io_uring/io_uring.c +1 -3
index 23e283e65eeb..21b70ad0edc4 100644
@@ -1657,7 +1657,6 @@ static __cold void io_drain_req(struct io_kiocb *req)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_defer_entry *de;
-	int ret;
 	u32 seq = io_get_sequence(req);
 
 	/* Still need defer if there is pending req in defer list. */
@@ -1674,8 +1673,7 @@ static __cold void io_drain_req(struct io_kiocb *req)
 	io_prep_async_link(req);
 	de = kmalloc(sizeof(*de), GFP_KERNEL_ACCOUNT);
 	if (!de) {
-		ret = -ENOMEM;
-		io_req_defer_failed(req, ret);
+		io_req_defer_failed(req, -ENOMEM);
 		return;
 	}
 
Signature · 1 line
2.49.0
#3 lore raw

PATCH 3/6 io_uring: remove drain prealloc checks

In reply to Pavel Begunkov

Currently io_drain_req() has two steps. The first is fast path checking sequence numbers. The second is allocations, rechecking and actual queuing. Further simplify it by removing the first step.

---
 io_uring/io_uring.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)
io_uring/io_uring.c +3 -12
index 21b70ad0edc4..72ae350f4f8b 100644
@@ -1659,17 +1659,6 @@ static __cold void io_drain_req(struct io_kiocb *req)
 	struct io_defer_entry *de;
 	u32 seq = io_get_sequence(req);
 
-	/* Still need defer if there is pending req in defer list. */
-	spin_lock(&ctx->completion_lock);
-	if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) {
-		spin_unlock(&ctx->completion_lock);
-queue:
-		ctx->drain_active = false;
-		io_req_task_queue(req);
-		return;
-	}
-	spin_unlock(&ctx->completion_lock);
-
 	io_prep_async_link(req);
 	de = kmalloc(sizeof(*de), GFP_KERNEL_ACCOUNT);
 	if (!de) {
@@ -1681,7 +1670,9 @@ static __cold void io_drain_req(struct io_kiocb *req)
 	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
 		spin_unlock(&ctx->completion_lock);
 		kfree(de);
-		goto queue;
+		ctx->drain_active = false;
+		io_req_task_queue(req);
+		return;
 	}
 
 	trace_io_uring_defer(req);
Signature · 1 line
2.49.0
#4 lore raw

PATCH 4/6 io_uring: consolidate drain seq checking

In reply to Pavel Begunkov

We check sequences when queuing drained requests as well when flushing them. Instead, always queue and immediately try to flush, so that all seq handling can be kept contained in the flushing code.

---
 io_uring/io_uring.c | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)
io_uring/io_uring.c +17 -17
index 72ae350f4f8b..e50c153d8edc 100644
@@ -559,9 +559,9 @@ void io_req_queue_iowq(struct io_kiocb *req)
 	io_req_task_work_add(req);
 }
 
-static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
+
+static __cold noinline void __io_queue_deferred(struct io_ring_ctx *ctx)
 {
-	spin_lock(&ctx->completion_lock);
 	while (!list_empty(&ctx->defer_list)) {
 		struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
 						struct io_defer_entry, list);
@@ -572,7 +572,12 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
 		io_req_task_queue(de->req);
 		kfree(de);
 	}
-	spin_unlock(&ctx->completion_lock);
+}
+
+static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
+{
+	guard(spinlock)(&ctx->completion_lock);
+	__io_queue_deferred(ctx);
 }
 
 void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
@@ -1657,29 +1662,24 @@ static __cold void io_drain_req(struct io_kiocb *req)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_defer_entry *de;
-	u32 seq = io_get_sequence(req);
 
-	io_prep_async_link(req);
 	de = kmalloc(sizeof(*de), GFP_KERNEL_ACCOUNT);
 	if (!de) {
 		io_req_defer_failed(req, -ENOMEM);
 		return;
 	}
 
-	spin_lock(&ctx->completion_lock);
-	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
-		spin_unlock(&ctx->completion_lock);
-		kfree(de);
-		ctx->drain_active = false;
-		io_req_task_queue(req);
-		return;
-	}
-
+	io_prep_async_link(req);
 	trace_io_uring_defer(req);
 	de->req = req;
-	de->seq = seq;
-	list_add_tail(&de->list, &ctx->defer_list);
-	spin_unlock(&ctx->completion_lock);
+	de->seq = io_get_sequence(req);
+
+	scoped_guard(spinlock, &ctx->completion_lock) {
+		list_add_tail(&de->list, &ctx->defer_list);
+		__io_queue_deferred(ctx);
+		if (list_empty(&ctx->defer_list))
+			ctx->drain_active = false;
+	}
 }
 
 static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
Signature · 1 line
2.49.0
#6 lore raw

PATCH 5/6 io_uring/net: move CONFIG_NET guards to Makefile

In reply to Pavel Begunkov

Instruct Makefile to never try to compile net.c without CONFIG_NET and kill ifdefs in the file.

---
 io_uring/Makefile | 4 ++--
 io_uring/net.c    | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)
io_uring/Makefile +2 -2
index 75e0ca795685..11a739927a62 100644
@@ -7,7 +7,7 @@ GCOV_PROFILE := y
 endif
 
 obj-$(CONFIG_IO_URING)		+= io_uring.o opdef.o kbuf.o rsrc.o notif.o \
-					tctx.o filetable.o rw.o net.o poll.o \
+					tctx.o filetable.o rw.o poll.o \
 					eventfd.o uring_cmd.o openclose.o \
 					sqpoll.o xattr.o nop.o fs.o splice.o \
 					sync.o msg_ring.o advise.o openclose.o \
@@ -19,4 +19,4 @@ obj-$(CONFIG_IO_WQ)		+= io-wq.o
 obj-$(CONFIG_FUTEX)		+= futex.o
 obj-$(CONFIG_EPOLL)		+= epoll.o
 obj-$(CONFIG_NET_RX_BUSY_POLL)	+= napi.o
-obj-$(CONFIG_NET) += cmd_net.o
+obj-$(CONFIG_NET) += net.o cmd_net.o
io_uring/net.c +0 -2
index b3a643675ce8..1fbdb2bbb3f3 100644
@@ -18,7 +18,6 @@
 #include "rsrc.h"
 #include "zcrx.h"
 
-#if defined(CONFIG_NET)
 struct io_shutdown {
 	struct file			*file;
 	int				how;
@@ -1836,4 +1835,3 @@ void io_netmsg_cache_free(const void *entry)
 	io_vec_free(&kmsg->vec);
 	kfree(kmsg);
 }
-#endif
Signature · 1 line
2.49.0
#7 lore raw

PATCH 6/6 io_uring: add lockdep asserts to io_add_aux_cqe

In reply to Pavel Begunkov

io_add_aux_cqe() can only be called for rings with uring_lock protected completion queues, add a couple of assertions in regards to that.

---
 io_uring/io_uring.c | 3 +++
 1 file changed, 3 insertions(+)
io_uring/io_uring.c +3 -0
index e50c153d8edc..503205ced136 100644
@@ -853,6 +853,9 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
  */
 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 {
+	lockdep_assert_held(&ctx->uring_lock);
+	lockdep_assert(ctx->lockless_cq);
+
 	if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
 		spin_lock(&ctx->completion_lock);
 		io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
Signature · 1 line
2.49.0
#5 lore raw

Re: [PATCH 4/6] io_uring: consolidate drain seq checking

In reply to Pavel Begunkov
Quoted 3 lines · Pavel Begunkov

We check sequences when queuing drained requests as well when flushing them. Instead, always queue and immediately try to flush, so that all seq handling can be kept contained in the flushing code.

That might be not entirely correct, I'll need to remake it.

Signature · 1 line
Pavel Begunkov