Skip to contentSkip to conversation

Allow non-atomic allocs for overflows

PATCH Superseded Jens Axboe 3 participants 10 messages May 16, 2025 io_uring

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

cover Allow non-atomic allocs for overflows

Hi,

This is heavily inspired by the series that Pavel posted here:

https://lore.kernel.org/io-uring/[email protected]/

since I do think that potentially increasing the reliability of overflow handling is a worthy endeavour. It's just somewhat simpler as it doesn't move anything around really, it just does the split of allocating the overflow entry separately from adding it to the io_ring_ctx context.

Further cleanups could be done on top of this, obviously.

 io_uring/io_uring.c | 79 +++++++++++++++++++++++++++++----------------
 1 file changed, 52 insertions(+), 27 deletions(-)
Signature · 1 line
Jens Axboe
author maintainer
#1 lore raw

PATCH 1/2 io_uring: split alloc and add of overflow

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

---
 io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++----------------
 1 file changed, 48 insertions(+), 27 deletions(-)
io_uring/io_uring.c +48 -27
index 9a9b8d35349b..2519fab303c4 100644
@@ -718,20 +718,11 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
 	}
 }
 
-static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
-				     s32 res, u32 cflags, u64 extra1, u64 extra2)
+static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
+				   struct io_overflow_cqe *ocqe)
 {
-	struct io_overflow_cqe *ocqe;
-	size_t ocq_size = sizeof(struct io_overflow_cqe);
-	bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
-
 	lockdep_assert_held(&ctx->completion_lock);
 
-	if (is_cqe32)
-		ocq_size += sizeof(struct io_uring_cqe);
-
-	ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
-	trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
 	if (!ocqe) {
 		struct io_rings *r = ctx->rings;
 
@@ -749,22 +740,44 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
 		atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
 
 	}
-	ocqe->cqe.user_data = user_data;
-	ocqe->cqe.res = res;
-	ocqe->cqe.flags = cflags;
-	if (is_cqe32) {
-		ocqe->cqe.big_cqe[0] = extra1;
-		ocqe->cqe.big_cqe[1] = extra2;
-	}
 	list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
 	return true;
 }
 
-static void io_req_cqe_overflow(struct io_kiocb *req)
+static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
+					     u64 user_data, s32 res, u32 cflags,
+					     u64 extra1, u64 extra2, gfp_t gfp)
+{
+	struct io_overflow_cqe *ocqe;
+	size_t ocq_size = sizeof(struct io_overflow_cqe);
+	bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
+
+	if (is_cqe32)
+		ocq_size += sizeof(struct io_uring_cqe);
+
+	ocqe = kmalloc(ocq_size, gfp | __GFP_ACCOUNT);
+	trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
+	if (ocqe) {
+		ocqe->cqe.user_data = user_data;
+		ocqe->cqe.res = res;
+		ocqe->cqe.flags = cflags;
+		if (is_cqe32) {
+			ocqe->cqe.big_cqe[0] = extra1;
+			ocqe->cqe.big_cqe[1] = extra2;
+		}
+	}
+	return ocqe;
+}
+
+static void io_req_cqe_overflow(struct io_kiocb *req, gfp_t gfp)
 {
-	io_cqring_event_overflow(req->ctx, req->cqe.user_data,
-				req->cqe.res, req->cqe.flags,
-				req->big_cqe.extra1, req->big_cqe.extra2);
+	struct io_ring_ctx *ctx = req->ctx;
+	struct io_overflow_cqe *ocqe;
+
+	ocqe = io_alloc_ocqe(ctx, req->cqe.user_data, req->cqe.res,
+			     req->cqe.flags, req->big_cqe.extra1,
+			     req->big_cqe.extra2, gfp);
+	io_cqring_add_overflow(ctx, ocqe);
 	memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 }
 
@@ -832,8 +845,12 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 
 	io_cq_lock(ctx);
 	filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
-	if (!filled)
-		filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+	if (unlikely(!filled)) {
+		struct io_overflow_cqe *ocqe;
+
+		ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_ATOMIC);
+		filled = io_cqring_add_overflow(ctx, ocqe);
+	}
 	io_cq_unlock_post(ctx);
 	return filled;
 }
@@ -848,8 +865,11 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 	lockdep_assert(ctx->lockless_cq);
 
 	if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
+		struct io_overflow_cqe *ocqe;
+
+		ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_KERNEL);
 		spin_lock(&ctx->completion_lock);
-		io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+		io_cqring_add_overflow(ctx, ocqe);
 		spin_unlock(&ctx->completion_lock);
 	}
 	ctx->submit_state.cq_flush = true;
@@ -1442,10 +1462,11 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 		    unlikely(!io_fill_cqe_req(ctx, req))) {
 			if (ctx->lockless_cq) {
 				spin_lock(&ctx->completion_lock);
-				io_req_cqe_overflow(req);
+				io_req_cqe_overflow(req, GFP_ATOMIC);
 				spin_unlock(&ctx->completion_lock);
 			} else {
-				io_req_cqe_overflow(req);
+				gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;
+				io_req_cqe_overflow(req, gfp);
 			}
 		}
 	}
Signature · 1 line
2.49.0
#2 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

Quoted 124 lines · Jens Axboe

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

---
 io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++----------------
 1 file changed, 48 insertions(+), 27 deletions(-)
io_uring/io_uring.c +45 -25
index 9a9b8d35349b..2519fab303c4 100644
@@ -718,20 +718,11 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
        }
 }
 
-static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
-                                    s32 res, u32 cflags, u64 extra1, u64 extra2)
+static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
+                                  struct io_overflow_cqe *ocqe)
 {
-       struct io_overflow_cqe *ocqe;
-       size_t ocq_size = sizeof(struct io_overflow_cqe);
-       bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
-
        lockdep_assert_held(&ctx->completion_lock);
 
-       if (is_cqe32)
-               ocq_size += sizeof(struct io_uring_cqe);
-
-       ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
-       trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
        if (!ocqe) {
                struct io_rings *r = ctx->rings;
 
@@ -749,22 +740,44 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
                atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
 
        }
-       ocqe->cqe.user_data = user_data;
-       ocqe->cqe.res = res;
-       ocqe->cqe.flags = cflags;
-       if (is_cqe32) {
-               ocqe->cqe.big_cqe[0] = extra1;
-               ocqe->cqe.big_cqe[1] = extra2;
-       }
        list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
        return true;
 }
 
-static void io_req_cqe_overflow(struct io_kiocb *req)
+static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
+                                            u64 user_data, s32 res, u32 cflags,
+                                            u64 extra1, u64 extra2, gfp_t gfp)
+{
+       struct io_overflow_cqe *ocqe;
+       size_t ocq_size = sizeof(struct io_overflow_cqe);
+       bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
+
+       if (is_cqe32)
+               ocq_size += sizeof(struct io_uring_cqe);
+
+       ocqe = kmalloc(ocq_size, gfp | __GFP_ACCOUNT);
+       trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
+       if (ocqe) {
+               ocqe->cqe.user_data = user_data;
+               ocqe->cqe.res = res;
+               ocqe->cqe.flags = cflags;
+               if (is_cqe32) {
+                       ocqe->cqe.big_cqe[0] = extra1;
+                       ocqe->cqe.big_cqe[1] = extra2;
+               }
+       }
+       return ocqe;
+}
+
+static void io_req_cqe_overflow(struct io_kiocb *req, gfp_t gfp)
 {
-       io_cqring_event_overflow(req->ctx, req->cqe.user_data,
-                               req->cqe.res, req->cqe.flags,
-                               req->big_cqe.extra1, req->big_cqe.extra2);
+       struct io_ring_ctx *ctx = req->ctx;
+       struct io_overflow_cqe *ocqe;
+
+       ocqe = io_alloc_ocqe(ctx, req->cqe.user_data, req->cqe.res,
+                            req->cqe.flags, req->big_cqe.extra1,
+                            req->big_cqe.extra2, gfp);
+       io_cqring_add_overflow(ctx, ocqe);
        memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 }
 
@@ -832,8 +845,12 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 
        io_cq_lock(ctx);
        filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
-       if (!filled)
-               filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+       if (unlikely(!filled)) {
+               struct io_overflow_cqe *ocqe;
+
+               ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_ATOMIC);
+               filled = io_cqring_add_overflow(ctx, ocqe);
+       }
        io_cq_unlock_post(ctx);
        return filled;
 }
@@ -848,8 +865,11 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
        lockdep_assert(ctx->lockless_cq);
 
        if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
+               struct io_overflow_cqe *ocqe;
+
+               ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_KERNEL);
                spin_lock(&ctx->completion_lock);
-               io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+               io_cqring_add_overflow(ctx, ocqe);
                spin_unlock(&ctx->completion_lock);
        }
        ctx->submit_state.cq_flush = true;

io_uring/io_uring.c void __io_submit_flush_completions(struct io_ring_ctx *ctx) · line 1468

Quoted 9 lines
io_uring/io_uring.c +2 -2
@@ -1442,10 +1462,11 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
                    unlikely(!io_fill_cqe_req(ctx, req))) {
                        if (ctx->lockless_cq) {
                                spin_lock(&ctx->completion_lock);
-                               io_req_cqe_overflow(req);
+                               io_req_cqe_overflow(req, GFP_ATOMIC);
                                spin_unlock(&ctx->completion_lock);
                        } else {
-                               io_req_cqe_overflow(req);
+                               gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;

This is in the else case of an if (ctx->lockless_cq). Isn't ctx->lockless_cq known to be false?

Best,
Caleb

Quoted 8 lines
io_uring/io_uring.c +1 -1
+                               io_req_cqe_overflow(req, gfp);
                        }
                }
        }
--

2.49.0

author maintainer
#3 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

Quoted 5 lines · Caleb Sander Mateos
Quoted 5 lines
io_uring/io_uring.c +0 -1
@@ -1442,10 +1462,11 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
                    unlikely(!io_fill_cqe_req(ctx, req))) {
                        if (ctx->lockless_cq) {
                                spin_lock(&ctx->completion_lock);
-                               io_req_cqe_overflow(req);
Quoted 8 lines

io_uring/io_uring.c void __io_submit_flush_completions(struct io_ring_ctx *ctx) · line 1468

Quoted 5 lines
io_uring/io_uring.c +2 -1
+                               io_req_cqe_overflow(req, GFP_ATOMIC);
                                spin_unlock(&ctx->completion_lock);
                        } else {
-                               io_req_cqe_overflow(req);
+                               gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;

This is in the else case of an if (ctx->lockless_cq). Isn't ctx->lockless_cq known to be false?

Indeed! Actually this part needs to be split too. I'll redo this one.

Signature · 1 line
Jens Axboe
#4 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

In reply to Jens Axboe
Quoted 6 lines · Jens Axboe

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

I didn't suggest that. If anything, it complicates CQE posting helpers when we should be moving in the opposite direction.

Quoted 10 lines
---
  io_uring/io_uring.c | 75 +++++++++++++++++++++++++++++----------------
  1 file changed, 48 insertions(+), 27 deletions(-)
io_uring/io_uring.c +0 -0
index 9a9b8d35349b..2519fab303c4 100644
@@ -718,20 +718,11 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)

...

io_uring/io_uring.c void __io_submit_flush_completions(struct io_ring_ctx *ctx) · line 1468

Quoted 11 lines
  	ctx->submit_state.cq_flush = true;
io_uring/io_uring.c +2 -2
@@ -1442,10 +1462,11 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
  		    unlikely(!io_fill_cqe_req(ctx, req))) {
  			if (ctx->lockless_cq) {
  				spin_lock(&ctx->completion_lock);
-				io_req_cqe_overflow(req);
+				io_req_cqe_overflow(req, GFP_ATOMIC);
  				spin_unlock(&ctx->completion_lock);
  			} else {
-				io_req_cqe_overflow(req);
+				gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;

if (!ctx->lockless_cq)

	gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;
Signature · 1 line
Pavel Begunkov
author maintainer
#5 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

Quoted 10 lines · Pavel Begunkov
Quoted 6 lines · Jens Axboe

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

I didn't suggest that. If anything, it complicates CQE posting helpers when we should be moving in the opposite direction.

I'll kill the attribution then - it's not meant to mean the approach, but the concept of being able to use GFP_KERNEL when we can.

Quoted 12 lines

io_uring/io_uring.c void __io_submit_flush_completions(struct io_ring_ctx *ctx) · line 1468

Quoted 10 lines
io_uring/io_uring.c +2 -2
@@ -1442,10 +1462,11 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
              unlikely(!io_fill_cqe_req(ctx, req))) {
              if (ctx->lockless_cq) {
                  spin_lock(&ctx->completion_lock);
-                io_req_cqe_overflow(req);
+                io_req_cqe_overflow(req, GFP_ATOMIC);
                  spin_unlock(&ctx->completion_lock);
              } else {
-                io_req_cqe_overflow(req);
+                gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;

if (!ctx->lockless_cq)

Quoted 2 lines
    gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;

Yeah see other reply to Caleb. I'll just slurp in your patch 1/4 as this makes it simpler.

Signature · 1 line
Jens Axboe
#6 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 5 lines · Jens Axboe

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

I didn't suggest that. If anything, it complicates CQE posting helpers when we should be moving in the opposite direction.

I'll kill the attribution then - it's not meant to mean the approach, but the concept of being able to use GFP_KERNEL when we can.

Sure, but that will be blurred by time, while the patch IMHO is making it worse and should never see the light.

Signature · 1 line
Pavel Begunkov
author maintainer
#7 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

Quoted 13 lines · Pavel Begunkov
Quoted 12 lines · Jens Axboe
Quoted 10 lines · Pavel Begunkov
Quoted 6 lines · Jens Axboe

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

I didn't suggest that. If anything, it complicates CQE posting helpers when we should be moving in the opposite direction.

Quoted 6 lines
Quoted 3 lines

I'll kill the attribution then - it's not meant to mean the approach, but the concept of being able to use GFP_KERNEL when we can.

Sure, but that will be blurred by time, while the patch IMHO is making it worse and should never see the light.

Well, you're certainly cheerful today.

Signature · 1 line
Jens Axboe
#8 lore raw

Re: [PATCH 1/2] io_uring: split alloc and add of overflow

Quoted 14 lines · Jens Axboe
Quoted 13 lines · Pavel Begunkov
Quoted 12 lines · Jens Axboe
Quoted 10 lines · Pavel Begunkov
Quoted 6 lines · Jens Axboe

Add a new helper, io_alloc_ocqe(), that simply allocates and fills an overflow entry. Then it can get done outside of the locking section, and hence use more appropriate gfp_t allocation flags rather than always default to GFP_ATOMIC.

I didn't suggest that. If anything, it complicates CQE posting helpers when we should be moving in the opposite direction.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

I'll kill the attribution then - it's not meant to mean the approach, but the concept of being able to use GFP_KERNEL when we can.

Sure, but that will be blurred by time, while the patch IMHO is making it worse and should never see the light.

Well, you're certainly cheerful today.

Only extending an opinion, which, it seems, you're going to shrug off. That's fine, I just need to learn my lessons and stop caring about improving the code base.

Signature · 1 line
Pavel Begunkov
author maintainer
#9 lore raw

PATCH 2/2 io_uring: make io_alloc_ocqe() take a struct io_cqe pointer

In reply to Jens Axboe

The number of arguments to io_alloc_ocqe() is a bit unwieldy. Make it take a struct io_cqe pointer rather than three sepearate CQE args. One path already has that readily available, add an io_init_cqe() helper for the remainding two.

---
 io_uring/io_uring.c | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)
io_uring/io_uring.c +14 -10
index 2519fab303c4..e3c8c19902e8 100644
@@ -745,8 +745,8 @@ static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
 }
 
 static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
-					     u64 user_data, s32 res, u32 cflags,
-					     u64 extra1, u64 extra2, gfp_t gfp)
+					     struct io_cqe *cqe, u64 extra1,
+					     u64 extra2, gfp_t gfp)
 {
 	struct io_overflow_cqe *ocqe;
 	size_t ocq_size = sizeof(struct io_overflow_cqe);
@@ -756,11 +756,11 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
 		ocq_size += sizeof(struct io_uring_cqe);
 
 	ocqe = kmalloc(ocq_size, gfp | __GFP_ACCOUNT);
-	trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
+	trace_io_uring_cqe_overflow(ctx, cqe->user_data, cqe->res, cqe->flags, ocqe);
 	if (ocqe) {
-		ocqe->cqe.user_data = user_data;
-		ocqe->cqe.res = res;
-		ocqe->cqe.flags = cflags;
+		ocqe->cqe.user_data = cqe->user_data;
+		ocqe->cqe.res = cqe->res;
+		ocqe->cqe.flags = cqe->flags;
 		if (is_cqe32) {
 			ocqe->cqe.big_cqe[0] = extra1;
 			ocqe->cqe.big_cqe[1] = extra2;
@@ -774,8 +774,7 @@ static void io_req_cqe_overflow(struct io_kiocb *req, gfp_t gfp)
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_overflow_cqe *ocqe;
 
-	ocqe = io_alloc_ocqe(ctx, req->cqe.user_data, req->cqe.res,
-			     req->cqe.flags, req->big_cqe.extra1,
+	ocqe = io_alloc_ocqe(ctx, &req->cqe, req->big_cqe.extra1,
 			     req->big_cqe.extra2, gfp);
 	io_cqring_add_overflow(ctx, ocqe);
 	memset(&req->big_cqe, 0, sizeof(req->big_cqe));
@@ -839,6 +838,9 @@ static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
 	return false;
 }
 
+#define io_init_cqe(user_data, res, cflags)	\
+	(struct io_cqe) { .user_data = user_data, .res = res, .flags = cflags }
+
 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 {
 	bool filled;
@@ -847,8 +849,9 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 	filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
 	if (unlikely(!filled)) {
 		struct io_overflow_cqe *ocqe;
+		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
 
-		ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_ATOMIC);
+		ocqe = io_alloc_ocqe(ctx, &cqe, 0, 0, GFP_ATOMIC);
 		filled = io_cqring_add_overflow(ctx, ocqe);
 	}
 	io_cq_unlock_post(ctx);
@@ -866,8 +869,9 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 
 	if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
 		struct io_overflow_cqe *ocqe;
+		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
 
-		ocqe = io_alloc_ocqe(ctx, user_data, res, cflags, 0, 0, GFP_KERNEL);
+		ocqe = io_alloc_ocqe(ctx, &cqe, 0, 0, GFP_KERNEL);
 		spin_lock(&ctx->completion_lock);
 		io_cqring_add_overflow(ctx, ocqe);
 		spin_unlock(&ctx->completion_lock);
Signature · 1 line
2.49.0