overflow CQE cleanups
cover overflow CQE cleanups
Refactoring for overflow CQE flushing and posting. The next related problem would be to make io_cqring_event_overflow()'s locking saner.
Pavel Begunkov (5):
io_uring: unexport io_req_cqe_overflow()
io_uring: remove extra SQPOLL overflow flush
io_uring: open code io_cqring_overflow_flush()
io_uring: always lock __io_cqring_overflow_flush
io_uring: consolidate overflow flushing
io_uring/io_uring.c | 60 +++++++++++++++++---------------------------- io_uring/io_uring.h | 1 - 2 files changed, 23 insertions(+), 38 deletions(-)
Signature · 1 line
2.44.0
PATCH 1/5 io_uring: unexport io_req_cqe_overflow()
There are no users of io_req_cqe_overflow() apart from io_uring.c, make it static.
--- io_uring/io_uring.c | 2 +- io_uring/io_uring.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-)
io_uring/io_uring.c +1 -1
@@ -819,7 +819,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,return true; } -void io_req_cqe_overflow (struct io_kiocb *req) +static void io_req_cqe_overflow (struct io_kiocb *req) { io_cqring_event_overflow(req->ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags,
io_uring/io_uring.h +0 -1
@@ -62,7 +62,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) }bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow); -void io_req_cqe_overflow (struct io_kiocb *req);int io_run_task_work_sig (struct io_ring_ctx *ctx);void io_req_defer_failed (struct io_kiocb *req,s32 res);bool io_post_aux_cqe (struct io_ring_ctx *ctx,u64 user_data,s32 res,u32 cflags);
Signature · 1 line
2.44.0
PATCH 2/5 io_uring: remove extra SQPOLL overflow flush
In reply to Pavel Begunkovc1edbf5f081be ("io_uring: flag SQPOLL busy condition to userspace") added an extra overflowed CQE flush in the SQPOLL submission path due to backpressure, which was later removed. Remove the flush and let io_cqring_wait() / iopoll handle it.
--- io_uring/io_uring.c | 2 -- 1 file changed, 2 deletions(-)
io_uring/io_uring.c +0 -2
@@ -3238,8 +3238,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,*/ ret =0 ;if (ctx->flags &IORING_SETUP_SQPOLL ) { -io_cqring_overflow_flush (ctx); -if (unlikely (ctx->sq_data->thread ==NULL )) { ret = -EOWNERDEAD ; goto out;
Signature · 1 line
2.44.0
PATCH 3/5 io_uring: open code io_cqring_overflow_flush()
In reply to Pavel BegunkovThere is only one caller of io_cqring_overflow_flush(), open code it
--- io_uring/io_uring.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-)
io_uring/io_uring.c +3 -8
@@ -726,12 +726,6 @@ static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)mutex_unlock (&ctx->uring_lock); } -static void io_cqring_overflow_flush (struct io_ring_ctx *ctx) -{ -if (test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq)) -io_cqring_do_overflow_flush (ctx); -} -/* can be called by any task */ static void io_put_task_remote (struct task_struct *task) {
@@ -2452,8 +2446,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,if (!llist_empty (&ctx->work_llist))io_run_local_work (ctx, min_events);io_run_task_work (); -io_cqring_overflow_flush (ctx); -/* if user messes with these they will just get an early return */ + +if (unlikely (test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq))) +io_cqring_do_overflow_flush (ctx);if (__io_cqring_events_user (ctx) >= min_events)return 0 ;
Signature · 1 line
2.44.0
PATCH 4/5 io_uring: always lock __io_cqring_overflow_flush
In reply to Pavel BegunkovConditional locking is never great, in case of __io_cqring_overflow_flush(), which is a slow path, it's not justified. Don't handle IOPOLL separately, always grab uring_lock for overflow flushing.
--- io_uring/io_uring.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
io_uring/io_uring.c +8 -5
@@ -673,6 +673,8 @@ static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)struct io_overflow_cqe *ocqe;LIST_HEAD (list); +lockdep_assert_held (&ctx->uring_lock); +spin_lock (&ctx->completion_lock);list_splice_init (&ctx->cq_overflow_list, &list);clear_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq);
@@ -689,6 +691,8 @@ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx) {size_t cqe_size =sizeof (struct io_uring_cqe ); +lockdep_assert_held (&ctx->uring_lock); +if (__io_cqring_events (ctx) == ctx->cq_entries)return ;
@@ -718,12 +722,9 @@ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx)static void io_cqring_do_overflow_flush (struct io_ring_ctx *ctx) { -/* iopoll syncs against uring_lock, not completion_lock */ -if (ctx->flags &IORING_SETUP_IOPOLL ) -mutex_lock (&ctx->uring_lock); +mutex_lock (&ctx->uring_lock);__io_cqring_overflow_flush (ctx); -if (ctx->flags &IORING_SETUP_IOPOLL ) -mutex_unlock (&ctx->uring_lock); +mutex_unlock (&ctx->uring_lock); }/* can be called by any task */
@@ -1522,6 +1523,8 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)unsigned int nr_events =0 ;unsigned long check_cq; +lockdep_assert_held (&ctx->uring_lock); +if (!io_allowed_run_tw (ctx))return -EEXIST ;
Signature · 1 line
2.44.0
PATCH 5/5 io_uring: consolidate overflow flushing
In reply to Pavel BegunkovConsolidate __io_cqring_overflow_flush and io_cqring_overflow_kill() into a single function as it once was, it's easier to work with it this way.
--- io_uring/io_uring.c | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-)
io_uring/io_uring.c +15 -25
@@ -668,26 +668,7 @@ static void io_cq_unlock_post(struct io_ring_ctx *ctx)io_commit_cqring_flush (ctx); } -static void io_cqring_overflow_kill (struct io_ring_ctx *ctx) -{ -struct io_overflow_cqe *ocqe; -LIST_HEAD (list); - -lockdep_assert_held (&ctx->uring_lock); - -spin_lock (&ctx->completion_lock); -list_splice_init (&ctx->cq_overflow_list, &list); -clear_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq); -spin_unlock (&ctx->completion_lock); - -while (!list_empty (&list)) { - ocqe = list_first_entry(&list, struct io_overflow_cqe, list); -list_del (&ocqe->list); -kfree (ocqe); - } -} - -static void __io_cqring_overflow_flush (struct io_ring_ctx *ctx) +static void __io_cqring_overflow_flush (struct io_ring_ctx *ctx,bool dying) {size_t cqe_size =sizeof (struct io_uring_cqe );
@@ -704,11 +685,14 @@ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx)struct io_uring_cqe *cqe;struct io_overflow_cqe *ocqe; -if (!io_get_cqe_overflow (ctx, &cqe, true)) -break ; ocqe =list_first_entry (&ctx->cq_overflow_list, struct io_overflow_cqe, list); -memcpy (cqe, &ocqe->cqe, cqe_size); + +if (!dying) { +if (!io_get_cqe_overflow (ctx, &cqe, true)) +break ; +memcpy (cqe, &ocqe->cqe, cqe_size); + }list_del (&ocqe->list);kfree (ocqe); }
@@ -720,10 +704,16 @@ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx)io_cq_unlock_post (ctx); } +static void io_cqring_overflow_kill (struct io_ring_ctx *ctx) +{ +if (ctx->rings) +__io_cqring_overflow_flush (ctx, true); +} +static void io_cqring_do_overflow_flush (struct io_ring_ctx *ctx) {mutex_lock (&ctx->uring_lock); -__io_cqring_overflow_flush (ctx); +__io_cqring_overflow_flush (ctx, false);mutex_unlock (&ctx->uring_lock); }
@@ -1531,7 +1521,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) check_cq =READ_ONCE (ctx->check_cq);if (unlikely (check_cq)) {if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) -__io_cqring_overflow_flush (ctx); +__io_cqring_overflow_flush (ctx, false);/* * Similarly do not spin if we have not informed the user of any * dropped CQE.
Signature · 1 line
2.44.0
Quoted 11 lines · Pavel Begunkov
Refactoring for overflow CQE flushing and posting. The next related problem would be to make io_cqring_event_overflow()'s locking saner.
Pavel Begunkov (5):
io_uring: unexportio_req_cqe_overflow()
io_uring: remove extra SQPOLL overflow flush
io_uring: open codeio_cqring_overflow_flush()
io_uring: always lock__io_cqring_overflow_flush
io_uring: consolidate overflow flushing[...]
Applied, thanks!
[1/5] io_uring: unexport io_req_cqe_overflow()
commit: 3de3cc01f18fc7f6c9a5f8f28d97c5e36912e78b
[2/5] io_uring: remove extra SQPOLL overflow flush
commit: 2aa2ddefbe584264ee618e15b1a0d1183e8e37b8
[3/5] io_uring: open code io_cqring_overflow_flush()
commit: bd08cb7a6f5b05bc1b122117a922da21c081c58e
[4/5] io_uring: always lock __io_cqring_overflow_flush
commit: 678b1aa58dffc01d9359a3fc093192746350f137
[5/5] io_uring: consolidate overflow flushing
commit: ed50ebf24b391a6a3b17a7f6bf968303f0277bb7Best regards,
Signature · 1 line
Jens Axboe