bunch of random fixes
v2 cover bunch of random fixes
v2:
- add 3/6 to trigger eventfd for iopoll
- add 5/6, yet another iopoll sync fix
- rework 6/6 (Jens)
Pavel Begunkov (6):
io_uring: drop file refs after task cancel
io_uring: cancel more aggressively in exit_work
io_uring: trigger eventfd for IOPOLL
io_uring: dont kill fasync under completion_lock
io_uring: synchronise IOPOLL on task_submit fail
io_uring: patch up IOPOLL overflow_flush sync
fs/io_uring.c | 146 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 87 insertions(+), 59 deletions(-)
Signature · 1 line
2.24.0
PATCH v2 1/6 io_uring: drop file refs after task cancel
io_uring fds marked O_CLOEXEC and we explicitly cancel all requests before going through exec, so we don't want to leave task's file references to not our anymore io_uring instances.
--- fs/io_uring.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-)
fs/io_uring.c +16 -9
@@ -8948,6 +8948,15 @@ static void io_uring_attempt_task_drop(struct file *file)io_uring_del_task_file (file); } +static void io_uring_remove_task_files (struct io_uring_task *tctx) +{ +struct file *file; +unsigned long index; + +xa_for_each (&tctx->xa, index, file) +io_uring_del_task_file (file); +} +void __io_uring_files_cancel (struct files_struct *files) {struct io_uring_task *tctx = current->io_uring;
@@ -8956,16 +8965,12 @@ void __io_uring_files_cancel(struct files_struct *files)/* make sure overflow events are dropped */ atomic_inc (&tctx->in_idle); - -xa_for_each (&tctx->xa, index, file) { -struct io_ring_ctx *ctx = file->private_data; - -io_uring_cancel_task_requests (ctx, files); -if (files) -io_uring_del_task_file (file); - } - +xa_for_each (&tctx->xa, index, file) +io_uring_cancel_task_requests (file->private_data, files);atomic_dec (&tctx->in_idle); + +if (files) +io_uring_remove_task_files (tctx); }static s64 tctx_inflight (struct io_uring_task *tctx)
@@ -9028,6 +9033,8 @@ void __io_uring_task_cancel(void) }while (1 );atomic_dec (&tctx->in_idle); + +io_uring_remove_task_files (tctx); }static int io_uring_flush (struct file *file,void *data)
Signature · 1 line
2.24.0
PATCH v2 2/6 io_uring: cancel more aggressively in exit_work
In reply to Pavel BegunkovWhile io_ring_exit_work() is running new requests of all sorts may be issued, so it should do a bit more to cancel them, otherwise they may just get stuck. e.g. in io-wq, in poll lists, etc.
--- fs/io_uring.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
@@ -992,6 +992,9 @@ enum io_mem_account { ACCT_PINNED, }; +static void __io_uring_cancel_task_requests (struct io_ring_ctx *ctx, +struct task_struct *task); +static void destroy_fixed_file_ref_node (struct fixed_file_ref_node *ref_node);static struct fixed_file_ref_node *alloc_fixed_file_ref_node (struct io_ring_ctx *ctx);
@@ -8663,7 +8666,7 @@ static void io_ring_exit_work(struct work_struct *work)* as nobody else will be looking for them. */ do { -io_iopoll_try_reap_events (ctx); +__io_uring_cancel_task_requests (ctx,NULL ); }while (!wait_for_completion_timeout (&ctx->ref_comp,HZ /20 ));io_ring_ctx_free (ctx); }
@@ -8818,9 +8821,11 @@ static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,enum io_wq_cancel cret;bool ret = false; - cret =io_wq_cancel_cb (ctx->io_wq, io_cancel_task_cb, &cancel, true); -if (cret !=IO_WQ_CANCEL_NOTFOUND ) - ret = true; +if (ctx->io_wq) { + cret =io_wq_cancel_cb (ctx->io_wq, io_cancel_task_cb, + &cancel, true); + ret |= (cret !=IO_WQ_CANCEL_NOTFOUND ); + }/* SQPOLL thread does its own polling */ if (!(ctx->flags &IORING_SETUP_SQPOLL )) {
Signature · 1 line
2.24.0
PATCH v2 3/6 io_uring: trigger eventfd for IOPOLL
In reply to Pavel BegunkovMake sure io_iopoll_complete() tries to wake up eventfd, which currently is skipped together with io_cqring_ev_posted() for non-SQPOLL IOPOLL.
Add an iopoll version of io_cqring_ev_posted(), duplicates a bit of code, but they actually use different sets of wait queues may be for better.
--- fs/io_uring.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
fs/io_uring.c +11 -2
@@ -1715,6 +1715,16 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx)eventfd_signal (ctx->cq_ev_fd,1 ); } +static void io_cqring_ev_posted_iopoll (struct io_ring_ctx *ctx) +{ +if (ctx->flags &IORING_SETUP_SQPOLL ) { +if (waitqueue_active (&ctx->wait)) +wake_up (&ctx->wait); + } +if (io_should_trigger_evfd (ctx)) +eventfd_signal (ctx->cq_ev_fd,1 ); +} +/* Returns true if there are no backlogged entries after the flush */ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, struct task_struct *tsk,
@@ -2427,8 +2437,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, }io_commit_cqring (ctx); -if (ctx->flags &IORING_SETUP_SQPOLL ) -io_cqring_ev_posted (ctx); +io_cqring_ev_posted_iopoll (ctx);io_req_free_batch_finish (ctx, &rb);if (!list_empty (&again))
Signature · 1 line
2.24.0
PATCH v2 4/6 io_uring: dont kill fasync under completion_lock
In reply to Pavel Begunkov CPU0 CPU1
---- ----
lock(&new->fa_lock);
local_irq_disable();
lock(&ctx->completion_lock);
lock(&new->fa_lock);
<Interrupt>
lock(&ctx->completion_lock);*** DEADLOCK ***
Move kill_fasync() out of io_commit_cqring() to io_cqring_ev_posted(), so it doesn't hold completion_lock while doing it. That saves from the reported deadlock, and it's just nice to shorten the locking time and untangle nested locks (compl_lock -> wq_head::lock).
--- fs/io_uring.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
fs/io_uring.c +8 -5
@@ -1345,11 +1345,6 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx)/* order cqe stores with ring update */ smp_store_release (&rings->cq.tail, ctx->cached_cq_tail); - -if (wq_has_sleeper (&ctx->cq_wait)) { -wake_up_interruptible (&ctx->cq_wait); -kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); - } }static void io_put_identity (struct io_uring_task *tctx,struct io_kiocb *req)
@@ -1713,6 +1708,10 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx)wake_up (&ctx->sq_data->wait);if (io_should_trigger_evfd (ctx))eventfd_signal (ctx->cq_ev_fd,1 ); +if (wq_has_sleeper (&ctx->cq_wait)) { +wake_up_interruptible (&ctx->cq_wait); +kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); + } }static void io_cqring_ev_posted_iopoll (struct io_ring_ctx *ctx)
@@ -1723,6 +1722,10 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) }if (io_should_trigger_evfd (ctx))eventfd_signal (ctx->cq_ev_fd,1 ); +if (wq_has_sleeper (&ctx->cq_wait)) { +wake_up_interruptible (&ctx->cq_wait); +kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); + } }/* Returns true if there are no backlogged entries after the flush */
Signature · 1 line
2.24.0
PATCH v2 5/6 io_uring: synchronise IOPOLL on task_submit fail
In reply to Pavel Begunkovio_req_task_submit() might be called for IOPOLL, do the fail path under uring_lock to comply with IOPOLL synchronisation based solely on it.
--- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/io_uring.c +2 -2
@@ -2143,14 +2143,14 @@ static void __io_req_task_submit(struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx; +mutex_lock (&ctx->uring_lock);if (!__io_sq_thread_acquire_mm (ctx) && !__io_sq_thread_acquire_files (ctx)) { -mutex_lock (&ctx->uring_lock);__io_queue_sqe (req,NULL ); -mutex_unlock (&ctx->uring_lock); }else {__io_req_task_cancel (req, -EFAULT ); } +mutex_unlock (&ctx->uring_lock); }static void io_req_task_submit (struct callback_head *cb)
Signature · 1 line
2.24.0
PATCH v2 6/6 io_uring: patch up IOPOLL overflow_flush sync
In reply to Pavel BegunkovIOPOLL skips completion locking but keeps it under uring_lock, thus io_cqring_overflow_flush() and so io_cqring_events() need additional locking with uring_lock in some cases for IOPOLL.
Remove __io_cqring_overflow_flush() from io_cqring_events(), introduce a wrapper around flush doing needed synchronisation and call it by hand.
--- fs/io_uring.c | 78 +++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 37 deletions(-)
fs/io_uring.c +41 -37
@@ -1729,9 +1729,9 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) }/* Returns true if there are no backlogged entries after the flush */ -static bool io_cqring_overflow_flush (struct io_ring_ctx *ctx,bool force, -struct task_struct *tsk, -struct files_struct *files) +static bool __io_cqring_overflow_flush (struct io_ring_ctx *ctx,bool force, +struct task_struct *tsk, +struct files_struct *files) {struct io_rings *rings = ctx->rings;struct io_kiocb *req, *tmp;
@@ -1784,6 +1784,20 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force,return all_flushed; } +static void io_cqring_overflow_flush (struct io_ring_ctx *ctx,bool force, +struct task_struct *tsk, +struct files_struct *files) +{ +if (test_bit (0 , &ctx->cq_check_overflow)) { +/* iopoll syncs against uring_lock, not completion_lock */ +if (ctx->flags &IORING_SETUP_IOPOLL ) +mutex_lock (&ctx->uring_lock); +__io_cqring_overflow_flush (ctx, force, tsk, files); +if (ctx->flags &IORING_SETUP_IOPOLL ) +mutex_unlock (&ctx->uring_lock); + } +} +static void __io_cqring_fill_event (struct io_kiocb *req,long res,long cflags) {struct io_ring_ctx *ctx = req->ctx;
@@ -2329,20 +2343,8 @@ static void io_double_put_req(struct io_kiocb *req)io_free_req (req); } -static unsigned io_cqring_events (struct io_ring_ctx *ctx,bool noflush) +static unsigned io_cqring_events (struct io_ring_ctx *ctx) { -if (test_bit (0 , &ctx->cq_check_overflow)) { -/* -* noflush == true is from the waitqueue handler, just ensure -* we wake up the task, and the next invocation will flush the -* entries. We cannot safely to it from here. -*/ -if (noflush) -return -1U ; - -io_cqring_overflow_flush (ctx, false,NULL ,NULL ); - } -/* See comment at the top of this file */ smp_rmb ();return __io_cqring_events (ctx);
@@ -2566,7 +2568,8 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)* If we do, we can potentially be spinning for commands that * already triggered a CQE (eg in error). */ -if (io_cqring_events (ctx, false)) +__io_cqring_overflow_flush (ctx, false,NULL ,NULL ); +if (io_cqring_events (ctx))break ;/*
@@ -6841,7 +6844,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)/* if we have a backlog and couldn't flush it all, return BUSY */ if (test_bit (0 , &ctx->sq_check_overflow)) { -if (!io_cqring_overflow_flush (ctx, false,NULL ,NULL )) +if (!__io_cqring_overflow_flush (ctx, false,NULL ,NULL ))return -EBUSY ; }
@@ -7104,7 +7107,7 @@ struct io_wait_queue {unsigned nr_timeouts; }; -static inline bool io_should_wake (struct io_wait_queue *iowq,bool noflush) +static inline bool io_should_wake (struct io_wait_queue *iowq) {struct io_ring_ctx *ctx = iowq->ctx;
@@ -7113,7 +7116,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)* started waiting. For timeouts, we always want to return to userspace, * regardless of event count. */ -return io_cqring_events (ctx, noflush) >= iowq->to_wait || +return io_cqring_events (ctx) >= iowq->to_wait ||atomic_read (&ctx->cq_timeouts) != iowq->nr_timeouts; }
@@ -7123,11 +7126,13 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, wq); -/* use noflush == true, as we can't safely rely on locking context */ -if (!io_should_wake (iowq, true)) -return -1 ; - -return autoremove_wake_function (curr, mode, wake_flags, key); +/* +* Cannot safely flush overflowed CQEs from here, ensure we wake up +* the task, and the next invocation will do it. +*/ +if (io_should_wake (iowq) ||test_bit (0 , &iowq->ctx->cq_check_overflow)) +return autoremove_wake_function (curr, mode, wake_flags, key); +return -1 ; }static int io_run_task_work_sig (void )
@@ -7164,7 +7169,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,int ret =0 ; do { - if (io_cqring_events(ctx, false) >= min_events) + io_cqring_overflow_flush(ctx, false, NULL, NULL); + if (io_cqring_events(ctx) >= min_events) return 0; if (!io_run_task_work()) break;
@@ -7192,6 +7198,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, iowq.nr_timeouts =atomic_read (&ctx->cq_timeouts);trace_io_uring_cqring_wait (ctx, min_events); do { + io_cqring_overflow_flush(ctx, false, NULL, NULL); prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, TASK_INTERRUPTIBLE); /* make sure we run task_work before checking for signals */
@@ -7200,8 +7207,11 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,continue ; else if (ret < 0) break; - if (io_should_wake(&iowq, false)) + if (io_should_wake(&iowq)) break; + if (test_bit(0, &ctx->cq_check_overflow)) + continue; + if (uts) { timeout =schedule_timeout (timeout);if (timeout ==0 ) {
@@ -8639,7 +8649,8 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait)smp_rmb ();if (!io_sqring_full (ctx)) mask |=EPOLLOUT |EPOLLWRNORM ; -if (io_cqring_events (ctx, false)) +io_cqring_overflow_flush (ctx, false,NULL ,NULL ); +if (io_cqring_events (ctx)) mask |=EPOLLIN |EPOLLRDNORM ;return mask;
@@ -8697,7 +8708,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)/* if force is set, the ring is going away. always drop after that */ ctx->cq_overflow_flushed =1 ;if (ctx->rings) -io_cqring_overflow_flush (ctx, true,NULL ,NULL ); +__io_cqring_overflow_flush (ctx, true,NULL ,NULL );mutex_unlock (&ctx->uring_lock);io_kill_timeouts (ctx,NULL ,NULL );
@@ -8873,9 +8884,7 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, }io_cancel_defer_files (ctx, task, files); -io_ring_submit_lock (ctx, (ctx->flags &IORING_SETUP_IOPOLL ));io_cqring_overflow_flush (ctx, true, task, files); -io_ring_submit_unlock (ctx, (ctx->flags &IORING_SETUP_IOPOLL ));if (!files)__io_uring_cancel_task_requests (ctx, task);
@@ -9218,13 +9227,8 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,*/ ret =0 ;if (ctx->flags &IORING_SETUP_SQPOLL ) { -if (!list_empty_careful (&ctx->cq_overflow_list)) { -bool needs_lock = ctx->flags &IORING_SETUP_IOPOLL ; +io_cqring_overflow_flush (ctx, false,NULL ,NULL ); -io_ring_submit_lock (ctx, needs_lock); -io_cqring_overflow_flush (ctx, false,NULL ,NULL ); -io_ring_submit_unlock (ctx, needs_lock); - }if (flags &IORING_ENTER_SQ_WAKEUP )wake_up (&ctx->sq_data->wait);if (flags &IORING_ENTER_SQ_WAIT )
Signature · 1 line
2.24.0
Quoted 4 lines · Pavel Begunkov
v2:
- add 3/6 to trigger eventfd for iopoll
- add 5/6, yet another iopoll sync fix
- rework 6/6 (Jens)
I'll go with v3, probably splitting the series. Apart from small changes I want to add, during long runs, it kills tests with SIGINT, mainly 500f9fbadef8-test. Any ideas what it can be?
Quoted 12 lines
Pavel Begunkov (6):
io_uring: drop file refs after task cancel
io_uring: cancel more aggressively inexit_work
io_uring: trigger eventfd for IOPOLL
io_uring: dont kill fasync undercompletion_lock
io_uring: synchronise IOPOLL ontask_submitfail
io_uring: patch up IOPOLLoverflow_flushsyncfs/io_uring.c | 146 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 87 insertions(+), 59 deletions(-)
Signature · 1 line
Pavel Begunkov