cleanup task/files cancel
cover cleanup task/files cancel
That unifies cancellation/matching/etc., so we can kill all that going out of hand zoo of functions.
Jens, [3/6] changes the behaviour, but as last time it's less restrictive and doesn't kill what we don't need to kill. Though, it'd prefer you to check the assumption in light of the cancel changes you've done.
Based on for-5.11/io_uring + "io_uring: fix link lookup racing with link timeout", should apply ok after you merge everything.
Pavel Begunkov (6):
io_uring: simplify io_task_match()
io_uring: add a {task,files} pair matching helper
io_uring: cancel only requests of current task
io_uring: don't iterate io_uring_cancel_files()
io_uring: pass files into kill timeouts/poll
io_uring: always batch cancel in *cancel_files()
fs/io-wq.c | 10 -- fs/io-wq.h | 1 - fs/io_uring.c | 260 ++++++++++++++------------------------------------ 3 files changed, 69 insertions(+), 202 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/6 io_uring: simplify io_task_match()
If IORING_SETUP_SQPOLL is set all requests belong to the corresponding SQPOLL task, so skip task checking in that case and always match.
--- fs/io_uring.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
fs/io_uring.c +1 -5
@@ -1562,11 +1562,7 @@ static bool io_task_match(struct io_kiocb *req, struct task_struct *tsk)if (!tsk || req->task == tsk)return true; -if (ctx->flags &IORING_SETUP_SQPOLL ) { -if (ctx->sq_data && req->task == ctx->sq_data->thread) -return true; - } -return false; +return (ctx->flags &IORING_SETUP_SQPOLL ); }/*
Signature · 1 line
2.24.0
PATCH 2/6 io_uring: add a {task,files} pair matching helper
In reply to Pavel BegunkovAdd io_match_task() that matches both task and files.
--- fs/io_uring.c | 48 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 26 deletions(-)
fs/io_uring.c +22 -26
@@ -1037,6 +1037,26 @@ static inline void io_clean_op(struct io_kiocb *req)__io_clean_op (req); } +static bool io_match_task (struct io_kiocb *head, +struct task_struct *task, +struct files_struct *files) +{ +struct io_kiocb *req; + +if (task && head->task != task) +return false; +if (!files) +return true; + +io_for_each_link (req, head) { +if ((req->flags &REQ_F_WORK_INITIALIZED ) && + (req->work.flags &IO_WQ_WORK_FILES ) && + req->work.identity->files == files) +return true; + } +return false; +} +static void io_sq_thread_drop_mm_files (void ) {struct files_struct *files = current->files;
@@ -1686,27 +1706,6 @@ static void io_cqring_mark_overflow(struct io_ring_ctx *ctx) } } -static inline bool __io_match_files (struct io_kiocb *req, -struct files_struct *files) -{ -return ((req->flags &REQ_F_WORK_INITIALIZED ) && - (req->work.flags &IO_WQ_WORK_FILES )) && - req->work.identity->files == files; -} - -static bool io_match_files (struct io_kiocb *head,struct files_struct *files) -{ -struct io_kiocb *req; - -if (!files) -return true; -io_for_each_link (req, head) { -if (__io_match_files (req, files)) -return true; - } -return false; -} -/* Returns true if there are no backlogged entries after the flush */ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, struct task_struct *tsk,
@@ -1734,9 +1733,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, cqe =NULL ;list_for_each_entry_safe (req, tmp, &ctx->cq_overflow_list, compl.list) { -if (tsk && req->task != tsk) -continue ; -if (!io_match_files (req, files)) +if (!io_match_task (req, tsk, files))continue ; cqe =io_get_cqring (ctx);
@@ -8776,8 +8773,7 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx,spin_lock_irq (&ctx->completion_lock);list_for_each_entry_reverse (de, &ctx->defer_list, list) { -if (io_task_match (de->req, task) && -io_match_files (de->req, files)) { +if (io_match_task (de->req, task, files)) {list_cut_position (&list, &ctx->defer_list, &de->list);break ; }
Signature · 1 line
2.24.0
PATCH 3/6 io_uring: cancel only requests of current task
In reply to Pavel Begunkovio_uring_cancel_files() cancels all request that match files regardless of task. There is no real need in that, cancel only requests of the specified task. That also handles SQPOLL case as it already changes task to it.
--- fs/io_uring.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-)
fs/io_uring.c +5 -18
@@ -8654,14 +8654,6 @@ static int io_uring_release(struct inode *inode, struct file *file)return 0 ; } -static bool io_wq_files_match (struct io_wq_work *work,void *data) -{ -struct files_struct *files = data; - -return !files || ((work->flags &IO_WQ_WORK_FILES ) && - work->identity->files == files); -} -/* * Returns true if 'preq' is the link parent of 'req' */
@@ -8794,21 +8786,20 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx,* Returns true if we found and killed one or more files pinning requests */ static bool io_uring_cancel_files (struct io_ring_ctx *ctx, +struct task_struct *task,struct files_struct *files) {if (list_empty_careful (&ctx->inflight_list))return false; -/* cancel all at once, should be faster than doing it one by one*/ -io_wq_cancel_cb (ctx->io_wq, io_wq_files_match, files, true); -while (!list_empty_careful (&ctx->inflight_list)) {struct io_kiocb *cancel_req =NULL , *req;DEFINE_WAIT (wait);spin_lock_irq (&ctx->inflight_lock);list_for_each_entry (req, &ctx->inflight_list, inflight_entry) { -if (files && (req->work.flags &IO_WQ_WORK_FILES ) && +if (req->task == task && + (req->work.flags &IO_WQ_WORK_FILES ) && req->work.identity->files != files)continue ;/* req is being completed, ignore */
@@ -8851,7 +8842,7 @@ static bool __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, {bool ret; - ret =io_uring_cancel_files (ctx, files); + ret =io_uring_cancel_files (ctx, task, files);if (!files) {enum io_wq_cancel cret;
@@ -8890,11 +8881,7 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,io_sq_thread_park (ctx->sq_data); } -if (files) -io_cancel_defer_files (ctx,NULL , files); -else -io_cancel_defer_files (ctx, task,NULL ); - +io_cancel_defer_files (ctx, task, files);io_cqring_overflow_flush (ctx, true, task, files);while (__io_uring_cancel_task_requests (ctx, task, files)) {
Signature · 1 line
2.24.0
PATCH 4/6 io_uring: don't iterate io_uring_cancel_files()
In reply to Pavel Begunkovio_uring_cancel_files() guarantees to cancel all matching requests, that's not necessary to do that in a loop. Move it up in the callchain into io_uring_cancel_task_requests().
--- fs/io_uring.c | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-)
fs/io_uring.c +12 -22
@@ -8782,16 +8782,10 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, } } -/* -* Returns true if we found and killed one or more files pinning requests -*/ -static bool io_uring_cancel_files (struct io_ring_ctx *ctx, +static void io_uring_cancel_files (struct io_ring_ctx *ctx,struct task_struct *task,struct files_struct *files) { -if (list_empty_careful (&ctx->inflight_list)) -return false; -while (!list_empty_careful (&ctx->inflight_list)) {struct io_kiocb *cancel_req =NULL , *req;DEFINE_WAIT (wait);
@@ -8824,8 +8818,6 @@ static bool io_uring_cancel_files(struct io_ring_ctx *ctx,schedule ();finish_wait (&ctx->inflight_wait, &wait); } - -return true; }static bool io_cancel_task_cb (struct io_wq_work *work,void *data)
@@ -8836,15 +8828,12 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data)return io_task_match (req, task); } -static bool __io_uring_cancel_task_requests (struct io_ring_ctx *ctx, -struct task_struct *task, -struct files_struct *files) +static void __io_uring_cancel_task_requests (struct io_ring_ctx *ctx, +struct task_struct *task) { -bool ret; - - ret =io_uring_cancel_files (ctx, task, files); -if (!files) { +while (1 ) {enum io_wq_cancel cret; +bool ret = false; cret =io_wq_cancel_cb (ctx->io_wq, io_cancel_task_cb, task, true);if (cret !=IO_WQ_CANCEL_NOTFOUND )
@@ -8860,9 +8849,11 @@ static bool __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, ret |=io_poll_remove_all (ctx, task); ret |=io_kill_timeouts (ctx, task); +if (!ret) +break ; +io_run_task_work (); +cond_resched (); } - -return ret; }/*
@@ -8883,11 +8874,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,io_cancel_defer_files (ctx, task, files);io_cqring_overflow_flush (ctx, true, task, files); +io_uring_cancel_files (ctx, task, files); -while (__io_uring_cancel_task_requests (ctx, task, files)) { -io_run_task_work (); -cond_resched (); - } +if (!files) +__io_uring_cancel_task_requests (ctx, task);if ((ctx->flags &IORING_SETUP_SQPOLL ) && ctx->sq_data) {atomic_dec (&task->io_uring->in_idle);
Signature · 1 line
2.24.0
PATCH 5/6 io_uring: pass files into kill timeouts/poll
In reply to Pavel BegunkovMake io_poll_remove_all() and io_kill_timeouts() to match against files as well. A preparation patch, effectively not used by now.
--- fs/io_uring.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-)
fs/io_uring.c +10 -8
@@ -1588,14 +1588,15 @@ static bool io_task_match(struct io_kiocb *req, struct task_struct *tsk)/* * Returns true if we found and killed one or more timeouts */ -static bool io_kill_timeouts (struct io_ring_ctx *ctx,struct task_struct *tsk) +static bool io_kill_timeouts (struct io_ring_ctx *ctx,struct task_struct *tsk, +struct files_struct *files) {struct io_kiocb *req, *tmp;int canceled =0 ;spin_lock_irq (&ctx->completion_lock);list_for_each_entry_safe (req, tmp, &ctx->timeout_list, timeout.list) { -if (io_task_match (req, tsk)) { +if (io_match_task (req, tsk, files)) {io_kill_timeout (req); canceled++; }
@@ -5469,7 +5470,8 @@ static bool io_poll_remove_one(struct io_kiocb *req)/* * Returns true if we found and killed one or more poll requests */ -static bool io_poll_remove_all (struct io_ring_ctx *ctx,struct task_struct *tsk) +static bool io_poll_remove_all (struct io_ring_ctx *ctx,struct task_struct *tsk, +struct files_struct *files) {struct hlist_node *tmp;struct io_kiocb *req;
@@ -5481,7 +5483,7 @@ static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk) list = &ctx->cancel_hash[i];hlist_for_each_entry_safe (req, tmp, list, hash_node) { -if (io_task_match (req, tsk)) +if (io_match_task (req, tsk, files)) posted +=io_poll_remove_one (req); } }
@@ -8615,8 +8617,8 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)percpu_ref_kill (&ctx->refs);mutex_unlock (&ctx->uring_lock); -io_kill_timeouts (ctx,NULL ); -io_poll_remove_all (ctx,NULL ); +io_kill_timeouts (ctx,NULL ,NULL ); +io_poll_remove_all (ctx,NULL ,NULL );if (ctx->io_wq)io_wq_cancel_all (ctx->io_wq);
@@ -8847,8 +8849,8 @@ static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, } } - ret |=io_poll_remove_all (ctx, task); - ret |=io_kill_timeouts (ctx, task); + ret |=io_poll_remove_all (ctx, task,NULL ); + ret |=io_kill_timeouts (ctx, task,NULL );if (!ret)break ;io_run_task_work ();
Signature · 1 line
2.24.0
PATCH 6/6 io_uring: always batch cancel in *cancel_files()
In reply to Pavel BegunkovInstead of iterating over each request and cancelling it individually in io_uring_cancel_files(), try to cancel all matching requests and use ->inflight_list only to check if there anything left.
In many cases it should be faster, and we can reuse a lot of code from task cancellation.
--- fs/io-wq.c | 10 ---- fs/io-wq.h | 1 - fs/io_uring.c | 135 ++++++++------------------------------------------ 3 files changed, 21 insertions(+), 125 deletions(-)
fs/io-wq.c +0 -10
@@ -1078,16 +1078,6 @@ enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,return IO_WQ_CANCEL_NOTFOUND ; } -static bool io_wq_io_cb_cancel_data (struct io_wq_work *work,void *data) -{ -return work == data; -} - -enum io_wq_cancel io_wq_cancel_work (struct io_wq *wq,struct io_wq_work *cwork) -{ -return io_wq_cancel_cb (wq, io_wq_io_cb_cancel_data, (void *)cwork, false); -} -struct io_wq *io_wq_create (unsigned bounded,struct io_wq_data *data) {int ret = -ENOMEM , node;
fs/io-wq.h +0 -1
@@ -129,7 +129,6 @@ static inline bool io_wq_is_hashed(struct io_wq_work *work) }void io_wq_cancel_all (struct io_wq *wq); -enum io_wq_cancel io_wq_cancel_work (struct io_wq *wq,struct io_wq_work *cwork);typedef bool (work_cancel_fn )(struct io_wq_work *,void *);
fs/io_uring.c +21 -114
197 lines not on this page · raw message
Signature · 1 line
2.24.0
Quoted 19 lines · Pavel Begunkov
That unifies cancellation/matching/etc., so we can kill all that going out of hand zoo of functions.
Jens, [3/6] changes the behaviour, but as last time it's less restrictive and doesn't kill what we don't need to kill. Though, it'd prefer you to check the assumption in light of the cancel changes you've done.
Based on for-5.11/io_uring + "
io_uring: fix link lookup racing with link timeout", should apply ok after you merge everything.Pavel Begunkov (6):
io_uring: simplifyio_task_match()
io_uring: add a {task,files} pair matching helper
io_uring: cancel only requests of current task
io_uring: don't iterateio_uring_cancel_files()
io_uring: pass files into kill timeouts/poll
io_uring: always batch cancel in*cancel_files()
Quoted 4 lines
fs/io-wq.c | 10 -- fs/io-wq.h | 1 - fs/io_uring.c | 260 ++++++++++++++------------------------------------ 3 files changed, 69 insertions(+), 202 deletions(-)
Applied, thanks.
Signature · 1 line
Jens Axboe