iopoll fixes
cover iopoll fixes
Did more comprehensive iopoll testing and found some more problems.
[1] is from the previous series. Actually, v2 for this one, addressing the double-reissue bug found by Jens. It maybe not as efficient, but simple and easy to backport.
[2,3] current->mm NULL deref
Pavel Begunkov (3):
io_uring: fix hanging iopoll in case of -EAGAIN
io_uring: fix current->mm NULL dereference on exit
io_uring: fix NULL-mm for linked reqs
fs/io_uring.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/3 io_uring: fix hanging iopoll in case of -EAGAIN
io_do_iopoll() won't do anything with a request unless req->iopoll_completed is set. So io_complete_rw_iopoll() has to set it, otherwise io_do_iopoll() will poll a file again and again even though the request of interest was completed long time ago.
Also, remove -EAGAIN check from io_issue_sqe() as it races with the changed lines. The request will take the long way and be resubmitted from io_iopoll*().
io_kiocb's result and iopoll_completed")
--- fs/io_uring.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
fs/io_uring.c +2 -7
@@ -2104,10 +2104,8 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)WRITE_ONCE (req->result, res);/* order with io_poll_complete() checking ->result */ -if (res != -EAGAIN ) { -smp_wmb (); -WRITE_ONCE (req->iopoll_completed,1 ); - } +smp_wmb (); +WRITE_ONCE (req->iopoll_completed,1 ); }/*
@@ -5592,9 +5590,6 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,if ((ctx->flags &IORING_SETUP_IOPOLL ) && req->file) {const bool in_async =io_wq_current_is_worker (); -if (req->result == -EAGAIN ) -return -EAGAIN ; -/* workqueue context doesn't hold uring_lock, grab it now */ if (in_async)mutex_lock (&ctx->uring_lock);
Signature · 1 line
2.24.0
PATCH 2/3 io_uring: fix current->mm NULL dereference on exit
In reply to Pavel BegunkovDon't reissue requests from io_iopoll_reap_events(), the task may not have mm, which ends up with NULL. It's better to kill everything off on exit anyway.
[ 677.734670] RIP: 0010:io_iopoll_complete+0x27e/0x630 ... [ 677.734679] Call Trace: [ 677.734695] ? __send_signal+0x1f2/0x420 [ 677.734698] ? _raw_spin_unlock_irqrestore+0x24/0x40 [ 677.734699] ? send_signal+0xf5/0x140 [ 677.734700] io_iopoll_getevents+0x12f/0x1a0 [ 677.734702] io_iopoll_reap_events.part.0+0x5e/0xa0 [ 677.734703] io_ring_ctx_wait_and_kill+0x132/0x1c0 [ 677.734704] io_uring_release+0x20/0x30 [ 677.734706] __fput+0xcd/0x230 [ 677.734707] ____fput+0xe/0x10 [ 677.734709] task_work_run+0x67/0xa0 [ 677.734710] do_exit+0x35d/0xb70 [ 677.734712] do_group_exit+0x43/0xa0 [ 677.734713] get_signal+0x140/0x900 [ 677.734715] do_signal+0x37/0x780 [ 677.734717] ? enqueue_hrtimer+0x41/0xb0 [ 677.734718] ? recalibrate_cpu_khz+0x10/0x10 [ 677.734720] ? ktime_get+0x3e/0xa0 [ 677.734721] ? lapic_next_deadline+0x26/0x30 [ 677.734723] ? tick_program_event+0x4d/0x90 [ 677.734724] ? __hrtimer_get_next_event+0x4d/0x80 [ 677.734726] __prepare_exit_to_usermode+0x126/0x1c0 [ 677.734741] prepare_exit_to_usermode+0x9/0x40 [ 677.734742] idtentry_exit_cond_rcu+0x4c/0x60 [ 677.734743] sysvec_reschedule_ipi+0x92/0x160 [ 677.734744] ? asm_sysvec_reschedule_ipi+0xa/0x20 [ 677.734745] asm_sysvec_reschedule_ipi+0x12/0x20
--- fs/io_uring.c | 9 +++++++++ 1 file changed, 9 insertions(+)
fs/io_uring.c +9 -0
@@ -884,6 +884,7 @@ enum io_mem_account { ACCT_PINNED, }; +static void io_complete_rw_common (struct kiocb *kiocb,long res);static void io_wq_submit_work (struct io_wq_work **workptr);static void io_cqring_fill_event (struct io_kiocb *req,long res);static void io_put_req (struct io_kiocb *req);
@@ -1756,6 +1757,14 @@ static void io_iopoll_queue(struct list_head *again)do { req = list_first_entry(again, struct io_kiocb, list);list_del (&req->list); + +/* shouldn't happen unless io_uring is dying, cancel reqs */ +if (unlikely (!current->mm)) { +io_complete_rw_common (&req->rw.kiocb, -EAGAIN ); +io_put_req (req); +continue ; + } +refcount_inc (&req->refs);io_queue_async_work (req); }while (!list_empty (again));
Signature · 1 line
2.24.0
PATCH 3/3 io_uring: fix NULL-mm for linked reqs
In reply to Pavel Begunkov__io_queue_sqe() tries to handle all request of a link, so it's not enough to grab mm in io_sq_thread_acquire_mm() based just on the head.
Don't check req->needs_mm and do it always.
--- fs/io_uring.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-)
fs/io_uring.c +11 -4
@@ -2000,10 +2000,9 @@ static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx) } } -static int io_sq_thread_acquire_mm (struct io_ring_ctx *ctx, -struct io_kiocb *req) +static int __io_sq_thread_acquire_mm (struct io_ring_ctx *ctx) { -if (io_op_defs[req->opcode].needs_mm && !current->mm) { +if (!current->mm) {if (unlikely (!mmget_not_zero (ctx->sqo_mm)))return -EFAULT ;kthread_use_mm (ctx->sqo_mm);
@@ -2012,6 +2011,14 @@ static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,return 0 ; } +static int io_sq_thread_acquire_mm (struct io_ring_ctx *ctx, +struct io_kiocb *req) +{ +if (!io_op_defs[req->opcode].needs_mm) +return 0 ; +return __io_sq_thread_acquire_mm (ctx); +} +#ifdef CONFIG_BLOCK static bool io_resubmit_prep (struct io_kiocb *req,int error) {
@@ -2788,7 +2795,7 @@ static void io_async_buf_retry(struct callback_head *cb) ctx = req->ctx;__set_current_state (TASK_RUNNING ); -if (!io_sq_thread_acquire_mm (ctx, req)) { +if (!__io_sq_thread_acquire_mm (ctx)) {mutex_lock (&ctx->uring_lock);__io_queue_sqe (req,NULL );mutex_unlock (&ctx->uring_lock);
Signature · 1 line
2.24.0
Quoted 7 lines · Pavel Begunkov
Did more comprehensive iopoll testing and found some more problems.
[1] is from the previous series. Actually, v2 for this one, addressing the double-reissue bug found by Jens. It maybe not as efficient, but simple and easy to backport.
[2,3]
current->mmNULL deref
And yet there are issues left... I'll resend
Quoted 9 lines
Pavel Begunkov (3):
io_uring: fix hanging iopoll in case of-EAGAIN
io_uring: fixcurrent->mmNULL dereference on exit
io_uring: fix NULL-mm for linked reqsfs/io_uring.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-)
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerQuoted 10 lines · Pavel Begunkov
Quoted 7 lines · Pavel Begunkov
Did more comprehensive iopoll testing and found some more problems.
[1] is from the previous series. Actually, v2 for this one, addressing the double-reissue bug found by Jens. It maybe not as efficient, but simple and easy to backport.
[2,3]
current->mmNULL derefAnd yet there are issues left... I'll resend
Can you also please have them separated in terms of what's for 5.8 and what's for 5.9?
Signature · 1 line
Jens Axboe