for-5.12 stuff
Subsystems (3)
cover for-5.12 stuff
1/6 is a for-stable fix (syzbot).
Others are the first part of 5.12 hardening and random cleanups.
Pavel Begunkov (6):
io_uring: fix inconsistent lock state
io_uring: kill not used needs_file_no_error
io_uring: inline io_req_drop_files()
io_uring: remove work flags after cleanup
io_uring: deduplicate adding to REQ_F_INFLIGHT
io_uring: simplify do_read return parsing
fs/io_uring.c | 119 ++++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 66 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/6 io_uring: fix inconsistent lock state
WARNING: inconsistent lock state
inconsistent {HARDIRQ-ON-W} -> {IN-HARDIRQ-W} usage.
syz-executor217/8450 [HC1[1]:SC0[0]:HE0:SE1] takes:ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: io_req_clean_work fs/io_uring.c:1398 [inline] ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: io_dismantle_req+0x66f/0xf60 fs/io_uring.c:2029
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(&fs->lock);
<Interrupt>
lock(&fs->lock);*** DEADLOCK ***
1 lock held by syz-executor217/8450:
#0: ffff88802417c3e8 (&ctx->uring_lock){+.+.}-{3:3}, at: __do_sys_io_uring_enter+0x1071/0x1f30 fs/io_uring.c:9442
stack backtrace: CPU: 1 PID: 8450 Comm: syz-executor217 Not tainted 5.11.0-rc5-next-20210129-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: <IRQ> [...] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:354 [inline] io_req_clean_work fs/io_uring.c:1398 [inline] io_dismantle_req+0x66f/0xf60 fs/io_uring.c:2029 __io_free_req+0x3d/0x2e0 fs/io_uring.c:2046 io_free_req fs/io_uring.c:2269 [inline] io_double_put_req fs/io_uring.c:2392 [inline] io_put_req+0xf9/0x570 fs/io_uring.c:2388 io_link_timeout_fn+0x30c/0x480 fs/io_uring.c:6497 __run_hrtimer kernel/time/hrtimer.c:1519 [inline] __hrtimer_run_queues+0x609/0xe40 kernel/time/hrtimer.c:1583 hrtimer_interrupt+0x334/0x940 kernel/time/hrtimer.c:1645 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1085 [inline] __sysvec_apic_timer_interrupt+0x146/0x540 arch/x86/kernel/apic/apic.c:1102 asm_call_irq_on_stack+0xf/0x20 </IRQ> __run_sysvec_on_irqstack arch/x86/include/asm/irq_stack.h:37 [inline] run_sysvec_on_irqstack_cond arch/x86/include/asm/irq_stack.h:89 [inline] sysvec_apic_timer_interrupt+0xbd/0x100 arch/x86/kernel/apic/apic.c:1096 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:629 RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:169 [inline] RIP: 0010:_raw_spin_unlock_irq+0x25/0x40 kernel/locking/spinlock.c:199 spin_unlock_irq include/linux/spinlock.h:404 [inline] io_queue_linked_timeout+0x194/0x1f0 fs/io_uring.c:6525 __io_queue_sqe+0x328/0x1290 fs/io_uring.c:6594 io_queue_sqe+0x631/0x10d0 fs/io_uring.c:6639 io_queue_link_head fs/io_uring.c:6650 [inline] io_submit_sqe fs/io_uring.c:6697 [inline] io_submit_sqes+0x19b5/0x2720 fs/io_uring.c:6960 __do_sys_io_uring_enter+0x107d/0x1f30 fs/io_uring.c:9443 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9
Don't free requests from under hrtimer context (softirq) as it may sleep or take spinlocks improperly (e.g. non-irq versions).
--- fs/io_uring.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-)
fs/io_uring.c +11 -4
@@ -1886,8 +1886,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res)__io_cqring_fill_event (req, res,0 ); } -static void io_req_complete_nostate (struct io_kiocb *req,long res, -unsigned int cflags) +static void io_req_complete_post (struct io_kiocb *req,long res, +unsigned int cflags) {struct io_ring_ctx *ctx = req->ctx;unsigned long flags;
@@ -1898,6 +1898,12 @@ static void io_req_complete_nostate(struct io_kiocb *req, long res,spin_unlock_irqrestore (&ctx->completion_lock, flags);io_cqring_ev_posted (ctx); +} + +static inline void io_req_complete_nostate (struct io_kiocb *req,long res, +unsigned int cflags) +{ +io_req_complete_post (req, res, cflags);io_put_req (req); }
@@ -6489,9 +6495,10 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)if (prev) {req_set_fail_links (prev);io_async_find_and_cancel (ctx, req, prev->user_data, -ETIME ); -io_put_req (prev); +io_put_req_deferred (prev,1 ); }else { -io_req_complete (req, -ETIME ); +io_req_complete_post (req, -ETIME ,0 ); +io_put_req_deferred (req,1 ); }return HRTIMER_NORESTART ; }
Signature · 1 line
2.24.0
PATCH 2/6 io_uring: kill not used needs_file_no_error
In reply to Pavel BegunkovWe have no request types left using needs_file_no_error, remove it.
--- fs/io_uring.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-)
fs/io_uring.c +1 -4
@@ -788,8 +788,6 @@ struct io_submit_state {struct io_op_def {/* needs req->file assigned */ unsigned needs_file :1 ; -/* don't fail if file grab fails */ -unsigned needs_file_no_error :1 ;/* hash wq insertion if file is a regular file */ unsigned hash_reg_file :1 ;/* unbound wq insertion if file is a non-regular file */
@@ -6896,8 +6894,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,bool fixed = req->flags &REQ_F_FIXED_FILE ; req->file =io_file_get (state, req,READ_ONCE (sqe->fd), fixed); -if (unlikely (!req->file && - !io_op_defs[req->opcode].needs_file_no_error)) +if (unlikely (!req->file)) ret = -EBADF ; }
Signature · 1 line
2.24.0
PATCH 3/6 io_uring: inline io_req_drop_files()
In reply to Pavel Begunkovreq->files now have same lifetime as all other iowq-work resources, inline io_req_drop_files() for consistency. Moreover, since REQ_F_INFLIGHT is no more files specific, the function name became very confusing.
--- fs/io_uring.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-)
fs/io_uring.c +17 -22
@@ -1036,7 +1036,6 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req,static int io_setup_async_rw (struct io_kiocb *req,const struct iovec *iovec,const struct iovec *fast_iov,struct iov_iter *iter,bool force); -static void io_req_drop_files (struct io_kiocb *req);static void io_req_task_queue (struct io_kiocb *req);static struct kmem_cache *req_cachep;
@@ -1402,8 +1401,23 @@ static void io_req_clean_work(struct io_kiocb *req)free_fs_struct (fs); req->work.flags &= ~IO_WQ_WORK_FS ; } -if (req->flags &REQ_F_INFLIGHT ) -io_req_drop_files (req); +if (req->work.flags &IO_WQ_WORK_FILES ) { +put_files_struct (req->work.identity->files); +put_nsproxy (req->work.identity->nsproxy); + req->work.flags &= ~IO_WQ_WORK_FILES ; + } +if (req->flags &REQ_F_INFLIGHT ) { +struct io_ring_ctx *ctx = req->ctx; +struct io_uring_task *tctx = req->task->io_uring; +unsigned long flags; + +spin_lock_irqsave (&ctx->inflight_lock, flags); +list_del (&req->inflight_entry); +spin_unlock_irqrestore (&ctx->inflight_lock, flags); + req->flags &= ~REQ_F_INFLIGHT ; +if (atomic_read (&tctx->in_idle)) +wake_up (&tctx->wait); + }io_put_identity (req->task->io_uring, req); }
@@ -6164,25 +6178,6 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -EIOCBQUEUED ; } -static void io_req_drop_files (struct io_kiocb *req) -{ -struct io_ring_ctx *ctx = req->ctx; -struct io_uring_task *tctx = req->task->io_uring; -unsigned long flags; - -if (req->work.flags &IO_WQ_WORK_FILES ) { -put_files_struct (req->work.identity->files); -put_nsproxy (req->work.identity->nsproxy); - } -spin_lock_irqsave (&ctx->inflight_lock, flags); -list_del (&req->inflight_entry); -spin_unlock_irqrestore (&ctx->inflight_lock, flags); - req->flags &= ~REQ_F_INFLIGHT ; - req->work.flags &= ~IO_WQ_WORK_FILES ; -if (atomic_read (&tctx->in_idle)) -wake_up (&tctx->wait); -} -static void __io_clean_op (struct io_kiocb *req) {if (req->flags &REQ_F_BUFFER_SELECTED ) {
Signature · 1 line
2.24.0
PATCH 4/6 io_uring: remove work flags after cleanup
In reply to Pavel BegunkovShouldn't be a problem now, but it's better to clean REQ_F_WORK_INITIALIZED and work->flags only after relevant resources are killed, so cancellation see them.
--- fs/io_uring.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-)
fs/io_uring.c +6 -13
@@ -1374,22 +1374,14 @@ static void io_req_clean_work(struct io_kiocb *req)if (!(req->flags &REQ_F_WORK_INITIALIZED ))return ; - req->flags &= ~REQ_F_WORK_INITIALIZED ; - -if (req->work.flags &IO_WQ_WORK_MM ) { +if (req->work.flags &IO_WQ_WORK_MM )mmdrop (req->work.identity->mm); - req->work.flags &= ~IO_WQ_WORK_MM ; - }#ifdef CONFIG_BLK_CGROUP -if (req->work.flags &IO_WQ_WORK_BLKCG ) { +if (req->work.flags &IO_WQ_WORK_BLKCG )css_put (req->work.identity->blkcg_css); - req->work.flags &= ~IO_WQ_WORK_BLKCG ; - }#endif -if (req->work.flags &IO_WQ_WORK_CREDS ) { +if (req->work.flags &IO_WQ_WORK_CREDS )put_cred (req->work.identity->creds); - req->work.flags &= ~IO_WQ_WORK_CREDS ; - }if (req->work.flags &IO_WQ_WORK_FS ) {struct fs_struct *fs = req->work.identity->fs;
@@ -1399,12 +1391,10 @@ static void io_req_clean_work(struct io_kiocb *req)spin_unlock (&req->work.identity->fs->lock);if (fs)free_fs_struct (fs); - req->work.flags &= ~IO_WQ_WORK_FS ; }if (req->work.flags &IO_WQ_WORK_FILES ) {put_files_struct (req->work.identity->files);put_nsproxy (req->work.identity->nsproxy); - req->work.flags &= ~IO_WQ_WORK_FILES ; }if (req->flags &REQ_F_INFLIGHT ) {struct io_ring_ctx *ctx = req->ctx;
@@ -1419,6 +1409,9 @@ static void io_req_clean_work(struct io_kiocb *req)wake_up (&tctx->wait); } + req->flags &= ~REQ_F_WORK_INITIALIZED ; + req->work.flags &= ~(IO_WQ_WORK_MM |IO_WQ_WORK_BLKCG |IO_WQ_WORK_FS | +IO_WQ_WORK_CREDS |IO_WQ_WORK_FILES );io_put_identity (req->task->io_uring, req); }
Signature · 1 line
2.24.0
PATCH 5/6 io_uring: deduplicate adding to REQ_F_INFLIGHT
In reply to Pavel BegunkovWe don't know for how long REQ_F_INFLIGHT is going to stay, cleaner to extract a helper for marking requests as so.
--- fs/io_uring.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-)
fs/io_uring.c +17 -19
@@ -1460,11 +1460,24 @@ static bool io_identity_cow(struct io_kiocb *req)return true; } +static void io_req_track_inflight (struct io_kiocb *req) +{ +struct io_ring_ctx *ctx = req->ctx; + +if (!(req->flags &REQ_F_INFLIGHT )) { +io_req_init_async (req); + req->flags |=REQ_F_INFLIGHT ; + +spin_lock_irq (&ctx->inflight_lock); +list_add (&req->inflight_entry, &ctx->inflight_list); +spin_unlock_irq (&ctx->inflight_lock); + } +} +static bool io_grab_identity (struct io_kiocb *req) {const struct io_op_def *def = &io_op_defs[req->opcode];struct io_identity *id = req->work.identity; -struct io_ring_ctx *ctx = req->ctx;if (def->work_flags &IO_WQ_WORK_FSIZE ) {if (id->fsize !=rlimit (RLIMIT_FSIZE ))
@@ -1520,15 +1533,8 @@ static bool io_grab_identity(struct io_kiocb *req)return false;atomic_inc (&id->files->count);get_nsproxy (id->nsproxy); - -if (!(req->flags &REQ_F_INFLIGHT )) { - req->flags |=REQ_F_INFLIGHT ; - -spin_lock_irq (&ctx->inflight_lock); -list_add (&req->inflight_entry, &ctx->inflight_list); -spin_unlock_irq (&ctx->inflight_lock); - } req->work.flags |=IO_WQ_WORK_FILES ; +io_req_track_inflight (req); }if (!(req->work.flags &IO_WQ_WORK_MM ) && (def->work_flags &IO_WQ_WORK_MM )) {
@@ -6443,16 +6449,8 @@ static struct file *io_file_get(struct io_submit_state *state, file =__io_file_get (state, fd); } -if (file && file->f_op == &io_uring_fops && - !(req->flags &REQ_F_INFLIGHT )) { -io_req_init_async (req); - req->flags |=REQ_F_INFLIGHT ; - -spin_lock_irq (&ctx->inflight_lock); -list_add (&req->inflight_entry, &ctx->inflight_list); -spin_unlock_irq (&ctx->inflight_lock); - } - +if (file &&unlikely (file->f_op == &io_uring_fops)) +io_req_track_inflight (req);return file; }
Signature · 1 line
2.24.0
PATCH 6/6 io_uring: simplify do_read return parsing
In reply to Pavel Begunkovdo_read() returning 0 bytes read (not -EAGAIN/etc.) is too important of a case to prioritise it. Fold it into ret<0 check, so we get rid of an extra if and make it a bit more readable.
--- fs/io_uring.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)
fs/io_uring.c +2 -5
@@ -3526,7 +3526,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock,else kiocb->ki_flags |=IOCB_NOWAIT ; -/* If the file doesn't support async, just async punt */ no_async = force_nonblock && !io_file_supports_async (req->file,READ );if (no_async)
@@ -3538,9 +3537,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, ret =io_iter_do_read (req, iter); -if (!ret) { - goto done; - }else if (ret == -EIOCBQUEUED ) { +if (ret == -EIOCBQUEUED ) { ret =0 ; goto out_free; }else if (ret == -EAGAIN ) {
@@ -3554,7 +3551,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock,iov_iter_revert (iter, io_size -iov_iter_count (iter)); ret =0 ; goto copy_iov; - }else if (ret <0 ) { + }else if (ret <=0 ) {/* make sure -ERESTARTSYS -> -EINTR is done */ goto done; }
Signature · 1 line
2.24.0
Jens Axboe
maintainerQuoted 12 lines · Pavel Begunkov
1/6 is a for-stable fix (syzbot).
Others are the first part of 5.12 hardening and random cleanups.
Pavel Begunkov (6):
io_uring: fix inconsistent lock state
io_uring: kill not usedneeds_file_no_error
io_uring: inlineio_req_drop_files()
io_uring: remove work flags after cleanup
io_uring: deduplicate adding toREQ_F_INFLIGHT
io_uring: simplifydo_readreturn parsing
Quoted 2 lines
fs/io_uring.c | 119 ++++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 66 deletions(-)
All looks good to me, thanks!
Signature · 1 line
Jens Axboe