io_uring patches queued up for 5.10
Jens Axboe
maintainercover io_uring patches queued up for 5.10
Hi,
A few patches that are queued up for this release. All address issues introduced in this cycle.
- Ensure SQPOLL cancelations are done in the same fashion as for "normal" rings.
- syzbot reported mm deference oops
- syzbot reported use-after-free for registered io_identity lookup and COW
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 io-wq: cancel request if it's asking for files and we don't have them
This can't currently happen, but will be possible shortly. Handle missing files just like we do not being able to grab a needed mm, and mark the request as needing cancelation.
--- fs/io-wq.c | 4 ++++ 1 file changed, 4 insertions(+)
fs/io-wq.c +4 -0
@@ -482,6 +482,10 @@ static void io_impersonate_work(struct io_worker *worker, current->files = work->identity->files; current->nsproxy = work->identity->nsproxy;task_unlock (current); +if (!work->identity->files) { +/* failed grabbing files, ensure work gets cancelled */ + work->flags |=IO_WQ_WORK_CANCEL ; + } }if ((work->flags &IO_WQ_WORK_FS ) && current->fs != work->identity->fs) current->fs = work->identity->fs;
Signature · 1 line
2.29.2
Jens Axboe
maintainerPATCH 2/4 io_uring: properly handle SQPOLL request cancelations
In reply to Jens AxboeTrack if a given task io_uring context contains SQPOLL instances, so we can iterate those for cancelation (and request counts). This ensures that we properly wait on SQPOLL contexts, and find everything that needs canceling.
--- fs/io_uring.c | 77 +++++++++++++++++++++++++++++++++------- include/linux/io_uring.h | 3 +- 2 files changed, 67 insertions(+), 13 deletions(-)
fs/io_uring.c +65 -12
@@ -1668,7 +1668,8 @@ static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)WRITE_ONCE (cqe->user_data, req->user_data);WRITE_ONCE (cqe->res, res);WRITE_ONCE (cqe->flags, cflags); - }else if (ctx->cq_overflow_flushed || req->task->io_uring->in_idle) { + }else if (ctx->cq_overflow_flushed || +atomic_read (&req->task->io_uring->in_idle)) {/* * If we're in ring overflow flush mode, or in task cancel mode, * then we cannot store the request for later flushing, we need
@@ -1838,7 +1839,7 @@ static void __io_free_req(struct io_kiocb *req)io_dismantle_req (req);percpu_counter_dec (&tctx->inflight); -if (tctx->in_idle) +if (atomic_read (&tctx->in_idle))wake_up (&tctx->wait);put_task_struct (req->task);
@@ -7694,7 +7695,8 @@ static int io_uring_alloc_task_context(struct task_struct *task)xa_init (&tctx->xa);init_waitqueue_head (&tctx->wait); tctx->last =NULL ; - tctx->in_idle =0 ; +atomic_set (&tctx->in_idle,0 ); + tctx->sqpoll = false;io_init_identity (&tctx->__identity); tctx->identity = &tctx->__identity; task->io_uring = tctx;
@@ -8597,8 +8599,11 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, {struct task_struct *task = current; -if ((ctx->flags &IORING_SETUP_SQPOLL ) && ctx->sq_data) +if ((ctx->flags &IORING_SETUP_SQPOLL ) && ctx->sq_data) { task = ctx->sq_data->thread; +atomic_inc (&task->io_uring->in_idle); +io_sq_thread_park (ctx->sq_data); + }io_cqring_overflow_flush (ctx, true, task, files);
@@ -8606,12 +8611,23 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,io_run_task_work ();cond_resched (); } + +if ((ctx->flags &IORING_SETUP_SQPOLL ) && ctx->sq_data) { +atomic_dec (&task->io_uring->in_idle); +/* +* If the files that are going away are the ones in the thread +* identity, clear them out. +*/ +if (task->io_uring->identity->files == files) + task->io_uring->identity->files =NULL ; +io_sq_thread_unpark (ctx->sq_data); + } }/* * Note that this task has used io_uring. We use it for cancelation purposes. */ -static int io_uring_add_task_file (struct file *file) +static int io_uring_add_task_file (struct io_ring_ctx *ctx,struct file *file) {struct io_uring_task *tctx = current->io_uring;
@@ -8633,6 +8649,14 @@ static int io_uring_add_task_file(struct file *file) tctx->last = file; } +/* +* This is race safe in that the task itself is doing this, hence it +* cannot be going through the exit/cancel paths at the same time. +* This cannot be modified while exit/cancel is running. +*/ +if (!tctx->sqpoll && (ctx->flags &IORING_SETUP_SQPOLL )) + tctx->sqpoll = true; +return 0 ; }
@@ -8674,7 +8698,7 @@ void __io_uring_files_cancel(struct files_struct *files)unsigned long index;/* make sure overflow events are dropped */ - tctx->in_idle = true; +atomic_inc (&tctx->in_idle);xa_for_each (&tctx->xa, index, file) {struct io_ring_ctx *ctx = file->private_data;
@@ -8683,6 +8707,35 @@ void __io_uring_files_cancel(struct files_struct *files)if (files)io_uring_del_task_file (file); } + +atomic_dec (&tctx->in_idle); +} + +static s64 tctx_inflight (struct io_uring_task *tctx) +{ +unsigned long index; +struct file *file; +s64 inflight; + + inflight =percpu_counter_sum (&tctx->inflight); +if (!tctx->sqpoll) +return inflight; + +/* +* If we have SQPOLL rings, then we need to iterate and find them, and +* add the pending count for those. +*/ +xa_for_each (&tctx->xa, index, file) { +struct io_ring_ctx *ctx = file->private_data; + +if (ctx->flags &IORING_SETUP_SQPOLL ) { +struct io_uring_task *__tctx = ctx->sqo_task->io_uring; + + inflight +=percpu_counter_sum (&__tctx->inflight); + } + } + +return inflight; }/*
@@ -8696,11 +8749,11 @@ void __io_uring_task_cancel(void)s64 inflight;/* make sure overflow events are dropped */ - tctx->in_idle = true; +atomic_inc (&tctx->in_idle); do { /* read completions before cancelations */ - inflight = percpu_counter_sum(&tctx->inflight); + inflight = tctx_inflight(tctx); if (!inflight) break; __io_uring_files_cancel(NULL);
@@ -8711,13 +8764,13 @@ void __io_uring_task_cancel(void)* If we've seen completions, retry. This avoids a race where * a completion comes in before we did prepare_to_wait(). */ -if (inflight !=percpu_counter_sum (&tctx->inflight)) +if (inflight !=tctx_inflight (tctx))continue ;schedule (); }while (1 );finish_wait (&tctx->wait, &wait); - tctx->in_idle = false; +atomic_dec (&tctx->in_idle); }static int io_uring_flush (struct file *file,void *data)
@@ -8862,7 +8915,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,io_sqpoll_wait_sq (ctx); submitted = to_submit; }else if (to_submit) { - ret =io_uring_add_task_file (f.file); + ret =io_uring_add_task_file (ctx, f.file);if (unlikely (ret)) goto out;mutex_lock (&ctx->uring_lock);
@@ -9091,7 +9144,7 @@ static int io_uring_get_fd(struct io_ring_ctx *ctx)#if defined(CONFIG_UNIX ) ctx->ring_sock->file = file;#endif -if (unlikely (io_uring_add_task_file (file))) { +if (unlikely (io_uring_add_task_file (ctx, file))) { file =ERR_PTR (-ENOMEM ); goto err_fd; }
include/linux/io_uring.h +2 -1
@@ -30,7 +30,8 @@ struct io_uring_task {struct percpu_counter inflight;struct io_identity __identity;struct io_identity *identity; -bool in_idle; +atomic_t in_idle; +bool sqpoll; };#if defined(CONFIG_IO_URING )
Signature · 1 line
2.29.2
Jens Axboe
maintainerPATCH 3/4 io_uring: ensure consistent view of original task ->mm from SQPOLL
In reply to Jens AxboeEnsure we get a valid view of the task mm, by using task_lock() when attempting to grab the original task mm.
--- fs/io_uring.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-)
fs/io_uring.c +20 -7
@@ -995,20 +995,33 @@ static void io_sq_thread_drop_mm(void)if (mm) {kthread_unuse_mm (mm);mmput (mm); + current->mm =NULL ; } }static int __io_sq_thread_acquire_mm (struct io_ring_ctx *ctx) { -if (!current->mm) { -if (unlikely (!(ctx->flags &IORING_SETUP_SQPOLL ) || - !ctx->sqo_task->mm || - !mmget_not_zero (ctx->sqo_task->mm))) -return -EFAULT ; -kthread_use_mm (ctx->sqo_task->mm); +struct mm_struct *mm; + +if (current->mm) +return 0 ; + +/* Should never happen */ +if (unlikely (!(ctx->flags &IORING_SETUP_SQPOLL ))) +return -EFAULT ; + +task_lock (ctx->sqo_task); + mm = ctx->sqo_task->mm; +if (unlikely (!mm || !mmget_not_zero (mm))) + mm =NULL ; +task_unlock (ctx->sqo_task); + +if (mm) { +kthread_use_mm (mm); +return 0 ; } -return 0 ; +return -EFAULT ; } static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
Signature · 1 line
2.29.2
Jens Axboe
maintainerPATCH 4/4 io_uring: drop req/tctx io_identity separately
In reply to Jens AxboeWe can't bundle this into one operation, as the identity may not have originated from the tctx to begin with. Drop one ref for each of them separately, if they don't match the static assignment. If we don't, then if the identity is a lookup from registered credentials, we could be freeing that identity as we're dropping a reference assuming it came from the tctx. syzbot reports this as a use-after-free, as the identity is still referencable from idr lookup:
==================================================================
BUG: KASAN: use-after-free in instrument_atomic_read_write include/linux/instrumented.h:101 [inline] BUG: KASAN: use-after-free in atomic_fetch_add_relaxed include/asm-generic/atomic-instrumented.h:142 [inline] BUG: KASAN: use-after-free in __refcount_add include/linux/refcount.h:193 [inline] BUG: KASAN: use-after-free in __refcount_inc include/linux/refcount.h:250 [inline] BUG: KASAN: use-after-free in refcount_inc include/linux/refcount.h:267 [inline] BUG: KASAN: use-after-free in io_init_req fs/io_uring.c:6700 [inline] BUG: KASAN: use-after-free in io_submit_sqes+0x15a9/0x25f0 fs/io_uring.c:6774 Write of size 4 at addr ffff888011e08e48 by task syz-executor165/8487 CPU: 1 PID: 8487 Comm: syz-executor165 Not tainted 5.10.0-rc1-next-20201102-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x107/0x163 lib/dump_stack.c:118 print_address_description.constprop.0.cold+0xae/0x4c8 mm/kasan/report.c:385 __kasan_report mm/kasan/report.c:545 [inline] kasan_report.cold+0x1f/0x37 mm/kasan/report.c:562 check_memory_region_inline mm/kasan/generic.c:186 [inline] check_memory_region+0x13d/0x180 mm/kasan/generic.c:192 instrument_atomic_read_write include/linux/instrumented.h:101 [inline] atomic_fetch_add_relaxed include/asm-generic/atomic-instrumented.h:142 [inline] __refcount_add include/linux/refcount.h:193 [inline] __refcount_inc include/linux/refcount.h:250 [inline] refcount_inc include/linux/refcount.h:267 [inline] io_init_req fs/io_uring.c:6700 [inline] io_submit_sqes+0x15a9/0x25f0 fs/io_uring.c:6774 __do_sys_io_uring_enter+0xc8e/0x1b50 fs/io_uring.c:9159 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x440e19 Code: 18 89 d0 c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 0f 83 eb 0f fc ff c3 66 2e 0f 1f 84 00 00 00 00 RSP: 002b:00007fff644ff178 EFLAGS: 00000246 ORIG_RAX: 00000000000001aa RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 0000000000440e19 RDX: 0000000000000000 RSI: 000000000000450c RDI: 0000000000000003 RBP: 0000000000000004 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00000000022b4850 R13: 0000000000000010 R14: 0000000000000000 R15: 0000000000000000 Allocated by task 8487: kasan_save_stack+0x1b/0x40 mm/kasan/common.c:48 kasan_set_track mm/kasan/common.c:56 [inline] __kasan_kmalloc.constprop.0+0xc2/0xd0 mm/kasan/common.c:461 kmalloc include/linux/slab.h:552 [inline] io_register_personality fs/io_uring.c:9638 [inline] __io_uring_register fs/io_uring.c:9874 [inline] __do_sys_io_uring_register+0x10f0/0x40a0 fs/io_uring.c:9924 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Freed by task 8487: kasan_save_stack+0x1b/0x40 mm/kasan/common.c:48 kasan_set_track+0x1c/0x30 mm/kasan/common.c:56 kasan_set_free_info+0x1b/0x30 mm/kasan/generic.c:355 __kasan_slab_free+0x102/0x140 mm/kasan/common.c:422 slab_free_hook mm/slub.c:1544 [inline] slab_free_freelist_hook+0x5d/0x150 mm/slub.c:1577 slab_free mm/slub.c:3140 [inline] kfree+0xdb/0x360 mm/slub.c:4122 io_identity_cow fs/io_uring.c:1380 [inline] io_prep_async_work+0x903/0xbc0 fs/io_uring.c:1492 io_prep_async_link fs/io_uring.c:1505 [inline] io_req_defer fs/io_uring.c:5999 [inline] io_queue_sqe+0x212/0xed0 fs/io_uring.c:6448 io_submit_sqe fs/io_uring.c:6542 [inline] io_submit_sqes+0x14f6/0x25f0 fs/io_uring.c:6784 __do_sys_io_uring_enter+0xc8e/0x1b50 fs/io_uring.c:9159 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 The buggy address belongs to the object at ffff888011e08e00 which belongs to the cache kmalloc-96 of size 96 The buggy address is located 72 bytes inside of 96-byte region [ffff888011e08e00, ffff888011e08e60) The buggy address belongs to the page: page:00000000a7104751 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x11e08 flags: 0xfff00000000200(slab) raw: 00fff00000000200 ffffea00004f8540 0000001f00000002 ffff888010041780 raw: 0000000000000000 0000000080200020 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff888011e08d00: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc ffff888011e08d80: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc
Quoted 1 line
ffff888011e08e00: fa fb fb fb fb fb fb fb fb fb fb fb fc fc fc fc
^
ffff888011e08e80: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc
ffff888011e08f00: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc
==================================================================
--- fs/io_uring.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
fs/io_uring.c +6 -3
@@ -1287,9 +1287,12 @@ static bool io_identity_cow(struct io_kiocb *req)/* add one for this request */ refcount_inc (&id->count); -/* drop old identity, assign new one. one ref for req, one for tctx */ -if (req->work.identity != tctx->identity && -refcount_sub_and_test (2 , &req->work.identity->count)) +/* drop tctx and req identity references, if needed */ +if (tctx->identity != &tctx->__identity && +refcount_dec_and_test (&tctx->identity->count)) +kfree (tctx->identity); +if (req->work.identity != &tctx->__identity && +refcount_dec_and_test (&req->work.identity->count))kfree (req->work.identity); req->work.identity = id;
Signature · 1 line
2.29.2