io_uring: avoid whole io_wq_work copy for requests completed inline
PATCH v4 1/2 io_uring: avoid whole io_wq_work copy for requests completed inline
If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.
I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120
before this patch:
Run status group 0 (all jobs):
READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msec
With this patch:
Run status group 0 (all jobs):
READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msec
About 5% improvement.
---
V4:
add io_req_init_async() helper
--- fs/io-wq.h | 5 ---- fs/io_uring.c | 66 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 21 deletions(-)
fs/io-wq.h +0 -5
@@ -94,11 +94,6 @@ struct io_wq_work {pid_t task_pid; }; -#define INIT_IO_WORK (work, _func) \ - do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ - } while (0) \ -static inline struct io_wq_work *wq_next_work (struct io_wq_work *work) {if (!work->list.next)
fs/io_uring.c +50 -16
172 lines not on this page · raw message
Signature · 1 line
2.17.2
PATCH v4 2/2 io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Basically IORING_OP_POLL_ADD command and async armed poll handlers for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED is set, can we do io_wq_work copy and restore.
---
V3:
drop the REQ_F_WORK_NEED_RESTORE flag introduced in V2 patch, just use REQ_F_WORK_INITIALIZED to control whether to do io_wq_work copy and restore.
--- fs/io_uring.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
@@ -4513,7 +4513,8 @@ static void io_async_task_func(struct callback_head *cb)spin_unlock_irq (&ctx->completion_lock);/* restore ->work in case we need to retry again */ -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work,sizeof (req->work));kfree (apoll);if (!canceled) {
@@ -4610,7 +4611,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)return false; req->flags |=REQ_F_POLLED ; -memcpy (&apoll->work, &req->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&apoll->work, &req->work,sizeof (req->work)); had_io = req->io !=NULL ;get_task_struct (current);
@@ -4635,7 +4637,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)if (!had_io)io_poll_remove_double (req);spin_unlock_irq (&ctx->completion_lock); -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work,sizeof (req->work));kfree (apoll);return false; }
@@ -4680,7 +4683,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)* io_req_work_drop_env below when dropping the * final reference. */ -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work, +sizeof (req->work));kfree (apoll); } }
Signature · 1 line
2.17.2
Jens Axboe
maintainerQuoted 17 lines · Xiaoguang Wang
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines
With this patch:
Run status group 0 (all jobs):READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx
and also saw this:
[ 168.208940] ================================================================== [ 168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000 [ 168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [ 168.209987] [ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318 [ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace: [ 168.210991] dump_stack+0x97/0xe0 [ 168.211164] print_address_description.constprop.0+0x1a/0x210 [ 168.211446] ? __lock_acquire+0x8bf/0x3000 [ 168.211649] __kasan_report.cold+0x20/0x39 [ 168.211851] ? __lock_acquire+0x8bf/0x3000 [ 168.212051] kasan_report+0x30/0x40 [ 168.212226] __lock_acquire+0x8bf/0x3000 [ 168.212432] ? ret_from_fork+0x24/0x30 [ 168.212623] ? stack_trace_save+0x81/0xa0 [ 168.212821] ? lockdep_hardirqs_on+0x270/0x270 [ 168.213039] ? save_stack+0x32/0x40 [ 168.213212] lock_acquire+0x122/0x570 [ 168.213398] ? __close_fd_get_file+0x40/0x150 [ 168.213615] ? lock_release+0x3f0/0x3f0 [ 168.213814] ? __lock_acquire+0x87e/0x3000 [ 168.214016] _raw_spin_lock+0x2c/0x40 [ 168.214196] ? __close_fd_get_file+0x40/0x150 [ 168.214408] __close_fd_get_file+0x40/0x150 [ 168.214618] io_issue_sqe+0x57f/0x22f0 [ 168.214803] ? lockdep_hardirqs_on+0x270/0x270 [ 168.215019] ? mark_held_locks+0x24/0x90 [ 168.215211] ? quarantine_put+0x6f/0x190 [ 168.215404] ? io_assign_current_work+0x59/0x80 [ 168.215623] ? __ia32_sys_io_uring_setup+0x30/0x30 [ 168.215855] ? find_held_lock+0xcb/0x100 [ 168.216054] ? io_worker_handle_work+0x289/0x980 [ 168.216280] ? lock_downgrade+0x340/0x340 [ 168.216476] ? io_wq_submit_work+0x5d/0x140 [ 168.216679] ? _raw_spin_unlock_irq+0x24/0x40 [ 168.216890] io_wq_submit_work+0x5d/0x140 [ 168.217087] io_worker_handle_work+0x30a/0x980 [ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70 [ 168.217537] ? do_raw_spin_lock+0x100/0x180 [ 168.217742] ? rwlock_bug.part.0+0x60/0x60 [ 168.217943] io_wqe_worker+0x5fd/0x780 [ 168.218126] ? lock_downgrade+0x340/0x340 [ 168.218323] ? io_worker_handle_work+0x980/0x980 [ 168.218546] ? lockdep_hardirqs_on+0x17d/0x270 [ 168.218765] ? __kthread_parkme+0xca/0xe0 [ 168.218961] ? io_worker_handle_work+0x980/0x980 [ 168.219186] kthread+0x1f0/0x220 [ 168.219346] ? kthread_create_worker_on_cpu+0xb0/0xb0 [ 168.219590] ret_from_fork+0x24/0x30 [ 168.219768] [ 168.219846] Allocated by task 41758: [ 168.220021] save_stack+0x1b/0x40 [ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0 [ 168.220416] kmem_cache_alloc+0xe0/0x290 [ 168.220607] dup_fd+0x4e/0x5a0 [ 168.220758] copy_process+0xe35/0x2bf0 [ 168.220942] _do_fork+0xd8/0x550 [ 168.221102] __do_sys_clone+0xb5/0xe0 [ 168.221282] do_syscall_64+0x5e/0xe0 [ 168.221457] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.221729] [ 168.221848] Freed by task 41759: [ 168.222088] save_stack+0x1b/0x40 [ 168.222336] __kasan_slab_free+0x12f/0x180 [ 168.222632] slab_free_freelist_hook+0x4d/0x120 [ 168.222959] kmem_cache_free+0x90/0x2e0 [ 168.223239] do_exit+0x5d2/0x12e0 [ 168.223482] do_group_exit+0x6f/0x130 [ 168.223754] __x64_sys_exit_group+0x28/0x30 [ 168.224061] do_syscall_64+0x5e/0xe0 [ 168.224326] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.224686]
which indicates that current->files is no longer valid.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerQuoted 106 lines · Jens Axboe
Quoted 17 lines · Xiaoguang Wang
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msecQuoted 6 lines
With this patch:
Run status group 0 (all jobs):READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd
send_recvsend_recveventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statxand also saw this:
[ 168.208940] ================================================================== [ 168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000 [ 168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [ 168.209987] [ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318 [ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace: [ 168.210991] dump_stack+0x97/0xe0 [ 168.211164] print_address_description.constprop.0+0x1a/0x210 [ 168.211446] ? __lock_acquire+0x8bf/0x3000 [ 168.211649] __kasan_report.cold+0x20/0x39 [ 168.211851] ? __lock_acquire+0x8bf/0x3000 [ 168.212051] kasan_report+0x30/0x40 [ 168.212226] __lock_acquire+0x8bf/0x3000 [ 168.212432] ? ret_from_fork+0x24/0x30 [ 168.212623] ? stack_trace_save+0x81/0xa0 [ 168.212821] ? lockdep_hardirqs_on+0x270/0x270 [ 168.213039] ? save_stack+0x32/0x40 [ 168.213212] lock_acquire+0x122/0x570 [ 168.213398] ? __close_fd_get_file+0x40/0x150 [ 168.213615] ? lock_release+0x3f0/0x3f0 [ 168.213814] ? __lock_acquire+0x87e/0x3000 [ 168.214016] _raw_spin_lock+0x2c/0x40 [ 168.214196] ? __close_fd_get_file+0x40/0x150 [ 168.214408] __close_fd_get_file+0x40/0x150 [ 168.214618] io_issue_sqe+0x57f/0x22f0 [ 168.214803] ? lockdep_hardirqs_on+0x270/0x270 [ 168.215019] ? mark_held_locks+0x24/0x90 [ 168.215211] ? quarantine_put+0x6f/0x190 [ 168.215404] ? io_assign_current_work+0x59/0x80 [ 168.215623] ? __ia32_sys_io_uring_setup+0x30/0x30 [ 168.215855] ? find_held_lock+0xcb/0x100 [ 168.216054] ? io_worker_handle_work+0x289/0x980 [ 168.216280] ? lock_downgrade+0x340/0x340 [ 168.216476] ? io_wq_submit_work+0x5d/0x140 [ 168.216679] ? _raw_spin_unlock_irq+0x24/0x40 [ 168.216890] io_wq_submit_work+0x5d/0x140 [ 168.217087] io_worker_handle_work+0x30a/0x980 [ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70 [ 168.217537] ? do_raw_spin_lock+0x100/0x180 [ 168.217742] ? rwlock_bug.part.0+0x60/0x60 [ 168.217943] io_wqe_worker+0x5fd/0x780 [ 168.218126] ? lock_downgrade+0x340/0x340 [ 168.218323] ? io_worker_handle_work+0x980/0x980 [ 168.218546] ? lockdep_hardirqs_on+0x17d/0x270 [ 168.218765] ? __kthread_parkme+0xca/0xe0 [ 168.218961] ? io_worker_handle_work+0x980/0x980 [ 168.219186] kthread+0x1f0/0x220 [ 168.219346] ? kthread_create_worker_on_cpu+0xb0/0xb0 [ 168.219590] ret_from_fork+0x24/0x30 [ 168.219768] [ 168.219846] Allocated by task 41758: [ 168.220021] save_stack+0x1b/0x40 [ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0 [ 168.220416] kmem_cache_alloc+0xe0/0x290 [ 168.220607] dup_fd+0x4e/0x5a0 [ 168.220758] copy_process+0xe35/0x2bf0 [ 168.220942] _do_fork+0xd8/0x550 [ 168.221102] __do_sys_clone+0xb5/0xe0 [ 168.221282] do_syscall_64+0x5e/0xe0 [ 168.221457] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.221729] [ 168.221848] Freed by task 41759: [ 168.222088] save_stack+0x1b/0x40 [ 168.222336] __kasan_slab_free+0x12f/0x180 [ 168.222632] slab_free_freelist_hook+0x4d/0x120 [ 168.222959] kmem_cache_free+0x90/0x2e0 [ 168.223239] do_exit+0x5d2/0x12e0 [ 168.223482] do_group_exit+0x6f/0x130 [ 168.223754] __x64_sys_exit_group+0x28/0x30 [ 168.224061] do_syscall_64+0x5e/0xe0 [ 168.224326] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.224686]
Quoted 1 line
which indicates that
current->filesis no longer valid.
Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:
ret = test_close(&ring, ring.ring_fd, 1);
This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.
Why isn't io_req_init_async() just doing:
static inline void io_req_init_async(struct io_kiocb *req,
void (*func)(struct io_wq_work **))
{
if (!(req->flags & REQ_F_WORK_INITIALIZED)) {
req->work = (struct io_wq_work){ .func = func };
req->flags |= REQ_F_WORK_INITIALIZED;
}
}?
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerQuoted 131 lines · Jens Axboe
Quoted 106 lines · Jens Axboe
Quoted 17 lines · Xiaoguang Wang
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msecQuoted 6 lines
With this patch:
Run status group 0 (all jobs):READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd
send_recvsend_recveventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statxand also saw this:
[ 168.208940] ================================================================== [ 168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000 [ 168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [ 168.209987] [ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318 [ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace: [ 168.210991] dump_stack+0x97/0xe0 [ 168.211164] print_address_description.constprop.0+0x1a/0x210 [ 168.211446] ? __lock_acquire+0x8bf/0x3000 [ 168.211649] __kasan_report.cold+0x20/0x39 [ 168.211851] ? __lock_acquire+0x8bf/0x3000 [ 168.212051] kasan_report+0x30/0x40 [ 168.212226] __lock_acquire+0x8bf/0x3000 [ 168.212432] ? ret_from_fork+0x24/0x30 [ 168.212623] ? stack_trace_save+0x81/0xa0 [ 168.212821] ? lockdep_hardirqs_on+0x270/0x270 [ 168.213039] ? save_stack+0x32/0x40 [ 168.213212] lock_acquire+0x122/0x570 [ 168.213398] ? __close_fd_get_file+0x40/0x150 [ 168.213615] ? lock_release+0x3f0/0x3f0 [ 168.213814] ? __lock_acquire+0x87e/0x3000 [ 168.214016] _raw_spin_lock+0x2c/0x40 [ 168.214196] ? __close_fd_get_file+0x40/0x150 [ 168.214408] __close_fd_get_file+0x40/0x150 [ 168.214618] io_issue_sqe+0x57f/0x22f0 [ 168.214803] ? lockdep_hardirqs_on+0x270/0x270 [ 168.215019] ? mark_held_locks+0x24/0x90 [ 168.215211] ? quarantine_put+0x6f/0x190 [ 168.215404] ? io_assign_current_work+0x59/0x80 [ 168.215623] ? __ia32_sys_io_uring_setup+0x30/0x30 [ 168.215855] ? find_held_lock+0xcb/0x100 [ 168.216054] ? io_worker_handle_work+0x289/0x980 [ 168.216280] ? lock_downgrade+0x340/0x340 [ 168.216476] ? io_wq_submit_work+0x5d/0x140 [ 168.216679] ? _raw_spin_unlock_irq+0x24/0x40 [ 168.216890] io_wq_submit_work+0x5d/0x140 [ 168.217087] io_worker_handle_work+0x30a/0x980 [ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70 [ 168.217537] ? do_raw_spin_lock+0x100/0x180 [ 168.217742] ? rwlock_bug.part.0+0x60/0x60 [ 168.217943] io_wqe_worker+0x5fd/0x780 [ 168.218126] ? lock_downgrade+0x340/0x340 [ 168.218323] ? io_worker_handle_work+0x980/0x980 [ 168.218546] ? lockdep_hardirqs_on+0x17d/0x270 [ 168.218765] ? __kthread_parkme+0xca/0xe0 [ 168.218961] ? io_worker_handle_work+0x980/0x980 [ 168.219186] kthread+0x1f0/0x220 [ 168.219346] ? kthread_create_worker_on_cpu+0xb0/0xb0 [ 168.219590] ret_from_fork+0x24/0x30 [ 168.219768] [ 168.219846] Allocated by task 41758: [ 168.220021] save_stack+0x1b/0x40 [ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0 [ 168.220416] kmem_cache_alloc+0xe0/0x290 [ 168.220607] dup_fd+0x4e/0x5a0 [ 168.220758] copy_process+0xe35/0x2bf0 [ 168.220942] _do_fork+0xd8/0x550 [ 168.221102] __do_sys_clone+0xb5/0xe0 [ 168.221282] do_syscall_64+0x5e/0xe0 [ 168.221457] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.221729] [ 168.221848] Freed by task 41759: [ 168.222088] save_stack+0x1b/0x40 [ 168.222336] __kasan_slab_free+0x12f/0x180 [ 168.222632] slab_free_freelist_hook+0x4d/0x120 [ 168.222959] kmem_cache_free+0x90/0x2e0 [ 168.223239] do_exit+0x5d2/0x12e0 [ 168.223482] do_group_exit+0x6f/0x130 [ 168.223754] __x64_sys_exit_group+0x28/0x30 [ 168.224061] do_syscall_64+0x5e/0xe0 [ 168.224326] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.224686]Quoted 1 line
which indicates that
current->filesis no longer valid.Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:
ret =
test_close(&ring, ring.ring_fd, 1);This seems to be because you do
io_req_init_async()after callingio_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for theio_uringas it has ->flush() defined.Why isn't
io_req_init_async()just doing:static inline void io_req_init_async(struct
io_kiocb*req,void (*func)(struct io_wq_work **)) { if (!(req->flags & REQ_F_WORK_INITIALIZED)) { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; } }
Quoted 1 line
?
I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?
Also please fix up missing braces. The cases of:
if (something) {
line 1
line 2
} else
line 3
should always includes braces, if one clause has it.
A v5 with those two things would be ready to commit.
Signature · 1 line
Jens Axboe
Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Jens Axboe
Quoted 17 lines · Xiaoguang Wang
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msecQuoted 6 lines
With this patch:
Run status group 0 (all jobs):READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd
send_recvsend_recveventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statxand also saw this:
[ 168.208940] ================================================================== [ 168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000 [ 168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [ 168.209987] [ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318 [ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace: [ 168.210991] dump_stack+0x97/0xe0 [ 168.211164] print_address_description.constprop.0+0x1a/0x210 [ 168.211446] ? __lock_acquire+0x8bf/0x3000 [ 168.211649] __kasan_report.cold+0x20/0x39 [ 168.211851] ? __lock_acquire+0x8bf/0x3000 [ 168.212051] kasan_report+0x30/0x40 [ 168.212226] __lock_acquire+0x8bf/0x3000 [ 168.212432] ? ret_from_fork+0x24/0x30 [ 168.212623] ? stack_trace_save+0x81/0xa0 [ 168.212821] ? lockdep_hardirqs_on+0x270/0x270 [ 168.213039] ? save_stack+0x32/0x40 [ 168.213212] lock_acquire+0x122/0x570 [ 168.213398] ? __close_fd_get_file+0x40/0x150 [ 168.213615] ? lock_release+0x3f0/0x3f0 [ 168.213814] ? __lock_acquire+0x87e/0x3000 [ 168.214016] _raw_spin_lock+0x2c/0x40 [ 168.214196] ? __close_fd_get_file+0x40/0x150 [ 168.214408] __close_fd_get_file+0x40/0x150 [ 168.214618] io_issue_sqe+0x57f/0x22f0 [ 168.214803] ? lockdep_hardirqs_on+0x270/0x270 [ 168.215019] ? mark_held_locks+0x24/0x90 [ 168.215211] ? quarantine_put+0x6f/0x190 [ 168.215404] ? io_assign_current_work+0x59/0x80 [ 168.215623] ? __ia32_sys_io_uring_setup+0x30/0x30 [ 168.215855] ? find_held_lock+0xcb/0x100 [ 168.216054] ? io_worker_handle_work+0x289/0x980 [ 168.216280] ? lock_downgrade+0x340/0x340 [ 168.216476] ? io_wq_submit_work+0x5d/0x140 [ 168.216679] ? _raw_spin_unlock_irq+0x24/0x40 [ 168.216890] io_wq_submit_work+0x5d/0x140 [ 168.217087] io_worker_handle_work+0x30a/0x980 [ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70 [ 168.217537] ? do_raw_spin_lock+0x100/0x180 [ 168.217742] ? rwlock_bug.part.0+0x60/0x60 [ 168.217943] io_wqe_worker+0x5fd/0x780 [ 168.218126] ? lock_downgrade+0x340/0x340 [ 168.218323] ? io_worker_handle_work+0x980/0x980 [ 168.218546] ? lockdep_hardirqs_on+0x17d/0x270 [ 168.218765] ? __kthread_parkme+0xca/0xe0 [ 168.218961] ? io_worker_handle_work+0x980/0x980 [ 168.219186] kthread+0x1f0/0x220 [ 168.219346] ? kthread_create_worker_on_cpu+0xb0/0xb0 [ 168.219590] ret_from_fork+0x24/0x30 [ 168.219768] [ 168.219846] Allocated by task 41758: [ 168.220021] save_stack+0x1b/0x40 [ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0 [ 168.220416] kmem_cache_alloc+0xe0/0x290 [ 168.220607] dup_fd+0x4e/0x5a0 [ 168.220758] copy_process+0xe35/0x2bf0 [ 168.220942] _do_fork+0xd8/0x550 [ 168.221102] __do_sys_clone+0xb5/0xe0 [ 168.221282] do_syscall_64+0x5e/0xe0 [ 168.221457] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.221729] [ 168.221848] Freed by task 41759: [ 168.222088] save_stack+0x1b/0x40 [ 168.222336] __kasan_slab_free+0x12f/0x180 [ 168.222632] slab_free_freelist_hook+0x4d/0x120 [ 168.222959] kmem_cache_free+0x90/0x2e0 [ 168.223239] do_exit+0x5d2/0x12e0 [ 168.223482] do_group_exit+0x6f/0x130 [ 168.223754] __x64_sys_exit_group+0x28/0x30 [ 168.224061] do_syscall_64+0x5e/0xe0 [ 168.224326] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.224686]Quoted 1 line
which indicates that
current->filesis no longer valid.Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:
ret =
test_close(&ring, ring.ring_fd, 1);This seems to be because you do
io_req_init_async()after callingio_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for theio_uringas it has ->flush() defined.Why isn't
io_req_init_async()just doing:static inline void io_req_init_async(struct
io_kiocb*req,void (*func)(struct io_wq_work **)) { if (!(req->flags & REQ_F_WORK_INITIALIZED)) { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; } }Quoted 1 line
?
I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to
io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?Also please fix up missing braces. The cases of:
if (something) {
line 1
line 2
} else
line 3
Quoted 4 lines
should always includes braces, if one clause has it.
A v5 with those two things would be ready to commit.
There is another thing:
io_submit_sqes()
-> io_close() (let ->flush == NULL)
-> __io_close_finish()
-> filp_close(req->close.put_file, *req->work.files*);where req->work.files is garbage.
Signature · 1 line
Pavel Begunkov
hi,
Quoted 153 lines · Jens Axboe
Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Xiaoguang Wang
Quoted 17 lines
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msecQuoted 6 lines
With this patch:
Run status group 0 (all jobs):READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd
send_recvsend_recveventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statxand also saw this:
[ 168.208940] ================================================================== [ 168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000 [ 168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [ 168.209987] [ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318 [ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace: [ 168.210991] dump_stack+0x97/0xe0 [ 168.211164] print_address_description.constprop.0+0x1a/0x210 [ 168.211446] ? __lock_acquire+0x8bf/0x3000 [ 168.211649] __kasan_report.cold+0x20/0x39 [ 168.211851] ? __lock_acquire+0x8bf/0x3000 [ 168.212051] kasan_report+0x30/0x40 [ 168.212226] __lock_acquire+0x8bf/0x3000 [ 168.212432] ? ret_from_fork+0x24/0x30 [ 168.212623] ? stack_trace_save+0x81/0xa0 [ 168.212821] ? lockdep_hardirqs_on+0x270/0x270 [ 168.213039] ? save_stack+0x32/0x40 [ 168.213212] lock_acquire+0x122/0x570 [ 168.213398] ? __close_fd_get_file+0x40/0x150 [ 168.213615] ? lock_release+0x3f0/0x3f0 [ 168.213814] ? __lock_acquire+0x87e/0x3000 [ 168.214016] _raw_spin_lock+0x2c/0x40 [ 168.214196] ? __close_fd_get_file+0x40/0x150 [ 168.214408] __close_fd_get_file+0x40/0x150 [ 168.214618] io_issue_sqe+0x57f/0x22f0 [ 168.214803] ? lockdep_hardirqs_on+0x270/0x270 [ 168.215019] ? mark_held_locks+0x24/0x90 [ 168.215211] ? quarantine_put+0x6f/0x190 [ 168.215404] ? io_assign_current_work+0x59/0x80 [ 168.215623] ? __ia32_sys_io_uring_setup+0x30/0x30 [ 168.215855] ? find_held_lock+0xcb/0x100 [ 168.216054] ? io_worker_handle_work+0x289/0x980 [ 168.216280] ? lock_downgrade+0x340/0x340 [ 168.216476] ? io_wq_submit_work+0x5d/0x140 [ 168.216679] ? _raw_spin_unlock_irq+0x24/0x40 [ 168.216890] io_wq_submit_work+0x5d/0x140 [ 168.217087] io_worker_handle_work+0x30a/0x980 [ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70 [ 168.217537] ? do_raw_spin_lock+0x100/0x180 [ 168.217742] ? rwlock_bug.part.0+0x60/0x60 [ 168.217943] io_wqe_worker+0x5fd/0x780 [ 168.218126] ? lock_downgrade+0x340/0x340 [ 168.218323] ? io_worker_handle_work+0x980/0x980 [ 168.218546] ? lockdep_hardirqs_on+0x17d/0x270 [ 168.218765] ? __kthread_parkme+0xca/0xe0 [ 168.218961] ? io_worker_handle_work+0x980/0x980 [ 168.219186] kthread+0x1f0/0x220 [ 168.219346] ? kthread_create_worker_on_cpu+0xb0/0xb0 [ 168.219590] ret_from_fork+0x24/0x30 [ 168.219768] [ 168.219846] Allocated by task 41758: [ 168.220021] save_stack+0x1b/0x40 [ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0 [ 168.220416] kmem_cache_alloc+0xe0/0x290 [ 168.220607] dup_fd+0x4e/0x5a0 [ 168.220758] copy_process+0xe35/0x2bf0 [ 168.220942] _do_fork+0xd8/0x550 [ 168.221102] __do_sys_clone+0xb5/0xe0 [ 168.221282] do_syscall_64+0x5e/0xe0 [ 168.221457] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.221729] [ 168.221848] Freed by task 41759: [ 168.222088] save_stack+0x1b/0x40 [ 168.222336] __kasan_slab_free+0x12f/0x180 [ 168.222632] slab_free_freelist_hook+0x4d/0x120 [ 168.222959] kmem_cache_free+0x90/0x2e0 [ 168.223239] do_exit+0x5d2/0x12e0 [ 168.223482] do_group_exit+0x6f/0x130 [ 168.223754] __x64_sys_exit_group+0x28/0x30 [ 168.224061] do_syscall_64+0x5e/0xe0 [ 168.224326] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.224686]Quoted 1 line
which indicates that
current->filesis no longer valid.Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:
ret =
test_close(&ring, ring.ring_fd, 1);This seems to be because you do
io_req_init_async()after callingio_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for theio_uringas it has ->flush() defined.Why isn't
io_req_init_async()just doing:static inline void io_req_init_async(struct
io_kiocb*req,void (*func)(struct io_wq_work **)) { if (!(req->flags & REQ_F_WORK_INITIALIZED)) { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; } }Quoted 1 line
?
I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to
io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?Also please fix up missing braces. The cases of:
if (something) {
line 1
line 2
} else
line 3Quoted 4 lines
should always includes braces, if one clause has it.
A v5 with those two things would be ready to commit.
Quoted 8 lines
There is another thing:
io_submit_sqes()-> io_close() (let ->flush == NULL) -> __io_close_finish() -> filp_close(req->close.put_file, *req->work.files*);where
req->work.filesis garbage.
I think this bug is independent of my patch. Without my patches, if close request will be submitted and completed inline, req->work.files will be NULL, it's still problematic, should we use current->files here?
Regards,
Xiaoguang Wang
Quoted 1 line
Quoted 9 lines · Xiaoguang Wang
Quoted 8 lines
There is another thing:
io_submit_sqes()
->io_close()(let ->flush == NULL)
->__io_close_finish()
->filp_close(req->close.put_file, *req->work.files*);where
req->work.filesis garbage.I think this bug is independent of my patch. Without my patches, if close request
It looks like it's ok to pass NULL, at least Jens did it here and I see an occurrence of ->flush(NULL). And it's usually not referenced or completely ignored. I'll check later.
Quoted 2 lines
will be submitted and completed inline,
req->work.fileswill be NULL, it's still problematic, should we usecurrent->fileshere?
Signature · 1 line
Pavel Begunkov
hi jens,
Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Xiaoguang Wang
Quoted 17 lines
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msecQuoted 6 lines
With this patch:
Run status group 0 (all jobs):READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd
send_recvsend_recveventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statxand also saw this:
[ 168.208940] ================================================================== [ 168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000 [ 168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [ 168.209987] [ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318 [ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace: [ 168.210991] dump_stack+0x97/0xe0 [ 168.211164] print_address_description.constprop.0+0x1a/0x210 [ 168.211446] ? __lock_acquire+0x8bf/0x3000 [ 168.211649] __kasan_report.cold+0x20/0x39 [ 168.211851] ? __lock_acquire+0x8bf/0x3000 [ 168.212051] kasan_report+0x30/0x40 [ 168.212226] __lock_acquire+0x8bf/0x3000 [ 168.212432] ? ret_from_fork+0x24/0x30 [ 168.212623] ? stack_trace_save+0x81/0xa0 [ 168.212821] ? lockdep_hardirqs_on+0x270/0x270 [ 168.213039] ? save_stack+0x32/0x40 [ 168.213212] lock_acquire+0x122/0x570 [ 168.213398] ? __close_fd_get_file+0x40/0x150 [ 168.213615] ? lock_release+0x3f0/0x3f0 [ 168.213814] ? __lock_acquire+0x87e/0x3000 [ 168.214016] _raw_spin_lock+0x2c/0x40 [ 168.214196] ? __close_fd_get_file+0x40/0x150 [ 168.214408] __close_fd_get_file+0x40/0x150 [ 168.214618] io_issue_sqe+0x57f/0x22f0 [ 168.214803] ? lockdep_hardirqs_on+0x270/0x270 [ 168.215019] ? mark_held_locks+0x24/0x90 [ 168.215211] ? quarantine_put+0x6f/0x190 [ 168.215404] ? io_assign_current_work+0x59/0x80 [ 168.215623] ? __ia32_sys_io_uring_setup+0x30/0x30 [ 168.215855] ? find_held_lock+0xcb/0x100 [ 168.216054] ? io_worker_handle_work+0x289/0x980 [ 168.216280] ? lock_downgrade+0x340/0x340 [ 168.216476] ? io_wq_submit_work+0x5d/0x140 [ 168.216679] ? _raw_spin_unlock_irq+0x24/0x40 [ 168.216890] io_wq_submit_work+0x5d/0x140 [ 168.217087] io_worker_handle_work+0x30a/0x980 [ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70 [ 168.217537] ? do_raw_spin_lock+0x100/0x180 [ 168.217742] ? rwlock_bug.part.0+0x60/0x60 [ 168.217943] io_wqe_worker+0x5fd/0x780 [ 168.218126] ? lock_downgrade+0x340/0x340 [ 168.218323] ? io_worker_handle_work+0x980/0x980 [ 168.218546] ? lockdep_hardirqs_on+0x17d/0x270 [ 168.218765] ? __kthread_parkme+0xca/0xe0 [ 168.218961] ? io_worker_handle_work+0x980/0x980 [ 168.219186] kthread+0x1f0/0x220 [ 168.219346] ? kthread_create_worker_on_cpu+0xb0/0xb0 [ 168.219590] ret_from_fork+0x24/0x30 [ 168.219768] [ 168.219846] Allocated by task 41758: [ 168.220021] save_stack+0x1b/0x40 [ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0 [ 168.220416] kmem_cache_alloc+0xe0/0x290 [ 168.220607] dup_fd+0x4e/0x5a0 [ 168.220758] copy_process+0xe35/0x2bf0 [ 168.220942] _do_fork+0xd8/0x550 [ 168.221102] __do_sys_clone+0xb5/0xe0 [ 168.221282] do_syscall_64+0x5e/0xe0 [ 168.221457] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.221729] [ 168.221848] Freed by task 41759: [ 168.222088] save_stack+0x1b/0x40 [ 168.222336] __kasan_slab_free+0x12f/0x180 [ 168.222632] slab_free_freelist_hook+0x4d/0x120 [ 168.222959] kmem_cache_free+0x90/0x2e0 [ 168.223239] do_exit+0x5d2/0x12e0 [ 168.223482] do_group_exit+0x6f/0x130 [ 168.223754] __x64_sys_exit_group+0x28/0x30 [ 168.224061] do_syscall_64+0x5e/0xe0 [ 168.224326] entry_SYSCALL_64_after_hwframe+0x49/0xb3 [ 168.224686]Quoted 1 line
which indicates that
current->filesis no longer valid.Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:
ret =
test_close(&ring, ring.ring_fd, 1);This seems to be because you do
io_req_init_async()after callingio_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for theio_uringas it has ->flush() defined.Why isn't
io_req_init_async()just doing:static inline void io_req_init_async(struct
io_kiocb*req,void (*func)(struct io_wq_work **)) { if (!(req->flags & REQ_F_WORK_INITIALIZED)) { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; } }Quoted 1 line
?
I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to
io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?Also please fix up missing braces. The cases of:
if (something) {
line 1
line 2
} else
line 3
Quoted 3 lines
should always includes braces, if one clause has it.
A v5 with those two things would be ready to commit.
First thanks for figuring out this bug.
Indeed, I had run test cases in liburing/test multiple rounds before sending V4 patches, and all test cases pass, no kernel issues occurred, here is my kernel build config, seems that I had kasan enabled.
[lege@localhost linux]$ cat .config | grep KASAN CONFIG_KASAN_SHADOW_OFFSET=0xdffffc0000000000 CONFIG_HAVE_ARCH_KASAN=yCONFIG_HAVE_ARCH_KASAN_VMALLOC=yCONFIG_CC_HAS_KASAN_GENERIC=yCONFIG_KASAN=yCONFIG_KASAN_GENERIC=yCONFIG_KASAN_OUTLINE=y
# CONFIG_KASAN_INLINE is not setCONFIG_KASAN_STACK=1
# CONFIG_KASAN_VMALLOC is not set
# CONFIG_TEST_KASAN is not set
But I don't know why I did't reproduce the bug, sorry.
Are you fine below changes?
fs/io_uring.c +9 -3
@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work);static inline void io_req_init_async (struct io_kiocb *req,void (*func)(struct io_wq_work **)) { -if (req->flags &REQ_F_WORK_INITIALIZED ) - req->work.func = func; -else { +if (req->flags &REQ_F_WORK_INITIALIZED ) { +if (!req->work.func) + req->work.func = func; + }else { req->work = (struct io_wq_work ){ .func = func }; req->flags |=REQ_F_WORK_INITIALIZED ; }
@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req,return ret; req->flags |=REQ_F_NEED_CLEANUP ; +/* Splice will be punted aync, so initialize io_wq_work firstly_*/ +io_req_init_async (req, io_wq_submit_work);if (!S_ISREG (file_inode (sp->file_in)->i_mode)) req->work.flags |=IO_WQ_WORK_UNBOUND ;
@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)static int io_close_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) { +/* Close may be punted aync, so initialize io_wq_work firstly */ +io_req_init_async (req, io_wq_submit_work); +/* * If we queue this for async, it must not be cancellable. That would * leave the 'file' in an undeterminate state.
Regards,
Xiaoguang Wang
Quoted 1 line
hi,
Quoted 199 lines
hi jens,
Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Xiaoguang Wang
Quoted 17 lines
If requests can be submitted and completed inline, we don't need to initialize whole
io_wq_workinio_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whetherio_wq_workis initialized.I use
/dev/nullb0to evaluate performance improvement in my physical machine:
modprobenull_blknr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120before this patch:
Run status group 0 (all jobs):
READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msecQuoted 6 lines
With this patch:
Run status group 0 (all jobs):
READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msecAbout 5% improvement.
There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:
Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd
send_recvsend_recveventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statxand also saw this:
[ 168.208940] ==================================================================
[ 168.209311] BUG: KASAN: use-after-free in__lock_acquire+0x8bf/0x3000
[ 168.209626] Read of size 8 at addrffff88806801c0d8by task io_wqe_worker-0/41761 [ 168.209987]
[ 168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[ 168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [ 168.210857] Call Trace:
[ 168.210991]dump_stack+0x97/0xe0
[ 168.211164] print_address_description.constprop.0+0x1a/0x210
[ 168.211446] ?__lock_acquire+0x8bf/0x3000
[ 168.211649]__kasan_report.cold+0x20/0x39
[ 168.211851] ?__lock_acquire+0x8bf/0x3000
[ 168.212051]kasan_report+0x30/0x40
[ 168.212226]__lock_acquire+0x8bf/0x3000
[ 168.212432] ?ret_from_fork+0x24/0x30
[ 168.212623] ?stack_trace_save+0x81/0xa0
[ 168.212821] ?lockdep_hardirqs_on+0x270/0x270
[ 168.213039] ?save_stack+0x32/0x40
[ 168.213212]lock_acquire+0x122/0x570
[ 168.213398] ?__close_fd_get_file+0x40/0x150
[ 168.213615] ?lock_release+0x3f0/0x3f0
[ 168.213814] ?__lock_acquire+0x87e/0x3000
[ 168.214016]_raw_spin_lock+0x2c/0x40
[ 168.214196] ?__close_fd_get_file+0x40/0x150
[ 168.214408]__close_fd_get_file+0x40/0x150
[ 168.214618]io_issue_sqe+0x57f/0x22f0
[ 168.214803] ?lockdep_hardirqs_on+0x270/0x270
[ 168.215019] ?mark_held_locks+0x24/0x90
[ 168.215211] ?quarantine_put+0x6f/0x190
[ 168.215404] ?io_assign_current_work+0x59/0x80
[ 168.215623] ?__ia32_sys_io_uring_setup+0x30/0x30
[ 168.215855] ?find_held_lock+0xcb/0x100
[ 168.216054] ?io_worker_handle_work+0x289/0x980
[ 168.216280] ?lock_downgrade+0x340/0x340
[ 168.216476] ?io_wq_submit_work+0x5d/0x140
[ 168.216679] ?_raw_spin_unlock_irq+0x24/0x40
[ 168.216890]io_wq_submit_work+0x5d/0x140
[ 168.217087]io_worker_handle_work+0x30a/0x980
[ 168.217305] ? io_wqe_dec_running.isra.0+0x70/0x70
[ 168.217537] ?do_raw_spin_lock+0x100/0x180
[ 168.217742] ? rwlock_bug.part.0+0x60/0x60
[ 168.217943]io_wqe_worker+0x5fd/0x780
[ 168.218126] ?lock_downgrade+0x340/0x340
[ 168.218323] ?io_worker_handle_work+0x980/0x980
[ 168.218546] ?lockdep_hardirqs_on+0x17d/0x270
[ 168.218765] ?__kthread_parkme+0xca/0xe0
[ 168.218961] ?io_worker_handle_work+0x980/0x980
[ 168.219186]kthread+0x1f0/0x220
[ 168.219346] ?kthread_create_worker_on_cpu+0xb0/0xb0
[ 168.219590]ret_from_fork+0x24/0x30
[ 168.219768]
[ 168.219846] Allocated by task 41758:
[ 168.220021]save_stack+0x1b/0x40
[ 168.220185] __kasan_kmalloc.constprop.0+0xc2/0xd0
[ 168.220416]kmem_cache_alloc+0xe0/0x290
[ 168.220607]dup_fd+0x4e/0x5a0
[ 168.220758]copy_process+0xe35/0x2bf0
[ 168.220942]_do_fork+0xd8/0x550
[ 168.221102]__do_sys_clone+0xb5/0xe0
[ 168.221282]do_syscall_64+0x5e/0xe0
[ 168.221457]entry_SYSCALL_64_after_hwframe+0x49/0xb3
[ 168.221729]
[ 168.221848] Freed by task 41759:
[ 168.222088]save_stack+0x1b/0x40
[ 168.222336]__kasan_slab_free+0x12f/0x180
[ 168.222632]slab_free_freelist_hook+0x4d/0x120
[ 168.222959]kmem_cache_free+0x90/0x2e0
[ 168.223239]do_exit+0x5d2/0x12e0
[ 168.223482]do_group_exit+0x6f/0x130
[ 168.223754]__x64_sys_exit_group+0x28/0x30
[ 168.224061]do_syscall_64+0x5e/0xe0
[ 168.224326]entry_SYSCALL_64_after_hwframe+0x49/0xb3
[ 168.224686]Quoted 1 line
which indicates that
current->filesis no longer valid.Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:
ret =
test_close(&ring, ring.ring_fd, 1);This seems to be because you do
io_req_init_async()after callingio_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for theio_uringas it has ->flush() defined.Why isn't
io_req_init_async()just doing:static inline void
io_req_init_async(struct io_kiocb *req, void (*func)(struct io_wq_work **)){
if (!(req->flags &REQ_F_WORK_INITIALIZED)) {req->work= (struct io_wq_work){ .func = func };req->flags|=REQ_F_WORK_INITIALIZED; }
}Quoted 1 line
?
I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to
io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?Also please fix up missing braces. The cases of:
if (something) {
line 1
line 2
} else
line 3Quoted 3 lines
should always includes braces, if one clause has it.
A v5 with those two things would be ready to commit.
First thanks for figuring out this bug.
Indeed, I had run test cases in liburing/test multiple rounds before sending V4 patches, and all test cases pass, no kernel issues occurred, here is my kernel build config, seems that I had kasan enabled.
[lege@localhost linux]$ cat .config | grep KASANCONFIG_KASAN_SHADOW_OFFSET=0xdffffc0000000000CONFIG_HAVE_ARCH_KASAN=yCONFIG_HAVE_ARCH_KASAN_VMALLOC=yCONFIG_CC_HAS_KASAN_GENERIC=yCONFIG_KASAN=yCONFIG_KASAN_GENERIC=yCONFIG_KASAN_OUTLINE=y
#CONFIG_KASAN_INLINEis not setCONFIG_KASAN_STACK=1
#CONFIG_KASAN_VMALLOCis not set
#CONFIG_TEST_KASANis not set
But I don't know why I did't reproduce the bug, sorry.Are you fine below changes?
fs/io_uring.c +6
-3@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work); static inline void io_req_init_async(struct io_kiocb *req, void (*func)(struct io_wq_work **)) { - if (req->flags & REQ_F_WORK_INITIALIZED) - req->work.func = func; - else { + if (req->flags & REQ_F_WORK_INITIALIZED) { + if (!req->work.func) + req->work.func = func; + } else { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; }@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req, return ret; req->flags |= REQ_F_NEED_CLEANUP; + /* Splice will be punted aync, so initialize io_wq_work firstly_*/ + io_req_init_async(req, io_wq_submit_work); if (!S_ISREG(file_inode(sp->file_in)->i_mode)) req->work.flags |= IO_WQ_WORK_UNBOUND;
fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3600
Quoted 6 lines
fs/io_uring.c +3
-0@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + /* Close may be punted aync, so initialize io_wq_work firstly */ + io_req_init_async(req, io_wq_submit_work); +
For splice and close requests, these two about io_req_init_async() calls should be io_req_init_async(req, NULL), because they change req->work.flags firstly.
Regards,
Xiaoguang Wang
Quoted 9 lines
fs/io_uring.c +0
-0/** If we queue this for async, it must not be cancellable. That would * leave the 'file' in an undeterminate state. Regards,
Xiaoguang WangQuoted 1 line
Quoted 29 lines
Quoted 29 lines
fs/io_uring.c +6
-3@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work); static inline void io_req_init_async(struct io_kiocb *req, void (*func)(struct io_wq_work **)) { - if (req->flags & REQ_F_WORK_INITIALIZED) - req->work.func = func; - else { + if (req->flags & REQ_F_WORK_INITIALIZED) { + if (!req->work.func) + req->work.func = func; + } else { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; }@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req, return ret; req->flags |= REQ_F_NEED_CLEANUP; + /* Splice will be punted aync, so initialize io_wq_work firstly_*/ + io_req_init_async(req, io_wq_submit_work); if (!S_ISREG(file_inode(sp->file_in)->i_mode)) req->work.flags |= IO_WQ_WORK_UNBOUND;@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)
Quoted 7 lines
fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3600
Quoted 5 lines
fs/io_uring.c +3
-0static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + /* Close may be punted aync, so initialize io_wq_work firstly */ + io_req_init_async(req, io_wq_submit_work); +For splice and close requests, these two about
io_req_init_async()calls should beio_req_init_async(req, NULL), because they changereq->work.flagsfirstly.
Please no. Such assumptions/dependencies are prone to break. It'll get us subtle bugs in no time.
BTW, why not io_wq_submit_work in place of NULL?
Signature · 1 line
Pavel Begunkov
hi,
Quoted 37 lines
Quoted 29 lines
Quoted 29 lines
fs/io_uring.c +6
-3@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work); static inline void io_req_init_async(struct io_kiocb *req, void (*func)(struct io_wq_work **)) { - if (req->flags & REQ_F_WORK_INITIALIZED) - req->work.func = func; - else { + if (req->flags & REQ_F_WORK_INITIALIZED) { + if (!req->work.func) + req->work.func = func; + } else { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; }@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req, return ret; req->flags |= REQ_F_NEED_CLEANUP; + /* Splice will be punted aync, so initialize io_wq_work firstly_*/ + io_req_init_async(req, io_wq_submit_work); if (!S_ISREG(file_inode(sp->file_in)->i_mode)) req->work.flags |= IO_WQ_WORK_UNBOUND;@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)Quoted 7 lines
fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3600
Quoted 5 lines
fs/io_uring.c +3
-0static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + /* Close may be punted aync, so initialize io_wq_work firstly */ + io_req_init_async(req, io_wq_submit_work); +For splice and close requests, these two about
io_req_init_async()calls should beio_req_init_async(req, NULL), because they changereq->work.flagsfirstly.
Quoted 4 lines
Please no. Such assumptions/dependencies are prone to break. It'll get us subtle bugs in no time.
BTW, why not
io_wq_submit_workin place of NULL?
In the begin of __io_splice_prep or io_close_prep, current io_uring mainline codes will modify req->work.flags firstly, so we need to call io_req_init_async to initialize io_wq_work before the work.flags modification.
For below codes:
static inline void io_req_init_async(struct io_kiocb *req,
void (*func)(struct io_wq_work **))
{
if (req->flags & REQ_F_WORK_INITIALIZED) {
if (!req->work.func)
req->work.func = func;
} else {
req->work = (struct io_wq_work){ .func = func };
req->flags |= REQ_F_WORK_INITIALIZED;
}
}if we not pass NULL to parameter 'func', e.g. pass io_wq_submit_work, then we can not use io_req_init_async() to pass io_close_finish again.
Now I'm confused how to write better codes based on current io_uring mainline codes :) If you have some free time, please have a deeper look, thanks.
Regards,
Xiaoguang Wang
Quoted 1 line
Quoted 16 lines · Xiaoguang Wang
In the begin of
__io_splice_preporio_close_prep, currentio_uringmainline codes will modifyreq->work.flagsfirstly, so we need to callio_req_init_asyncto initializeio_wq_workbefore the work.flags modification.
For below codes:
static inline void io_req_init_async(structio_kiocb*req,
void (*func)(structio_wq_work**))
{
if (req->flags&REQ_F_WORK_INITIALIZED) {
if (!req->work.func)
req->work.func= func;
} else {
req->work= (struct io_wq_work){ .func = func };req->flags|=REQ_F_WORK_INITIALIZED;
}
}
Quoted 2 lines
if we not pass NULL to parameter 'func', e.g. pass
io_wq_submit_work, then we can not useio_req_init_async()to passio_close_finishagain.
It's not as bad, just the thing you poked is overused and don't have strict rules. I have a feeling, that for it to be done right it'd need more fundamental refactoring with putting everything related to ->work closer to io_queue_async_work().
Quoted 3 lines
Now I'm confused how to write better codes based on current
io_uringmainline codes :) If you have some free time, please have a deeper look, thanks.
Signature · 1 line
Pavel Begunkov
hi,
Quoted 17 lines · Xiaoguang Wang
Quoted 16 lines
In the begin of
__io_splice_preporio_close_prep, currentio_uringmainline codes will modifyreq->work.flagsfirstly, so we need to callio_req_init_asyncto initializeio_wq_workbefore the work.flags modification.
For below codes:
static inline void io_req_init_async(structio_kiocb*req,
void (*func)(structio_wq_work**))
{
if (req->flags&REQ_F_WORK_INITIALIZED) {
if (!req->work.func)
req->work.func= func;
} else {
req->work= (struct io_wq_work){ .func = func };req->flags|=REQ_F_WORK_INITIALIZED;
}
}
Quoted 6 lines
Quoted 2 lines
if we not pass NULL to parameter 'func', e.g. pass
io_wq_submit_work, then we can not useio_req_init_async()to passio_close_finishagain.It's not as bad, just the thing you poked is overused and don't have strict rules. I have a feeling, that for it to be done right it'd need more fundamental refactoring with putting everything related to ->work closer to
io_queue_async_work().
Yes, I totally agree with you, that would be safer.
Regards,
Xiaoguang Wang
Quoted 5 lines
Quoted 4 lines
Now I'm confused how to write better codes based on current
io_uringmainline codes :) If you have some free time, please have a deeper look, thanks.
PATCH v5 1/2 io_uring: avoid whole io_wq_work copy for requests completed inline
In reply to Pavel BegunkovIf requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.
I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120
before this patch:
Run status group 0 (all jobs):
READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msec
With this patch:
Run status group 0 (all jobs):
READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msec
About 5% improvement.
---
V4:
add io_req_init_async() helper
V5:
refactor io_req_init_async() to io_init_req_work() and io_init_req_work_func in case we need to change io_wq_work.func separately.
--- fs/io-wq.h | 5 ---- fs/io_uring.c | 83 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 67 insertions(+), 21 deletions(-)
fs/io-wq.h +0 -5
@@ -94,11 +94,6 @@ struct io_wq_work {pid_t task_pid; }; -#define INIT_IO_WORK (work, _func) \ - do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ - } while (0) \ -static inline struct io_wq_work *wq_next_work (struct io_wq_work *work) {if (!work->list.next)
fs/io_uring.c +67 -16
201 lines not on this page · raw message
Signature · 1 line
2.17.2
PATCH v5 2/2 io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Basically IORING_OP_POLL_ADD command and async armed poll handlers for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED is set, can we do io_wq_work copy and restore.
---
V3:
drop the REQ_F_WORK_NEED_RESTORE flag introduced in V2 patch, just use REQ_F_WORK_INITIALIZED to control whether to do io_wq_work copy and restore.
--- fs/io_uring.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
@@ -4527,7 +4527,8 @@ static void io_async_task_func(struct callback_head *cb)spin_unlock_irq (&ctx->completion_lock);/* restore ->work in case we need to retry again */ -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work,sizeof (req->work));kfree (apoll);if (!canceled) {
@@ -4624,7 +4625,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)return false; req->flags |=REQ_F_POLLED ; -memcpy (&apoll->work, &req->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&apoll->work, &req->work,sizeof (req->work)); had_io = req->io !=NULL ;get_task_struct (current);
@@ -4649,7 +4651,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)if (!had_io)io_poll_remove_double (req);spin_unlock_irq (&ctx->completion_lock); -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work,sizeof (req->work));kfree (apoll);return false; }
@@ -4694,7 +4697,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)* io_req_work_drop_env below when dropping the * final reference. */ -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work, +sizeof (req->work));kfree (apoll); } }
Signature · 1 line
2.17.2
Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.
Regards,
Xiaoguang Wang
Quoted 60 lines
Basically
IORING_OP_POLL_ADDcommand and async armed poll handlers for regular commands don't touchio_wq_work, so onlyREQ_F_WORK_INITIALIZEDis set, can we doio_wq_workcopy and restore.Signed-off-by Xiaoguang Wang <[email protected]>---
V3:
drop theREQ_F_WORK_NEED_RESTOREflag introduced in V2 patch, just useREQ_F_WORK_INITIALIZEDto control whether to doio_wq_workcopy and restore.--- fs/io_uring.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-)fs/io_uring.c +9
-4@@ -4527,7 +4527,8 @@ static void io_async_task_func(struct callback_head *cb)spin_unlock_irq (&ctx->completion_lock);/* restore ->work in case we need to retry again */ -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work,sizeof (req->work));kfree (apoll);if (!canceled) {@@ -4624,7 +4625,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)return false; req->flags |=REQ_F_POLLED ; -memcpy (&apoll->work, &req->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&apoll->work, &req->work,sizeof (req->work)); had_io = req->io !=NULL ;get_task_struct (current);@@ -4649,7 +4651,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)if (!had_io)io_poll_remove_double (req);spin_unlock_irq (&ctx->completion_lock); -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work,sizeof (req->work));kfree (apoll);return false; }@@ -4694,7 +4697,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)* io_req_work_drop_env below when dropping the * final reference. */ -memcpy (&req->work, &apoll->work,sizeof (req->work)); +if (req->flags &REQ_F_WORK_INITIALIZED ) +memcpy (&req->work, &apoll->work, +sizeof (req->work));kfree (apoll); } }
Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 4 lines · Xiaoguang Wang
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related
io_wq_workfirstly.
It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.
Quoted 58 lines
Quoted 58 lines
Basically
IORING_OP_POLL_ADDcommand and async armed poll handlers for regular commands don't touchio_wq_work, so onlyREQ_F_WORK_INITIALIZEDis set, can we doio_wq_workcopy and restore.Signed-off-by Xiaoguang Wang <[email protected]>---
V3:
drop theREQ_F_WORK_NEED_RESTOREflag introduced in V2 patch, just useREQ_F_WORK_INITIALIZEDto control whether to doio_wq_workcopy and restore.
---
fs/io_uring.c| 13 +++++++++----1 file changed, 9 insertions(+), 4 deletions(-)fs/io_uring.c +9
-4@@ -4527,7 +4527,8 @@ static void io_async_task_func(struct callback_head *cb) spin_unlock_irq(&ctx->completion_lock); /* restore ->work in case we need to retry again */ - memcpy(&req->work, &apoll->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&req->work, &apoll->work, sizeof(req->work)); kfree(apoll); if (!canceled) {@@ -4624,7 +4625,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req) return false; req->flags |= REQ_F_POLLED; - memcpy(&apoll->work, &req->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&apoll->work, &req->work, sizeof(req->work)); had_io = req->io != NULL; get_task_struct(current);@@ -4649,7 +4651,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req) if (!had_io) io_poll_remove_double(req); spin_unlock_irq(&ctx->completion_lock); - memcpy(&req->work, &apoll->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&req->work, &apoll->work, sizeof(req->work)); kfree(apoll); return false; }@@ -4694,7 +4697,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)* io_req_work_drop_env below when dropping the * final reference. */ - memcpy(&req->work, &apoll->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&req->work, &apoll->work, + sizeof(req->work)); kfree(apoll); } }
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 8 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related
io_wq_workfirstly.It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.
Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.
Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.
Signature · 1 line
Jens Axboe
Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related
io_wq_workfirstly.
Quoted 9 lines
Quoted 2 lines
It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.
Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.
Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.
I just think it should not be done at expense of robustness.
e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space in io_kiocb (in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).
Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.
And then we can go and fix every other problem until this patch set looks good.
Signature · 1 line
Pavel Begunkov
Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 28 lines · Pavel Begunkov
Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related
io_wq_workfirstly.Quoted 9 lines
Quoted 2 lines
It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.
Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.
Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.
I just think it should not be done at expense of robustness.
e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with
io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space inio_kiocb(in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).
Quoted 3 lines
Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.
How about diff below? if split and cooked properly.
3 files changed, 73 insertions(+), 164 deletions(-)
fs/io-wq.c +6 -4
@@ -112,6 +112,7 @@ struct io_wq {unsigned long state;free_work_fn *free_work; +io_wq_work_fn *do_work;struct task_struct *manager;struct user_struct *user;
@@ -528,7 +529,7 @@ static void io_worker_handle_work(struct io_worker *worker) hash =io_get_work_hash (work); linked = old_work = work; - linked->func (&linked); + wq->do_work (&linked); linked = (old_work == linked) ?NULL : linked; work = next_hashed;
@@ -785,7 +786,7 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)struct io_wq_work *old_work = work; work->flags |=IO_WQ_WORK_CANCEL ; - work->func (&work); + wq->do_work (&work); work = (work == old_work) ?NULL : work; wq->free_work (old_work); }while (work);
@@ -1027,7 +1028,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)int ret = -ENOMEM , node;struct io_wq *wq; -if (WARN_ON_ONCE (!data->free_work)) +if (WARN_ON_ONCE (!data->free_work || !data->do_work))return ERR_PTR (-EINVAL ); wq =kzalloc (sizeof (*wq),GFP_KERNEL );
@@ -1041,6 +1042,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) } wq->free_work = data->free_work; + wq->do_work = data->do_work;/* caller must already hold a reference to this */ wq->user = data->user;
@@ -1097,7 +1099,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)bool io_wq_get (struct io_wq *wq,struct io_wq_data *data) { -if (data->free_work != wq->free_work) +if (data->free_work != wq->free_work || data->do_work != wq->do_work)return false;return refcount_inc_not_zero (&wq->use_refs);
fs/io-wq.h +4 -3
@@ -85,7 +85,6 @@ static inline void wq_list_del(struct io_wq_work_list *list,struct io_wq_work {struct io_wq_work_node list; -void (*func)(struct io_wq_work **);struct files_struct *files;struct mm_struct *mm;const struct cred *creds;
@@ -94,9 +93,9 @@ struct io_wq_work {pid_t task_pid; }; -#define INIT_IO_WORK (work, _func) \ +#define INIT_IO_WORK (work) \ do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ + *(work) = (struct io_wq_work){}; \ } while (0) \static inline struct io_wq_work *wq_next_work (struct io_wq_work *work)
@@ -108,10 +107,12 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) }typedef void (free_work_fn )(struct io_wq_work *); +typedef void (io_wq_work_fn )(struct io_wq_work **);struct io_wq_data {struct user_struct *user; +io_wq_work_fn *do_work;free_work_fn *free_work; };
fs/io_uring.c +63 -157
352 lines not on this page · raw message
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 38 lines · Pavel Begunkov
Quoted 28 lines · Pavel Begunkov
Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related
io_wq_workfirstly.Quoted 9 lines
Quoted 2 lines
It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.
Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.
Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.
I just think it should not be done at expense of robustness.
e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with
io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space inio_kiocb(in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).Quoted 3 lines
Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.
How about diff below? if split and cooked properly.
3 files changed, 73 insertions(+), 164 deletions(-)
fs/io-wq.h struct io_wq_work { · line 100
Quoted 10 lines
fs/io-wq.h +2
-2@@ -94,9 +93,9 @@ struct io_wq_work {pid_t task_pid; }; -#define INIT_IO_WORK (work, _func) \ +#define INIT_IO_WORK (work) \ do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ + *(work) = (struct io_wq_work){}; \ } while (0) \
Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.
From a quick look at this, otherwise looks great! Please do split and submit this.
Signature · 1 line
Jens Axboe
Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 12 lines · Jens Axboe
fs/io-wq.h struct io_wq_work { · line 99
Quoted 7 lines · Pavel Begunkov
fs/io-wq.h +2
-2-#define INIT_IO_WORK (work, _func) \ +#define INIT_IO_WORK (work) \ do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ + *(work) = (struct io_wq_work){}; \ } while (0) \Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.
Let's leave it to Xiaoguang and the series of the topic.
Quoted 3 lines
From a quick look at this, otherwise looks great! Please do split and submit this.
Sure. Have great time off!
Signature · 1 line
Pavel Begunkov
Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
hi,
Quoted 14 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov
fs/io-wq.h struct io_wq_work { · line 99
Quoted 7 lines
fs/io-wq.h +2
-2-#define INIT_IO_WORK (work, _func) \ +#define INIT_IO_WORK (work) \ do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ + *(work) = (struct io_wq_work){}; \ } while (0) \Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.
Quoted 1 line
Let's leave it to Xiaoguang and the series of the topic.
Yeah, I'd be glad to continue this job, thanks.
Regards,
Xiaoguang Wang
Quoted 8 lines
Quoted 4 lines
From a quick look at this, otherwise looks great! Please do split and
submit this.Sure. Have great time off!
Jens Axboe
maintainerRe: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
Quoted 16 lines · Xiaoguang Wang
hi,
Quoted 14 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov
fs/io-wq.h struct io_wq_work { · line 99
Quoted 7 lines
fs/io-wq.h +2
-2-#define INIT_IO_WORK (work, _func) \ +#define INIT_IO_WORK (work) \ do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ + *(work) = (struct io_wq_work){}; \ } while (0) \Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.
Quoted 2 lines
Quoted 1 line
Let's leave it to Xiaoguang and the series of the topic.
Yeah, I'd be glad to continue this job, thanks.
Perfect! I think io_uring-5.8 (as of now) will be a great base for that.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerRe: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature
In reply to Pavel BegunkovQuoted 11 lines · Pavel Begunkov
Quoted 10 lines · Jens Axboe
Quoted 8 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang
hi Jens, Pavel,
Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related
io_wq_workfirstly.It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.
Quoted 8 lines
Quoted 6 lines
Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.
Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.
I just think it should not be done at expense of robustness.
Oh I totally agree, which is why I've been holding back. I'm just saying that we're currently doing a number of semi expensive operations for slow path setup up front, which is annoying and would be nice NOT to do.
Quoted 7 lines
e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with
io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space inio_kiocb(in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).
Definitely sounds good.
Quoted 3 lines
Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.
Don't care too much about the extra lookups, it's a very minor cost compared to the wins.
Quoted 2 lines
And then we can go and fix every other problem until this patch set looks good.
I'll take a look at your patch.
Signature · 1 line
Jens Axboe
Quoted 10 lines · Xiaoguang Wang
Quoted 1 line
A v5 with those two things would be ready to commit.
First thanks for figuring out this bug.
Indeed, I had run test cases in liburing/test multiple rounds before sending V4 patches, and all test cases pass, no kernel issues occurred, here is my kernel build config, seems that I had kasan enabled.
[lege@localhost linux]$ cat .config | grep KASANCONFIG_KASAN_SHADOW_OFFSET=0xdffffc0000000000CONFIG_HAVE_ARCH_KASAN=yCONFIG_HAVE_ARCH_KASAN_VMALLOC=yCONFIG_CC_HAS_KASAN_GENERIC=y
Quoted 8 lines
CONFIG_KASAN=yCONFIG_KASAN_GENERIC=yCONFIG_KASAN_OUTLINE=y
#CONFIG_KASAN_INLINEis not setCONFIG_KASAN_STACK=1
#CONFIG_KASAN_VMALLOCis not set
#CONFIG_TEST_KASANis not set
But I don't know why I did't reproduce the bug, sorry.
Hmm, can't find an option to poison allocated mem. Try to fill req->work with garbage by hand (just after allocation), should help to reproduce.
Quoted 7 lines
Are you fine below changes?
fs/io_uring.c +0
-0
fs/io_uring.c static void io_file_put_work(struct work_struct *work); · line 911
Quoted 8 lines
fs/io_uring.c +2
-3@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work); static inline void io_req_init_async(struct io_kiocb *req, void (*func)(struct io_wq_work **)) { - if (req->flags & REQ_F_WORK_INITIALIZED) - req->work.func = func; - else { + if (req->flags & REQ_F_WORK_INITIALIZED) { + if (!req->work.func)
->func should be initialised after alloc then, if not already.
Quoted 15 lines
fs/io_uring.c +4
-0+ req->work.func = func; + } else { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; }@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req, return ret; req->flags |= REQ_F_NEED_CLEANUP; + /* Splice will be punted aync, so initialize io_wq_work firstly_*/ + io_req_init_async(req, io_wq_submit_work); if (!S_ISREG(file_inode(sp->file_in)->i_mode)) req->work.flags |= IO_WQ_WORK_UNBOUND;
fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3598
Quoted 4 lines
fs/io_uring.c +1
-0@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + /* Close may be punted aync, so initialize io_wq_work firstly */
s/aync/async/
Quoted 11 lines
fs/io_uring.c +2
-0+ io_req_init_async(req, io_wq_submit_work); + /** If we queue this for async, it must not be cancellable. That would * leave the 'file' in an undeterminate state. Regards,
Xiaoguang WangQuoted 1 line
Signature · 1 line
Pavel Begunkov