Skip to contentSkip to conversation

io_uring: avoid whole io_wq_work copy for requests completed inline

PATCH Superseded Xiaoguang Wang 3 participants 27 messages May 30, 2020 – Jun 8, 2020

Subsystems (2)

v4 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

PATCH v4 1/2 io_uring: avoid whole io_wq_work copy for requests completed inline

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):
READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msec

With this patch:
Run status group 0 (all jobs):
READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

---
V4:
add io_req_init_async() helper

---
 fs/io-wq.h    |  5 ----
 fs/io_uring.c | 66 ++++++++++++++++++++++++++++++++++++++-------------
 2 files changed, 50 insertions(+), 21 deletions(-)
fs/io-wq.h +0 -5
index 5ba12de7572f..3d85d365d764 100644
@@ -94,11 +94,6 @@ struct io_wq_work {
 	pid_t task_pid;
 };
 
-#define INIT_IO_WORK(work, _func)				\
-	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
-	} while (0)						\
-
 static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
 {
 	if (!work->list.next)
fs/io_uring.c +50 -16

172 lines not on this page · raw message

Signature · 1 line
2.17.2
#1 lore raw

PATCH v4 2/2 io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Basically IORING_OP_POLL_ADD command and async armed poll handlers for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED is set, can we do io_wq_work copy and restore.

---
V3:
drop the REQ_F_WORK_NEED_RESTORE flag introduced in V2 patch, just use REQ_F_WORK_INITIALIZED to control whether to do io_wq_work copy and restore.

---
 fs/io_uring.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
index 12296ce3e8b9..a7d61f3366a5 100644
@@ -4513,7 +4513,8 @@ static void io_async_task_func(struct callback_head *cb)
 	spin_unlock_irq(&ctx->completion_lock);
 
 	/* restore ->work in case we need to retry again */
-	memcpy(&req->work, &apoll->work, sizeof(req->work));
+	if (req->flags & REQ_F_WORK_INITIALIZED)
+		memcpy(&req->work, &apoll->work, sizeof(req->work));
 	kfree(apoll);
 
 	if (!canceled) {
@@ -4610,7 +4611,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
 		return false;
 
 	req->flags |= REQ_F_POLLED;
-	memcpy(&apoll->work, &req->work, sizeof(req->work));
+	if (req->flags & REQ_F_WORK_INITIALIZED)
+		memcpy(&apoll->work, &req->work, sizeof(req->work));
 	had_io = req->io != NULL;
 
 	get_task_struct(current);
@@ -4635,7 +4637,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
 		if (!had_io)
 			io_poll_remove_double(req);
 		spin_unlock_irq(&ctx->completion_lock);
-		memcpy(&req->work, &apoll->work, sizeof(req->work));
+		if (req->flags & REQ_F_WORK_INITIALIZED)
+			memcpy(&req->work, &apoll->work, sizeof(req->work));
 		kfree(apoll);
 		return false;
 	}
@@ -4680,7 +4683,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)
 			 * io_req_work_drop_env below when dropping the
 			 * final reference.
 			 */
-			memcpy(&req->work, &apoll->work, sizeof(req->work));
+			if (req->flags & REQ_F_WORK_INITIALIZED)
+				memcpy(&req->work, &apoll->work,
+				       sizeof(req->work));
 			kfree(apoll);
 		}
 	}
Signature · 1 line
2.17.2
maintainer
#2 lore raw
In reply to Xiaoguang Wang
Quoted 17 lines · Xiaoguang Wang

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):

   READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s),
   io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines

With this patch:
Run status group 0 (all jobs):

   READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s),
   io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761
[  168.209987] 
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014
[  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768] 
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729] 
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686] 

which indicates that current->files is no longer valid.

Signature · 1 line
Jens Axboe
maintainer
#3 lore raw
Quoted 106 lines · Jens Axboe
Quoted 17 lines · Xiaoguang Wang

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):

   READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s),
   io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines

With this patch:
Run status group 0 (all jobs):

   READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s),
   io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761
[  168.209987] 
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014
[  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768] 
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729] 
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686] 
Quoted 1 line

which indicates that current->files is no longer valid.

Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:

ret = test_close(&ring, ring.ring_fd, 1);

This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.

Why isn't io_req_init_async() just doing:

static inline void io_req_init_async(struct io_kiocb *req,

                         void (*func)(struct io_wq_work **))
{                                                                               
        if (!(req->flags & REQ_F_WORK_INITIALIZED)) {
                req->work = (struct io_wq_work){ .func = func };
                req->flags |= REQ_F_WORK_INITIALIZED;
        }
}

?

Signature · 1 line
Jens Axboe
maintainer
#4 lore raw
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Jens Axboe
Quoted 17 lines · Xiaoguang Wang

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):

   READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s),
   io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines

With this patch:
Run status group 0 (all jobs):

   READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s),
   io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761
[  168.209987] 
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014
[  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768] 
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729] 
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686] 
Quoted 1 line

which indicates that current->files is no longer valid.

Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:

ret = test_close(&ring, ring.ring_fd, 1);

This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.

Why isn't io_req_init_async() just doing:

static inline void io_req_init_async(struct io_kiocb *req,

                         void (*func)(struct io_wq_work **))
{                                                                               
        if (!(req->flags & REQ_F_WORK_INITIALIZED)) {
                req->work = (struct io_wq_work){ .func = func };
                req->flags |= REQ_F_WORK_INITIALIZED;
        }
}
Quoted 1 line

?

I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?

Also please fix up missing braces. The cases of:

if (something) {
line 1
line 2
} else
line 3

should always includes braces, if one clause has it.

A v5 with those two things would be ready to commit.

Signature · 1 line
Jens Axboe
#5 lore raw
Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Jens Axboe
Quoted 17 lines · Xiaoguang Wang

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):

   READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s),
   io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines

With this patch:
Run status group 0 (all jobs):

   READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s),
   io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761
[  168.209987] 
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014
[  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768] 
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729] 
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686] 
Quoted 1 line

which indicates that current->files is no longer valid.

Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:

ret = test_close(&ring, ring.ring_fd, 1);

This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.

Why isn't io_req_init_async() just doing:

static inline void io_req_init_async(struct io_kiocb *req,

                         void (*func)(struct io_wq_work **))
{                                                                               
        if (!(req->flags & REQ_F_WORK_INITIALIZED)) {
                req->work = (struct io_wq_work){ .func = func };
                req->flags |= REQ_F_WORK_INITIALIZED;
        }
}
Quoted 1 line

?

I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?

Also please fix up missing braces. The cases of:

if (something) {
line 1
line 2
} else
line 3

Quoted 4 lines

should always includes braces, if one clause has it.

A v5 with those two things would be ready to commit.

There is another thing:

io_submit_sqes()

    -> io_close() (let ->flush == NULL)
        -> __io_close_finish()
            -> filp_close(req->close.put_file, *req->work.files*);

where req->work.files is garbage.

Signature · 1 line
Pavel Begunkov
#6 lore raw

hi,

Quoted 153 lines · Jens Axboe
Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Xiaoguang Wang
Quoted 17 lines

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):

    READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s),
    io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines

With this patch:
Run status group 0 (all jobs):

    READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s),
    io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761
[  168.209987]
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014
[  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768]
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729]
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686]
Quoted 1 line

which indicates that current->files is no longer valid.

Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:

ret = test_close(&ring, ring.ring_fd, 1);

This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.

Why isn't io_req_init_async() just doing:

static inline void io_req_init_async(struct io_kiocb *req,

                          void (*func)(struct io_wq_work **))
{
         if (!(req->flags & REQ_F_WORK_INITIALIZED)) {
                 req->work = (struct io_wq_work){ .func = func };
                 req->flags |= REQ_F_WORK_INITIALIZED;
         }
}
Quoted 1 line

?

I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?

Also please fix up missing braces. The cases of:

if (something) {
line 1
line 2
} else
line 3

Quoted 4 lines

should always includes braces, if one clause has it.

A v5 with those two things would be ready to commit.

Quoted 8 lines

There is another thing:

io_submit_sqes()

     -> io_close() (let ->flush == NULL)
         -> __io_close_finish()
             -> filp_close(req->close.put_file, *req->work.files*);

where req->work.files is garbage.

I think this bug is independent of my patch. Without my patches, if close request will be submitted and completed inline, req->work.files will be NULL, it's still problematic, should we use current->files here?

Regards,
Xiaoguang Wang

Quoted 1 line
#7 lore raw
Quoted 9 lines · Xiaoguang Wang
Quoted 8 lines

There is another thing:

io_submit_sqes()
     -> io_close() (let ->flush == NULL)
         -> __io_close_finish()
             -> filp_close(req->close.put_file, *req->work.files*);

where req->work.files is garbage.

I think this bug is independent of my patch. Without my patches, if close request

It looks like it's ok to pass NULL, at least Jens did it here and I see an occurrence of ->flush(NULL). And it's usually not referenced or completely ignored. I'll check later.

Quoted 2 lines

will be submitted and completed inline, req->work.files will be NULL, it's still problematic, should we use current->files here?

Signature · 1 line
Pavel Begunkov
#8 lore raw
In reply to Jens Axboe

hi jens,

Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Xiaoguang Wang
Quoted 17 lines

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):

    READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s),
    io=84.8GiB (91.1GB), run=120001-120001msec
Quoted 6 lines

With this patch:
Run status group 0 (all jobs):

    READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s),
    io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed: ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761
[  168.209987]
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014
[  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768]
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729]
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686]
Quoted 1 line

which indicates that current->files is no longer valid.

Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:

ret = test_close(&ring, ring.ring_fd, 1);

This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.

Why isn't io_req_init_async() just doing:

static inline void io_req_init_async(struct io_kiocb *req,

                          void (*func)(struct io_wq_work **))
{
         if (!(req->flags & REQ_F_WORK_INITIALIZED)) {
                 req->work = (struct io_wq_work){ .func = func };
                 req->flags |= REQ_F_WORK_INITIALIZED;
         }
}
Quoted 1 line

?

I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?

Also please fix up missing braces. The cases of:

if (something) {
line 1
line 2
} else
line 3

Quoted 3 lines

should always includes braces, if one clause has it.

A v5 with those two things would be ready to commit.

First thanks for figuring out this bug.
Indeed, I had run test cases in liburing/test multiple rounds before sending V4 patches, and all test cases pass, no kernel issues occurred, here is my kernel build config, seems that I had kasan enabled.
[lege@localhost linux]$ cat .config | grep KASAN CONFIG_KASAN_SHADOW_OFFSET=0xdffffc0000000000 CONFIG_HAVE_ARCH_KASAN=y
CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
CONFIG_CC_HAS_KASAN_GENERIC=y
CONFIG_KASAN=y
CONFIG_KASAN_GENERIC=y
CONFIG_KASAN_OUTLINE=y
# CONFIG_KASAN_INLINE is not set
CONFIG_KASAN_STACK=1
# CONFIG_KASAN_VMALLOC is not set
# CONFIG_TEST_KASAN is not set
But I don't know why I did't reproduce the bug, sorry.

Are you fine below changes?

fs/io_uring.c +9 -3
index 12296ce3e8b9..2a3a02838f7b 100644
@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work);
 static inline void io_req_init_async(struct io_kiocb *req,
                        void (*func)(struct io_wq_work **))
 {
-       if (req->flags & REQ_F_WORK_INITIALIZED)
-               req->work.func = func;
-       else {
+       if (req->flags & REQ_F_WORK_INITIALIZED) {
+               if (!req->work.func)
+                       req->work.func = func;
+       } else {
                req->work = (struct io_wq_work){ .func = func };
                req->flags |= REQ_F_WORK_INITIALIZED;
        }
@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req,
                return ret;
        req->flags |= REQ_F_NEED_CLEANUP;
 
+       /* Splice will be punted aync, so initialize io_wq_work firstly_*/
+       io_req_init_async(req, io_wq_submit_work);
        if (!S_ISREG(file_inode(sp->file_in)->i_mode))
                req->work.flags |= IO_WQ_WORK_UNBOUND;
 
@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)
 
 static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
+        /* Close may be punted aync, so initialize io_wq_work firstly */
+       io_req_init_async(req, io_wq_submit_work);
+
        /*
         * If we queue this for async, it must not be cancellable. That would
         * leave the 'file' in an undeterminate state.

Regards,
Xiaoguang Wang

Quoted 1 line
#9 lore raw

hi,

Quoted 199 lines

hi jens,

Quoted 147 lines · Jens Axboe
Quoted 131 lines · Jens Axboe
Quoted 106 lines · Xiaoguang Wang
Quoted 17 lines

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
   modprobe null_blk nr_devices=1 completion_nsec=0
   sudo taskset -c 60 fio  -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):
    READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msec

Quoted 6 lines

With this patch:
Run status group 0 (all jobs):
    READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

There's something funky going on here. I ran the liburing test suite on this, and get a lot of left behind workers:

Tests _maybe_ failed:  ring-leak open-close open-close file-update file-update accept-reuse accept-reuse poll-v-poll poll-v-poll fadvise fadvise madvise madvise short-read short-read openat2 openat2 probe probe shared-wq shared-wq personality personality eventfd eventfd send_recv send_recv eventfd-ring eventfd-ring across-fork across-fork sq-poll-kthread sq-poll-kthread splice splice lfs-openat lfs-openat lfs-openat-write lfs-openat-write iopoll iopoll d4ae271dfaae-test d4ae271dfaae-test eventfd-disable eventfd-disable write-file write-file buf-rw buf-rw statx statx

and also saw this:

[  168.208940] ==================================================================
[  168.209311] BUG: KASAN: use-after-free in __lock_acquire+0x8bf/0x3000
[  168.209626] Read of size 8 at addr ffff88806801c0d8 by task io_wqe_worker-0/41761 [  168.209987]
[  168.210069] CPU: 0 PID: 41761 Comm: io_wqe_worker-0 Not tainted 5.7.0-rc7+ #6318
[  168.210424] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 [  168.210857] Call Trace:
[  168.210991]  dump_stack+0x97/0xe0
[  168.211164]  print_address_description.constprop.0+0x1a/0x210
[  168.211446]  ? __lock_acquire+0x8bf/0x3000
[  168.211649]  __kasan_report.cold+0x20/0x39
[  168.211851]  ? __lock_acquire+0x8bf/0x3000
[  168.212051]  kasan_report+0x30/0x40
[  168.212226]  __lock_acquire+0x8bf/0x3000
[  168.212432]  ? ret_from_fork+0x24/0x30
[  168.212623]  ? stack_trace_save+0x81/0xa0
[  168.212821]  ? lockdep_hardirqs_on+0x270/0x270
[  168.213039]  ? save_stack+0x32/0x40
[  168.213212]  lock_acquire+0x122/0x570
[  168.213398]  ? __close_fd_get_file+0x40/0x150
[  168.213615]  ? lock_release+0x3f0/0x3f0
[  168.213814]  ? __lock_acquire+0x87e/0x3000
[  168.214016]  _raw_spin_lock+0x2c/0x40
[  168.214196]  ? __close_fd_get_file+0x40/0x150
[  168.214408]  __close_fd_get_file+0x40/0x150
[  168.214618]  io_issue_sqe+0x57f/0x22f0
[  168.214803]  ? lockdep_hardirqs_on+0x270/0x270
[  168.215019]  ? mark_held_locks+0x24/0x90
[  168.215211]  ? quarantine_put+0x6f/0x190
[  168.215404]  ? io_assign_current_work+0x59/0x80
[  168.215623]  ? __ia32_sys_io_uring_setup+0x30/0x30
[  168.215855]  ? find_held_lock+0xcb/0x100
[  168.216054]  ? io_worker_handle_work+0x289/0x980
[  168.216280]  ? lock_downgrade+0x340/0x340
[  168.216476]  ? io_wq_submit_work+0x5d/0x140
[  168.216679]  ? _raw_spin_unlock_irq+0x24/0x40
[  168.216890]  io_wq_submit_work+0x5d/0x140
[  168.217087]  io_worker_handle_work+0x30a/0x980
[  168.217305]  ? io_wqe_dec_running.isra.0+0x70/0x70
[  168.217537]  ? do_raw_spin_lock+0x100/0x180
[  168.217742]  ? rwlock_bug.part.0+0x60/0x60
[  168.217943]  io_wqe_worker+0x5fd/0x780
[  168.218126]  ? lock_downgrade+0x340/0x340
[  168.218323]  ? io_worker_handle_work+0x980/0x980
[  168.218546]  ? lockdep_hardirqs_on+0x17d/0x270
[  168.218765]  ? __kthread_parkme+0xca/0xe0
[  168.218961]  ? io_worker_handle_work+0x980/0x980
[  168.219186]  kthread+0x1f0/0x220
[  168.219346]  ? kthread_create_worker_on_cpu+0xb0/0xb0
[  168.219590]  ret_from_fork+0x24/0x30
[  168.219768]
[  168.219846] Allocated by task 41758:
[  168.220021]  save_stack+0x1b/0x40
[  168.220185]  __kasan_kmalloc.constprop.0+0xc2/0xd0
[  168.220416]  kmem_cache_alloc+0xe0/0x290
[  168.220607]  dup_fd+0x4e/0x5a0
[  168.220758]  copy_process+0xe35/0x2bf0
[  168.220942]  _do_fork+0xd8/0x550
[  168.221102]  __do_sys_clone+0xb5/0xe0
[  168.221282]  do_syscall_64+0x5e/0xe0
[  168.221457]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.221729]
[  168.221848] Freed by task 41759:
[  168.222088]  save_stack+0x1b/0x40
[  168.222336]  __kasan_slab_free+0x12f/0x180
[  168.222632]  slab_free_freelist_hook+0x4d/0x120
[  168.222959]  kmem_cache_free+0x90/0x2e0
[  168.223239]  do_exit+0x5d2/0x12e0
[  168.223482]  do_group_exit+0x6f/0x130
[  168.223754]  __x64_sys_exit_group+0x28/0x30
[  168.224061]  do_syscall_64+0x5e/0xe0
[  168.224326]  entry_SYSCALL_64_after_hwframe+0x49/0xb3
[  168.224686]

Quoted 1 line

which indicates that current->files is no longer valid.

Narrowed it down to the test/open-close test, and in particular where it closes the ring itself:

ret = test_close(&ring, ring.ring_fd, 1);

This seems to be because you do io_req_init_async() after calling io_issue_sqe(), and the command handler may have set something else for ->func at that point. Hence we never call the right handler if the close needs to be deferred, as it needs to for the io_uring as it has ->flush() defined.

Why isn't io_req_init_async() just doing:

static inline void io_req_init_async(struct io_kiocb *req, void (*func)(struct io_wq_work **)) {
         if (!(req->flags & REQ_F_WORK_INITIALIZED)) { req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED; }
}

Quoted 1 line

?

I guess that won't always work, if the request has been deferred and we're now setting a new work func. So we really want the 'reset to io_wq_submit_work' to only happen if the opcode hasn't already set a private handler. Can you make that change?

Also please fix up missing braces. The cases of:

if (something) {
    line 1
    line 2
} else
    line 3

Quoted 3 lines

should always includes braces, if one clause has it.

A v5 with those two things would be ready to commit.

First thanks for figuring out this bug.
Indeed, I had run test cases in liburing/test multiple rounds before sending V4 patches, and all test cases pass, no kernel issues occurred, here is my kernel build config, seems that I had kasan enabled.
[lege@localhost linux]$ cat .config | grep KASAN CONFIG_KASAN_SHADOW_OFFSET=0xdffffc0000000000 CONFIG_HAVE_ARCH_KASAN=y
CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
CONFIG_CC_HAS_KASAN_GENERIC=y
CONFIG_KASAN=y
CONFIG_KASAN_GENERIC=y
CONFIG_KASAN_OUTLINE=y
# CONFIG_KASAN_INLINE is not set
CONFIG_KASAN_STACK=1
# CONFIG_KASAN_VMALLOC is not set
# CONFIG_TEST_KASAN is not set
But I don't know why I did't reproduce the bug, sorry.

Are you fine below changes?

fs/io_uring.c +6 -3
index 12296ce3e8b9..2a3a02838f7b 100644
@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work);
  static inline void io_req_init_async(struct io_kiocb *req,
                         void (*func)(struct io_wq_work **))
  {
-       if (req->flags & REQ_F_WORK_INITIALIZED)
-               req->work.func = func;
-       else {
+       if (req->flags & REQ_F_WORK_INITIALIZED) {
+               if (!req->work.func)
+                       req->work.func = func;
+       } else {
                 req->work = (struct io_wq_work){ .func = func };
                 req->flags |= REQ_F_WORK_INITIALIZED;
         }
@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req,
                 return ret;
         req->flags |= REQ_F_NEED_CLEANUP;
 
+       /* Splice will be punted aync, so initialize io_wq_work firstly_*/
+       io_req_init_async(req, io_wq_submit_work);
         if (!S_ISREG(file_inode(sp->file_in)->i_mode))
                 req->work.flags |= IO_WQ_WORK_UNBOUND;
 

fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3600

Quoted 6 lines
fs/io_uring.c +3 -0
@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)
 
  static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
  {
+        /* Close may be punted aync, so initialize io_wq_work firstly */
+       io_req_init_async(req, io_wq_submit_work);
+

For splice and close requests, these two about io_req_init_async() calls should be io_req_init_async(req, NULL), because they change req->work.flags firstly.

Regards,
Xiaoguang Wang

Quoted 9 lines
fs/io_uring.c +0 -0
         /*
          * If we queue this for async, it must not be cancellable. That would
          * leave the 'file' in an undeterminate state.

Regards,
Xiaoguang Wang

Quoted 1 line
#10 lore raw
Quoted 29 lines
Quoted 29 lines
fs/io_uring.c +6 -3
index 12296ce3e8b9..2a3a02838f7b 100644
@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work);
   static inline void io_req_init_async(struct io_kiocb *req,
                          void (*func)(struct io_wq_work **))
   {
-       if (req->flags & REQ_F_WORK_INITIALIZED)
-               req->work.func = func;
-       else {
+       if (req->flags & REQ_F_WORK_INITIALIZED) {
+               if (!req->work.func)
+                       req->work.func = func;
+       } else {
                  req->work = (struct io_wq_work){ .func = func };
                  req->flags |= REQ_F_WORK_INITIALIZED;
          }
@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req,
                  return ret;
          req->flags |= REQ_F_NEED_CLEANUP;
 
+       /* Splice will be punted aync, so initialize io_wq_work firstly_*/
+       io_req_init_async(req, io_wq_submit_work);
          if (!S_ISREG(file_inode(sp->file_in)->i_mode))
                  req->work.flags |= IO_WQ_WORK_UNBOUND;
 
@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)
 
Quoted 7 lines

fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3600

Quoted 5 lines
fs/io_uring.c +3 -0
   static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
   {
+        /* Close may be punted aync, so initialize io_wq_work firstly */
+       io_req_init_async(req, io_wq_submit_work);
+

For splice and close requests, these two about io_req_init_async() calls should be io_req_init_async(req, NULL), because they change req->work.flags firstly.

Please no. Such assumptions/dependencies are prone to break. It'll get us subtle bugs in no time.

BTW, why not io_wq_submit_work in place of NULL?

Signature · 1 line
Pavel Begunkov
#11 lore raw

hi,

Quoted 37 lines
Quoted 29 lines
Quoted 29 lines
fs/io_uring.c +6 -3
index 12296ce3e8b9..2a3a02838f7b 100644
@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work);
   static inline void io_req_init_async(struct io_kiocb *req,
                          void (*func)(struct io_wq_work **))
   {
-       if (req->flags & REQ_F_WORK_INITIALIZED)
-               req->work.func = func;
-       else {
+       if (req->flags & REQ_F_WORK_INITIALIZED) {
+               if (!req->work.func)
+                       req->work.func = func;
+       } else {
                  req->work = (struct io_wq_work){ .func = func };
                  req->flags |= REQ_F_WORK_INITIALIZED;
          }
@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req,
                  return ret;
          req->flags |= REQ_F_NEED_CLEANUP;
 
+       /* Splice will be punted aync, so initialize io_wq_work firstly_*/
+       io_req_init_async(req, io_wq_submit_work);
          if (!S_ISREG(file_inode(sp->file_in)->i_mode))
                  req->work.flags |= IO_WQ_WORK_UNBOUND;
 
@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)
 
Quoted 7 lines

fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3600

Quoted 5 lines
fs/io_uring.c +3 -0
   static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
   {
+        /* Close may be punted aync, so initialize io_wq_work firstly */
+       io_req_init_async(req, io_wq_submit_work);
+

For splice and close requests, these two about io_req_init_async() calls should be io_req_init_async(req, NULL), because they change req->work.flags firstly.

Quoted 4 lines

Please no. Such assumptions/dependencies are prone to break. It'll get us subtle bugs in no time.

BTW, why not io_wq_submit_work in place of NULL?

In the begin of __io_splice_prep or io_close_prep, current io_uring mainline codes will modify req->work.flags firstly, so we need to call io_req_init_async to initialize io_wq_work before the work.flags modification.
For below codes:
static inline void io_req_init_async(struct io_kiocb *req,

                        void (*func)(struct io_wq_work **))
{
        if (req->flags & REQ_F_WORK_INITIALIZED) {
                if (!req->work.func)
                        req->work.func = func;
        } else {
                req->work = (struct io_wq_work){ .func = func };
                req->flags |= REQ_F_WORK_INITIALIZED;
        }
}

if we not pass NULL to parameter 'func', e.g. pass io_wq_submit_work, then we can not use io_req_init_async() to pass io_close_finish again.

Now I'm confused how to write better codes based on current io_uring mainline codes :) If you have some free time, please have a deeper look, thanks.

Regards,
Xiaoguang Wang

Quoted 1 line
#12 lore raw
Quoted 16 lines · Xiaoguang Wang

In the begin of __io_splice_prep or io_close_prep, current io_uring mainline codes will modify req->work.flags firstly, so we need to call io_req_init_async to initialize io_wq_work before the work.flags modification.
For below codes:
static inline void io_req_init_async(struct io_kiocb *req,
                        void (*func)(struct io_wq_work **))
{
        if (req->flags & REQ_F_WORK_INITIALIZED) {
                if (!req->work.func)
                        req->work.func = func;
        } else {
                req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED;
        }
}

Quoted 2 lines

if we not pass NULL to parameter 'func', e.g. pass io_wq_submit_work, then we can not use io_req_init_async() to pass io_close_finish again.

It's not as bad, just the thing you poked is overused and don't have strict rules. I have a feeling, that for it to be done right it'd need more fundamental refactoring with putting everything related to ->work closer to io_queue_async_work().

Quoted 3 lines

Now I'm confused how to write better codes based on current io_uring mainline codes :) If you have some free time, please have a deeper look, thanks.

Signature · 1 line
Pavel Begunkov
#13 lore raw

hi,

Quoted 17 lines · Xiaoguang Wang
Quoted 16 lines

In the begin of __io_splice_prep or io_close_prep, current io_uring mainline codes will modify req->work.flags firstly, so we need to call io_req_init_async to initialize io_wq_work before the work.flags modification.
For below codes:
static inline void io_req_init_async(struct io_kiocb *req,
                        void (*func)(struct io_wq_work **))
{
        if (req->flags & REQ_F_WORK_INITIALIZED) {
                if (!req->work.func)
                        req->work.func = func;
        } else {
                req->work = (struct io_wq_work){ .func = func }; req->flags |= REQ_F_WORK_INITIALIZED;
        }
}

Quoted 6 lines
Quoted 2 lines

if we not pass NULL to parameter 'func', e.g. pass io_wq_submit_work, then we can not use io_req_init_async() to pass io_close_finish again.

It's not as bad, just the thing you poked is overused and don't have strict rules. I have a feeling, that for it to be done right it'd need more fundamental refactoring with putting everything related to ->work closer to io_queue_async_work().

Yes, I totally agree with you, that would be safer.

Regards,
Xiaoguang Wang

Quoted 5 lines
Quoted 4 lines

Now I'm confused how to write better codes based on current io_uring mainline codes :) If you have some free time, please have a deeper look, thanks.

#14 lore raw

PATCH v5 1/2 io_uring: avoid whole io_wq_work copy for requests completed inline

In reply to Pavel Begunkov

If requests can be submitted and completed inline, we don't need to initialize whole io_wq_work in io_init_req(), which is an expensive operation, add a new 'REQ_F_WORK_INITIALIZED' to control whether io_wq_work is initialized.

I use /dev/nullb0 to evaluate performance improvement in my physical machine:
modprobe null_blk nr_devices=1 completion_nsec=0
sudo taskset -c 60 fio -name=fiotest -filename=/dev/nullb0 -iodepth=128 -thread -rw=read -ioengine=io_uring -direct=1 -bs=4k -size=100G -numjobs=1 -time_based -runtime=120

before this patch:
Run status group 0 (all jobs):
READ: bw=724MiB/s (759MB/s), 724MiB/s-724MiB/s (759MB/s-759MB/s), io=84.8GiB (91.1GB), run=120001-120001msec

With this patch:
Run status group 0 (all jobs):
READ: bw=761MiB/s (798MB/s), 761MiB/s-761MiB/s (798MB/s-798MB/s), io=89.2GiB (95.8GB), run=120001-120001msec

About 5% improvement.

---
V4:
add io_req_init_async() helper

V5:
refactor io_req_init_async() to io_init_req_work() and io_init_req_work_func in case we need to change io_wq_work.func separately.

---
 fs/io-wq.h    |  5 ----
 fs/io_uring.c | 83 +++++++++++++++++++++++++++++++++++++++++----------
 2 files changed, 67 insertions(+), 21 deletions(-)
fs/io-wq.h +0 -5
index 5ba12de7572f..3d85d365d764 100644
@@ -94,11 +94,6 @@ struct io_wq_work {
 	pid_t task_pid;
 };
 
-#define INIT_IO_WORK(work, _func)				\
-	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
-	} while (0)						\
-
 static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
 {
 	if (!work->list.next)
fs/io_uring.c +67 -16

201 lines not on this page · raw message

Signature · 1 line
2.17.2
#15 lore raw

PATCH v5 2/2 io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Basically IORING_OP_POLL_ADD command and async armed poll handlers for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED is set, can we do io_wq_work copy and restore.

---
V3:
drop the REQ_F_WORK_NEED_RESTORE flag introduced in V2 patch, just use REQ_F_WORK_INITIALIZED to control whether to do io_wq_work copy and restore.

---
 fs/io_uring.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
index 8e022d0f0c86..b761ef7366f9 100644
@@ -4527,7 +4527,8 @@ static void io_async_task_func(struct callback_head *cb)
 	spin_unlock_irq(&ctx->completion_lock);
 
 	/* restore ->work in case we need to retry again */
-	memcpy(&req->work, &apoll->work, sizeof(req->work));
+	if (req->flags & REQ_F_WORK_INITIALIZED)
+		memcpy(&req->work, &apoll->work, sizeof(req->work));
 	kfree(apoll);
 
 	if (!canceled) {
@@ -4624,7 +4625,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
 		return false;
 
 	req->flags |= REQ_F_POLLED;
-	memcpy(&apoll->work, &req->work, sizeof(req->work));
+	if (req->flags & REQ_F_WORK_INITIALIZED)
+		memcpy(&apoll->work, &req->work, sizeof(req->work));
 	had_io = req->io != NULL;
 
 	get_task_struct(current);
@@ -4649,7 +4651,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
 		if (!had_io)
 			io_poll_remove_double(req);
 		spin_unlock_irq(&ctx->completion_lock);
-		memcpy(&req->work, &apoll->work, sizeof(req->work));
+		if (req->flags & REQ_F_WORK_INITIALIZED)
+			memcpy(&req->work, &apoll->work, sizeof(req->work));
 		kfree(apoll);
 		return false;
 	}
@@ -4694,7 +4697,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)
 			 * io_req_work_drop_env below when dropping the
 			 * final reference.
 			 */
-			memcpy(&req->work, &apoll->work, sizeof(req->work));
+			if (req->flags & REQ_F_WORK_INITIALIZED)
+				memcpy(&req->work, &apoll->work,
+				       sizeof(req->work));
 			kfree(apoll);
 		}
 	}
Signature · 1 line
2.17.2
#16 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

Regards,
Xiaoguang Wang

Quoted 60 lines

Basically IORING_OP_POLL_ADD command and async armed poll handlers for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED is set, can we do io_wq_work copy and restore.

---
V3:
drop the REQ_F_WORK_NEED_RESTORE flag introduced in V2 patch, just use REQ_F_WORK_INITIALIZED to control whether to do io_wq_work copy and restore.

---
  fs/io_uring.c | 13 +++++++++----
  1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
index 8e022d0f0c86..b761ef7366f9 100644
@@ -4527,7 +4527,8 @@ static void io_async_task_func(struct callback_head *cb)
  	spin_unlock_irq(&ctx->completion_lock);
 
  	/* restore ->work in case we need to retry again */
-	memcpy(&req->work, &apoll->work, sizeof(req->work));
+	if (req->flags & REQ_F_WORK_INITIALIZED)
+		memcpy(&req->work, &apoll->work, sizeof(req->work));
  	kfree(apoll);
 
  	if (!canceled) {
@@ -4624,7 +4625,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
  		return false;
 
  	req->flags |= REQ_F_POLLED;
-	memcpy(&apoll->work, &req->work, sizeof(req->work));
+	if (req->flags & REQ_F_WORK_INITIALIZED)
+		memcpy(&apoll->work, &req->work, sizeof(req->work));
  	had_io = req->io != NULL;
 
  	get_task_struct(current);
@@ -4649,7 +4651,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
  		if (!had_io)
  			io_poll_remove_double(req);
  		spin_unlock_irq(&ctx->completion_lock);
-		memcpy(&req->work, &apoll->work, sizeof(req->work));
+		if (req->flags & REQ_F_WORK_INITIALIZED)
+			memcpy(&req->work, &apoll->work, sizeof(req->work));
  		kfree(apoll);
  		return false;
  	}
@@ -4694,7 +4697,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)
  			 * io_req_work_drop_env below when dropping the
  			 * final reference.
  			 */
-			memcpy(&req->work, &apoll->work, sizeof(req->work));
+			if (req->flags & REQ_F_WORK_INITIALIZED)
+				memcpy(&req->work, &apoll->work,
+				       sizeof(req->work));
  			kfree(apoll);
  		}
  	}
#17 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 4 lines · Xiaoguang Wang

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.

Quoted 58 lines
Quoted 58 lines

Basically IORING_OP_POLL_ADD command and async armed poll handlers for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED is set, can we do io_wq_work copy and restore.

---
V3:
   drop the REQ_F_WORK_NEED_RESTORE flag introduced in V2 patch, just use REQ_F_WORK_INITIALIZED to control whether to do io_wq_work copy and restore.
---
  fs/io_uring.c | 13 +++++++++----

  1 file changed, 9 insertions(+), 4 deletions(-)
fs/io_uring.c +9 -4
index 8e022d0f0c86..b761ef7366f9 100644
@@ -4527,7 +4527,8 @@ static void io_async_task_func(struct callback_head *cb)
       spin_unlock_irq(&ctx->completion_lock);
         /* restore ->work in case we need to retry again */
-    memcpy(&req->work, &apoll->work, sizeof(req->work));
+    if (req->flags & REQ_F_WORK_INITIALIZED)
+        memcpy(&req->work, &apoll->work, sizeof(req->work));
       kfree(apoll);
         if (!canceled) {
@@ -4624,7 +4625,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
           return false;
         req->flags |= REQ_F_POLLED;
-    memcpy(&apoll->work, &req->work, sizeof(req->work));
+    if (req->flags & REQ_F_WORK_INITIALIZED)
+        memcpy(&apoll->work, &req->work, sizeof(req->work));
       had_io = req->io != NULL;
         get_task_struct(current);
@@ -4649,7 +4651,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
           if (!had_io)
               io_poll_remove_double(req);
           spin_unlock_irq(&ctx->completion_lock);
-        memcpy(&req->work, &apoll->work, sizeof(req->work));
+        if (req->flags & REQ_F_WORK_INITIALIZED)
+            memcpy(&req->work, &apoll->work, sizeof(req->work));
           kfree(apoll);
           return false;
       }
@@ -4694,7 +4697,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)
                * io_req_work_drop_env below when dropping the
                * final reference.
                */
-            memcpy(&req->work, &apoll->work, sizeof(req->work));
+            if (req->flags & REQ_F_WORK_INITIALIZED)
+                memcpy(&req->work, &apoll->work,
+                       sizeof(req->work));
               kfree(apoll);
           }
       }
Signature · 1 line
Pavel Begunkov
maintainer
#18 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 8 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.

Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.

Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.

Signature · 1 line
Jens Axboe
#19 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

Quoted 9 lines
Quoted 2 lines

It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.

Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.

Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.

I just think it should not be done at expense of robustness.

e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space in io_kiocb (in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).

Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.

And then we can go and fix every other problem until this patch set looks good.

Signature · 1 line
Pavel Begunkov
#20 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 28 lines · Pavel Begunkov
Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

Quoted 9 lines
Quoted 2 lines

It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.

Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.

Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.

I just think it should not be done at expense of robustness.

e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space in io_kiocb (in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).

Quoted 3 lines

Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.

How about diff below? if split and cooked properly.

3 files changed, 73 insertions(+), 164 deletions(-)
fs/io-wq.c +6 -4
index 2bfa9117bc28..a44ad3b98886 100644
@@ -112,6 +112,7 @@ struct io_wq {
 	unsigned long state;
 
 	free_work_fn *free_work;
+	io_wq_work_fn *do_work;
 
 	struct task_struct *manager;
 	struct user_struct *user;
@@ -528,7 +529,7 @@ static void io_worker_handle_work(struct io_worker *worker)
 
 			hash = io_get_work_hash(work);
 			linked = old_work = work;
-			linked->func(&linked);
+			wq->do_work(&linked);
 			linked = (old_work == linked) ? NULL : linked;
 
 			work = next_hashed;
@@ -785,7 +786,7 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)
 		struct io_wq_work *old_work = work;
 
 		work->flags |= IO_WQ_WORK_CANCEL;
-		work->func(&work);
+		wq->do_work(&work);
 		work = (work == old_work) ? NULL : work;
 		wq->free_work(old_work);
 	} while (work);
@@ -1027,7 +1028,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
 	int ret = -ENOMEM, node;
 	struct io_wq *wq;
 
-	if (WARN_ON_ONCE(!data->free_work))
+	if (WARN_ON_ONCE(!data->free_work || !data->do_work))
 		return ERR_PTR(-EINVAL);
 
 	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
@@ -1041,6 +1042,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
 	}
 
 	wq->free_work = data->free_work;
+	wq->do_work = data->do_work;
 
 	/* caller must already hold a reference to this */
 	wq->user = data->user;
@@ -1097,7 +1099,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
 
 bool io_wq_get(struct io_wq *wq, struct io_wq_data *data)
 {
-	if (data->free_work != wq->free_work)
+	if (data->free_work != wq->free_work || data->do_work != wq->do_work)
 		return false;
 
 	return refcount_inc_not_zero(&wq->use_refs);
fs/io-wq.h +4 -3
index df8a4cd3236d..f3bb596f5a3f 100644
@@ -85,7 +85,6 @@ static inline void wq_list_del(struct io_wq_work_list *list,
 
 struct io_wq_work {
 	struct io_wq_work_node list;
-	void (*func)(struct io_wq_work **);
 	struct files_struct *files;
 	struct mm_struct *mm;
 	const struct cred *creds;
@@ -94,9 +93,9 @@ struct io_wq_work {
 	pid_t task_pid;
 };
 
-#define INIT_IO_WORK(work, _func)				\
+#define INIT_IO_WORK(work)					\
 	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
+		*(work) = (struct io_wq_work){};		\
 	} while (0)						\
 
 static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
@@ -108,10 +107,12 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
 }
 
 typedef void (free_work_fn)(struct io_wq_work *);
+typedef void (io_wq_work_fn)(struct io_wq_work **);
 
 struct io_wq_data {
 	struct user_struct *user;
 
+	io_wq_work_fn *do_work;
 	free_work_fn *free_work;
 };
 
fs/io_uring.c +63 -157

352 lines not on this page · raw message

Signature · 1 line
Pavel Begunkov
maintainer
#21 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 38 lines · Pavel Begunkov
Quoted 28 lines · Pavel Begunkov
Quoted 7 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

Quoted 9 lines
Quoted 2 lines

It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.

Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.

Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.

I just think it should not be done at expense of robustness.

e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space in io_kiocb (in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).

Quoted 3 lines

Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.

How about diff below? if split and cooked properly.

3 files changed, 73 insertions(+), 164 deletions(-)

fs/io-wq.h struct io_wq_work { · line 100

Quoted 10 lines
fs/io-wq.h +2 -2
@@ -94,9 +93,9 @@ struct io_wq_work {
 	pid_t task_pid;
 };
 
-#define INIT_IO_WORK(work, _func)				\
+#define INIT_IO_WORK(work)					\
 	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
+		*(work) = (struct io_wq_work){};		\
 	} while (0)						\
 

Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.

From a quick look at this, otherwise looks great! Please do split and submit this.

Signature · 1 line
Jens Axboe
#22 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 12 lines · Jens Axboe

fs/io-wq.h struct io_wq_work { · line 99

Quoted 7 lines · Pavel Begunkov
fs/io-wq.h +2 -2
-#define INIT_IO_WORK(work, _func)				\
+#define INIT_IO_WORK(work)					\
 	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
+		*(work) = (struct io_wq_work){};		\
 	} while (0)						\

Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.

Let's leave it to Xiaoguang and the series of the topic.

Quoted 3 lines

From a quick look at this, otherwise looks great! Please do split and submit this.

Sure. Have great time off!

Signature · 1 line
Pavel Begunkov
#23 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

hi,

Quoted 14 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov

fs/io-wq.h struct io_wq_work { · line 99

Quoted 7 lines
fs/io-wq.h +2 -2
-#define INIT_IO_WORK(work, _func)				\
+#define INIT_IO_WORK(work)					\
  	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
+		*(work) = (struct io_wq_work){};		\
  	} while (0)						\

Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.

Quoted 1 line

Let's leave it to Xiaoguang and the series of the topic.

Yeah, I'd be glad to continue this job, thanks.

Regards,
Xiaoguang Wang

Quoted 8 lines
Quoted 4 lines

From a quick look at this, otherwise looks great! Please do split and
submit this.

Sure. Have great time off!

maintainer
#24 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Quoted 16 lines · Xiaoguang Wang

hi,

Quoted 14 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov

fs/io-wq.h struct io_wq_work { · line 99

Quoted 7 lines
fs/io-wq.h +2 -2
-#define INIT_IO_WORK(work, _func)				\
+#define INIT_IO_WORK(work)					\
  	do {							\
-		*(work) = (struct io_wq_work){ .func = _func };	\
+		*(work) = (struct io_wq_work){};		\
  	} while (0)						\

Would be nice to optimize this one, it's a lot of clearing for something we'll generally not use at all in the fast path. Or at least keep it only for before when we submit the work for async execution.

Quoted 2 lines
Quoted 1 line

Let's leave it to Xiaoguang and the series of the topic.

Yeah, I'd be glad to continue this job, thanks.

Perfect! I think io_uring-5.8 (as of now) will be a great base for that.

Signature · 1 line
Jens Axboe
maintainer
#25 lore raw

Re: [PATCH v5 2/2] io_uring: avoid unnecessary io_wq_work copy for fast poll feature

In reply to Pavel Begunkov
Quoted 11 lines · Pavel Begunkov
Quoted 10 lines · Jens Axboe
Quoted 8 lines · Pavel Begunkov
Quoted 4 lines · Xiaoguang Wang

hi Jens, Pavel,

Will you have a look at this V5 version? Or we hold on this patchset, and do the refactoring work related io_wq_work firstly.

It's entirely up to Jens, but frankly, I think it'll bring more bugs than merits in the current state of things.

Quoted 8 lines
Quoted 6 lines

Well, I'd really like to reduce the overhead where we can, particularly when the overhead just exists to cater to the slow path.

Planning on taking the next week off and not do too much, but I'll see if I can get some testing in with the current patches.

I just think it should not be done at expense of robustness.

Oh I totally agree, which is why I've been holding back. I'm just saying that we're currently doing a number of semi expensive operations for slow path setup up front, which is annoying and would be nice NOT to do.

Quoted 7 lines

e.g. instead of having tons of if's around ->func, we can get rid of it and issue everything with io_wq_submit_work(). And there are plenty of pros of doing that:
- freeing some space in io_kiocb (in req.work in particular)
- removing much of stuff with nice negative diffstat
- helping this series
- even safer than now -- can't be screwed with memcpy(req).

Definitely sounds good.

Quoted 3 lines

Extra switch-lookup in io-wq shouldn't even be noticeable considering punting overhead. And even though io-wq loses some flexibility, as for me that's fine as long as there is only 1 user.

Don't care too much about the extra lookups, it's a very minor cost compared to the wins.

Quoted 2 lines

And then we can go and fix every other problem until this patch set looks good.

I'll take a look at your patch.

Signature · 1 line
Jens Axboe
#26 lore raw
In reply to Xiaoguang Wang
Quoted 10 lines · Xiaoguang Wang
Quoted 1 line

A v5 with those two things would be ready to commit.

First thanks for figuring out this bug.
Indeed, I had run test cases in liburing/test multiple rounds before sending V4 patches, and all test cases pass, no kernel issues occurred, here is my kernel build config, seems that I had kasan enabled.
[lege@localhost linux]$ cat .config | grep KASAN CONFIG_KASAN_SHADOW_OFFSET=0xdffffc0000000000 CONFIG_HAVE_ARCH_KASAN=y
CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
CONFIG_CC_HAS_KASAN_GENERIC=y

Quoted 8 lines

CONFIG_KASAN=y
CONFIG_KASAN_GENERIC=y
CONFIG_KASAN_OUTLINE=y
# CONFIG_KASAN_INLINE is not set CONFIG_KASAN_STACK=1
# CONFIG_KASAN_VMALLOC is not set
# CONFIG_TEST_KASAN is not set
But I don't know why I did't reproduce the bug, sorry.

Hmm, can't find an option to poison allocated mem. Try to fill req->work with garbage by hand (just after allocation), should help to reproduce.

Quoted 7 lines

Are you fine below changes?

fs/io_uring.c +0 -0
index 12296ce3e8b9..2a3a02838f7b 100644

fs/io_uring.c static void io_file_put_work(struct work_struct *work); · line 911

Quoted 8 lines
fs/io_uring.c +2 -3
@@ -907,9 +907,10 @@ static void io_file_put_work(struct work_struct *work);
  static inline void io_req_init_async(struct io_kiocb *req,
                         void (*func)(struct io_wq_work **))
  {
-       if (req->flags & REQ_F_WORK_INITIALIZED)
-               req->work.func = func;
-       else {
+       if (req->flags & REQ_F_WORK_INITIALIZED) {
+               if (!req->work.func)

->func should be initialised after alloc then, if not already.

Quoted 15 lines
fs/io_uring.c +4 -0
+                       req->work.func = func;
+       } else {
                 req->work = (struct io_wq_work){ .func = func };
                 req->flags |= REQ_F_WORK_INITIALIZED;
         }
@@ -2920,6 +2921,8 @@ static int __io_splice_prep(struct io_kiocb *req,
                 return ret;
         req->flags |= REQ_F_NEED_CLEANUP;
 
+       /* Splice will be punted aync, so initialize io_wq_work firstly_*/
+       io_req_init_async(req, io_wq_submit_work);
         if (!S_ISREG(file_inode(sp->file_in)->i_mode))
                 req->work.flags |= IO_WQ_WORK_UNBOUND;
 

fs/io_uring.c static int io_statx(struct io_kiocb *req, bool force_nonblock) · line 3598

Quoted 4 lines
fs/io_uring.c +1 -0
@@ -3592,6 +3595,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock)
 
  static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
  {
+        /* Close may be punted aync, so initialize io_wq_work firstly */

s/aync/async/

Quoted 11 lines
fs/io_uring.c +2 -0
+       io_req_init_async(req, io_wq_submit_work);
+
         /*
          * If we queue this for async, it must not be cancellable. That would
          * leave the 'file' in an undeterminate state.

Regards,
Xiaoguang Wang

Quoted 1 line
Signature · 1 line
Pavel Begunkov