Fixups/improvements for iopoll passthrough
Jens Axboe
maintainerv2 cover Fixups/improvements for iopoll passthrough
Hi,
A mix of cleanups and fixes for the passthrough iopoll support.
1) Cleanup the io_uring iopoll checking, making sure we never mix types.
2) Prep patch for getting local task_work run efficiently for iopoll.
3) Fixup io_uring iopoll for local task_work.
4) Let's not add an ->uring_cmd_iopoll() handler with a type we know we have to change, once we support batching. And more importantly, pass in the poll flags.
Since v1:
- Fix some of patch 4 bleeding into patch 1
- Drop nvme patch, add the two io_uring iopoll patches instead
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v2 1/4 io_uring: cleanly separate request types for iopoll
After the addition of iopoll support for passthrough, there's a bit of a mixup here. Clean it up and get rid of the casting for the passthrough command type.
--- io_uring/rw.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
io_uring/rw.c +9 -6
@@ -994,7 +994,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)wq_list_for_each (pos, start, &ctx->iopoll_list) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +struct file *file = req->file;int ret;/*
@@ -1006,12 +1006,15 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)break ;if (req->opcode ==IORING_OP_URING_CMD ) { -struct io_uring_cmd *ioucmd = (struct io_uring_cmd *)rw; +struct io_uring_cmd *ioucmd; - ret = req->file->f_op->uring_cmd_iopoll (ioucmd); - }else - ret = rw->kiocb.ki_filp->f_op->iopoll (&rw->kiocb, &iob, - poll_flags); + ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); + ret = file->f_op->uring_cmd_iopoll (ioucmd); + }else { + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + + ret = file->f_op->iopoll (&rw->kiocb, &iob, poll_flags); + }if (unlikely (ret <0 ))return ret;else if (ret)
Signature · 1 line
2.35.1
Re: [PATCH 1/4] io_uring: cleanly separate request types for iopoll
Quoted 3 lines · Jens Axboe
After the addition of iopoll support for passthrough, there's a bit of a mixup here. Clean it up and get rid of the casting for the passthrough command type.
Jens Axboe
maintainerPATCH v2 2/4 io_uring: add local task_work run helper that is entered locked
In reply to Jens AxboeWe have a few spots that drop the mutex just to run local task_work, which immediately tries to grab it again. Add a helper that just passes in whether we're locked already.
--- io_uring/io_uring.c | 23 ++++++++++++++++------- io_uring/io_uring.h | 1 + 2 files changed, 17 insertions(+), 7 deletions(-)
io_uring/io_uring.c +16 -7
@@ -1161,9 +1161,8 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) } } -int io_run_local_work (struct io_ring_ctx *ctx) +int __io_run_local_work (struct io_ring_ctx *ctx,bool locked) { -bool locked;struct llist_node *node;struct llist_node fake;struct llist_node *current_final =NULL ;
@@ -1178,8 +1177,6 @@ int io_run_local_work(struct io_ring_ctx *ctx)return -EEXIST ; } - locked =mutex_trylock (&ctx->uring_lock); - node =io_llist_xchg (&ctx->work_llist, &fake); ret =0 ; again:
@@ -1204,12 +1201,24 @@ int io_run_local_work(struct io_ring_ctx *ctx) goto again; } -if (locked) { +if (locked)io_submit_flush_completions (ctx); -mutex_unlock (&ctx->uring_lock); - }trace_io_uring_local_work_run (ctx, ret, loops);return ret; + +} + +int io_run_local_work (struct io_ring_ctx *ctx) +{ +bool locked; +int ret; + + locked =mutex_trylock (&ctx->uring_lock); + ret =__io_run_local_work (ctx, locked); +if (locked) +mutex_unlock (&ctx->uring_lock); + +return ret; }static void io_req_tw_post (struct io_kiocb *req,bool *locked)
io_uring/io_uring.h +1 -0
@@ -27,6 +27,7 @@ enum {struct io_uring_cqe *__io_get_cqe (struct io_ring_ctx *ctx);bool io_req_cqe_overflow (struct io_kiocb *req);int io_run_task_work_sig (struct io_ring_ctx *ctx); +int __io_run_local_work (struct io_ring_ctx *ctx,bool locked);int io_run_local_work (struct io_ring_ctx *ctx);void io_req_complete_failed (struct io_kiocb *req,s32 res);void __io_req_complete (struct io_kiocb *req,unsigned issue_flags);
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH v2 3/4 io_uring: ensure iopoll runs local task work as well
In reply to Jens AxboeCombine the two checks we have for task_work running and whether or not we need to shuffle the mutex into one, so we unify how task_work is run in the iopoll loop. This helps ensure that local task_work is run when needed, and also optimizes that path to avoid a mutex shuffle if it's not needed.
--- io_uring/io_uring.c | 39 ++++++++++++++++++++------------------- io_uring/io_uring.h | 6 ++++++ 2 files changed, 26 insertions(+), 19 deletions(-)
io_uring/io_uring.c +20 -19
@@ -1428,25 +1428,26 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)* forever, while the workqueue is stuck trying to acquire the * very same mutex. */ -if (wq_list_empty (&ctx->iopoll_list)) { -u32 tail = ctx->cached_cq_tail; - -mutex_unlock (&ctx->uring_lock); - ret =io_run_task_work_ctx (ctx); -mutex_lock (&ctx->uring_lock); -if (ret <0 ) -break ; - -/* some requests don't go through iopoll_list */ -if (tail != ctx->cached_cq_tail || -wq_list_empty (&ctx->iopoll_list)) -break ; - } - -if (task_work_pending (current)) { -mutex_unlock (&ctx->uring_lock); -io_run_task_work (); -mutex_lock (&ctx->uring_lock); +if (wq_list_empty (&ctx->iopoll_list) || +io_task_work_pending (ctx)) { +if (!llist_empty (&ctx->work_llist)) +__io_run_local_work (ctx, true); +if (task_work_pending (current) || +wq_list_empty (&ctx->iopoll_list)) { +u32 tail = ctx->cached_cq_tail; + +mutex_unlock (&ctx->uring_lock); + ret =io_run_task_work (); +mutex_lock (&ctx->uring_lock); + +if (ret <0 ) +break ; + +/* some requests don't go through iopoll_list */ +if (tail != ctx->cached_cq_tail || +wq_list_empty (&ctx->iopoll_list)) +break ; + } } ret =io_do_iopoll (ctx, !min);if (ret <0 )
io_uring/io_uring.h +6 -0
@@ -236,6 +236,12 @@ static inline int io_run_task_work(void)return 0 ; } +static inline bool io_task_work_pending (struct io_ring_ctx *ctx) +{ +return test_thread_flag (TIF_NOTIFY_SIGNAL ) || + !wq_list_empty (&ctx->work_llist); +} +static inline int io_run_task_work_ctx (struct io_ring_ctx *ctx) {int ret =0 ;
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH v2 4/4 fs: add batch and poll flags to the uring_cmd_iopoll() handler
In reply to Jens AxboeWe need the poll_flags to know how to poll for the IO, and we should have the batch structure in preparation for supporting batched completions with iopoll.
--- drivers/nvme/host/ioctl.c | 12 ++++++++---- drivers/nvme/host/nvme.h | 6 ++++-- include/linux/fs.h | 3 ++- io_uring/rw.c | 3 ++- 4 files changed, 16 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +8 -4
@@ -623,7 +623,9 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)return nvme_ns_uring_cmd (ns, ioucmd, issue_flags); } -int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd) +int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob, +unsigned int poll_flags) {struct bio *bio;int ret =0 ;
@@ -636,7 +638,7 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd) struct nvme_ns, cdev); q = ns->queue;if (test_bit (QUEUE_FLAG_POLL , &q->queue_flags) && bio && bio->bi_bdev) - ret =bio_poll (bio,NULL ,0 ); + ret =bio_poll (bio, iob, poll_flags);rcu_read_unlock ();return ret; }
@@ -722,7 +724,9 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,return ret; } -int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd) +int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob, +unsigned int poll_flags) {struct cdev *cdev =file_inode (ioucmd->file)->i_cdev; struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
@@ -738,7 +742,7 @@ int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd) q = ns->queue;if (test_bit (QUEUE_FLAG_POLL , &q->queue_flags) && bio && bio->bi_bdev) - ret =bio_poll (bio,NULL ,0 ); + ret =bio_poll (bio, iob, poll_flags);rcu_read_unlock (); }srcu_read_unlock (&head->srcu, srcu_idx);
drivers/nvme/host/nvme.h +4 -2
@@ -821,8 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg);long nvme_dev_ioctl (struct file *file,unsigned int cmd,unsigned long arg); -int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd); -int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd); +int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob,unsigned int poll_flags); +int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob,unsigned int poll_flags);int nvme_ns_chr_uring_cmd (struct io_uring_cmd *ioucmd,unsigned int issue_flags); int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
include/linux/fs.h +2 -1
@@ -2132,7 +2132,8 @@ struct file_operations { loff_t len, unsigned int remap_flags);int (*fadvise)(struct file *,loff_t ,loff_t ,int );int (*uring_cmd)(struct io_uring_cmd *ioucmd,unsigned int issue_flags); -int (*uring_cmd_iopoll)(struct io_uring_cmd *ioucmd); +int (*uring_cmd_iopoll)(struct io_uring_cmd *,struct io_comp_batch *, +unsigned int poll_flags); } __randomize_layout;struct inode_operations {
io_uring/rw.c +2 -1
@@ -1009,7 +1009,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)struct io_uring_cmd *ioucmd; ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); - ret = file->f_op->uring_cmd_iopoll (ioucmd); + ret = file->f_op->uring_cmd_iopoll (ioucmd, &iob, + poll_flags); }else { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
Signature · 1 line
2.35.1
Re: [PATCH 4/4] fs: add batch and poll flags to the uring_cmd_iopoll() handler
On Sat, Sep 03, 2022 at 10:52:34AM -0600, Jens Axboe wrote:
nvme missing in title i.e. "fs,nvme: xyz".