Fixups/improvements for iopoll passthrough
Jens Axboe
maintainercover Fixups/improvements for iopoll passthrough
Hi,
A mix of cleanups and fixes for the passthrough iopoll support.
1) Cleanup the io_uring iopoll checking, making sure we never mix types.
2) Use a separate handler for the uring_cmd completion. This fixes a hang with iopoll passthrough and no poll queues.
3) Let's not add an ->uring_cmd_iopoll() handler with a type we know we have to change, once we support batching. And more importantly, pass in the poll flags.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/3 io_uring: cleanly separate request types for iopoll
After the addition of iopoll support for passthrough, there's a bit of a mixup here. Clean it up and get rid of the casting for the passthrough command type.
--- io_uring/rw.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
io_uring/rw.c +9 -6
@@ -994,7 +994,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)wq_list_for_each (pos, start, &ctx->iopoll_list) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +struct file *file = req->file;int ret;/*
@@ -1006,12 +1006,15 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)break ;if (req->opcode ==IORING_OP_URING_CMD ) { -struct io_uring_cmd *ioucmd = (struct io_uring_cmd *)rw; +struct io_uring_cmd *ioucmd; - ret = req->file->f_op->uring_cmd_iopoll (ioucmd); - }else - ret = rw->kiocb.ki_filp->f_op->iopoll (&rw->kiocb, &iob, - poll_flags); + ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); + ret = file->f_op->uring_cmd_iopoll (ioucmd, poll_flags); + }else { + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + + ret = file->f_op->iopoll (&rw->kiocb, &iob, poll_flags); + }if (unlikely (ret <0 ))return ret;else if (ret)
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 2/3 nvme: use separate end IO handler for IOPOLL
In reply to Jens AxboeDon't need to rely on the cookie or request type, set the right handler based on how we're handling the IO.
--- drivers/nvme/host/ioctl.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +22 -8
@@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd)io_uring_cmd_done (ioucmd, status, result); } -static void nvme_uring_cmd_end_io (struct request *req,blk_status_t err) +static void nvme_uring_iopoll_cmd_end_io (struct request *req,blk_status_t err) {struct io_uring_cmd *ioucmd = req->end_io_data;struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd);/* extract bio before reusing the same field for request */ struct bio *bio = pdu->bio; -void *cookie =READ_ONCE (ioucmd->cookie); pdu->req = req; req->bio = bio;/* * For iopoll, complete it directly. -* Otherwise, move the completion to task work. */ -if (cookie !=NULL &&blk_rq_is_poll (req)) -nvme_uring_task_cb (ioucmd); -else -io_uring_cmd_complete_in_task (ioucmd, nvme_uring_task_cb); +nvme_uring_task_cb (ioucmd); +} + +static void nvme_uring_cmd_end_io (struct request *req,blk_status_t err) +{ +struct io_uring_cmd *ioucmd = req->end_io_data; +struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd); +/* extract bio before reusing the same field for request */ +struct bio *bio = pdu->bio; + + pdu->req = req; + req->bio = bio; + +/* +* Move the completion to task work. +*/ +io_uring_cmd_complete_in_task (ioucmd, nvme_uring_task_cb); } static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
@@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, blk_flags);if (IS_ERR (req))return PTR_ERR (req); - req->end_io = nvme_uring_cmd_end_io; +if (issue_flags &IO_URING_F_IOPOLL ) + req->end_io = nvme_uring_iopoll_cmd_end_io; +else + req->end_io = nvme_uring_cmd_end_io; req->end_io_data = ioucmd;if (issue_flags &IO_URING_F_IOPOLL && rq_flags &REQ_POLLED ) {
Signature · 1 line
2.35.1
Re: [PATCH 2/3] nvme: use separate end IO handler for IOPOLL
Quoted 57 lines · Jens Axboe
Don't need to rely on the cookie or request type, set the right handler based on how we're handling the IO.
Signed-off-by Jens Axboe <[email protected]>---
drivers/nvme/host/ioctl.c| 30 ++++++++++++++++++++++--------1 file changed, 22 insertions(+), 8 deletions(-)drivers/nvme/host/ioctl.c +18
-7@@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd)io_uring_cmd_done (ioucmd, status, result); } -static void nvme_uring_cmd_end_io (struct request *req,blk_status_t err) +static void nvme_uring_iopoll_cmd_end_io (struct request *req,blk_status_t err) {struct io_uring_cmd *ioucmd = req->end_io_data;struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd);/* extract bio before reusing the same field for request */ struct bio *bio = pdu->bio; -void *cookie =READ_ONCE (ioucmd->cookie); pdu->req = req; req->bio = bio;/* * For iopoll, complete it directly. -* Otherwise, move the completion to task work. */ -if (cookie !=NULL &&blk_rq_is_poll (req)) -nvme_uring_task_cb (ioucmd); -else -io_uring_cmd_complete_in_task (ioucmd, nvme_uring_task_cb); +nvme_uring_task_cb (ioucmd); +} + +static void nvme_uring_cmd_end_io (struct request *req,blk_status_t err) +{ +struct io_uring_cmd *ioucmd = req->end_io_data; +struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd); +/* extract bio before reusing the same field for request */ +struct bio *bio = pdu->bio; + + pdu->req = req; + req->bio = bio; + +/* +* Move the completion to task work. +*/ +io_uring_cmd_complete_in_task (ioucmd, nvme_uring_task_cb); } static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
drivers/nvme/host/ioctl.c static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · line 481
Quoted 8 lines
drivers/nvme/host/ioctl.c +4
-1@@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, blk_flags);if (IS_ERR (req))return PTR_ERR (req); - req->end_io = nvme_uring_cmd_end_io; +if (issue_flags &IO_URING_F_IOPOLL ) + req->end_io = nvme_uring_iopoll_cmd_end_io; +else + req->end_io = nvme_uring_cmd_end_io;
The polled handler (nvme_uring_iopoll_cmd_end_io) may get called in irq
context (some swapper/kworker etc.) too. And in that case will it be
safe to call nvme_uring_task_cb directly. We don't touch the user-fields in cmd (thanks to Big CQE) so that part is sorted. But there is blk_rq_unmap_user call - can that or anything else
inside io_req_complete_post() cause trouble.
* A matching blk_rq_unmap_user() must be issued at the end of I/O, while
* still in process context.
*/
int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
struct rq_map_data *map_data, const struct iov_iter *iter, gfp_t gfp_mask)
Jens Axboe
maintainerRe: [PATCH 2/3] nvme: use separate end IO handler for IOPOLL
Quoted 65 lines · Kanchan Joshi
Quoted 64 lines · Jens Axboe
Don't need to rely on the cookie or request type, set the right handler based on how we're handling the IO.
Signed-off-by Jens Axboe <[email protected]>---
drivers/nvme/host/ioctl.c| 30 ++++++++++++++++++++++--------1 file changed, 22 insertions(+), 8 deletions(-)drivers/nvme/host/ioctl.c +1
-1@@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd) ????io_uring_cmd_done(ioucmd, status, result); } -static void nvme_uring_cmd_end_io (struct request *req,blk_status_t err) +static void nvme_uring_iopoll_cmd_end_io (struct request *req,blk_status_t err) { ????struct io_uring_cmd *ioucmd = req->end_io_data; ????struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);????/* extract bio before reusing the same field for request */ ????struct bio *bio =
pdu->bio;
-??? void *cookie =READ_ONCE(ioucmd->cookie);????pdu->req = req;
????req->bio = bio;????/*
???? * For iopoll, complete it directly.
-???? * Otherwise, move the completion to task work. ???? */
-??? if (cookie != NULL &&blk_rq_is_poll(req))
-???????nvme_uring_task_cb(ioucmd);
-??? else
-???????io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); +???nvme_uring_task_cb(ioucmd);drivers/nvme/host/ioctl.c +16
-0+} + +static void nvme_uring_cmd_end_io (struct request *req,blk_status_t err) +{ +??? struct io_uring_cmd *ioucmd = req->end_io_data; +??? struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); +??? /* extract bio before reusing the same field for request */ +??? struct bio *bio = pdu->bio; + +??? pdu->req = req; +??? req->bio = bio; + +??? /* +???? * Move the completion to task work. +???? */ +??? io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);}
static int nvme_uring_cmd_io(struct
nvme_ctrl*ctrl, structnvme_ns*ns,drivers/nvme/host/ioctl.c +3
-1@@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, ??????????? blk_flags); ????if (IS_ERR(req)) ??????? return PTR_ERR(req); -??? req->end_io = nvme_uring_cmd_end_io; +??? if (issue_flags & IO_URING_F_IOPOLL) +??????? req->end_io = nvme_uring_iopoll_cmd_end_io; +??? else
Quoted 8 lines
drivers/nvme/host/ioctl.c static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · line 481
Quoted 1 line
drivers/nvme/host/ioctl.c +1
-0+??????? req->end_io = nvme_uring_cmd_end_io;The polled handler (
nvme_uring_iopoll_cmd_end_io) may get called in irq context (some swapper/kworker etc.) too. And in that case will it be safe to callnvme_uring_task_cbdirectly. We don't touch the user-fields in cmd (thanks to Big CQE) so that part is sorted. But there isblk_rq_unmap_usercall - can that or anything else insideio_req_complete_post()cause trouble.
The unmap might be problematic if the data wasn't mapped. That's a slow path and unexpected, however. Might be better to just leave the unified completion path and ensure that nvme_uring_task_cb() checks for polled as well. I'll give it a quick spin.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 3/3 fs: add batch and poll flags to the uring_cmd_iopoll() handler
In reply to Jens AxboeWe need the poll_flags to know how to poll for the IO, and we should have the batch structure in preparation for supporting batched completions with iopoll.
--- drivers/nvme/host/ioctl.c | 12 ++++++++---- drivers/nvme/host/nvme.h | 6 ++++-- include/linux/fs.h | 3 ++- io_uring/rw.c | 3 ++- 4 files changed, 16 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +8 -4
@@ -637,7 +637,9 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)return nvme_ns_uring_cmd (ns, ioucmd, issue_flags); } -int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd) +int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob, +unsigned int poll_flags) {struct bio *bio;int ret =0 ;
@@ -650,7 +652,7 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd) struct nvme_ns, cdev); q = ns->queue;if (test_bit (QUEUE_FLAG_POLL , &q->queue_flags) && bio && bio->bi_bdev) - ret =bio_poll (bio,NULL ,0 ); + ret =bio_poll (bio, iob, poll_flags);rcu_read_unlock ();return ret; }
@@ -736,7 +738,9 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,return ret; } -int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd) +int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob, +unsigned int poll_flags) {struct cdev *cdev =file_inode (ioucmd->file)->i_cdev; struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
@@ -752,7 +756,7 @@ int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd) q = ns->queue;if (test_bit (QUEUE_FLAG_POLL , &q->queue_flags) && bio && bio->bi_bdev) - ret =bio_poll (bio,NULL ,0 ); + ret =bio_poll (bio, iob, poll_flags);rcu_read_unlock (); }srcu_read_unlock (&head->srcu, srcu_idx);
drivers/nvme/host/nvme.h +4 -2
@@ -821,8 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg);long nvme_dev_ioctl (struct file *file,unsigned int cmd,unsigned long arg); -int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd); -int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd); +int nvme_ns_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob,unsigned int poll_flags); +int nvme_ns_head_chr_uring_cmd_iopoll (struct io_uring_cmd *ioucmd, +struct io_comp_batch *iob,unsigned int poll_flags);int nvme_ns_chr_uring_cmd (struct io_uring_cmd *ioucmd,unsigned int issue_flags); int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
include/linux/fs.h +2 -1
@@ -2132,7 +2132,8 @@ struct file_operations { loff_t len, unsigned int remap_flags);int (*fadvise)(struct file *,loff_t ,loff_t ,int );int (*uring_cmd)(struct io_uring_cmd *ioucmd,unsigned int issue_flags); -int (*uring_cmd_iopoll)(struct io_uring_cmd *ioucmd); +int (*uring_cmd_iopoll)(struct io_uring_cmd *,struct io_comp_batch *, +unsigned int poll_flags); } __randomize_layout;struct inode_operations {
io_uring/rw.c +2 -1
@@ -1009,7 +1009,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)struct io_uring_cmd *ioucmd; ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); - ret = file->f_op->uring_cmd_iopoll (ioucmd, poll_flags); + ret = file->f_op->uring_cmd_iopoll (ioucmd, &iob, + poll_flags); }else { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
Signature · 1 line
2.35.1