Skip to contentSkip to conversation

Fixups/improvements for iopoll passthrough

PATCH Superseded Jens Axboe 2 participants 6 messages Sep 2, 2022 – Sep 3, 2022

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

cover Fixups/improvements for iopoll passthrough

Hi,

A mix of cleanups and fixes for the passthrough iopoll support.

1) Cleanup the io_uring iopoll checking, making sure we never mix types.

2) Use a separate handler for the uring_cmd completion. This fixes a hang with iopoll passthrough and no poll queues.

3) Let's not add an ->uring_cmd_iopoll() handler with a type we know we have to change, once we support batching. And more importantly, pass in the poll flags.

Signature · 1 line
Jens Axboe
author maintainer
#1 lore raw

PATCH 1/3 io_uring: cleanly separate request types for iopoll

After the addition of iopoll support for passthrough, there's a bit of a mixup here. Clean it up and get rid of the casting for the passthrough command type.

---
 io_uring/rw.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)
io_uring/rw.c +9 -6
index 9698a789b3d5..3f03b6d2a5a3 100644
@@ -994,7 +994,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
 
 	wq_list_for_each(pos, start, &ctx->iopoll_list) {
 		struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);
-		struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+		struct file *file = req->file;
 		int ret;
 
 		/*
@@ -1006,12 +1006,15 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
 			break;
 
 		if (req->opcode == IORING_OP_URING_CMD) {
-			struct io_uring_cmd *ioucmd = (struct io_uring_cmd *)rw;
+			struct io_uring_cmd *ioucmd;
 
-			ret = req->file->f_op->uring_cmd_iopoll(ioucmd);
-		} else
-			ret = rw->kiocb.ki_filp->f_op->iopoll(&rw->kiocb, &iob,
-							poll_flags);
+			ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
+			ret = file->f_op->uring_cmd_iopoll(ioucmd, poll_flags);
+		} else {
+			struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+
+			ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);
+		}
 		if (unlikely(ret < 0))
 			return ret;
 		else if (ret)
Signature · 1 line
2.35.1
author maintainer
#2 lore raw

PATCH 2/3 nvme: use separate end IO handler for IOPOLL

In reply to Jens Axboe

Don't need to rely on the cookie or request type, set the right handler based on how we're handling the IO.

---
 drivers/nvme/host/ioctl.c | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +22 -8
index 7756b439a688..f34abe95821e 100644
@@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd)
 	io_uring_cmd_done(ioucmd, status, result);
 }
 
-static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err)
+static void nvme_uring_iopoll_cmd_end_io(struct request *req, blk_status_t err)
 {
 	struct io_uring_cmd *ioucmd = req->end_io_data;
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 	/* extract bio before reusing the same field for request */
 	struct bio *bio = pdu->bio;
-	void *cookie = READ_ONCE(ioucmd->cookie);
 
 	pdu->req = req;
 	req->bio = bio;
 
 	/*
 	 * For iopoll, complete it directly.
-	 * Otherwise, move the completion to task work.
 	 */
-	if (cookie != NULL && blk_rq_is_poll(req))
-		nvme_uring_task_cb(ioucmd);
-	else
-		io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
+	nvme_uring_task_cb(ioucmd);
+}
+
+static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err)
+{
+	struct io_uring_cmd *ioucmd = req->end_io_data;
+	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+	/* extract bio before reusing the same field for request */
+	struct bio *bio = pdu->bio;
+
+	pdu->req = req;
+	req->bio = bio;
+
+	/*
+	 * Move the completion to task work.
+	 */
+	io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
 }
 
 static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
@@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 			blk_flags);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
-	req->end_io = nvme_uring_cmd_end_io;
+	if (issue_flags & IO_URING_F_IOPOLL)
+		req->end_io = nvme_uring_iopoll_cmd_end_io;
+	else
+		req->end_io = nvme_uring_cmd_end_io;
 	req->end_io_data = ioucmd;
 
 	if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) {
Signature · 1 line
2.35.1
#3 lore raw

Re: [PATCH 2/3] nvme: use separate end IO handler for IOPOLL

Quoted 57 lines · Jens Axboe

Don't need to rely on the cookie or request type, set the right handler based on how we're handling the IO.

---
drivers/nvme/host/ioctl.c | 30 ++++++++++++++++++++++--------

1 file changed, 22 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +18 -7
index 7756b439a688..f34abe95821e 100644
@@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd)
 	io_uring_cmd_done(ioucmd, status, result);
 }
 
-static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err)
+static void nvme_uring_iopoll_cmd_end_io(struct request *req, blk_status_t err)
 {
 	struct io_uring_cmd *ioucmd = req->end_io_data;
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 	/* extract bio before reusing the same field for request */
 	struct bio *bio = pdu->bio;
-	void *cookie = READ_ONCE(ioucmd->cookie);
 
 	pdu->req = req;
 	req->bio = bio;
 
 	/*
 	 * For iopoll, complete it directly.
-	 * Otherwise, move the completion to task work.
 	 */
-	if (cookie != NULL && blk_rq_is_poll(req))
-		nvme_uring_task_cb(ioucmd);
-	else
-		io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
+	nvme_uring_task_cb(ioucmd);
+}
+
+static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err)
+{
+	struct io_uring_cmd *ioucmd = req->end_io_data;
+	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+	/* extract bio before reusing the same field for request */
+	struct bio *bio = pdu->bio;
+
+	pdu->req = req;
+	req->bio = bio;
+
+	/*
+	 * Move the completion to task work.
+	 */
+	io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
 }
 
 static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,

drivers/nvme/host/ioctl.c static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · line 481

Quoted 8 lines
drivers/nvme/host/ioctl.c +4 -1
@@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 			blk_flags);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
-	req->end_io = nvme_uring_cmd_end_io;
+	if (issue_flags & IO_URING_F_IOPOLL)
+		req->end_io = nvme_uring_iopoll_cmd_end_io;
+	else
+		req->end_io = nvme_uring_cmd_end_io;

The polled handler (nvme_uring_iopoll_cmd_end_io) may get called in irq
context (some swapper/kworker etc.) too. And in that case will it be
safe to call nvme_uring_task_cb directly. We don't touch the user-fields in cmd (thanks to Big CQE) so that part is sorted. But there is blk_rq_unmap_user call - can that or anything else
inside io_req_complete_post() cause trouble.

* A matching blk_rq_unmap_user() must be issued at the end of I/O, while
* still in process context.
*/
int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
struct rq_map_data *map_data, const struct iov_iter *iter, gfp_t gfp_mask)

author maintainer
#4 lore raw

Re: [PATCH 2/3] nvme: use separate end IO handler for IOPOLL

Quoted 65 lines · Kanchan Joshi
Quoted 64 lines · Jens Axboe

Don't need to rely on the cookie or request type, set the right handler based on how we're handling the IO.

---
drivers/nvme/host/ioctl.c | 30 ++++++++++++++++++++++--------

1 file changed, 22 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +1 -1
index 7756b439a688..f34abe95821e 100644
@@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd)
 ????io_uring_cmd_done(ioucmd, status, result);
 }
 
-static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err)
+static void nvme_uring_iopoll_cmd_end_io(struct request *req, blk_status_t err)
 {
 ????struct io_uring_cmd *ioucmd = req->end_io_data;
 ????struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);

????/* extract bio before reusing the same field for request */ ????struct bio *bio = pdu->bio;
-??? void *cookie = READ_ONCE(ioucmd->cookie);

????pdu->req = req;
????req->bio = bio;

????/*
???? * For iopoll, complete it directly.
-???? * Otherwise, move the completion to task work. ???? */
-??? if (cookie != NULL && blk_rq_is_poll(req))
-??????? nvme_uring_task_cb(ioucmd);
-??? else
-??????? io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); +??? nvme_uring_task_cb(ioucmd);

drivers/nvme/host/ioctl.c +16 -0
+}
+
+static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err)
+{
+??? struct io_uring_cmd *ioucmd = req->end_io_data;
+??? struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+??? /* extract bio before reusing the same field for request */
+??? struct bio *bio = pdu->bio;
+
+??? pdu->req = req;
+??? req->bio = bio;
+
+??? /*
+???? * Move the completion to task work.
+???? */
+??? io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);

}

static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,

drivers/nvme/host/ioctl.c +3 -1
@@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 ??????????? blk_flags);
 ????if (IS_ERR(req))
 ??????? return PTR_ERR(req);
-??? req->end_io = nvme_uring_cmd_end_io;
+??? if (issue_flags & IO_URING_F_IOPOLL)
+??????? req->end_io = nvme_uring_iopoll_cmd_end_io;
+??? else
Quoted 8 lines

drivers/nvme/host/ioctl.c static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · line 481

Quoted 1 line
drivers/nvme/host/ioctl.c +1 -0
+??????? req->end_io = nvme_uring_cmd_end_io;

The polled handler (nvme_uring_iopoll_cmd_end_io) may get called in irq context (some swapper/kworker etc.) too. And in that case will it be safe to call nvme_uring_task_cb directly. We don't touch the user-fields in cmd (thanks to Big CQE) so that part is sorted. But there is blk_rq_unmap_user call - can that or anything else inside io_req_complete_post() cause trouble.

The unmap might be problematic if the data wasn't mapped. That's a slow path and unexpected, however. Might be better to just leave the unified completion path and ensure that nvme_uring_task_cb() checks for polled as well. I'll give it a quick spin.

Signature · 1 line
Jens Axboe
author maintainer
#5 lore raw

PATCH 3/3 fs: add batch and poll flags to the uring_cmd_iopoll() handler

In reply to Jens Axboe

We need the poll_flags to know how to poll for the IO, and we should have the batch structure in preparation for supporting batched completions with iopoll.

---
 drivers/nvme/host/ioctl.c | 12 ++++++++----
 drivers/nvme/host/nvme.h  |  6 ++++--
 include/linux/fs.h        |  3 ++-
 io_uring/rw.c             |  3 ++-
 4 files changed, 16 insertions(+), 8 deletions(-)
drivers/nvme/host/ioctl.c +8 -4
index f34abe95821e..7a0b12ef49ae 100644
@@ -637,7 +637,9 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
 	return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
 }
 
-int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
+int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+				 struct io_comp_batch *iob,
+				 unsigned int poll_flags)
 {
 	struct bio *bio;
 	int ret = 0;
@@ -650,7 +652,7 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
 			struct nvme_ns, cdev);
 	q = ns->queue;
 	if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)
-		ret = bio_poll(bio, NULL, 0);
+		ret = bio_poll(bio, iob, poll_flags);
 	rcu_read_unlock();
 	return ret;
 }
@@ -736,7 +738,9 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
 	return ret;
 }
 
-int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
+int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+				      struct io_comp_batch *iob,
+				      unsigned int poll_flags)
 {
 	struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
 	struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
@@ -752,7 +756,7 @@ int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
 		q = ns->queue;
 		if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio
 				&& bio->bi_bdev)
-			ret = bio_poll(bio, NULL, 0);
+			ret = bio_poll(bio, iob, poll_flags);
 		rcu_read_unlock();
 	}
 	srcu_read_unlock(&head->srcu, srcu_idx);
drivers/nvme/host/nvme.h +4 -2
index fdcbc93dea21..216acbe953b3 100644
@@ -821,8 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
 		unsigned long arg);
 long nvme_dev_ioctl(struct file *file, unsigned int cmd,
 		unsigned long arg);
-int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd);
-int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd);
+int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+		struct io_comp_batch *iob, unsigned int poll_flags);
+int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+		struct io_comp_batch *iob, unsigned int poll_flags);
 int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd,
 		unsigned int issue_flags);
 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
include/linux/fs.h +2 -1
index d6badd19784f..01681d061a6a 100644
@@ -2132,7 +2132,8 @@ struct file_operations {
 				   loff_t len, unsigned int remap_flags);
 	int (*fadvise)(struct file *, loff_t, loff_t, int);
 	int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
-	int (*uring_cmd_iopoll)(struct io_uring_cmd *ioucmd);
+	int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
+				unsigned int poll_flags);
 } __randomize_layout;
 
 struct inode_operations {
io_uring/rw.c +2 -1
index 3f03b6d2a5a3..4a061326c664 100644
@@ -1009,7 +1009,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
 			struct io_uring_cmd *ioucmd;
 
 			ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
-			ret = file->f_op->uring_cmd_iopoll(ioucmd, poll_flags);
+			ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob,
+								poll_flags);
 		} else {
 			struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
 
Signature · 1 line
2.35.1