Skip to contentSkip to conversation

io_uring passthru: set result on blk-mq request completion

PATCH Superseded Caleb Sander Mateos 4 participants 8 messages Sep 2 – Sep 9

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover io_uring passthru: set result on blk-mq request completion

io_uring NVMe passthru currently receives the NVMe status and result in the blk-mq request completion callback nvme_uring_cmd_end_io() but doesn't post the io_uring CQE until the io_uring task work callback nvme_uring_task_cb(). The status and result must be plumbed through struct nvme_uring_cmd_pdu, taking up 16 bytes of the 32 available.

Store the status and result on the io_uring request in nvme_uring_cmd_end_io() instead of nvme_uring_task_cb() so it doesn't need to be passed through struct nvme_uring_cmd_pdu.

v2:
- Keep io_uring_cmd_done{,32}() wrappers to reduce changes in uring_cmd implementations (Joanne)

v1: https://lore.kernel.org/io-uring/[email protected]/

Caleb Sander Mateos (2):
io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done() nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

 drivers/nvme/host/ioctl.c    | 19 +++++++++----------
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 3 files changed, 44 insertions(+), 31 deletions(-)
Signature · 1 line
2.55.0
#1 lore raw

PATCH v2 1/2 io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

In preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.

---
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 21 deletions(-)
include/linux/io_uring/cmd.h +17 -6
index 331dcbefe72f..67f2ef700c43 100644
@@ -50,19 +50,21 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 				  const struct iovec __user *uvec,
 				  size_t uvec_segs,
 				  int ddir, struct iov_iter *iter,
 				  unsigned issue_flags);
 
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-			 unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 			    io_req_tw_func_t task_work_cb,
 			    unsigned flags);
 
@@ -105,12 +107,19 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 						int ddir, struct iov_iter *iter,
 						unsigned issue_flags)
 {
 	return -EOPNOTSUPP;
 }
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
-		u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+}
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret,
+					  u64 res2)
+{
+}
+static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
+				       unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 			    io_req_tw_func_t task_work_cb, unsigned flags)
 {
@@ -171,17 +180,19 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
 }
 
 static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
 				     unsigned issue_flags)
 {
-	return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
+	io_uring_cmd_set_res(ioucmd, ret);
+	__io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
 				       u64 res2, unsigned issue_flags)
 {
-	return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
+	io_uring_cmd_set_res32(ioucmd, ret, res2);
+	__io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
 			    void (*release)(void *), unsigned int index,
 			    unsigned int issue_flags);
io_uring/uring_cmd.c +18 -15
index 726a659f38c3..917b32a921e6 100644
@@ -136,40 +136,43 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 	req->io_task_work.func = task_work_cb;
 	__io_req_task_work_add(req, flags);
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
 
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
-					  u64 extra1, u64 extra2)
+void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
 {
-	req->big_cqe.extra1 = extra1;
-	req->big_cqe.extra2 = extra2;
+	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+	if (ret < 0)
+		req_set_fail(req);
+	io_req_set_res(req, ret, 0);
 }
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res);
+
+void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+	if (ret < 0)
+		req_set_fail(req);
+	io_req_set_res32(req, ret, 0, res2, 0);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res32);
 
 /*
  * Called by consumers of io_uring_cmd, if they originally returned
  * -EIOCBQUEUED upon receiving the command.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
-		       unsigned issue_flags, bool is_cqe32)
+void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, unsigned issue_flags)
 {
 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
 
 	if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
 		return;
 
 	io_uring_cmd_del_cancelable(ioucmd, issue_flags);
 
-	if (ret < 0)
-		req_set_fail(req);
-
-	io_req_set_res(req, ret, 0);
-	if (is_cqe32) {
-		if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
-			req->cqe.flags |= IORING_CQE_F_32;
-		io_req_set_cqe32_extra(req, res2, 0);
-	}
 	io_req_uring_cleanup(req, issue_flags);
 	if (req->flags & REQ_F_IOPOLL) {
 		/* order with io_do_iopoll() checking ->iopoll_completed */
 		smp_store_release(&req->iopoll_completed, 1);
 	} else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
Signature · 1 line
2.55.0
#2 lore raw

Re: [PATCH v2 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

Quoted 109 lines · Caleb Sander Mateos

In preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.

---
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 21 deletions(-)
include/linux/io_uring/cmd.h +17 -6
index 331dcbefe72f..67f2ef700c43 100644
@@ -50,19 +50,21 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                  const struct iovec __user *uvec,
                                  size_t uvec_segs,
                                  int ddir, struct iov_iter *iter,
                                  unsigned issue_flags);
 
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-                        unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
                            io_req_tw_func_t task_work_cb,
                            unsigned flags);
 
@@ -105,12 +107,19 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                                int ddir, struct iov_iter *iter,
                                                unsigned issue_flags)
 {
        return -EOPNOTSUPP;
 }
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
-               u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+}
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret,
+                                         u64 res2)
+{
+}
+static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
+                                      unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
                            io_req_tw_func_t task_work_cb, unsigned flags)
 {
@@ -171,17 +180,19 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
 }
 
 static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
                                     unsigned issue_flags)
 {
-       return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
+       io_uring_cmd_set_res(ioucmd, ret);
+       __io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
                                       u64 res2, unsigned issue_flags)
 {
-       return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
+       io_uring_cmd_set_res32(ioucmd, ret, res2);
+       __io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
                            void (*release)(void *), unsigned int index,
                            unsigned int issue_flags);
io_uring/uring_cmd.c +7 -4
index 726a659f38c3..917b32a921e6 100644
@@ -136,40 +136,43 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
        req->io_task_work.func = task_work_cb;
        __io_req_task_work_add(req, flags);
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
 
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
-                                         u64 extra1, u64 extra2)
+void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
 {
-       req->big_cqe.extra1 = extra1;
-       req->big_cqe.extra2 = extra2;
+       struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+       if (ret < 0)
+               req_set_fail(req);
+       io_req_set_res(req, ret, 0);
 }
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res);

io_uring/uring_cmd.c void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, · line 159

Quoted 10 lines
io_uring/uring_cmd.c +10 -0
+
+void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+       struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+       if (ret < 0)
+               req_set_fail(req);
+       io_req_set_res32(req, ret, 0, res2, 0);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res32);

Both the two helpers can be inline, can't they?

Thanks,

#3 lore raw

Re: [PATCH v2 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

Quoted 116 lines · Ming Lei
Quoted 113 lines · Caleb Sander Mateos

In preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.

---
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 21 deletions(-)
include/linux/io_uring/cmd.h +17 -6
index 331dcbefe72f..67f2ef700c43 100644
@@ -50,19 +50,21 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                  const struct iovec __user *uvec,
                                  size_t uvec_segs,
                                  int ddir, struct iov_iter *iter,
                                  unsigned issue_flags);
 
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-                        unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
                            io_req_tw_func_t task_work_cb,
                            unsigned flags);
 
@@ -105,12 +107,19 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                                int ddir, struct iov_iter *iter,
                                                unsigned issue_flags)
 {
        return -EOPNOTSUPP;
 }
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
-               u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+}
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret,
+                                         u64 res2)
+{
+}
+static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
+                                      unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
                            io_req_tw_func_t task_work_cb, unsigned flags)
 {
@@ -171,17 +180,19 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
 }
 
 static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
                                     unsigned issue_flags)
 {
-       return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
+       io_uring_cmd_set_res(ioucmd, ret);
+       __io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
                                       u64 res2, unsigned issue_flags)
 {
-       return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
+       io_uring_cmd_set_res32(ioucmd, ret, res2);
+       __io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
                            void (*release)(void *), unsigned int index,
                            unsigned int issue_flags);
io_uring/uring_cmd.c +11 -4
index 726a659f38c3..917b32a921e6 100644
@@ -136,40 +136,43 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
        req->io_task_work.func = task_work_cb;
        __io_req_task_work_add(req, flags);
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
 
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
-                                         u64 extra1, u64 extra2)
+void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
 {
-       req->big_cqe.extra1 = extra1;
-       req->big_cqe.extra2 = extra2;
+       struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+       if (ret < 0)
+               req_set_fail(req);
+       io_req_set_res(req, ret, 0);
 }
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res);
+
+void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+       struct io_kiocb *req = cmd_to_io_kiocb(cmd);
Quoted 8 lines

io_uring/uring_cmd.c void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, · line 159

Quoted 6 lines
io_uring/uring_cmd.c +6 -0
+
+       if (ret < 0)
+               req_set_fail(req);
+       io_req_set_res32(req, ret, 0, res2, 0);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res32);

Both the two helpers can be inline, can't they?

I also wanted to do that, but req_set_*() are defined in io_uring/io_uring.h, an io_uring-private header, whereas include/linux/io_uring/cmd.h is a public header. Perhaps the inline functions could be moved to include/linux/io_uring_types.h or include/linux/io_uring.h. If not, I think we'll have to rely on LTO to inline these calls. I'd be curious to hear Jens's thoughts.

Best,
Caleb

maintainer
#4 lore raw

Re: [PATCH v2 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

Quoted 126 lines · Caleb Sander Mateos
Quoted 116 lines · On Wed, Sep 2, 2026 at 6:02?PM Ming Lei <[email protected]> wrote:
Quoted 113 lines · On Wed, Sep 2, 2026 at 6:05?PM Caleb Sander Mateos <[email protected]>…

In preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.

---
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 21 deletions(-)
include/linux/io_uring/cmd.h +17 -6
index 331dcbefe72f..67f2ef700c43 100644
@@ -50,19 +50,21 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                  const struct iovec __user *uvec,
                                  size_t uvec_segs,
                                  int ddir, struct iov_iter *iter,
                                  unsigned issue_flags);
 
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-                        unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
                            io_req_tw_func_t task_work_cb,
                            unsigned flags);
 
@@ -105,12 +107,19 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                                int ddir, struct iov_iter *iter,
                                                unsigned issue_flags)
 {
        return -EOPNOTSUPP;
 }
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
-               u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+}
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret,
+                                         u64 res2)
+{
+}
+static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
+                                      unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
                            io_req_tw_func_t task_work_cb, unsigned flags)
 {
@@ -171,17 +180,19 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
 }
 
 static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
                                     unsigned issue_flags)
 {
-       return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
+       io_uring_cmd_set_res(ioucmd, ret);
+       __io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
                                       u64 res2, unsigned issue_flags)
 {
-       return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
+       io_uring_cmd_set_res32(ioucmd, ret, res2);
+       __io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
                            void (*release)(void *), unsigned int index,
                            unsigned int issue_flags);
io_uring/uring_cmd.c +11 -4
index 726a659f38c3..917b32a921e6 100644
@@ -136,40 +136,43 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
        req->io_task_work.func = task_work_cb;
        __io_req_task_work_add(req, flags);
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
 
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
-                                         u64 extra1, u64 extra2)
+void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
 {
-       req->big_cqe.extra1 = extra1;
-       req->big_cqe.extra2 = extra2;
+       struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+       if (ret < 0)
+               req_set_fail(req);
+       io_req_set_res(req, ret, 0);
 }
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res);
+
+void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+       struct io_kiocb *req = cmd_to_io_kiocb(cmd);
Quoted 8 lines

io_uring/uring_cmd.c void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, · line 159

Quoted 6 lines
io_uring/uring_cmd.c +6 -0
+
+       if (ret < 0)
+               req_set_fail(req);
+       io_req_set_res32(req, ret, 0, res2, 0);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res32);

Both the two helpers can be inline, can't they?

Quoted 6 lines

I also wanted to do that, but req_set_*() are defined in io_uring/io_uring.h, an io_uring-private header, whereas include/linux/io_uring/cmd.h is a public header. Perhaps the inline functions could be moved to include/linux/io_uring_types.h or include/linux/io_uring.h. If not, I think we'll have to rely on LTO to inline these calls. I'd be curious to hear Jens's thoughts.

Let's just move them to io_uring.h instead.

Signature · 1 line
Jens Axboe
Reviewed-by on v2 1/2
#5 lore raw

Re: [PATCH v2 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

In reply to Caleb Sander Mateos
Quoted 9 lines · Caleb Sander Mateos

In preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.

Quoted 10 lines
---
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 21 deletions(-)
include/linux/io_uring/cmd.h +0 -0
index 331dcbefe72f..67f2ef700c43 100644

include/linux/io_uring/cmd.h int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, · line 56

Quoted 7 lines
include/linux/io_uring/cmd.h +2 -0
@@ -50,19 +50,21 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                  const struct iovec __user *uvec,
                                  size_t uvec_segs,
                                  int ddir, struct iov_iter *iter,
                                  unsigned issue_flags);
 
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);

nit: it doesn't look like anywhere else in the header omits the param name for the cmd, so maybe worth also having it here (and below) for consistency?

include/linux/io_uring/cmd.h int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, · line 65

Quoted 11 lines
include/linux/io_uring/cmd.h +2 -2
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-                        unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);

Maybe worth adding a comment that any caller using this api needs to set the res themselves before calling this?

Thanks,
Joanne

#6 lore raw

Re: [PATCH v2 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

Quoted 30 lines · Joanne Koong
Quoted 9 lines · Caleb Sander Mateos

In preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.

Quoted 15 lines
---
 include/linux/io_uring/cmd.h | 23 +++++++++++++++++------
 io_uring/uring_cmd.c         | 33 ++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 21 deletions(-)
include/linux/io_uring/cmd.h +0 -0
index 331dcbefe72f..67f2ef700c43 100644
@@ -50,19 +50,21 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
                                  const struct iovec __user *uvec,
                                  size_t uvec_segs,
                                  int ddir, struct iov_iter *iter,
                                  unsigned issue_flags);
 
Quoted 6 lines

include/linux/io_uring/cmd.h int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, · line 56

Quoted 2 lines
include/linux/io_uring/cmd.h +2 -0
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);

nit: it doesn't look like anywhere else in the header omits the param name for the cmd, so maybe worth also having it here (and below) for consistency?

IMO the name "cmd" or "ioucmd" doesn't add anything on top of the argument type. But I can include it if people prefer it.

Quoted 15 lines

include/linux/io_uring/cmd.h int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, · line 65

Quoted 12 lines
include/linux/io_uring/cmd.h +2 -2
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-                        unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);

Maybe worth adding a comment that any caller using this api needs to set the res themselves before calling this?

Sure, that's a good idea.

Thanks,
Caleb

#7 lore raw

PATCH v2 2/2 nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

In reply to Caleb Sander Mateos

io_uring_cmd_set_res32() only performs loads and stores to the io_uring request state, so it's safe to call in interrupt context. Move the call from the nvme_uring_task_cb() task work to nvme_uring_cmd_end_io(). This unifies the 2 places setting the NVMe status and result on the uring_cmd and removes the need to pass them through struct nvme_uring_cmd_pdu, saving 16 bytes and a couple memory accesses.

---
 drivers/nvme/host/ioctl.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
drivers/nvme/host/ioctl.c +9 -10
index 6539d4750098..3aa5d2f2dfbb 100644
@@ -421,12 +421,10 @@ struct nvme_uring_data {
  * Expect build errors if this grows larger than that.
  */
 struct nvme_uring_cmd_pdu {
 	struct request *req;
 	struct bio *bio;
-	u64 result;
-	int status;
 };
 
 static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
 		struct io_uring_cmd *ioucmd)
 {
@@ -438,29 +436,30 @@ static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 	struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 
 	if (pdu->bio)
 		blk_rq_unmap_user(pdu->bio);
-	io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
-			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	__io_uring_cmd_done(ioucmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 						blk_status_t err,
 						const struct io_comp_batch *iob)
 {
 	struct io_uring_cmd *ioucmd = req->end_io_data;
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+	u64 result = le64_to_cpu(nvme_req(req)->result.u64);
+	int status;
 
 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
-		pdu->status = -EINTR;
+		status = -EINTR;
 	} else {
-		pdu->status = nvme_req(req)->status;
-		if (!pdu->status)
-			pdu->status = blk_status_to_errno(err);
+		status = nvme_req(req)->status;
+		if (!status)
+			status = blk_status_to_errno(err);
 	}
-	pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
+	io_uring_cmd_set_res32(ioucmd, status, result);
 
 	/*
 	 * For IOPOLL, check if this completion is happening in the context
 	 * of the same io_ring that owns the request (local context). If so,
 	 * we can complete inline without task_work overhead. Otherwise, we
@@ -469,11 +468,11 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 	 */
 	if (blk_rq_is_poll(req) && iob &&
 	    iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
 		if (pdu->bio)
 			blk_rq_unmap_user(pdu->bio);
-		io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0);
+		__io_uring_cmd_done(ioucmd, 0);
 	} else {
 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
 	}
 	return RQ_END_IO_FREE;
 }
Signature · 1 line
2.55.0