io_uring passthru: set result on blk-mq request completion
v3 cover io_uring passthru: set result on blk-mq request completion
io_uring NVMe passthru currently receives the NVMe status and result in the blk-mq request completion callback nvme_uring_cmd_end_io() but doesn't post the io_uring CQE until the io_uring task work callback nvme_uring_task_cb(). The status and result must be plumbed through struct nvme_uring_cmd_pdu, taking up 16 bytes of the 32 available.
Store the status and result on the io_uring request in nvme_uring_cmd_end_io() instead of nvme_uring_task_cb() so it doesn't need to be passed through struct nvme_uring_cmd_pdu.
v3:
- Inline io_uring_cmd_set_res*() (Ming, Jens)
- Document that io_uring_cmd_set_res*() must be called before __io_uring_cmd_done() (Joanne)
- Add Reviewed-by tag (Joanne)
v2: https://lore.kernel.org/io-uring/[email protected]/
- Keep io_uring_cmd_done{,32}() wrappers to reduce changes in uring_cmd implementations (Joanne)
v1: https://lore.kernel.org/io-uring/[email protected]/
Caleb Sander Mateos (3):
io_uring: move req_set_*() to public header io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done() nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()
drivers/nvme/host/ioctl.c | 19 +++++++++---------- include/linux/io_uring.h | 32 ++++++++++++++++++++++++++++++++ include/linux/io_uring/cmd.h | 34 ++++++++++++++++++++++++++++------ io_uring/io_uring.h | 31 ------------------------------- io_uring/uring_cmd.c | 19 +------------------ 5 files changed, 70 insertions(+), 65 deletions(-)
Signature · 1 line
2.55.0
PATCH v3 1/3 io_uring: move req_set_*() to public header
In preparation for using req_set_fail(), req_set_res(), and req_set_res32() in include/linux/io_uring/cmd.h, move them from the private header io_uring/io_uring.h to include/linux/io_uring.h.
--- include/linux/io_uring.h | 32 ++++++++++++++++++++++++++++++++ io_uring/io_uring.h | 31 ------------------------------- 2 files changed, 32 insertions(+), 31 deletions(-)
include/linux/io_uring.h +32 -0
@@ -1,13 +1,45 @@/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _LINUX_IO_URING_H#define _LINUX_IO_URING_H +#include <linux/io_uring_types.h> #include <linux/sched.h> #include <linux/xarray.h> #include <uapi/linux/io_uring.h> +static inline void req_set_fail (struct io_kiocb *req) +{ + req->flags |=REQ_F_FAIL ; +if (req->flags &REQ_F_CQE_SKIP ) { + req->flags &= ~REQ_F_CQE_SKIP ; + req->flags |=REQ_F_SKIP_LINK_CQES ; + } +} + +static inline void io_req_set_res (struct io_kiocb *req,s32 res,u32 cflags) +{ + req->cqe.res = res; + req->cqe.flags = cflags; +} + +static inline u32 ctx_cqe32_flags (struct io_ring_ctx *ctx) +{ +if (ctx->flags &IORING_SETUP_CQE_MIXED ) +return IORING_CQE_F_32 ; +return 0 ; +} + +static inline void io_req_set_res32 (struct io_kiocb *req,s32 res,u32 cflags, +__u64 extra1,__u64 extra2) +{ + req->cqe.res = res; + req->cqe.flags = cflags |ctx_cqe32_flags (req->ctx); + req->big_cqe.extra1 = extra1; + req->big_cqe.extra2 = extra2; +} +#if defined(CONFIG_IO_URING )void __io_uring_cancel (bool cancel_all);void __io_uring_free (struct task_struct *tsk);void io_uring_unreg_ringfd (void );const char *io_uring_get_opcode (u8 opcode);
io_uring/io_uring.h +0 -31
@@ -316,41 +316,10 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx,if (trace_io_uring_complete_enabled ())trace_call__io_uring_complete (req->ctx, req, cqe);return true; } -static inline void req_set_fail (struct io_kiocb *req) -{ - req->flags |=REQ_F_FAIL ; -if (req->flags &REQ_F_CQE_SKIP ) { - req->flags &= ~REQ_F_CQE_SKIP ; - req->flags |=REQ_F_SKIP_LINK_CQES ; - } -} - -static inline void io_req_set_res (struct io_kiocb *req,s32 res,u32 cflags) -{ - req->cqe.res = res; - req->cqe.flags = cflags; -} - -static inline u32 ctx_cqe32_flags (struct io_ring_ctx *ctx) -{ -if (ctx->flags &IORING_SETUP_CQE_MIXED ) -return IORING_CQE_F_32 ; -return 0 ; -} - -static inline void io_req_set_res32 (struct io_kiocb *req,s32 res,u32 cflags, -__u64 extra1,__u64 extra2) -{ - req->cqe.res = res; - req->cqe.flags = cflags |ctx_cqe32_flags (req->ctx); - req->big_cqe.extra1 = extra1; - req->big_cqe.extra2 = extra2; -} -static inline void *io_uring_alloc_async_data (struct io_alloc_cache *cache,struct io_kiocb *req) {if (cache) { req->async_data =io_cache_alloc (cache,GFP_KERNEL );
Signature · 1 line
2.55.0
PATCH v3 2/3 io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()
In reply to Caleb Sander MateosIn preparation for setting the io_uring NVMe passthru CQE results from the blk-mq request completion rather than the task work callback, split out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done(). io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then __io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made CQE-size-agnostic, with 3 fewer arguments.
--- include/linux/io_uring/cmd.h | 34 ++++++++++++++++++++++++++++------ io_uring/uring_cmd.c | 19 +------------------ 2 files changed, 29 insertions(+), 24 deletions(-)
include/linux/io_uring/cmd.h +28 -6
@@ -1,10 +1,11 @@/* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _LINUX_IO_URING_CMD_H#define _LINUX_IO_URING_CMD_H#include <uapi/linux/io_uring.h> +#include <linux/io_uring.h> #include <linux/io_uring_types.h> #include <linux/blk-mq.h> /* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */ #define IORING_URING_CMD_CANCELABLE (1U << 30)
@@ -39,10 +40,28 @@ static inline void io_uring_cmd_private_sz_check(size_t cmd_sz)#define io_uring_cmd_to_pdu (cmd, pdu_type) ( \ io_uring_cmd_private_sz_check(sizeof(pdu_type)), \ ((pdu_type *)&(cmd)->pdu) \ ) +static inline void io_uring_cmd_set_res (struct io_uring_cmd *cmd,s32 ret) +{ +struct io_kiocb *req =cmd_to_io_kiocb (cmd); + +if (ret <0 ) +req_set_fail (req); +io_req_set_res (req, ret,0 ); +} + +static inline void io_uring_cmd_set_res32 (struct io_uring_cmd *cmd,s32 ret,u64 res2) +{ +struct io_kiocb *req =cmd_to_io_kiocb (cmd); + +if (ret <0 ) +req_set_fail (req); +io_req_set_res32 (req, ret,0 , res2,0 ); +} +#if defined(CONFIG_IO_URING )int io_uring_cmd_import_fixed (u64 ubuf,unsigned long len,int rw,struct iov_iter *iter,struct io_uring_cmd *ioucmd,unsigned int issue_flags);
@@ -54,15 +73,16 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,/* * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd * and the corresponding io_uring request. * +* io_uring_cmd_set_res()/io_uring_cmd_set_res32() must be called first. +* * Note: the caller should never hard code @issue_flags and is only allowed * to pass the mask provided by the core io_uring code. */ -void __io_uring_cmd_done (struct io_uring_cmd *cmd,s32 ret,u64 res2, -unsigned issue_flags,bool is_cqe32); +void __io_uring_cmd_done (struct io_uring_cmd *,unsigned issue_flags);void __io_uring_cmd_do_in_task (struct io_uring_cmd *ioucmd,io_req_tw_func_t task_work_cb,unsigned flags);
@@ -105,12 +125,12 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, int ddir, struct iov_iter *iter, unsigned issue_flags) { return -EOPNOTSUPP; } -static inline void __io_uring_cmd_done (struct io_uring_cmd *cmd,s32 ret, -u64 ret2,unsigned issue_flags,bool is_cqe32) +static inline void __io_uring_cmd_done (struct io_uring_cmd *cmd, +unsigned issue_flags) { }static inline void __io_uring_cmd_do_in_task (struct io_uring_cmd *ioucmd,io_req_tw_func_t task_work_cb,unsigned flags) {
@@ -171,17 +191,19 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd) }static inline void io_uring_cmd_done (struct io_uring_cmd *ioucmd,s32 ret,unsigned issue_flags) { -return __io_uring_cmd_done (ioucmd, ret,0 , issue_flags, false); +io_uring_cmd_set_res (ioucmd, ret); +__io_uring_cmd_done (ioucmd, issue_flags); }static inline void io_uring_cmd_done32 (struct io_uring_cmd *ioucmd,s32 ret,u64 res2,unsigned issue_flags) { -return __io_uring_cmd_done (ioucmd, ret, res2, issue_flags, true); +io_uring_cmd_set_res32 (ioucmd, ret, res2); +__io_uring_cmd_done (ioucmd, issue_flags); }int io_buffer_register_bvec (struct io_uring_cmd *cmd,struct request *rq,void (*release)(void *),unsigned int index,unsigned int issue_flags);
io_uring/uring_cmd.c +1 -18
@@ -136,40 +136,23 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, req->io_task_work.func = task_work_cb;__io_req_task_work_add (req, flags); }EXPORT_SYMBOL_GPL (__io_uring_cmd_do_in_task); -static inline void io_req_set_cqe32_extra (struct io_kiocb *req, -u64 extra1,u64 extra2) -{ - req->big_cqe.extra1 = extra1; - req->big_cqe.extra2 = extra2; -} -/* * Called by consumers of io_uring_cmd, if they originally returned * -EIOCBQUEUED upon receiving the command. */ -void __io_uring_cmd_done (struct io_uring_cmd *ioucmd,s32 ret,u64 res2, -unsigned issue_flags,bool is_cqe32) +void __io_uring_cmd_done (struct io_uring_cmd *ioucmd,unsigned issue_flags) {struct io_kiocb *req =cmd_to_io_kiocb (ioucmd);if (WARN_ON_ONCE (req->flags &REQ_F_APOLL_MULTISHOT ))return ;io_uring_cmd_del_cancelable (ioucmd, issue_flags); -if (ret <0 ) -req_set_fail (req); - -io_req_set_res (req, ret,0 ); -if (is_cqe32) { -if (req->ctx->flags &IORING_SETUP_CQE_MIXED ) - req->cqe.flags |=IORING_CQE_F_32 ; -io_req_set_cqe32_extra (req, res2,0 ); - }io_req_uring_cleanup (req, issue_flags);if (req->flags &REQ_F_IOPOLL ) {/* order with io_do_iopoll() checking ->iopoll_completed */ smp_store_release (&req->iopoll_completed,1 ); }else if (issue_flags &IO_URING_F_COMPLETE_DEFER ) {
Signature · 1 line
2.55.0
PATCH v3 3/3 nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()
In reply to Caleb Sander Mateosio_uring_cmd_set_res32() only performs loads and stores to the io_uring request state, so it's safe to call in interrupt context. Move the call from the nvme_uring_task_cb() task work to nvme_uring_cmd_end_io(). This unifies the 2 places setting the NVMe status and result on the uring_cmd and removes the need to pass them through struct nvme_uring_cmd_pdu, saving 16 bytes and a couple memory accesses.
--- drivers/nvme/host/ioctl.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
drivers/nvme/host/ioctl.c +9 -10
@@ -421,12 +421,10 @@ struct nvme_uring_data {* Expect build errors if this grows larger than that. */ struct nvme_uring_cmd_pdu {struct request *req;struct bio *bio; -u64 result; -int status; };static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu (struct io_uring_cmd *ioucmd) {
@@ -438,29 +436,30 @@ static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)struct io_uring_cmd *ioucmd =io_uring_cmd_from_tw (tw_req);struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd);if (pdu->bio)blk_rq_unmap_user (pdu->bio); -io_uring_cmd_done32 (ioucmd, pdu->status, pdu->result, -IO_URING_CMD_TASK_WORK_ISSUE_FLAGS ); +__io_uring_cmd_done (ioucmd,IO_URING_CMD_TASK_WORK_ISSUE_FLAGS ); }static enum rq_end_io_ret nvme_uring_cmd_end_io (struct request *req,blk_status_t err,const struct io_comp_batch *iob) {struct io_uring_cmd *ioucmd = req->end_io_data;struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd); +u64 result =le64_to_cpu (nvme_req (req)->result.u64); +int status;if (nvme_req (req)->flags &NVME_REQ_CANCELLED ) { - pdu->status = -EINTR ; + status = -EINTR ; }else { - pdu->status =nvme_req (req)->status; -if (!pdu->status) - pdu->status =blk_status_to_errno (err); + status =nvme_req (req)->status; +if (!status) + status =blk_status_to_errno (err); } - pdu->result =le64_to_cpu (nvme_req (req)->result.u64); +io_uring_cmd_set_res32 (ioucmd, status, result);/* * For IOPOLL, check if this completion is happening in the context * of the same io_ring that owns the request (local context). If so, * we can complete inline without task_work overhead. Otherwise, we
@@ -469,11 +468,11 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,*/ if (blk_rq_is_poll (req) && iob && iob->poll_ctx ==io_uring_cmd_ctx_handle (ioucmd)) {if (pdu->bio)blk_rq_unmap_user (pdu->bio); -io_uring_cmd_done32 (ioucmd, pdu->status, pdu->result,0 ); +__io_uring_cmd_done (ioucmd,0 ); }else {io_uring_cmd_do_in_task_lazy (ioucmd, nvme_uring_task_cb); }return RQ_END_IO_FREE ; }
Signature · 1 line
2.55.0
Quoted 11 lines · Caleb Sander Mateos
io_uringNVMe passthru currently receives the NVMe status and result in the blk-mq request completion callbacknvme_uring_cmd_end_io()but doesn't post theio_uringCQE until theio_uringtask work callbacknvme_uring_task_cb(). The status and result must be plumbed through structnvme_uring_cmd_pdu, taking up 16 bytes of the 32 available.Store the status and result on the
io_uringrequest innvme_uring_cmd_end_io()instead ofnvme_uring_task_cb()so it doesn't need to be passed through structnvme_uring_cmd_pdu.[...]
Applied, thanks!
[1/3] io_uring: move req_set_*() to public header
commit: 7acb6aa7916023e072eadb9e45e34a4b3f5801bb
[2/3] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()
commit: c54cf3476674a6a01c61da85fa32aee9f07c0741
[3/3] nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()
commit: 11773ae6da9c9d92f5d1def78f7e70a9f1fa7b1cBest regards,
Signature · 1 line
Jens Axboe