io_uring: fix nvme's 32b cqes on mixed cq
PATCH 1/1 io_uring: fix nvme's 32b cqes on mixed cq
From: Keith Busch <[email protected]>
The nvme uring_cmd only uses 32b CQEs. If the ring uses a mixed CQ, then we need to make sure we flag the completion as a 32b CQE.
On the other hand, if nvme uring_cmd was using a dedicated 32b CQE, the posting was missing the extra memcpy because it only applied to bit CQEs on a mixed CQ.
---
Testing this is actually a bit difficult. The nvme uring command only accepts IO commands, and none of the usual nvme IO commands return anything in the big cqe extra fields. So had to pre-fill the CQ with bogus data and see if it was being overwritten as expected.
drivers/nvme/host/ioctl.c | 2 +- include/linux/io_uring/cmd.h | 20 ++++++++++++++++---- io_uring/io_uring.h | 2 +- io_uring/uring_cmd.c | 11 +++++++---- 4 files changed, 25 insertions(+), 10 deletions(-)
drivers/nvme/host/ioctl.c +1 -1
@@ -405,7 +405,7 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,if (pdu->bio)blk_rq_unmap_user (pdu->bio); -io_uring_cmd_done (ioucmd, pdu->status, pdu->result, issue_flags); +io_uring_cmd_done32 (ioucmd, pdu->status, pdu->result, issue_flags); } static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
include/linux/io_uring/cmd.h +16 -4
@@ -56,8 +56,8 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,* Note: the caller should never hard code @issue_flags and is only allowed * to pass the mask provided by the core io_uring code. */ -void io_uring_cmd_done (struct io_uring_cmd *cmd,s32 ret,u64 res2, -unsigned issue_flags); +void __io_uring_cmd_done (struct io_uring_cmd *cmd,s32 ret,u64 res2, +unsigned issue_flags,bool is_cqe32); void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, io_uring_cmd_tw_t task_work_cb,
@@ -104,8 +104,8 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, {return -EOPNOTSUPP ; } -static inline void io_uring_cmd_done (struct io_uring_cmd *cmd,s32 ret, -u64 ret2,unsigned issue_flags) +static inline void __io_uring_cmd_done (struct io_uring_cmd *cmd,s32 ret, +u64 ret2,unsigned issue_flags,bool is_cqe32) { } static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
@@ -159,6 +159,18 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)return cmd_to_io_kiocb (cmd)->ctx; } +static inline void io_uring_cmd_done (struct io_uring_cmd *ioucmd,s32 ret, +u64 res2,unsigned issue_flags) +{ +return __io_uring_cmd_done (ioucmd, ret, res2, issue_flags, false); +} + +static inline void io_uring_cmd_done32 (struct io_uring_cmd *ioucmd,s32 ret, +u64 res2,unsigned issue_flags) +{ +return __io_uring_cmd_done (ioucmd, ret, res2, issue_flags, true); +} +int io_buffer_register_bvec (struct io_uring_cmd *cmd,struct request *rq,void (*release)(void *),unsigned int index,unsigned int issue_flags);
io_uring/io_uring.h +1 -1
@@ -276,7 +276,7 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx,return false;memcpy (cqe, &req->cqe,sizeof (*cqe)); -if (is_cqe32) { +if (ctx->flags &IORING_SETUP_CQE32 || is_cqe32) {memcpy (cqe->big_cqe, &req->big_cqe,sizeof (*cqe));memset (&req->big_cqe,0 ,sizeof (req->big_cqe)); }
io_uring/uring_cmd.c +7 -4
@@ -151,8 +151,8 @@ static inline void io_req_set_cqe32_extra(struct io_kiocb *req,* Called by consumers of io_uring_cmd, if they originally returned * -EIOCBQUEUED upon receiving the command. */ -void io_uring_cmd_done (struct io_uring_cmd *ioucmd,s32 ret,u64 res2, -unsigned issue_flags) +void __io_uring_cmd_done (struct io_uring_cmd *ioucmd,s32 ret,u64 res2, +unsigned issue_flags,bool is_cqe32) {struct io_kiocb *req =cmd_to_io_kiocb (ioucmd);
@@ -165,8 +165,11 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,req_set_fail (req);io_req_set_res (req, ret,0 ); -if (req->ctx->flags &IORING_SETUP_CQE32 ) +if (is_cqe32) { +if (req->ctx->flags &IORING_SETUP_CQE_MIXED ) + req->cqe.flags |=IORING_CQE_F_32 ;io_req_set_cqe32_extra (req, res2,0 ); + }io_req_uring_cleanup (req, issue_flags);if (req->ctx->flags &IORING_SETUP_IOPOLL ) {/* order with io_iopoll_req_issued() checking ->iopoll_complete */
@@ -180,7 +183,7 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,io_req_task_work_add (req); } } -EXPORT_SYMBOL_GPL (io_uring_cmd_done); +EXPORT_SYMBOL_GPL (__io_uring_cmd_done);int io_uring_cmd_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) {
Signature · 1 line
2.47.3
Quoted 8 lines · Keith Busch
The nvme
uring_cmdonly uses 32b CQEs. If the ring uses a mixed CQ, then we need to make sure we flag the completion as a 32b CQE.On the other hand, if nvme
uring_cmdwas using a dedicated 32b CQE, the posting was missing the extra memcpy because it only applied to bit CQEs on a mixed CQ.[...]
Applied, thanks!
[1/1] io_uring: fix nvme's 32b cqes on mixed cq
commit: 79525b51acc1c8e331ab47eb131a99f5370a76c2
Best regards,
Signature · 1 line
Jens Axboe