ublk: simplify NEED_GET_DATA handling and request lookup
Subsystems (2)
v2 cover ublk: simplify NEED_GET_DATA handling and request lookup
Remove accesses to ublk_io's cmd field after the I/O request is posted to the ublk server. This allows the cmd field to be overlapped with a pointer to the struct request, avoiding several blk_mq_tag_to_rq() lookups.
Fix a couple of typos noticed along the way.
Caleb Sander Mateos (8):
ublk: fix "immepdately" typo in comment
ublk: remove misleading "ubq" in "ubq_complete_io_cmd()"
ublk: take const ubq pointer in ublk_get_iod()
ublk: don't log uring_cmd cmd_op in ublk_dispatch_req()
ublk: factor out ublk_start_io() helper
ublk: don't call ublk_dispatch_req() for NEED_GET_DATA
ublk: check UBLK_IO_FLAG_OWNED_BY_SRV in ublk_abort_queue()
ublk: store request pointer in ublk_io
Uday Shankar (1):
ublk: factor out ublk_commit_and_fetch
drivers/block/ublk_drv.c | 252 ++++++++++++++++++++------------------- 1 file changed, 131 insertions(+), 121 deletions(-)
v2:
- Don't complete uring_cmd if ublk_map_io() returns 0 (Ming)
- Take const ubq pointers
Signature · 1 line
2.45.2
PATCH v2 1/9 ublk: factor out ublk_commit_and_fetch
From: Uday Shankar <[email protected]>
Move the logic for the UBLK_IO_COMMIT_AND_FETCH_REQ opcode into its own function. This also allows us to mark ublk_queue pointers as const for that operation, which can help prevent data races since we may allow concurrent operation on one ublk_queue in the future. Also open code ublk_commit_completion in ublk_commit_and_fetch to reduce the number of parameters/avoid a redundant lookup.
[Restore __ublk_ch_uring_cmd() req variable used in commit d6aa0c178bf8 ("ublk: call ublk_dispatch_req() for handling UBLK_U_IO_NEED_GET_DATA")]
--- drivers/block/ublk_drv.c | 90 +++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 47 deletions(-)
drivers/block/ublk_drv.c +43 -47
@@ -1587,34 +1587,10 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) pfn =virt_to_phys (ublk_queue_cmd_buf (ub, q_id)) >>PAGE_SHIFT ;return remap_pfn_range (vma, vma->vm_start, pfn, sz, vma->vm_page_prot); } -static void ublk_commit_completion (struct ublk_device *ub, -const struct ublksrv_io_cmd *ub_cmd) -{ -u32 qid = ub_cmd->q_id, tag = ub_cmd->tag; -struct ublk_queue *ubq =ublk_get_queue (ub, qid); -struct ublk_io *io = &ubq->ios[tag]; -struct request *req; - -/* now this cmd slot is owned by nbd driver */ - io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV ; - io->res = ub_cmd->result; - -/* find the io request and complete */ - req =blk_mq_tag_to_rq (ub->tag_set.tags[qid], tag); -if (WARN_ON_ONCE (unlikely (!req))) -return ; - -if (req_op (req) ==REQ_OP_ZONE_APPEND ) - req->__sector = ub_cmd->zone_append_lba; - -if (likely (!blk_should_fake_timeout (req->q))) -ublk_put_req_ref (ubq, req); -} -static void __ublk_fail_req (struct ublk_queue *ubq,struct ublk_io *io,struct request *req) {WARN_ON_ONCE (io->flags &UBLK_IO_FLAG_ACTIVE );
@@ -2019,10 +1995,51 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_queue *ubq, out:mutex_unlock (&ub->mutex);return ret; } +static int ublk_commit_and_fetch (const struct ublk_queue *ubq, +struct ublk_io *io,struct io_uring_cmd *cmd, +const struct ublksrv_io_cmd *ub_cmd) +{ +struct blk_mq_tags *tags = ubq->dev->tag_set.tags[ub_cmd->q_id]; +struct request *req =blk_mq_tag_to_rq (tags, ub_cmd->tag); + +if (!(io->flags &UBLK_IO_FLAG_OWNED_BY_SRV )) +return -EINVAL ; + +if (ublk_need_map_io (ubq)) { +/* +* COMMIT_AND_FETCH_REQ has to provide IO buffer if +* NEED GET DATA is not enabled or it is Read IO. +*/ +if (!ub_cmd->addr && (!ublk_need_get_data (ubq) || +req_op (req) ==REQ_OP_READ )) +return -EINVAL ; + }else if (req_op (req) !=REQ_OP_ZONE_APPEND && ub_cmd->addr) { +/* +* User copy requires addr to be unset when command is +* not zone append +*/ +return -EINVAL ; + } + +ublk_fill_io_cmd (io, cmd, ub_cmd->addr); + +/* now this cmd slot is owned by ublk driver */ + io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV ; + io->res = ub_cmd->result; + +if (req_op (req) ==REQ_OP_ZONE_APPEND ) + req->__sector = ub_cmd->zone_append_lba; + +if (likely (!blk_should_fake_timeout (req->q))) +ublk_put_req_ref (ubq, req); + +return 0 ; +} +static int __ublk_ch_uring_cmd (struct io_uring_cmd *cmd,unsigned int issue_flags,const struct ublksrv_io_cmd *ub_cmd) {struct ublk_device *ub = cmd->file->private_data;
@@ -2077,34 +2094,13 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, ret =ublk_fetch (cmd, ubq, io, ub_cmd->addr);if (ret) goto out;break ;case UBLK_IO_COMMIT_AND_FETCH_REQ : - req =blk_mq_tag_to_rq (ub->tag_set.tags[ub_cmd->q_id], tag); - -if (!(io->flags &UBLK_IO_FLAG_OWNED_BY_SRV )) - goto out; - -if (ublk_need_map_io (ubq)) { -/* -* COMMIT_AND_FETCH_REQ has to provide IO buffer if -* NEED GET DATA is not enabled or it is Read IO. -*/ -if (!ub_cmd->addr && (!ublk_need_get_data (ubq) || -req_op (req) ==REQ_OP_READ )) - goto out; - }else if (req_op (req) !=REQ_OP_ZONE_APPEND && ub_cmd->addr) { -/* -* User copy requires addr to be unset when command is -* not zone append -*/ - ret = -EINVAL ; + ret =ublk_commit_and_fetch (ubq, io, cmd, ub_cmd); +if (ret) goto out; - } - -ublk_fill_io_cmd (io, cmd, ub_cmd->addr); -ublk_commit_completion (ub, ub_cmd);break ;case UBLK_IO_NEED_GET_DATA :if (!(io->flags &UBLK_IO_FLAG_OWNED_BY_SRV )) goto out;ublk_fill_io_cmd (io, cmd, ub_cmd->addr);
Signature · 1 line
2.45.2
PATCH v2 2/9 ublk: fix "immepdately" typo in comment
In reply to Caleb Sander MateosLooks like "immediately" was intended.
--- drivers/block/ublk_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/block/ublk_drv.c +1 -1
@@ -1180,11 +1180,11 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, }if (ublk_need_get_data (ubq) &&ublk_need_map_req (req)) {/* * We have not handled UBLK_IO_NEED_GET_DATA command yet, -* so immepdately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv +* so immediately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv * and notify it. */ if (!(io->flags &UBLK_IO_FLAG_NEED_GET_DATA )) { io->flags |=UBLK_IO_FLAG_NEED_GET_DATA ; pr_devel("%s: need get data. op %d, qid %d tag %d io_flags %x\n",
Signature · 1 line
2.45.2
PATCH v2 3/9 ublk: remove misleading "ubq" in "ubq_complete_io_cmd()"
In reply to Caleb Sander Mateosubq_complete_io_cmd() doesn't interact with a ublk queue, so "ubq" in the name is confusing. Most likely "ubq" was meant to be "ublk".
--- drivers/block/ublk_drv.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/block/ublk_drv.c +5 -4
@@ -1123,12 +1123,12 @@ static void ublk_complete_rq(struct kref *ref)struct request *req =blk_mq_rq_from_pdu (data);__ublk_complete_rq (req); } -static void ubq_complete_io_cmd (struct ublk_io *io,int res, -unsigned issue_flags) +static void ublk_complete_io_cmd (struct ublk_io *io,int res, +unsigned issue_flags) {/* mark this cmd owned by ublksrv */ io->flags |=UBLK_IO_FLAG_OWNED_BY_SRV ;/*
@@ -1188,11 +1188,12 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,if (!(io->flags &UBLK_IO_FLAG_NEED_GET_DATA )) { io->flags |=UBLK_IO_FLAG_NEED_GET_DATA ;pr_devel ("%s: need get data. op %d, qid %d tag %d io_flags %x\n" , __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags); -ubq_complete_io_cmd (io,UBLK_IO_RES_NEED_GET_DATA , issue_flags); +ublk_complete_io_cmd (io,UBLK_IO_RES_NEED_GET_DATA , + issue_flags);return ; }/* * We have handled UBLK_IO_NEED_GET_DATA command, * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just
@@ -1227,11 +1228,11 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,ublk_get_iod (ubq, req->tag)->nr_sectors = mapped_bytes >>9 ; }ublk_init_req_ref (ubq, req); -ubq_complete_io_cmd (io,UBLK_IO_RES_OK , issue_flags); +ublk_complete_io_cmd (io,UBLK_IO_RES_OK , issue_flags); }static void ublk_cmd_tw_cb (struct io_uring_cmd *cmd,unsigned int issue_flags) {
Signature · 1 line
2.45.2
PATCH v2 4/9 ublk: take const ubq pointer in ublk_get_iod()
In reply to Caleb Sander Mateosublk_get_iod() doesn't modify the struct ublk_queue it is passed. Clarify that by making the argument a const pointer.
Move the function definition earlier in the file so it doesn't need a forward declaration.
--- drivers/block/ublk_drv.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
drivers/block/ublk_drv.c +7 -8
@@ -203,12 +203,17 @@ struct ublk_params_header {static void ublk_stop_dev_unlocked (struct ublk_device *ub);static void ublk_abort_queue (struct ublk_device *ub,struct ublk_queue *ubq);static inline struct request *__ublk_check_and_get_req (struct ublk_device *ub,struct ublk_queue *ubq,int tag,size_t offset);static inline unsigned int ublk_req_build_flags (struct request *req); -static inline struct ublksrv_io_desc *ublk_get_iod (struct ublk_queue *ubq, -int tag); + +static inline struct ublksrv_io_desc * +ublk_get_iod (const struct ublk_queue *ubq,unsigned tag) +{ +return &ubq->io_cmd_buf[tag]; +} +static inline bool ublk_dev_is_user_copy (const struct ublk_device *ub) {return ub->dev_info.flags & (UBLK_F_USER_COPY |UBLK_F_SUPPORT_ZERO_COPY ); }
@@ -692,16 +697,10 @@ static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,static inline bool ublk_rq_has_data (const struct request *rq) {return bio_has_data (rq->bio); } -static inline struct ublksrv_io_desc *ublk_get_iod (struct ublk_queue *ubq, -int tag) -{ -return &ubq->io_cmd_buf[tag]; -} -static inline struct ublksrv_io_desc *ublk_queue_cmd_buf (struct ublk_device *ub,int q_id) {return ublk_get_queue (ub, q_id)->io_cmd_buf; }
Signature · 1 line
2.45.2
Re: [PATCH v2 4/9] ublk: take const ubq pointer in ublk_get_iod()
Quoted 7 lines · Caleb Sander Mateos
ublk_get_iod()doesn't modify the structublk_queueit is passed. Clarify that by making the argument a const pointer.Move the function definition earlier in the file so it doesn't need a forward declaration.
Signed-off-by Caleb Sander Mateos <[email protected]>
thanks,
Ming
PATCH v2 5/9 ublk: don't log uring_cmd cmd_op in ublk_dispatch_req()
In reply to Caleb Sander Mateoscmd_op is either UBLK_U_IO_FETCH_REQ, UBLK_U_IO_COMMIT_AND_FETCH_REQ, or UBLK_U_IO_NEED_GET_DATA. Which one isn't particularly interesting and is already recorded by the log line in __ublk_ch_uring_cmd().
--- drivers/block/ublk_drv.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
drivers/block/ublk_drv.c +6 -7
@@ -1158,12 +1158,12 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, {int tag = req->tag;struct ublk_io *io = &ubq->ios[tag];unsigned int mapped_bytes; -pr_devel ("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n" , - __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags, +pr_devel ("%s: complete: qid %d tag %d io_flags %x addr %llx\n" , + __func__, ubq->q_id, req->tag, io->flags,ublk_get_iod (ubq, req->tag)->addr);/* * Task is exiting if either: *
@@ -1184,13 +1184,12 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,* so immediately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv * and notify it. */ if (!(io->flags &UBLK_IO_FLAG_NEED_GET_DATA )) { io->flags |=UBLK_IO_FLAG_NEED_GET_DATA ; -pr_devel ("%s: need get data. op %d, qid %d tag %d io_flags %x\n" , - __func__, io->cmd->cmd_op, ubq->q_id, - req->tag, io->flags); +pr_devel ("%s: need get data. qid %d tag %d io_flags %x\n" , + __func__, ubq->q_id, req->tag, io->flags);ublk_complete_io_cmd (io,UBLK_IO_RES_NEED_GET_DATA , issue_flags);return ; }/*
@@ -1199,12 +1198,12 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,* do the copy work. */ io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA ;/* update iod->addr because ublksrv may have passed a new io buffer */ ublk_get_iod (ubq, req->tag)->addr = io->addr; -pr_devel ("%s: update iod->addr: op %d, qid %d tag %d io_flags %x addr %llx\n" , - __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags, +pr_devel ("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n" , + __func__, ubq->q_id, req->tag, io->flags,ublk_get_iod (ubq, req->tag)->addr); } mapped_bytes =ublk_map_io (ubq, req, io);
Signature · 1 line
2.45.2
PATCH v2 6/9 ublk: factor out ublk_start_io() helper
In reply to Caleb Sander MateosIn preparation for calling it from outside ublk_dispatch_req(), factor out the code responsible for setting up an incoming ublk I/O request.
--- drivers/block/ublk_drv.c | 54 +++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 23 deletions(-)
drivers/block/ublk_drv.c +31 -23
@@ -1150,17 +1150,45 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq,blk_mq_requeue_request (rq, false); elseblk_mq_end_request (rq ,BLK_STS_IOERR ); } +static bool ublk_start_io (const struct ublk_queue *ubq,struct request *req, +struct ublk_io *io) +{ +unsigned mapped_bytes =ublk_map_io (ubq, req, io); + +/* partially mapped, update io descriptor */ +if (unlikely (mapped_bytes !=blk_rq_bytes (req))) { +/* +* Nothing mapped, retry until we succeed. +* +* We may never succeed in mapping any bytes here because +* of OOM. TODO: reserve one buffer with single page pinned +* for providing forward progress guarantee. +*/ +if (unlikely (!mapped_bytes)) { +blk_mq_requeue_request (req, false); +blk_mq_delay_kick_requeue_list (req->q, +UBLK_REQUEUE_DELAY_MS ); +return false; + } + +ublk_get_iod (ubq, req->tag)->nr_sectors = + mapped_bytes >>9 ; + } + +ublk_init_req_ref (ubq, req); +return true; +} +static void ublk_dispatch_req (struct ublk_queue *ubq,struct request *req,unsigned int issue_flags) {int tag = req->tag;struct ublk_io *io = &ubq->ios[tag]; -unsigned int mapped_bytes;pr_devel ("%s: complete: qid %d tag %d io_flags %x addr %llx\n" , __func__, ubq->q_id, req->tag, io->flags,ublk_get_iod (ubq, req->tag)->addr);
@@ -1203,33 +1231,13 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,pr_devel ("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n" , __func__, ubq->q_id, req->tag, io->flags,ublk_get_iod (ubq, req->tag)->addr); } - mapped_bytes =ublk_map_io (ubq, req, io); - -/* partially mapped, update io descriptor */ -if (unlikely (mapped_bytes !=blk_rq_bytes (req))) { -/* -* Nothing mapped, retry until we succeed. -* -* We may never succeed in mapping any bytes here because -* of OOM. TODO: reserve one buffer with single page pinned -* for providing forward progress guarantee. -*/ -if (unlikely (!mapped_bytes)) { -blk_mq_requeue_request (req, false); -blk_mq_delay_kick_requeue_list (req->q, -UBLK_REQUEUE_DELAY_MS ); -return ; - } - -ublk_get_iod (ubq, req->tag)->nr_sectors = - mapped_bytes >>9 ; - } +if (!ublk_start_io (ubq, req, io)) +return ; -ublk_init_req_ref (ubq, req);ublk_complete_io_cmd (io,UBLK_IO_RES_OK , issue_flags); }static void ublk_cmd_tw_cb (struct io_uring_cmd *cmd,unsigned int issue_flags)
Signature · 1 line
2.45.2
Re: [PATCH v2 6/9] ublk: factor out ublk_start_io() helper
Quoted 4 lines · Caleb Sander Mateos
In preparation for calling it from outside
ublk_dispatch_req(), factor out the code responsible for setting up an incoming ublk I/O request.Signed-off-by Caleb Sander Mateos <[email protected]>
Thanks,
Ming
PATCH v2 7/9 ublk: don't call ublk_dispatch_req() for NEED_GET_DATA
In reply to Caleb Sander Mateosublk_dispatch_req() currently handles 3 different cases: incoming ublk requests that don't need to wait for a data buffer, incoming requests that do need to wait for a buffer, and resuming those requests once the buffer is provided. But the call site that provides a data buffer (UBLK_IO_NEED_GET_DATA) is separate from those for incoming requests.
So simplify the function by splitting the UBLK_IO_NEED_GET_DATA case into its own function ublk_get_data(). This avoids several redundant checks in the UBLK_IO_NEED_GET_DATA case, and streamlines the incoming request cases.
Don't call ublk_fill_io_cmd() for UBLK_IO_NEED_GET_DATA, as it's no longer necessary to set io->cmd or the UBLK_IO_FLAG_ACTIVE flag for ublk_dispatch_req().
Since UBLK_IO_NEED_GET_DATA no longer relies on ublk_dispatch_req() calling io_uring_cmd_done(), return the UBLK_IO_RES_OK status directly from the ->uring_cmd() handler. If ublk_start_io() fails, don't complete the UBLK_IO_NEED_GET_DATA command, matching the existing behavior.
--- drivers/block/ublk_drv.c | 51 +++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 22 deletions(-)
drivers/block/ublk_drv.c +29 -22
@@ -1210,29 +1210,16 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,/* * We have not handled UBLK_IO_NEED_GET_DATA command yet, * so immediately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv * and notify it. */ -if (!(io->flags &UBLK_IO_FLAG_NEED_GET_DATA )) { - io->flags |=UBLK_IO_FLAG_NEED_GET_DATA ; -pr_devel ("%s: need get data. qid %d tag %d io_flags %x\n" , - __func__, ubq->q_id, req->tag, io->flags); -ublk_complete_io_cmd (io,UBLK_IO_RES_NEED_GET_DATA , - issue_flags); -return ; - } -/* -* We have handled UBLK_IO_NEED_GET_DATA command, -* so clear UBLK_IO_FLAG_NEED_GET_DATA now and just -* do the copy work. -*/ - io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA ; -/* update iod->addr because ublksrv may have passed a new io buffer */ -ublk_get_iod (ubq, req->tag)->addr = io->addr; -pr_devel ("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n" , - __func__, ubq->q_id, req->tag, io->flags, -ublk_get_iod (ubq, req->tag)->addr); + io->flags |=UBLK_IO_FLAG_NEED_GET_DATA ; +pr_devel ("%s: need get data. qid %d tag %d io_flags %x\n" , + __func__, ubq->q_id, req->tag, io->flags); +ublk_complete_io_cmd (io,UBLK_IO_RES_NEED_GET_DATA , + issue_flags); +return ; }if (!ublk_start_io (ubq, req, io))return ;
@@ -2043,10 +2030,28 @@ static int ublk_commit_and_fetch(const struct ublk_queue *ubq,ublk_put_req_ref (ubq, req);return 0 ; } +static bool ublk_get_data (const struct ublk_queue *ubq,struct ublk_io *io, +struct request *req) +{ +/* +* We have handled UBLK_IO_NEED_GET_DATA command, +* so clear UBLK_IO_FLAG_NEED_GET_DATA now and just +* do the copy work. +*/ + io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA ; +/* update iod->addr because ublksrv may have passed a new io buffer */ +ublk_get_iod (ubq, req->tag)->addr = io->addr; +pr_devel ("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n" , + __func__, ubq->q_id, req->tag, io->flags, +ublk_get_iod (ubq, req->tag)->addr); + +return ublk_start_io (ubq, req, io); +} +static int __ublk_ch_uring_cmd (struct io_uring_cmd *cmd,unsigned int issue_flags,const struct ublksrv_io_cmd *ub_cmd) {struct ublk_device *ub = cmd->file->private_data;
@@ -2108,14 +2113,16 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, goto out;break ;case UBLK_IO_NEED_GET_DATA :if (!(io->flags &UBLK_IO_FLAG_OWNED_BY_SRV )) goto out; -ublk_fill_io_cmd (io, cmd, ub_cmd->addr); + io->addr = ub_cmd->addr; req =blk_mq_tag_to_rq (ub->tag_set.tags[ub_cmd->q_id], tag); -ublk_dispatch_req (ubq, req, issue_flags); -return -EIOCBQUEUED ; +if (!ublk_get_data (ubq, io, req)) +return -EIOCBQUEUED ; + +return UBLK_IO_RES_OK ;default : goto out; }ublk_prep_cancel (cmd, issue_flags, ubq, tag);return -EIOCBQUEUED ;
Signature · 1 line
2.45.2
PATCH v2 8/9 ublk: check UBLK_IO_FLAG_OWNED_BY_SRV in ublk_abort_queue()
In reply to Caleb Sander Mateosublk_abort_queue() currently checks whether the UBLK_IO_FLAG_ACTIVE flag is cleared to tell whether to abort each ublk_io in the queue. But it's possible for a ublk_io to not be ACTIVE but also not have a request in flight, such as when no fetch request has yet been submitted for a tag or when a fetch request is cancelled. So ublk_abort_queue() must additionally check for an inflight request.
Simplify this code by checking for UBLK_IO_FLAG_OWNED_BY_SRV instead, which indicates precisely whether a request is currently inflight.
--- drivers/block/ublk_drv.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
drivers/block/ublk_drv.c +2 -7
@@ -1609,20 +1609,15 @@ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)int i;for (i =0 ; i < ubq->q_depth; i++) {struct ublk_io *io = &ubq->ios[i]; -if (!(io->flags &UBLK_IO_FLAG_ACTIVE )) { +if (io->flags &UBLK_IO_FLAG_OWNED_BY_SRV ) {struct request *rq; -/* -* Either we fail the request or ublk_rq_task_work_cb -* will do it -*/ rq =blk_mq_tag_to_rq (ub->tag_set.tags[ubq->q_id], i); -if (rq &&blk_mq_request_started (rq)) -__ublk_fail_req (ubq, io, rq); +__ublk_fail_req (ubq, io, rq); } } }/* Must be called when queue is frozen */
Signature · 1 line
2.45.2
PATCH v2 9/9 ublk: store request pointer in ublk_io
In reply to Caleb Sander MateosA ublk_io is converted to a request in several places in the I/O path by using blk_mq_tag_to_rq() to look up the (qid, tag) on the ublk device's tagset. This involves a bunch of dereferences and a tag bounds check.
To make this conversion cheaper, store the request pointer in ublk_io. Overlap this storage with the io_uring_cmd pointer. This is safe because the io_uring_cmd pointer is only valid if UBLK_IO_FLAG_ACTIVE is set on the ublk_io, the request pointer is valid if UBLK_IO_FLAG_OWNED_BY_SRV, and these flags are mutually exclusive.
--- drivers/block/ublk_drv.c | 43 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 19 deletions(-)
drivers/block/ublk_drv.c +24 -19
@@ -140,11 +140,16 @@ struct ublk_io {/* userspace buffer address from io cmd */ __u64 addr;unsigned int flags;int res; -struct io_uring_cmd *cmd; +union { +/* valid if UBLK_IO_FLAG_ACTIVE is set */ +struct io_uring_cmd *cmd; +/* valid if UBLK_IO_FLAG_OWNED_BY_SRV is set */ +struct request *req; + }; };struct ublk_queue {int q_id;int q_depth;
@@ -1122,24 +1127,29 @@ static void ublk_complete_rq(struct kref *ref)struct request *req =blk_mq_rq_from_pdu (data);__ublk_complete_rq (req); } -static void ublk_complete_io_cmd (struct ublk_io *io,int res, -unsigned issue_flags) +static void ublk_complete_io_cmd (struct ublk_io *io,struct request *req, +int res,unsigned issue_flags) { +/* read cmd first because req will overwrite it */ +struct io_uring_cmd *cmd = io->cmd; +/* mark this cmd owned by ublksrv */ io->flags |=UBLK_IO_FLAG_OWNED_BY_SRV ;/* * clear ACTIVE since we are done with this sqe/cmd slot * We can only accept io cmd in case of being not active. */ io->flags &= ~UBLK_IO_FLAG_ACTIVE ; + io->req = req; +/* tell ublksrv one io request is coming */ -io_uring_cmd_done (io->cmd, res,0 , issue_flags); +io_uring_cmd_done (cmd, res,0 , issue_flags); }#define UBLK_REQUEUE_DELAY_MS 3 static inline void __ublk_abort_rq(struct ublk_queue *ubq,
@@ -1213,19 +1223,19 @@ static void ublk_dispatch_req(struct ublk_queue *ubq,* and notify it. */ io->flags |=UBLK_IO_FLAG_NEED_GET_DATA ;pr_devel ("%s: need get data. qid %d tag %d io_flags %x\n" , __func__, ubq->q_id, req->tag, io->flags); -ublk_complete_io_cmd (io,UBLK_IO_RES_NEED_GET_DATA , +ublk_complete_io_cmd (io, req,UBLK_IO_RES_NEED_GET_DATA , issue_flags);return ; }if (!ublk_start_io (ubq, req, io))return ; -ublk_complete_io_cmd (io,UBLK_IO_RES_OK , issue_flags); +ublk_complete_io_cmd (io, req,UBLK_IO_RES_OK , issue_flags); }static void ublk_cmd_tw_cb (struct io_uring_cmd *cmd,unsigned int issue_flags) {
@@ -1609,16 +1619,12 @@ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)int i;for (i =0 ; i < ubq->q_depth; i++) {struct ublk_io *io = &ubq->ios[i]; -if (io->flags &UBLK_IO_FLAG_OWNED_BY_SRV ) { -struct request *rq; - - rq =blk_mq_tag_to_rq (ub->tag_set.tags[ubq->q_id], i); -__ublk_fail_req (ubq, io, rq); - } +if (io->flags &UBLK_IO_FLAG_OWNED_BY_SRV ) +__ublk_fail_req (ubq, io, io->req); } }/* Must be called when queue is frozen */ static void ublk_mark_queue_canceling (struct ublk_queue *ubq)
@@ -1988,16 +1994,16 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_queue *ubq,static int ublk_commit_and_fetch (const struct ublk_queue *ubq,struct ublk_io *io,struct io_uring_cmd *cmd,const struct ublksrv_io_cmd *ub_cmd) { -struct blk_mq_tags *tags = ubq->dev->tag_set.tags[ub_cmd->q_id]; -struct request *req =blk_mq_tag_to_rq (tags, ub_cmd->tag); +struct request *req;if (!(io->flags &UBLK_IO_FLAG_OWNED_BY_SRV ))return -EINVAL ; + req = io->req;if (ublk_need_map_io (ubq)) {/* * COMMIT_AND_FETCH_REQ has to provide IO buffer if * NEED GET DATA is not enabled or it is Read IO. */
@@ -2025,13 +2031,14 @@ static int ublk_commit_and_fetch(const struct ublk_queue *ubq,ublk_put_req_ref (ubq, req);return 0 ; } -static bool ublk_get_data (const struct ublk_queue *ubq,struct ublk_io *io, -struct request *req) +static bool ublk_get_data (const struct ublk_queue *ubq,struct ublk_io *io) { +struct request *req = io->req; +/* * We have handled UBLK_IO_NEED_GET_DATA command, * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just * do the copy work. */
@@ -2053,11 +2060,10 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,struct ublk_queue *ubq;struct ublk_io *io;u32 cmd_op = cmd->cmd_op;unsigned tag = ub_cmd->tag;int ret = -EINVAL ; -struct request *req;pr_devel ("%s: received: cmd op %d queue %d tag %d result %d\n" , __func__, cmd->cmd_op, ub_cmd->q_id, tag, ub_cmd->result);
@@ -2109,12 +2115,11 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,break ;case UBLK_IO_NEED_GET_DATA :if (!(io->flags &UBLK_IO_FLAG_OWNED_BY_SRV )) goto out; io->addr = ub_cmd->addr; - req =blk_mq_tag_to_rq (ub->tag_set.tags[ub_cmd->q_id], tag); -if (!ublk_get_data (ubq, io, req)) +if (!ublk_get_data (ubq, io))return -EIOCBQUEUED ;return UBLK_IO_RES_OK ;default : goto out;
Signature · 1 line
2.45.2
Quoted 16 lines · Caleb Sander Mateos
Remove accesses to ublk_io's cmd field after the I/O request is posted to the ublk server. This allows the cmd field to be overlapped with a pointer to the struct request, avoiding several
blk_mq_tag_to_rq()lookups.Fix a couple of typos noticed along the way.
Caleb Sander Mateos (8):
ublk: fix "immepdately" typo in comment
ublk: remove misleading "ubq" in "ubq_complete_io_cmd()"
ublk: take const ubq pointer inublk_get_iod()
ublk: don't loguring_cmdcmd_opinublk_dispatch_req()
ublk: factor outublk_start_io()helper
ublk: don't callublk_dispatch_req()forNEED_GET_DATA
ublk: checkUBLK_IO_FLAG_OWNED_BY_SRVinublk_abort_queue()
ublk: store request pointer inublk_io
Quoted 1 line
[...]
Applied, thanks!
[1/9] ublk: factor out ublk_commit_and_fetch
commit: 0cb39afd2a4d438b16d743952eb68c683c663a1e
[2/9] ublk: fix "immepdately" typo in comment
commit: 80c0789a7ddc89adb726e407ca22d9689608b710
[3/9] ublk: remove misleading "ubq" in "ubq_complete_io_cmd()"
commit: 5a43d93588dca88af0333ce18dedcfb0276ec4fe
[4/9] ublk: take const ubq pointer in ublk_get_iod()
commit: 2a86eec6396ff6d40e46d5cc11f99f1e7bbec78e
[5/9] ublk: don't log uring_cmd cmd_op in ublk_dispatch_req()
commit: 551270690dd6c751a6d62f272cf068bfef9a1729
[6/9] ublk: factor out ublk_start_io() helper
commit: 2fcb88bdf2e62a6aae211c8603341fd2e7b16a69
[7/9] ublk: don't call ublk_dispatch_req() for NEED_GET_DATA
commit: 9810362a57cb633d808cdfd51e8cae21243f0660
[8/9] ublk: check UBLK_IO_FLAG_OWNED_BY_SRV in ublk_abort_queue()
commit: 8ed95b54701a8e946b86ffdeeac8603316e0a59d
[9/9] ublk: store request pointer in ublk_io
commit: 64d1dc522b349a257256f636feb335441cca2571Best regards,
Signature · 1 line
Jens Axboe