nvme completion handling refactor and fix v3
Subsystems (5)
Christoph Hellwig
maintainerHi all,
the first two patches refactor the common nvme completion code to be a little less obsfucated, and the third one is the fix from Chao to redirect to a different path ported on top of these changes, which now make the fix very simple.
Changes since v2:
- fix a comment typo
- drop incorrect comment masking in nvme_decide_disposition
- add a comment to nvme_is_path_error
Changes since v1:
- add a new patch to handle the path related errors using the SCT class
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerPATCH 1/4 nvme: rename and document nvme_end_request
nvme_end_request is a bit misnamed, as it wraps around the blk_mq_complete_* API. It's semantics also are non-trivial, so give it a more descriptive name and add a comment explaining the semantics.
--- Documentation/fault-injection/nvme-fault-injection.rst | 2 +- drivers/nvme/host/fc.c | 2 +- drivers/nvme/host/nvme.h | 8 +++++++- drivers/nvme/host/pci.c | 2 +- drivers/nvme/host/rdma.c | 2 +- drivers/nvme/host/tcp.c | 4 ++-- drivers/nvme/target/loop.c | 2 +- 7 files changed, 14 insertions(+), 8 deletions(-)
Documentation/fault-injection/nvme-fault-injection.rst +1 -1
@@ -3,7 +3,7 @@ NVMe Fault Injection Linux's fault injection framework provides a systematic way to support error injection via debugfs in the /sys/kernel/debug directory. When enabled, the default NVME_SC_INVALID_OPCODE with no retry will be -injected into the nvme_end_request. Users can change the default status +injected into the nvme_try_complete_req. Users can change the default status code and no retry flag via the debugfs. The list of Generic Command Status can be found in include/linux/nvme.h
drivers/nvme/host/fc.c +1 -1
@@ -2035,7 +2035,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) }__nvme_fc_fcpop_chk_teardowns (ctrl, op, opstate); -if (!nvme_end_request (rq, status, result)) +if (!nvme_try_complete_req (rq, status, result))nvme_fc_complete_rq (rq); check_error:
drivers/nvme/host/nvme.h +7 -1
@@ -523,7 +523,13 @@ static inline u32 nvme_bytes_to_numd(size_t len)return (len >>2 ) -1 ; } -static inline bool nvme_end_request (struct request *req,__le16 status, +/* +* Fill in the status and result information from the CQE, and then figure out +* if blk-mq will need to use IPI magic to complete the request, and if yes do +* so. If not let the caller complete the request without an indirect function +* call. +*/ +static inline bool nvme_try_complete_req (struct request *req,__le16 status,union nvme_result result) {struct nvme_request *rq =nvme_req (req);
drivers/nvme/host/pci.c +1 -1
@@ -961,7 +961,7 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) req =blk_mq_tag_to_rq (nvme_queue_tagset (nvmeq), cqe->command_id);trace_nvme_sq (req, cqe->sq_head, nvmeq->sq_tail); -if (!nvme_end_request (req, cqe->status, cqe->result)) +if (!nvme_try_complete_req (req, cqe->status, cqe->result))nvme_pci_complete_rq (req); }
drivers/nvme/host/rdma.c +1 -1
@@ -1189,7 +1189,7 @@ static void nvme_rdma_end_request(struct nvme_rdma_request *req)if (!refcount_dec_and_test (&req->ref))return ; -if (!nvme_end_request (rq, req->status, req->result)) +if (!nvme_try_complete_req (rq, req->status, req->result))nvme_rdma_complete_rq (rq); }
drivers/nvme/host/tcp.c +2 -2
@@ -481,7 +481,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,return -EINVAL ; } -if (!nvme_end_request (rq, cqe->status, cqe->result)) +if (!nvme_try_complete_req (rq, cqe->status, cqe->result))nvme_complete_rq (rq); queue->nr_cqe++;
@@ -672,7 +672,7 @@ static inline void nvme_tcp_end_request(struct request *rq, u16 status) {union nvme_result res = {}; -if (!nvme_end_request (rq,cpu_to_le16 (status <<1 ), res)) +if (!nvme_try_complete_req (rq,cpu_to_le16 (status <<1 ), res))nvme_complete_rq (rq); }
drivers/nvme/target/loop.c +1 -1
@@ -115,7 +115,7 @@ static void nvme_loop_queue_response(struct nvmet_req *req)return ; } -if (!nvme_end_request (rq, cqe->status, cqe->result)) +if (!nvme_try_complete_req (rq, cqe->status, cqe->result))nvme_loop_complete_rq (rq); } }
Signature · 7 lines
2.28.0 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerPATCH 2/4 nvme: refactor command completion
In reply to Christoph HellwigLift all the code to decide the dispostition of a completed command from nvme_complete_rq and nvme_failover_req into a new helper, which returns an emum of the potential actions. nvme_complete_rq then just switches on those and calls the proper helper for the action.
--- drivers/nvme/host/core.c | 75 ++++++++++++++++++++++------------- drivers/nvme/host/multipath.c | 47 ++++++---------------- drivers/nvme/host/nvme.h | 31 +++++++++++++-- 3 files changed, 89 insertions(+), 64 deletions(-)
drivers/nvme/host/core.c +48 -27
@@ -241,17 +241,6 @@ static blk_status_t nvme_error_status(u16 status) } } -static inline bool nvme_req_needs_retry (struct request *req) -{ -if (blk_noretry_request (req)) -return false; -if (nvme_req (req)->status &NVME_SC_DNR ) -return false; -if (nvme_req (req)->retries >= nvme_max_retries) -return false; -return true; -} -static void nvme_retry_req (struct request *req) {struct nvme_ns *ns = req->q->queuedata;
@@ -268,34 +257,66 @@ static void nvme_retry_req(struct request *req)blk_mq_delay_kick_requeue_list (req->q, delay); } -void nvme_complete_rq (struct request *req) +enum nvme_disposition { +COMPLETE , +RETRY , +FAILOVER , +}; + +static inline enum nvme_disposition nvme_decide_disposition (struct request *req) { -blk_status_t status =nvme_error_status (nvme_req (req)->status); +if (likely (nvme_req (req)->status ==0 )) +return COMPLETE ; -trace_nvme_complete_rq (req); +if (blk_noretry_request (req) || + (nvme_req (req)->status &NVME_SC_DNR ) || +nvme_req (req)->retries >= nvme_max_retries) +return COMPLETE ; -nvme_cleanup_cmd (req); +if (req->cmd_flags &REQ_NVME_MPATH ) { +if (nvme_is_path_error (nvme_req (req)->status)) +return FAILOVER ; + } -if (nvme_req (req)->ctrl->kas) -nvme_req (req)->ctrl->comp_seen = true; +if (blk_queue_dying (req->q)) +return COMPLETE ; -if (unlikely (status !=BLK_STS_OK &&nvme_req_needs_retry (req))) { -if ((req->cmd_flags &REQ_NVME_MPATH ) &&nvme_failover_req (req)) -return ; +return RETRY ; +} -if (!blk_queue_dying (req->q)) { -nvme_retry_req (req); -return ; - } - }else if (IS_ENABLED (CONFIG_BLK_DEV_ZONED ) && -req_op (req) ==REQ_OP_ZONE_APPEND ) { +static inline void nvme_end_req (struct request *req) +{ +blk_status_t status =nvme_error_status (nvme_req (req)->status); + +if (IS_ENABLED (CONFIG_BLK_DEV_ZONED ) && +req_op (req) ==REQ_OP_ZONE_APPEND ) req->__sector =nvme_lba_to_sect (req->q->queuedata,le64_to_cpu (nvme_req (req)->result.u64)); - }nvme_trace_bio_complete (req, status);blk_mq_end_request (req, status); } + +void nvme_complete_rq (struct request *req) +{ +trace_nvme_complete_rq (req); +nvme_cleanup_cmd (req); + +if (nvme_req (req)->ctrl->kas) +nvme_req (req)->ctrl->comp_seen = true; + +switch (nvme_decide_disposition (req)) { +case COMPLETE : +nvme_end_req (req); +return ; +case RETRY : +nvme_retry_req (req); +return ; +case FAILOVER : +nvme_failover_req (req); +return ; + } +}EXPORT_SYMBOL_GPL (nvme_complete_rq);bool nvme_cancel_request (struct request *req,void *data,bool reserved)
drivers/nvme/host/multipath.c +13 -34
64 lines not on this page · raw message
drivers/nvme/host/nvme.h +28 -3
@@ -523,6 +523,32 @@ static inline u32 nvme_bytes_to_numd(size_t len)return (len >>2 ) -1 ; } +static inline bool nvme_is_ana_error (u16 status) +{ +switch (status &0x7ff ) { +case NVME_SC_ANA_TRANSITION : +case NVME_SC_ANA_INACCESSIBLE : +case NVME_SC_ANA_PERSISTENT_LOSS : +return true; +default : +return false; + } +} + +static inline bool nvme_is_path_error (u16 status) +{ +switch (status &0x7ff ) { +case NVME_SC_HOST_PATH_ERROR : +case NVME_SC_HOST_ABORTED_CMD : +case NVME_SC_ANA_TRANSITION : +case NVME_SC_ANA_INACCESSIBLE : +case NVME_SC_ANA_PERSISTENT_LOSS : +return true; +default : +return false; + } +} +/* * Fill in the status and result information from the CQE, and then figure out * if blk-mq will need to use IPI magic to complete the request, and if yes do
@@ -635,7 +661,7 @@ void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys);void nvme_mpath_start_freeze (struct nvme_subsystem *subsys);void nvme_set_disk_name (char *disk_name,struct nvme_ns *ns,struct nvme_ctrl *ctrl,int *flags); -bool nvme_failover_req (struct request *req); +void nvme_failover_req (struct request *req);void nvme_kick_requeue_lists (struct nvme_ctrl *ctrl);int nvme_mpath_alloc_disk (struct nvme_ctrl *ctrl,struct nvme_ns_head *head);void nvme_mpath_add_disk (struct nvme_ns *ns,struct nvme_id_ns *id);
@@ -694,9 +720,8 @@ static inline void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,sprintf (disk_name,"nvme%dn%d" , ctrl->instance, ns->head->instance); } -static inline bool nvme_failover_req (struct request *req) +static inline void nvme_failover_req (struct request *req) { -return false; }static inline void nvme_kick_requeue_lists (struct nvme_ctrl *ctrl) {
Signature · 7 lines
2.28.0 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerPATCH 3/4 nvme: just check the status code type in nvme_is_path_error
In reply to Christoph HellwigCheck the SCT sub-field for a path related status instead of enumerating invididual status code. As of NVMe 1.4 this adds "Internal Path Error" and "Controller Pathing Error" to the list, but it also future proofs for additional status codes added to the category.
--- drivers/nvme/host/nvme.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-)
drivers/nvme/host/nvme.h +2 -10
@@ -537,16 +537,8 @@ static inline bool nvme_is_ana_error(u16 status)static inline bool nvme_is_path_error (u16 status) { -switch (status &0x7ff ) { -case NVME_SC_HOST_PATH_ERROR : -case NVME_SC_HOST_ABORTED_CMD : -case NVME_SC_ANA_TRANSITION : -case NVME_SC_ANA_INACCESSIBLE : -case NVME_SC_ANA_PERSISTENT_LOSS : -return true; -default : -return false; - } +/* check for a status code type of 'path related status' */ +return (status &0x700 ) ==0x300 ; }/*
Signature · 7 lines
2.28.0 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 3/4] nvme: just check the status code type in nvme_is_path_error
Quoted 8 lines · Christoph Hellwig
Check the SCT sub-field for a path related status instead of enumerating invididual status code. As of NVMe 1.4 this adds "Internal Path Error" and "Controller Pathing Error" to the list, but it also future proofs for additional status codes added to the category.
Suggested-by Chao Leng<[email protected]>Signed-off-by Christoph Hellwig<[email protected]>Reviewed-by Mike Snitzer<[email protected]>
Looks good.
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerPATCH 4/4 nvme: redirect commands on dying queue
In reply to Christoph HellwigFrom: Chao Leng <[email protected]>
If a command send through nvme-multipath failed on a dying queue, resend it on another path.
--- drivers/nvme/host/core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +5 -4
@@ -274,13 +274,14 @@ static inline enum nvme_disposition nvme_decide_disposition(struct request *req)return COMPLETE ;if (req->cmd_flags &REQ_NVME_MPATH ) { -if (nvme_is_path_error (nvme_req (req)->status)) +if (nvme_is_path_error (nvme_req (req)->status) || +blk_queue_dying (req->q))return FAILOVER ; + }else { +if (blk_queue_dying (req->q)) +return COMPLETE ; } -if (blk_queue_dying (req->q)) -return COMPLETE ; -return RETRY ; }
Signature · 7 lines
2.28.0 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 4/4] nvme: redirect commands on dying queue
Quoted 26 lines · Christoph Hellwig
From: Chao Leng <[email protected]>
If a command send through nvme-multipath failed on a dying queue, resend it on another path.
Signed-off-by Chao Leng <[email protected]>[hch: rebased on top of the completion refactoring]Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Reviewed-by Mike Snitzer <[email protected]>--- drivers/nvme/host/core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)drivers/nvme/host/core.c +2
-1@@ -274,13 +274,14 @@ static inline enum nvme_disposition nvme_decide_disposition(struct request *req)return COMPLETE ;if (req->cmd_flags &REQ_NVME_MPATH ) { -if (nvme_is_path_error (nvme_req (req)->status)) +if (nvme_is_path_error (nvme_req (req)->status) || +blk_queue_dying (req->q))return FAILOVER ;
drivers/nvme/host/core.c static inline enum nvme_disposition nvme_decide_disposition(struct request *req) · line 286
Quoted 10 lines
drivers/nvme/host/core.c +3
-3+ }else { +if (blk_queue_dying (req->q)) +return COMPLETE ; } -if (blk_queue_dying (req->q)) -return COMPLETE ; -return RETRY ; }
Suggestion:
Do not use blk_noretry_request. The local retry mechanism, which is defined by nvme protocol, is conflicted with REQ_FAILFAST_TRANSPORT. blk_noretry_request is not a good choice for nvme, even for SCSI, this is not a good choice too, so SCSI does not use this MACRO. We can seperate REQ_FAILFAST_TRANSPORT with other FAILFAST flag. For REQ_FAILFAST_TRANSPORT, do local retry for non path error. For other FAILFAST flag, complete request with error code immediately.
--- drivers/nvme/host/core.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
drivers/nvme/host/core.c +11 -1
@@ -257,6 +257,16 @@ static void nvme_retry_req(struct request *req)blk_mq_delay_kick_requeue_list (req->q, delay); } +static inline bool nvme_noretry_request (struct request *req) +{ +if (req->cmd_flags & (REQ_FAILFAST_DRIVER |REQ_FAILFAST_DEV )) +return true; +if ((req->cmd_flags &REQ_FAILFAST_TRANSPORT ) && +nvme_is_path_error (nvme_req (req)->status)) +return true; +return false; +} +enum nvme_disposition {COMPLETE ,RETRY ,
@@ -268,7 +278,7 @@ static inline enum nvme_disposition nvme_decide_disposition(struct request *req)if (likely (nvme_req (req)->status ==0 ))return COMPLETE ; -if (blk_noretry_request (req) || +if (nvme_noretry_request (req) || (nvme_req (req)->status &NVME_SC_DNR ) ||nvme_req (req)->retries >= nvme_max_retries)return COMPLETE ;
Signature · 4 lines
_______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerRe: [PATCH 4/4] nvme: redirect commands on dying queue
Quoted 7 lines · Chao Leng
Do not use
blk_noretry_request. The local retry mechanism, which is defined by nvme protocol, is conflicted withREQ_FAILFAST_TRANSPORT.blk_noretry_requestis not a good choice for nvme, even for SCSI, this is not a good choice too, so SCSI does not use this MACRO. We can seperateREQ_FAILFAST_TRANSPORTwith other FAILFAST flag. ForREQ_FAILFAST_TRANSPORT, do local retry for non path error. For other FAILFAST flag, complete request with error code immediately.
As said before - please prepare a series removing blk_noretry_request, and fixing up any lose ends, with a good explanation of what in-tree supported use case this fixes for you. Out of tree modules are completely unsupported, as is ANA with dm-multipath.
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerLooks good, queued to nvme-5.9-rc
Quoted 7 lines · Christoph Hellwig
Hi all,
the first two patches refactor the common nvme completion code to be a little less obsfucated, and the third one is the fix from Chao to redirect to a different path ported on top of these changes, which now make the fix very simple.
Quoted 9 lines
Changes since v2:
- fix a comment typo
- drop incorrect comment masking innvme_decide_disposition
- add a comment tonvme_is_path_errorChanges since v1:
- add a new patch to handle the path related errors using the SCT class
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme