nvme: enable retries for authentication commands
Subsystems (2)
v3 cover nvme: enable retries for authentication commands
From: Hannes Reinecke <[email protected]>
Hi all,
authentitication commands are normal NVMe commands, and should be retried if returned with a status where the DNR bit is not set. But instead of having the caller retry the commaand this patchset enables the function __nvme_submit_sync_cmd() to not set the FASTFAIL flag in the request, thereby making the functionality generally available.
As usual, comments and reviews are welcome.
Hannes Reinecke (3):
nvme-auth: open-code single-use macros
nvme: simplify __nvme_submit_sync_cmd() calling conventions
nvme: enable retries for authentication commands
drivers/nvme/host/auth.c | 17 ++++++++--------- drivers/nvme/host/core.c | 21 ++++++++++++++------- drivers/nvme/host/fabrics.c | 18 +++++++++++------- drivers/nvme/host/nvme.h | 19 +++++++++++++++++-- 4 files changed, 50 insertions(+), 25 deletions(-)
Signature · 1 line
2.35.3
PATCH v3 2/3 nvme: simplify __nvme_submit_sync_cmd() calling conventions
From: Hannes Reinecke <[email protected]>
Instead of having two arguments for 'flags' and 'at_head' for __nvme_submit_sync_cmd() combine them into a single 'flags' argument and use function-specific values to indicate what should be set within the function.
--- drivers/nvme/host/auth.c | 7 +++---- drivers/nvme/host/core.c | 19 ++++++++++++------- drivers/nvme/host/fabrics.c | 18 +++++++++++------- drivers/nvme/host/nvme.h | 17 +++++++++++++++-- 4 files changed, 41 insertions(+), 20 deletions(-)
drivers/nvme/host/auth.c +3 -4
@@ -58,12 +58,12 @@ static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, void *data, size_t data_len, bool auth_send) { struct nvme_command cmd = {}; -blk_mq_req_flags_t flags =0 ; +nvme_submit_flags_t flags =0 ;struct request_queue *q = ctrl->fabrics_q;int ret;if (qid !=0 ) { - flags |=BLK_MQ_REQ_NOWAIT |BLK_MQ_REQ_RESERVED ; + flags |=NVME_SUBMIT_NOWAIT |NVME_SUBMIT_RESERVED ; q = ctrl->connect_q; }
@@ -80,8 +80,7 @@ static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, } ret =__nvme_submit_sync_cmd (q, &cmd,NULL , data, data_len, - qid ==0 ?NVME_QID_ANY : qid, -0 , flags); + qid ==0 ?NVME_QID_ANY : qid, flags);if (ret >0 )dev_warn (ctrl->device,"qid %d auth_send failed with status %d\n" , qid, ret);
drivers/nvme/host/core.c +12 -7
@@ -1049,15 +1049,20 @@ EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);*/ int __nvme_submit_sync_cmd (struct request_queue *q,struct nvme_command *cmd,union nvme_result *result,void *buffer,unsigned bufflen, -int qid,int at_head,blk_mq_req_flags_t flags) +int qid,nvme_submit_flags_t flags) {struct request *req;int ret; +blk_mq_req_flags_t blk_flags =0 ; +if (flags &NVME_SUBMIT_NOWAIT ) + blk_flags |=BLK_MQ_REQ_NOWAIT ; +if (flags &NVME_SUBMIT_RESERVED ) + blk_flags |=BLK_MQ_REQ_RESERVED ;if (qid ==NVME_QID_ANY ) - req =blk_mq_alloc_request (q,nvme_req_op (cmd), flags); + req =blk_mq_alloc_request (q,nvme_req_op (cmd), blk_flags);else - req =blk_mq_alloc_request_hctx (q,nvme_req_op (cmd), flags, + req =blk_mq_alloc_request_hctx (q,nvme_req_op (cmd), blk_flags, qid -1 );if (IS_ERR (req))
@@ -1070,7 +1075,7 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, goto out; } - ret =nvme_execute_rq (req, at_head); + ret =nvme_execute_rq (req, flags &NVME_SUBMIT_AT_HEAD );if (result && ret >=0 ) *result =nvme_req (req)->result; out:
@@ -1083,7 +1088,7 @@ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, void *buffer, unsigned bufflen) {return __nvme_submit_sync_cmd (q, cmd,NULL , buffer, bufflen, -NVME_QID_ANY ,0 ,0 ); +NVME_QID_ANY ,0 ); }EXPORT_SYMBOL_GPL (nvme_submit_sync_cmd);
@@ -1547,7 +1552,7 @@ static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid, c.features.dword11 =cpu_to_le32 (dword11); ret =__nvme_submit_sync_cmd (dev->admin_q, &c, &res, - buffer, buflen,NVME_QID_ANY ,0 ,0 ); + buffer, buflen,NVME_QID_ANY ,0 );if (ret >=0 && result) *result =le32_to_cpu (res.u32);return ret;
@@ -2149,7 +2154,7 @@ static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t l cmd.common.cdw11 =cpu_to_le32 (len);return __nvme_submit_sync_cmd (ctrl->admin_q, &cmd,NULL , buffer, len, -NVME_QID_ANY ,1 ,0 ); +NVME_QID_ANY ,NVME_SUBMIT_AT_HEAD ); }static void nvme_configure_opal (struct nvme_ctrl *ctrl,bool was_suspended)
drivers/nvme/host/fabrics.c +11 -7
@@ -180,7 +180,7 @@ int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) cmd.prop_get.offset =cpu_to_le32 (off); ret =__nvme_submit_sync_cmd (ctrl->fabrics_q, &cmd, &res,NULL ,0 , -NVME_QID_ANY ,0 ,0 ); +NVME_QID_ANY ,0 );if (ret >=0 ) *val =le64_to_cpu (res.u64);
@@ -226,7 +226,7 @@ int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) cmd.prop_get.offset =cpu_to_le32 (off); ret =__nvme_submit_sync_cmd (ctrl->fabrics_q, &cmd, &res,NULL ,0 , -NVME_QID_ANY ,0 ,0 ); +NVME_QID_ANY ,0 );if (ret >=0 ) *val =le64_to_cpu (res.u64);
@@ -271,7 +271,7 @@ int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) cmd.prop_set.value =cpu_to_le64 (val); ret =__nvme_submit_sync_cmd (ctrl->fabrics_q, &cmd,NULL ,NULL ,0 , -NVME_QID_ANY ,0 ,0 ); +NVME_QID_ANY ,0 );if (unlikely (ret)) dev_err(ctrl->device, "Property Set error: %d, offset %#x\n",
@@ -450,8 +450,10 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)return -ENOMEM ; ret =__nvme_submit_sync_cmd (ctrl->fabrics_q, &cmd, &res, - data,sizeof (*data),NVME_QID_ANY ,1 , -BLK_MQ_REQ_RESERVED |BLK_MQ_REQ_NOWAIT ); + data,sizeof (*data),NVME_QID_ANY , +NVME_SUBMIT_AT_HEAD | +NVME_SUBMIT_NOWAIT | +NVME_SUBMIT_RESERVED );if (ret) {nvmf_log_connect_error (ctrl, ret,le32_to_cpu (res.u32), &cmd, data);
@@ -525,8 +527,10 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)return -ENOMEM ; ret =__nvme_submit_sync_cmd (ctrl->connect_q, &cmd, &res, - data,sizeof (*data), qid,1 , -BLK_MQ_REQ_RESERVED |BLK_MQ_REQ_NOWAIT ); + data,sizeof (*data), qid, +NVME_SUBMIT_AT_HEAD | +NVME_SUBMIT_RESERVED | +NVME_SUBMIT_NOWAIT );if (ret) {nvmf_log_connect_error (ctrl, ret,le32_to_cpu (res.u32), &cmd, data);
drivers/nvme/host/nvme.h +15 -2
@@ -825,12 +825,25 @@ static inline bool nvme_is_unique_nsid(struct nvme_ctrl *ctrl, (ctrl->ctratt &NVME_CTRL_CTRATT_NVM_SETS ); } +/* +* Flags for __nvme_submit_sync_cmd() +*/ +typedef __u32 __bitwise nvme_submit_flags_t ; + +enum { +/* Insert request at the head of the queue */ +NVME_SUBMIT_AT_HEAD = (__force nvme_submit_flags_t)(1 <<0 ), +/* Set BLK_MQ_REQ_NOWAIT when allocating request */ +NVME_SUBMIT_NOWAIT = (__force nvme_submit_flags_t)(1 <<1 ), +/* Set BLK_MQ_REQ_RESERVED when allocating request */ +NVME_SUBMIT_RESERVED = (__force nvme_submit_flags_t)(1 <<2 ), +}; +int nvme_submit_sync_cmd (struct request_queue *q,struct nvme_command *cmd,void *buf,unsigned bufflen);int __nvme_submit_sync_cmd (struct request_queue *q,struct nvme_command *cmd,union nvme_result *result,void *buffer,unsigned bufflen, -int qid,int at_head, -blk_mq_req_flags_t flags); +int qid,nvme_submit_flags_t flags);int nvme_set_features (struct nvme_ctrl *dev,unsigned int fid,unsigned int dword11,void *buffer,size_t buflen,u32 *result);
Signature · 1 line
2.35.3
PATCH v3 3/3 nvme: enable retries for authentication commands
In reply to [email protected]From: Hannes Reinecke <[email protected]>
Authentication commands are normal NVMe commands, and as such can return a status where NVME_SC_DNR is not set, indicating that the command should be retried.
This patch enables retries by setting NVME_SUBMIT_RETRY for __nvme_submit_sync_cmd().
--- drivers/nvme/host/auth.c | 2 +- drivers/nvme/host/core.c | 2 ++ drivers/nvme/host/nvme.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-)
drivers/nvme/host/auth.c +1 -1
@@ -58,7 +58,7 @@ static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, void *data, size_t data_len, bool auth_send) { struct nvme_command cmd = {}; -nvme_submit_flags_t flags =0 ; +nvme_submit_flags_t flags =NVME_SUBMIT_RETRY ;struct request_queue *q = ctrl->fabrics_q;int ret;
drivers/nvme/host/core.c +2 -0
@@ -1068,6 +1068,8 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,if (IS_ERR (req))return PTR_ERR (req);nvme_init_request (req, cmd); +if (flags &NVME_SUBMIT_RETRY ) + req->cmd_flags &= ~REQ_FAILFAST_DRIVER ;if (buffer && bufflen) { ret =blk_rq_map_kern (q, req, buffer, bufflen,GFP_KERNEL );
drivers/nvme/host/nvme.h +2 -0
@@ -837,6 +837,8 @@ enum {NVME_SUBMIT_NOWAIT = (__force nvme_submit_flags_t)(1 <<1 ),/* Set BLK_MQ_REQ_RESERVED when allocating request */ NVME_SUBMIT_RESERVED = (__force nvme_submit_flags_t)(1 <<2 ), +/* Retry command when NVME_SC_DNR is not set in the result */ + NVME_SUBMIT_RETRY = (__force nvme_submit_flags_t)(1 << 3), }; int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
Signature · 1 line
2.35.3
PATCH v3 1/3 nvme-auth: open-code single-use macros
In reply to [email protected]From: Hannes Reinecke <[email protected]>
No point in having macros just for a single function nvme_auth_submit(). Open-code them into the caller.
--- drivers/nvme/host/auth.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
drivers/nvme/host/auth.c +7 -7
@@ -48,11 +48,6 @@ struct nvme_dhchap_queue_context {static struct workqueue_struct *nvme_auth_wq; -#define nvme_auth_flags_from_qid (qid) \ - (qid == 0) ? 0 : BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED -#define nvme_auth_queue_from_qid (ctrl, qid) \ - (qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q -static inline int ctrl_max_dhchaps (struct nvme_ctrl *ctrl) { return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +
@@ -63,10 +58,15 @@ static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, void *data, size_t data_len, bool auth_send) { struct nvme_command cmd = {}; -blk_mq_req_flags_t flags =nvme_auth_flags_from_qid (qid); -struct request_queue *q =nvme_auth_queue_from_qid (ctrl, qid); +blk_mq_req_flags_t flags =0 ; +struct request_queue *q = ctrl->fabrics_q;int ret; +if (qid !=0 ) { + flags |=BLK_MQ_REQ_NOWAIT |BLK_MQ_REQ_RESERVED ; + q = ctrl->connect_q; + } + cmd.auth_common.opcode = nvme_fabrics_command; cmd.auth_common.secp =NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER ; cmd.auth_common.spsp0 =0x01 ;
Signature · 1 line
2.35.3
Re: [PATCH 2/3] nvme: simplify __nvme_submit_sync_cmd() calling conventions
In reply to [email protected]Quoted 11 lines · On Sat, 27 Jan 2024, [email protected] wrote:
ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res,drivers/nvme/host/fabrics.c +4
-2- data,sizeof (*data),NVME_QID_ANY ,1 , - BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); + data,sizeof (*data),NVME_QID_ANY , +NVME_SUBMIT_AT_HEAD | +NVME_SUBMIT_NOWAIT | + NVME_SUBMIT_RESERVED);if (ret) { nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32), &cmd, data);
drivers/nvme/host/fabrics.c int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid) · line 533
Quoted 9 lines
drivers/nvme/host/fabrics.c +4
-2@@ -525,8 +527,10 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)return -ENOMEM ; ret =__nvme_submit_sync_cmd (ctrl->connect_q, &cmd, &res, - data,sizeof (*data), qid,1 , -BLK_MQ_REQ_RESERVED |BLK_MQ_REQ_NOWAIT ); + data,sizeof (*data), qid, +NVME_SUBMIT_AT_HEAD | +NVME_SUBMIT_RESERVED | +NVME_SUBMIT_NOWAIT );
Is it worth writing the new flags in the first hunk above in the same order as in the second (so NVME_SUBMIT_AT_HEAD | NVME_SUBMIT_RESERVED | NVME_SUBMIT_NOWAIT)? That way they're consistent with the old ones so might make reading the diff a little easier.
Nicky
Re: [PATCH 1/3] nvme-auth: open-code single-use macros
In reply to [email protected]Looks good:
Re: [PATCH 2/3] nvme: simplify __nvme_submit_sync_cmd() calling conventions
In reply to [email protected]Quoted 6 lines · On Sat, Jan 27, 2024 at 10:37:45AM +0100, [email protected] wrote:
From: Hannes Reinecke <[email protected]>
Instead of having two arguments for 'flags' and '
at_head' for__nvme_submit_sync_cmd()combine them into a single 'flags' argument and use function-specific values to indicate what should be set within the function.
I'm not sure it's simplifying per se, but changing the calling convention ahead of adding even more flags.
You can also use up to 73 characters for your commit log.
Otherwise looks good:
Christoph Hellwig
maintainerQuoted 5 lines · On Sat, Jan 27, 2024 at 10:37:46AM +0100, [email protected] wrote:
From: Hannes Reinecke <[email protected]>
Authentication commands are normal NVMe commands, and as such can return a status where
NVME_SC_DNRis not set, indicating that the command should be retried.
What command is not normal?
Quoted 2 lines
This patch enables retries by setting
NVME_SUBMIT_RETRYfor__nvme_submit_sync_cmd().
I think what should go into the commit log is why the retries are desirable here, and not for other admin commands.
Otherwise this looks good to me.
Hannes Reinecke
maintainerQuoted 9 lines · Christoph Hellwig
Quoted 5 lines · On Sat, Jan 27, 2024 at 10:37:46AM +0100, [email protected] wrote:
From: Hannes Reinecke <[email protected]>
Authentication commands are normal NVMe commands, and as such can return a status where
NVME_SC_DNRis not set, indicating that the command should be retried.What command is not normal?
Things like 'property set' or 'property get', which are used in egnvmf_reg_read() et al. They are used to retrieve or get register settings, and it's not quite clear if it's safe (or prudent) to retry them.
Quoted 6 lines
Quoted 2 lines
This patch enables retries by setting
NVME_SUBMIT_RETRYfor__nvme_submit_sync_cmd().I think what should go into the commit log is why the retries are desirable here, and not for other admin commands.
Okay.
Quoted 1 line
Otherwise this looks good to me.
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), GF: Ivo Totev, Andrew McDonald, Werner Knoblich
Quoted 17 lines · On 1/27/24 01:37, [email protected] wrote:
From: Hannes Reinecke <[email protected]>
Hi all,
authentitication commands are normal NVMe commands, and should be retried if returned with a status where the DNR bit is not set. But instead of having the caller retry the commaand this patchset enables the function
__nvme_submit_sync_cmd()to not set the FASTFAIL flag in the request, thereby making the functionality generally available.As usual, comments and reviews are welcome.
Hannes Reinecke (3):
nvme-auth: open-code single-use macros
nvme: simplify__nvme_submit_sync_cmd()calling conventions
nvme: enable retries for authentication commands
Quoted 6 lines
drivers/nvme/host/auth.c | 17 ++++++++--------- drivers/nvme/host/core.c | 21 ++++++++++++++------- drivers/nvme/host/fabrics.c | 18 +++++++++++------- drivers/nvme/host/nvme.h | 19 +++++++++++++++++-- 4 files changed, 50 insertions(+), 25 deletions(-)
this version addresses comments posted on V2, looks good to me, it'd be great
if we can somehow trigger retries through blktests ..
-ck
Re: [PATCH 1/3] nvme-auth: open-code single-use macros
In reply to [email protected]Re: [PATCH 1/3] nvme-auth: open-code single-use macros
In reply to [email protected]Quoted 6 lines · On Sat, Jan 27, 2024 at 10:37:44AM +0100, [email protected] wrote:
From: Hannes Reinecke <[email protected]>
No point in having macros just for a single function
nvme_auth_submit(). Open-code them into the caller.Signed-off-by Hannes Reinecke <[email protected]>
Thanks, applied for nvme-6.8.