Skip to contentSkip to conversation

nvme-fabrics: short-circuit connect retries

PATCH Superseded Daniel Wagner 2 participants 16 messages Apr 4, 2024 – Apr 5, 2024

Subsystems (3)

v4 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v4 cover nvme-fabrics: short-circuit connect retries

An rebase of Hannes two series which fix the TCP and RDMA transport to handle the DNR bit on connect attempts.

For testing I extended the nvme/045 test case. I'll update the test case later when the current batch of blktest changes are done. Also this change depends on the extension of the debugfs interface of nvmet, which is also not yet merged.

	echo "Renew host key on the controller and force reconnect"

	new_hostkey="$(nvme gen-dhchap-key -n ${def_subsysnqn} 2> /dev/null)"

	_set_nvmet_hostkey "${def_hostnqn}" "${new_hostkey}"

	# Force a reconnect
	nvmedev=$(_find_nvme_dev "${def_subsysnqn}")
	cntlid="$(nvme id-ctrl "/dev/${nvmedev}" | grep cntlid | awk '{print $3}')"
	echo "fatal" > /sys/kernel/debug/nvmet/"${def_subsysnqn}/ctrl$((${cntlid}))"/state
	nvmf_wait_for_ctrl_delete "${nvmedev}"

baseline:

run 1 loop (nvmet_blkdev_type file)
nvme/045 (Test re-authentication) [passed]

    runtime  2.690s  ...  2.777s

run 1 tcp (nvmet_blkdev_type file)
nvme/045 (Test re-authentication) [failed]
runtime 2.777s ... 8.030s
--- tests/nvme/045.out 2024-04-04 16:14:22.547250311 +0200
+++ /home/wagi/work/blktests/results/nodev/nvme/045.out.bad 2024-04-04 17:29:03.427799336 +0200 @@ -9,5 +9,6 @@
Change hash to hmac(sha512)
Re-authenticate with changed hash
Renew host key on the controller and force reconnect
-disconnected 0 controller(s)
+controller "nvme2" not deleted within 5 seconds
+disconnected 1 controller(s)
Test complete
run 1 rdma (nvmet_blkdev_type file)
nvme/045 (Test re-authentication) [failed]
runtime 8.030s ... 9.632s
--- tests/nvme/045.out 2024-04-04 16:14:22.547250311 +0200
+++ /home/wagi/work/blktests/results/nodev/nvme/045.out.bad 2024-04-04 17:29:15.017745115 +0200 @@ -9,5 +9,6 @@
Change hash to hmac(sha512)
Re-authenticate with changed hash
Renew host key on the controller and force reconnect
-disconnected 0 controller(s)
+controller "nvme2" not deleted within 5 seconds
+disconnected 1 controller(s)
Test complete
run 1 fc (nvmet_blkdev_type file)
nvme/045 (Test re-authentication) [passed]

    runtime  9.632s  ...  3.588s

patched:

run 1 loop (nvmet_blkdev_type file)
nvme/045 (Test re-authentication) [passed]

    runtime  6.816s  ...  2.492s
run 1 tcp (nvmet_blkdev_type file)
nvme/045 (Test re-authentication)                            [passed]
    runtime  2.492s  ...  3.663s
run 1 rdma (nvmet_blkdev_type file)
nvme/045 (Test re-authentication)                            [passed]
    runtime  3.663s  ...  3.795s
run 1 fc (nvmet_blkdev_type file)
nvme/045 (Test re-authentication)                            [passed]
    runtime  3.795s  ...  2.690s

changes:
v4:
- rebased
- added 'nvme: fixes for authentication errors' series

    https://lore.kernel.org/linux-nvme/[email protected]/

v3:
- added my SOB tag
- fixed indention
- https://lore.kernel.org/linux-nvme/[email protected]/

v2:
- refresh/rebase on current head
- extended blktests (nvme/045) to cover this case
(see separate post)
- https://lore.kernel.org/linux-nvme/[email protected]/

v1:
- initial version
- https://lore.kernel.org/linux-nvme/[email protected]/

*** BLURB HERE ***

Hannes Reinecke (5):
nvme: authentication error are always non-retryable
nvmet: lock config semaphore when accessing DH-HMAC-CHAP key
nvmet: return DHCHAP status codes from nvmet_setup_auth()
nvme-tcp: short-circuit reconnect retries
nvme-rdma: short-circuit reconnect retries

 drivers/nvme/host/core.c               |  6 +++---
 drivers/nvme/host/fabrics.c            | 29 +++++++++++++++-----------
 drivers/nvme/host/nvme.h               | 19 ++++++++++++++++-
 drivers/nvme/host/rdma.c               | 22 ++++++++++++-------
 drivers/nvme/host/tcp.c                | 23 +++++++++++++-------
 drivers/nvme/target/auth.c             | 20 ++++++++----------
 drivers/nvme/target/configfs.c         | 22 ++++++++++++++-----
 drivers/nvme/target/fabrics-cmd-auth.c | 11 +++++-----
 8 files changed, 100 insertions(+), 52 deletions(-)
Signature · 1 line
2.44.0
#1 lore raw

PATCH v4 1/5 nvme: authentication error are always non-retryable

From: Hannes Reinecke <[email protected]>

Any authentication errors which are generated internally are always non-retryable, so set the DNR bit to ensure they are not retried.

---
 drivers/nvme/host/core.c    |  6 +++---
 drivers/nvme/host/fabrics.c | 29 +++++++++++++++++------------
 drivers/nvme/host/nvme.h    |  2 +-
 3 files changed, 21 insertions(+), 16 deletions(-)
drivers/nvme/host/core.c +3 -3
index 504dc352c458..66387bcca8ae 100644
@@ -383,14 +383,14 @@ static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 	if (likely(nvme_req(req)->status == 0))
 		return COMPLETE;
 
-	if ((nvme_req(req)->status & 0x7ff) == NVME_SC_AUTH_REQUIRED)
-		return AUTHENTICATE;
-
 	if (blk_noretry_request(req) ||
 	    (nvme_req(req)->status & NVME_SC_DNR) ||
 	    nvme_req(req)->retries >= nvme_max_retries)
 		return COMPLETE;
 
+	if ((nvme_req(req)->status & 0x7ff) == NVME_SC_AUTH_REQUIRED)
+		return AUTHENTICATE;
+
 	if (req->cmd_flags & REQ_NVME_MPATH) {
 		if (nvme_is_path_error(nvme_req(req)->status) ||
 		    blk_queue_dying(req->q))
drivers/nvme/host/fabrics.c +17 -12
index 1f0ea1f32d22..309a69c24995 100644
@@ -467,7 +467,7 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 		if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 			dev_warn(ctrl->device,
 				 "qid 0: secure concatenation is not supported\n");
-			ret = NVME_SC_AUTH_REQUIRED;
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
 			goto out_free_data;
 		}
 		/* Authentication required */
@@ -475,14 +475,16 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 		if (ret) {
 			dev_warn(ctrl->device,
 				 "qid 0: authentication setup failed\n");
-			ret = NVME_SC_AUTH_REQUIRED;
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
 			goto out_free_data;
 		}
 		ret = nvme_auth_wait(ctrl, 0);
-		if (ret)
+		if (ret) {
 			dev_warn(ctrl->device,
-				 "qid 0: authentication failed\n");
-		else
+				 "qid 0: authentication failed, error %d\n",
+				 ret);
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
+		} else
 			dev_info(ctrl->device,
 				 "qid 0: authenticated\n");
 	}
@@ -542,7 +544,7 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 		if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 			dev_warn(ctrl->device,
 				 "qid 0: secure concatenation is not supported\n");
-			ret = NVME_SC_AUTH_REQUIRED;
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
 			goto out_free_data;
 		}
 		/* Authentication required */
@@ -550,12 +552,15 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 		if (ret) {
 			dev_warn(ctrl->device,
 				 "qid %d: authentication setup failed\n", qid);
-			ret = NVME_SC_AUTH_REQUIRED;
-		} else {
-			ret = nvme_auth_wait(ctrl, qid);
-			if (ret)
-				dev_warn(ctrl->device,
-					 "qid %u: authentication failed\n", qid);
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
+			goto out_free_data;
+		}
+		ret = nvme_auth_wait(ctrl, qid);
+		if (ret) {
+			dev_warn(ctrl->device,
+				 "qid %u: authentication failed, error %d\n",
+				 qid, ret);
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
 		}
 	}
 out_free_data:
drivers/nvme/host/nvme.h +1 -1
index d0ed64dc7380..882967365108 100644
@@ -1122,7 +1122,7 @@ static inline int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
 }
 static inline int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
 {
-	return NVME_SC_AUTH_REQUIRED;
+	return NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;
 }
 static inline void nvme_auth_free(struct nvme_ctrl *ctrl) {};
 #endif
Signature · 1 line
2.44.0
#2 lore raw

Re: [PATCH v4 1/5] nvme: authentication error are always non-retryable

drivers/nvme/host/fabrics.c int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl) · line 470

Quoted 8 lines
drivers/nvme/host/fabrics.c +1 -1
@@ -467,7 +467,7 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 		if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 			dev_warn(ctrl->device,
 				 "qid 0: secure concatenation is not supported\n");
-			ret = NVME_SC_AUTH_REQUIRED;
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;

This is still abusing on the wire status code for in-kernel return codes. Can we please sort this out properly?

#3 lore raw

Re: [PATCH v4 1/5] nvme: authentication error are always non-retryable

Quoted 11 lines · Christoph Hellwig

drivers/nvme/host/fabrics.c int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl) · line 470

Quoted 8 lines
drivers/nvme/host/fabrics.c +1 -1
@@ -467,7 +467,7 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 		if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 			dev_warn(ctrl->device,
 				 "qid 0: secure concatenation is not supported\n");
-			ret = NVME_SC_AUTH_REQUIRED;
+			ret = NVME_SC_AUTH_REQUIRED | NVME_SC_DNR;

This is still abusing on the wire status code for in-kernel return codes. Can we please sort this out properly?

Okay, though I am not really sure how to do it correctly.

So the current mapping is

ret < 0: kernel errors
ret = 0: all good
ret > 0: wire status incl DNR

In order to split the internal DNR away, we could attach it to the cmd. Is this what you had in mind? Or do you mean we should not return NVME_SC_AUTH_REQUIRED at all. Instead just a negative value and update the error handling on the callers?

#4 lore raw

Re: [PATCH v4 1/5] nvme: authentication error are always non-retryable

Quoted 10 lines · Daniel Wagner
Quoted 2 lines

This is still abusing on the wire status code for in-kernel return codes. Can we please sort this out properly?

Okay, though I am not really sure how to do it correctly.

So the current mapping is

ret < 0: kernel errors
ret = 0: all good
ret > 0: wire status incl DNR

Yes.

Quoted 4 lines

In order to split the internal DNR away, we could attach it to the cmd. Is this what you had in mind? Or do you mean we should not return NVME_SC_AUTH_REQUIRED at all. Instead just a negative value and update the error handling on the callers?

The latter.

#5 lore raw

PATCH v4 2/5 nvmet: lock config semaphore when accessing DH-HMAC-CHAP key

In reply to Daniel Wagner

From: Hannes Reinecke <[email protected]>

When the DH-HMAC-CHAP key is accessed via configfs we need to take the config semaphore as a reconnect might be running at the same time.

---
 drivers/nvme/target/auth.c     |  2 ++
 drivers/nvme/target/configfs.c | 22 +++++++++++++++++-----
 2 files changed, 19 insertions(+), 5 deletions(-)
drivers/nvme/target/auth.c +2 -0
index 3ddbc3880cac..9afc28f1ffac 100644
@@ -44,6 +44,7 @@ int nvmet_auth_set_key(struct nvmet_host *host, const char *secret,
 	dhchap_secret = kstrdup(secret, GFP_KERNEL);
 	if (!dhchap_secret)
 		return -ENOMEM;
+	down_write(&nvmet_config_sem);
 	if (set_ctrl) {
 		kfree(host->dhchap_ctrl_secret);
 		host->dhchap_ctrl_secret = strim(dhchap_secret);
@@ -53,6 +54,7 @@ int nvmet_auth_set_key(struct nvmet_host *host, const char *secret,
 		host->dhchap_secret = strim(dhchap_secret);
 		host->dhchap_key_hash = key_hash;
 	}
+	up_write(&nvmet_config_sem);
 	return 0;
 }
 
drivers/nvme/target/configfs.c +17 -5
index 77a6e817b315..7c28b9c0ee57 100644
@@ -1990,11 +1990,17 @@ static struct config_group nvmet_ports_group;
 static ssize_t nvmet_host_dhchap_key_show(struct config_item *item,
 		char *page)
 {
-	u8 *dhchap_secret = to_host(item)->dhchap_secret;
+	u8 *dhchap_secret;
+	ssize_t ret;
 
+	down_read(&nvmet_config_sem);
+	dhchap_secret = to_host(item)->dhchap_secret;
 	if (!dhchap_secret)
-		return sprintf(page, "\n");
-	return sprintf(page, "%s\n", dhchap_secret);
+		ret = sprintf(page, "\n");
+	else
+		ret = sprintf(page, "%s\n", dhchap_secret);
+	up_read(&nvmet_config_sem);
+	return ret;
 }
 
 static ssize_t nvmet_host_dhchap_key_store(struct config_item *item,
@@ -2018,10 +2024,16 @@ static ssize_t nvmet_host_dhchap_ctrl_key_show(struct config_item *item,
 		char *page)
 {
 	u8 *dhchap_secret = to_host(item)->dhchap_ctrl_secret;
+	ssize_t ret;
 
+	down_read(&nvmet_config_sem);
+	dhchap_secret = to_host(item)->dhchap_ctrl_secret;
 	if (!dhchap_secret)
-		return sprintf(page, "\n");
-	return sprintf(page, "%s\n", dhchap_secret);
+		ret = sprintf(page, "\n");
+	else
+		ret = sprintf(page, "%s\n", dhchap_secret);
+	up_read(&nvmet_config_sem);
+	return ret;
 }
 
 static ssize_t nvmet_host_dhchap_ctrl_key_store(struct config_item *item,
Signature · 1 line
2.44.0
#7 lore raw

PATCH v4 3/5 nvmet: return DHCHAP status codes from nvmet_setup_auth()

In reply to Daniel Wagner

From: Hannes Reinecke <[email protected]>

A failure in nvmet_setup_auth() does not mean that the NVMe authentication command failed, so we should rather return a protocol error with a 'failure1' response than an NVMe status.

---
 drivers/nvme/target/auth.c             | 18 +++++++-----------
 drivers/nvme/target/fabrics-cmd-auth.c | 11 ++++++-----
 2 files changed, 13 insertions(+), 16 deletions(-)
drivers/nvme/target/auth.c +7 -11
index 9afc28f1ffac..1079281a202e 100644
@@ -131,7 +131,6 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	int ret = 0;
 	struct nvmet_host_link *p;
 	struct nvmet_host *host = NULL;
-	const char *hash_name;
 
 	down_read(&nvmet_config_sem);
 	if (nvmet_is_disc_subsys(ctrl->subsys))
@@ -149,13 +148,16 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	}
 	if (!host) {
 		pr_debug("host %s not found\n", ctrl->hostnqn);
-		ret = -EPERM;
+		ret = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 		goto out_unlock;
 	}
 
 	ret = nvmet_setup_dhgroup(ctrl, host->dhchap_dhgroup_id);
-	if (ret < 0)
+	if (ret < 0) {
 		pr_warn("Failed to setup DH group");
+		ret = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
+		goto out_unlock;
+	}
 
 	if (!host->dhchap_secret) {
 		pr_debug("No authentication provided\n");
@@ -166,12 +168,6 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 		pr_debug("Re-use existing hash ID %d\n",
 			 ctrl->shash_id);
 	} else {
-		hash_name = nvme_auth_hmac_name(host->dhchap_hash_id);
-		if (!hash_name) {
-			pr_warn("Hash ID %d invalid\n", host->dhchap_hash_id);
-			ret = -EINVAL;
-			goto out_unlock;
-		}
 		ctrl->shash_id = host->dhchap_hash_id;
 	}
 
@@ -180,7 +176,7 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	ctrl->host_key = nvme_auth_extract_key(host->dhchap_secret + 10,
 					       host->dhchap_key_hash);
 	if (IS_ERR(ctrl->host_key)) {
-		ret = PTR_ERR(ctrl->host_key);
+		ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE;
 		ctrl->host_key = NULL;
 		goto out_free_hash;
 	}
@@ -198,7 +194,7 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	ctrl->ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret + 10,
 					       host->dhchap_ctrl_key_hash);
 	if (IS_ERR(ctrl->ctrl_key)) {
-		ret = PTR_ERR(ctrl->ctrl_key);
+		ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE;
 		ctrl->ctrl_key = NULL;
 		goto out_free_hash;
 	}
drivers/nvme/target/fabrics-cmd-auth.c +6 -5
index eb7785be0ca7..a95dc6606396 100644
@@ -240,12 +240,13 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
 			pr_debug("%s: ctrl %d qid %d reset negotiation\n", __func__,
 				 ctrl->cntlid, req->sq->qid);
 			if (!req->sq->qid) {
-				if (nvmet_setup_auth(ctrl) < 0) {
-					status = NVME_SC_INTERNAL;
-					pr_err("ctrl %d qid 0 failed to setup"
-					       "re-authentication",
+				status = nvmet_setup_auth(ctrl);
+				if (status) {
+					pr_err("ctrl %d qid 0 failed to setup re-authentication\n",
 					       ctrl->cntlid);
-					goto done_failure1;
+					req->sq->dhchap_status = status;
+					req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
+					goto done_kfree;
 				}
 			}
 			req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
Signature · 1 line
2.44.0
#8 lore raw

Re: [PATCH v4 3/5] nvmet: return DHCHAP status codes from nvmet_setup_auth()

Quoted 5 lines · Daniel Wagner

From: Hannes Reinecke <[email protected]>

A failure in nvmet_setup_auth() does not mean that the NVMe authentication command failed, so we should rather return a protocol error with a 'failure1' response than an NVMe status.

Nit: try to use up the 73 characters available for commit logs, this looks weirdly, um condensed.

Quoted 9 lines
drivers/nvme/target/auth.c +0 -1
@@ -131,7 +131,6 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	int ret = 0;
 	struct nvmet_host_link *p;
 	struct nvmet_host *host = NULL;
-	const char *hash_name;
 
 	down_read(&nvmet_config_sem);
 	if (nvmet_is_disc_subsys(ctrl->subsys))

drivers/nvme/target/auth.c int nvmet_setup_auth(struct nvmet_ctrl *ctrl) · line 152

Quoted 6 lines
drivers/nvme/target/auth.c +1 -1
@@ -149,13 +148,16 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	}
 	if (!host) {
 		pr_debug("host %s not found\n", ctrl->hostnqn);
-		ret = -EPERM;
+		ret = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 		goto out_unlock;

This is now returning returning random on the wire fields that aren't even the NVMe status codes from a function otherwise returning Linux errno values. I can't see how this works or is maintainable long term.

#9 lore raw

Re: [PATCH v4 3/5] nvmet: return DHCHAP status codes from nvmet_setup_auth()

Quoted 6 lines · Christoph Hellwig
Quoted 6 lines
drivers/nvme/target/auth.c +0 -1
@@ -131,7 +131,6 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	int ret = 0;
 	struct nvmet_host_link *p;
 	struct nvmet_host *host = NULL;
-	const char *hash_name;
 
Quoted 13 lines

drivers/nvme/target/auth.c int nvmet_setup_auth(struct nvmet_ctrl *ctrl) · line 152

Quoted 9 lines
drivers/nvme/target/auth.c +1 -1
 	down_read(&nvmet_config_sem);
 	if (nvmet_is_disc_subsys(ctrl->subsys))
@@ -149,13 +148,16 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	}
 	if (!host) {
 		pr_debug("host %s not found\n", ctrl->hostnqn);
-		ret = -EPERM;
+		ret = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 		goto out_unlock;

This is now returning returning random on the wire fields that aren't even the NVMe status codes from a function otherwise returning Linux errno values. I can't see how this works or is maintainable long term.

This is the target side and we generate the on wire return code here. Are you sure I should map this to errno codes and the back to NVME status codes? Sure, this is possible but don't really think it makes sense.

#10 lore raw

Re: [PATCH v4 3/5] nvmet: return DHCHAP status codes from nvmet_setup_auth()

Quoted 21 lines · Daniel Wagner
Quoted 6 lines · Christoph Hellwig
Quoted 6 lines
drivers/nvme/target/auth.c +0 -1
@@ -131,7 +131,6 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	int ret = 0;
 	struct nvmet_host_link *p;
 	struct nvmet_host *host = NULL;
-	const char *hash_name;
 
Quoted 13 lines

drivers/nvme/target/auth.c int nvmet_setup_auth(struct nvmet_ctrl *ctrl) · line 152

Quoted 9 lines
drivers/nvme/target/auth.c +1 -1
 	down_read(&nvmet_config_sem);
 	if (nvmet_is_disc_subsys(ctrl->subsys))
@@ -149,13 +148,16 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl)
 	}
 	if (!host) {
 		pr_debug("host %s not found\n", ctrl->hostnqn);
-		ret = -EPERM;
+		ret = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 		goto out_unlock;

This is now returning returning random on the wire fields that aren't even the NVMe status codes from a function otherwise returning Linux errno values. I can't see how this works or is maintainable long term.

Quoted 4 lines

This is the target side and we generate the on wire return code here. Are you sure I should map this to errno codes and the back to NVME status codes? Sure, this is possible but don't really think it makes sense.

I think I start to understand your feedback. I'll try to address it.

#11 lore raw

Re: [PATCH v4 3/5] nvmet: return DHCHAP status codes from nvmet_setup_auth()

In reply to Daniel Wagner
Quoted 11 lines · Daniel Wagner
Quoted 9 lines

drivers/nvme/target/auth.c int nvmet_setup_auth(struct nvmet_ctrl *ctrl) · line 152

Quoted 5 lines
drivers/nvme/target/auth.c +1 -1
 	if (!host) {
 		pr_debug("host %s not found\n", ctrl->hostnqn);
-		ret = -EPERM;
+		ret = NVME_AUTH_DHCHAP_FAILURE_FAILED;
 		goto out_unlock;

This is now returning returning random on the wire fields that aren't even the NVMe status codes from a function otherwise returning Linux errno values. I can't see how this works or is maintainable long term.

This is the target side and we generate the on wire return code here.

True.

Quoted 3 lines

Are you sure I should map this to errno codes and the back to NVME status codes? Sure, this is possible but don't really think it makes sense.

No, but we should not overload the return value. Pass in the req or sq, or add a new paramter for the auth fail reason.

#12 lore raw

Re: [PATCH v4 3/5] nvmet: return DHCHAP status codes from nvmet_setup_auth()

Quoted 6 lines · Christoph Hellwig
Quoted 3 lines

Are you sure I should map this to errno codes and the back to NVME status codes? Sure, this is possible but don't really think it makes sense.

No, but we should not overload the return value. Pass in the req or sq, or add a new paramter for the auth fail reason.

Or make sure all returns are the status codes, change the return value to a u8 and clearly document what values it returns if that works out.

And while you're at it please fix all the overly long lines.

#13 lore raw

PATCH v4 4/5 nvme-tcp: short-circuit reconnect retries

In reply to Daniel Wagner

From: Hannes Reinecke <[email protected]>

Returning an nvme status from nvme_tcp_setup_ctrl() indicates that the association was established and we have received a status from the controller; consequently we should honour the DNR bit. If not any future reconnect attempts will just return the same error, so we can short-circuit the reconnect attempts and fail the connection directly.

---
 drivers/nvme/host/nvme.h | 17 +++++++++++++++++
 drivers/nvme/host/tcp.c  | 23 +++++++++++++++--------
 2 files changed, 32 insertions(+), 8 deletions(-)
drivers/nvme/host/nvme.h +17 -0
index 882967365108..654d8d1a4d29 100644
@@ -701,6 +701,23 @@ static inline bool nvme_is_path_error(u16 status)
 	return (status & 0x700) == 0x300;
 }
 
+/*
+ * Evaluate the status information returned by the LLDD in order to
+ * decided if a reconnect attempt should be scheduled.
+ *
+ * There are two cases where no reconnect attempt should be attempted:
+ *
+ * 1) The LLDD reports an negative status. There was an error (e.g. no
+ *    memory) on the host side and thus abort the operation.
+ * 2) The DNR bit is set and the specification states no further
+ *    connect attempts with the same set of paramenters should be
+ *    attempted.
+ */
+static inline bool nvme_ctrl_reconnect(int status)
+{
+	return status > 0 && (status & NVME_SC_DNR) ? false : true;
+}
+
 /*
  * Fill in the status and result information from the CQE, and then figure out
  * if blk-mq will need to use IPI magic to complete the request, and if yes do
drivers/nvme/host/tcp.c +15 -8
index fdbcdcedcee9..7e25a96e9870 100644
@@ -2155,9 +2155,11 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
 	nvme_tcp_destroy_io_queues(ctrl, remove);
 }
 
-static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
+static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl,
+		int status)
 {
 	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
+	bool recon = nvme_ctrl_reconnect(status);
 
 	/* If we are resetting/deleting then do nothing */
 	if (state != NVME_CTRL_CONNECTING) {
@@ -2165,13 +2167,14 @@ static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
 		return;
 	}
 
-	if (nvmf_should_reconnect(ctrl)) {
+	if (recon && nvmf_should_reconnect(ctrl)) {
 		dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
 			ctrl->opts->reconnect_delay);
 		queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
 				ctrl->opts->reconnect_delay * HZ);
 	} else {
-		dev_info(ctrl->device, "Removing controller...\n");
+		dev_info(ctrl->device, "Removing controller (%d)...\n",
+			 status);
 		nvme_delete_ctrl(ctrl);
 	}
 }
@@ -2252,10 +2255,12 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
 	struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
 			struct nvme_tcp_ctrl, connect_work);
 	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
+	int ret;
 
 	++ctrl->nr_reconnects;
 
-	if (nvme_tcp_setup_ctrl(ctrl, false))
+	ret = nvme_tcp_setup_ctrl(ctrl, false);
+	if (ret)
 		goto requeue;
 
 	dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
@@ -2268,7 +2273,7 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
 requeue:
 	dev_info(ctrl->device, "Failed reconnect attempt %d\n",
 			ctrl->nr_reconnects);
-	nvme_tcp_reconnect_or_remove(ctrl);
+	nvme_tcp_reconnect_or_remove(ctrl, ret);
 }
 
 static void nvme_tcp_error_recovery_work(struct work_struct *work)
@@ -2295,7 +2300,7 @@ static void nvme_tcp_error_recovery_work(struct work_struct *work)
 		return;
 	}
 
-	nvme_tcp_reconnect_or_remove(ctrl);
+	nvme_tcp_reconnect_or_remove(ctrl, -ENOTCONN);
 }
 
 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
@@ -2315,6 +2320,7 @@ static void nvme_reset_ctrl_work(struct work_struct *work)
 {
 	struct nvme_ctrl *ctrl =
 		container_of(work, struct nvme_ctrl, reset_work);
+	int ret;
 
 	nvme_stop_ctrl(ctrl);
 	nvme_tcp_teardown_ctrl(ctrl, false);
@@ -2328,14 +2334,15 @@ static void nvme_reset_ctrl_work(struct work_struct *work)
 		return;
 	}
 
-	if (nvme_tcp_setup_ctrl(ctrl, false))
+	ret = nvme_tcp_setup_ctrl(ctrl, false);
+	if (ret)
 		goto out_fail;
 
 	return;
 
 out_fail:
 	++ctrl->nr_reconnects;
-	nvme_tcp_reconnect_or_remove(ctrl);
+	nvme_tcp_reconnect_or_remove(ctrl, ret);
 }
 
 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
Signature · 1 line
2.44.0
#14 lore raw

PATCH v4 5/5 nvme-rdma: short-circuit reconnect retries

In reply to Daniel Wagner

From: Hannes Reinecke <[email protected]>

Returning an nvme status from nvme_rdma_setup_ctrl() indicates that the association was established and we have received a status from the controller; consequently we should honour the DNR bit. If not any future reconnect attempts will just return the same error, so we can short-circuit the reconnect attempts and fail the connection directly.

---
 drivers/nvme/host/rdma.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
drivers/nvme/host/rdma.c +15 -7
index 366f0bb4ebfc..53d51df26ae1 100644
@@ -982,9 +982,11 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
 	kfree(ctrl);
 }
 
-static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
+static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl,
+		int status)
 {
 	enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl);
+	bool recon = nvme_ctrl_reconnect(status);
 
 	/* If we are resetting/deleting then do nothing */
 	if (state != NVME_CTRL_CONNECTING) {
@@ -992,12 +994,14 @@ static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
 		return;
 	}
 
-	if (nvmf_should_reconnect(&ctrl->ctrl)) {
+	if (recon && nvmf_should_reconnect(&ctrl->ctrl)) {
 		dev_info(ctrl->ctrl.device, "Reconnecting in %d seconds...\n",
 			ctrl->ctrl.opts->reconnect_delay);
 		queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
 				ctrl->ctrl.opts->reconnect_delay * HZ);
 	} else {
+		dev_info(ctrl->ctrl.device, "Removing controller (%d)...\n",
+			 status);
 		nvme_delete_ctrl(&ctrl->ctrl);
 	}
 }
@@ -1104,10 +1108,12 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 {
 	struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
 			struct nvme_rdma_ctrl, reconnect_work);
+	int ret;
 
 	++ctrl->ctrl.nr_reconnects;
 
-	if (nvme_rdma_setup_ctrl(ctrl, false))
+	ret = nvme_rdma_setup_ctrl(ctrl, false);
+	if (ret)
 		goto requeue;
 
 	dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
@@ -1120,7 +1126,7 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 requeue:
 	dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
 			ctrl->ctrl.nr_reconnects);
-	nvme_rdma_reconnect_or_remove(ctrl);
+	nvme_rdma_reconnect_or_remove(ctrl, ret);
 }
 
 static void nvme_rdma_error_recovery_work(struct work_struct *work)
@@ -1145,7 +1151,7 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
 		return;
 	}
 
-	nvme_rdma_reconnect_or_remove(ctrl);
+	nvme_rdma_reconnect_or_remove(ctrl, -ENOTCONN);
 }
 
 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
@@ -2169,6 +2175,7 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 {
 	struct nvme_rdma_ctrl *ctrl =
 		container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
+	int ret;
 
 	nvme_stop_ctrl(&ctrl->ctrl);
 	nvme_rdma_shutdown_ctrl(ctrl, false);
@@ -2179,14 +2186,15 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 		return;
 	}
 
-	if (nvme_rdma_setup_ctrl(ctrl, false))
+	ret = nvme_rdma_setup_ctrl(ctrl, false);
+	if (ret)
 		goto out_fail;
 
 	return;
 
 out_fail:
 	++ctrl->ctrl.nr_reconnects;
-	nvme_rdma_reconnect_or_remove(ctrl);
+	nvme_rdma_reconnect_or_remove(ctrl, ret);
 }
 
 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
Signature · 1 line
2.44.0