nvme: short-circuit connection retries
PATCH 1/2 nvme-tcp: short-circuit connect retries
In reply to Hannes ReineckeWhen a reconnect attempt fails with a non-retryable status (eg when the subsystem has been unprovisioned) there hardly is any reason to retry the reconnect attempt.
So pass the actual error status to nvme_tcp_reconnect_or_remove() and short-circuit retries if the DNR bit is set.
--- drivers/nvme/host/tcp.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/nvme/host/tcp.c +18 -7
@@ -2065,8 +2065,10 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,nvme_tcp_destroy_io_queues (ctrl, remove); } -static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl) +static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl,int status) { +bool recon = true; +/* If we are resetting/deleting then do nothing */ if (ctrl->state !=NVME_CTRL_CONNECTING ) { WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
@@ -2074,7 +2076,12 @@ static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)return ; } -if (nvmf_should_reconnect (ctrl)) { +if (status >0 && (status &NVME_SC_DNR )) { +dev_info (ctrl->device,"reconnect failure %d\n" , status); + recon = false; + } + +if (recon &&nvmf_should_reconnect (ctrl)) {dev_info (ctrl->device,"Reconnecting in %d seconds...\n" , ctrl->opts->reconnect_delay); queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
@@ -2162,10 +2169,12 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)struct nvme_tcp_ctrl *tcp_ctrl =container_of (to_delayed_work (work), struct nvme_tcp_ctrl, connect_work);struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; +int ret; ++ctrl->nr_reconnects; -if (nvme_tcp_setup_ctrl (ctrl, false)) + ret =nvme_tcp_setup_ctrl (ctrl, false); +if (ret) goto requeue; dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
@@ -2178,7 +2187,7 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) requeue:dev_info (ctrl->device,"Failed reconnect attempt %d\n" , ctrl->nr_reconnects); -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, ret); }static void nvme_tcp_error_recovery_work (struct work_struct *work)
@@ -2203,7 +2212,7 @@ static void nvme_tcp_error_recovery_work(struct work_struct *work)return ; } -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, -ENOTCONN ); }static void nvme_tcp_teardown_ctrl (struct nvme_ctrl *ctrl,bool shutdown)
@@ -2229,6 +2238,7 @@ static void nvme_reset_ctrl_work(struct work_struct *work) {struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, reset_work); +int ret;nvme_stop_ctrl (ctrl);nvme_tcp_teardown_ctrl (ctrl, false);
@@ -2240,14 +2250,15 @@ static void nvme_reset_ctrl_work(struct work_struct *work)return ; } -if (nvme_tcp_setup_ctrl (ctrl, false)) + ret =nvme_tcp_setup_ctrl (ctrl, false); +if (ret) goto out_fail;return ; out_fail: ++ctrl->nr_reconnects; -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, ret); }static void nvme_tcp_free_ctrl (struct nvme_ctrl *nctrl)
Signature · 1 line
2.29.2
cover nvme: short-circuit connection retries
Hi all,
here are two patches updating nvme-tcp and nvme-rdma transport to modifications we've already have in nvme-fc with commit f25f8ef70ce2 ("nvme-fc: short-circuit reconnect retries"). Gist is to short-circuit reconnection retries if we get a status back with the DNR bit set, in which case we shouldn't retry the connection attempt.
As usual, comments and reviews are welcome.
Hannes Reinecke (2):
nvme-tcp: short-circuit connect retries
nvme-rdma: short-circuit connect retries
drivers/nvme/host/rdma.c | 25 ++++++++++++++++++------- drivers/nvme/host/tcp.c | 25 ++++++++++++++++++------- 2 files changed, 36 insertions(+), 14 deletions(-)
Signature · 1 line
2.29.2
PATCH 2/2 nvme-rdma: short-circuit connect retries
When a reconnect attempt fails with a non-retryable status (eg when the subsystem has been unprovisioned) there hardly is any reason to retry the reconnect attempt.
So pass the actual error status to nvme_tcp_reconnect_or_remove() and short-circuit retries if the DNR bit is set.
--- drivers/nvme/host/rdma.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/nvme/host/rdma.c +18 -7
@@ -1065,8 +1065,10 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)kfree (ctrl); } -static void nvme_rdma_reconnect_or_remove (struct nvme_rdma_ctrl *ctrl) +static void nvme_rdma_reconnect_or_remove (struct nvme_rdma_ctrl *ctrl,int status) { +bool recon = true; +/* If we are resetting/deleting then do nothing */ if (ctrl->ctrl.state !=NVME_CTRL_CONNECTING ) { WARN_ON_ONCE(ctrl->ctrl.state == NVME_CTRL_NEW ||
@@ -1074,7 +1076,12 @@ static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)return ; } -if (nvmf_should_reconnect (&ctrl->ctrl)) { +if (status >0 && (status &NVME_SC_DNR )) { +dev_info (ctrl->ctrl.device,"reconnect failure %d\n" , status); + recon = false; + } + +if (recon &&nvmf_should_reconnect (&ctrl->ctrl)) {dev_info (ctrl->ctrl.device,"Reconnecting in %d seconds...\n" , ctrl->ctrl.opts->reconnect_delay); queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
@@ -1173,10 +1180,12 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) {struct nvme_rdma_ctrl *ctrl =container_of (to_delayed_work (work), struct nvme_rdma_ctrl, reconnect_work); +int ret; ++ctrl->ctrl.nr_reconnects; -if (nvme_rdma_setup_ctrl (ctrl, false)) + ret =nvme_rdma_setup_ctrl (ctrl, false); +if (ret) goto requeue; dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
@@ -1189,7 +1198,7 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) requeue:dev_info (ctrl->ctrl.device,"Failed reconnect attempt %d\n" , ctrl->ctrl.nr_reconnects); -nvme_rdma_reconnect_or_remove (ctrl); +nvme_rdma_reconnect_or_remove (ctrl, ret); }static void nvme_rdma_error_recovery_work (struct work_struct *work)
@@ -1212,7 +1221,7 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)return ; } -nvme_rdma_reconnect_or_remove (ctrl); +nvme_rdma_reconnect_or_remove (ctrl, -ENOTCONN ); }static void nvme_rdma_error_recovery (struct nvme_rdma_ctrl *ctrl)
@@ -2274,6 +2283,7 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work) {struct nvme_rdma_ctrl *ctrl = container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work); +int ret;nvme_stop_ctrl (&ctrl->ctrl);nvme_rdma_shutdown_ctrl (ctrl, false);
@@ -2284,14 +2294,15 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)return ; } -if (nvme_rdma_setup_ctrl (ctrl, false)) + ret =nvme_rdma_setup_ctrl (ctrl, false); +if (ret) goto out_fail;return ; out_fail: ++ctrl->ctrl.nr_reconnects; -nvme_rdma_reconnect_or_remove (ctrl); +nvme_rdma_reconnect_or_remove (ctrl, ret); }static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
Signature · 1 line
2.29.2
Sagi Grimberg
maintainerRe: [PATCH 1/2] nvme-tcp: short-circuit connect retries
In reply to Hannes ReineckeQuoted 29 lines
When a reconnect attempt fails with a non-retryable status (eg when the subsystem has been unprovisioned) there hardly is any reason to retry the reconnect attempt.
So pass the actual error status tonvme_tcp_reconnect_or_remove()and short-circuit retries if the DNR bit is set.Signed-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/host/tcp.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-)drivers/nvme/host/tcp.c +3
-1@@ -2065,8 +2065,10 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,nvme_tcp_destroy_io_queues (ctrl, remove); } -static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl) +static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl,int status) { +bool recon = true; +/* If we are resetting/deleting then do nothing */ if (ctrl->state !=NVME_CTRL_CONNECTING ) { WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
drivers/nvme/host/tcp.c static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) · line 2083
Quoted 9 lines
drivers/nvme/host/tcp.c +5
-1@@ -2074,7 +2076,12 @@ static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)return ; } -if (nvmf_should_reconnect (ctrl)) { +if (status >0 && (status &NVME_SC_DNR )) { +dev_info (ctrl->device,"reconnect failure %d\n" , status); + recon = false; + } +
Why should we call this if we need to remove for sure? I suggest we just change the call-site.
Quoted 6 lines
drivers/nvme/host/tcp.c +1
-0+if (recon &&nvmf_should_reconnect (ctrl)) {dev_info (ctrl->device,"Reconnecting in %d seconds...\n" , ctrl->opts->reconnect_delay); queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,@@ -2162,10 +2169,12 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
drivers/nvme/host/tcp.c static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) · line 2178
Quoted 10 lines
drivers/nvme/host/tcp.c +3
-1struct nvme_tcp_ctrl, connect_work);struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; +int ret; ++ctrl->nr_reconnects; -if (nvme_tcp_setup_ctrl (ctrl, false)) + ret =nvme_tcp_setup_ctrl (ctrl, false); +if (ret) goto requeue;
status = nvme_tcp_setup_ctrl(ctrl, false);
if (!status) {
dev_info(ctrl->device,
"Successfully reconnected (%d attempt)\n",
ctrl->nr_reconnects);
ctrl->nr_reconnects = 0;
} else {
dev_info(ctrl->device,
"Failed reconnect attempt %d (%d)\n",
ctrl->nr_reconnects, status);
if (status > 0 && (status & NVME_SC_DNR)) {
dev_info(ctrl->device,
"Removing controller...\n");
nvme_delete_ctrl(ctrl);
} else {
nvme_tcp_reconnect_or_remove(ctrl);
}
}
}Quoted 12 lines
drivers/nvme/host/tcp.c +1
-1dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",@@ -2178,7 +2187,7 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) requeue:dev_info (ctrl->device,"Failed reconnect attempt %d\n" , ctrl->nr_reconnects); -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, ret); }static void nvme_tcp_error_recovery_work (struct work_struct *work)
drivers/nvme/host/tcp.c static void nvme_tcp_error_recovery_work(struct work_struct *work) · line 2215
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-1@@ -2203,7 +2212,7 @@ static void nvme_tcp_error_recovery_work(struct work_struct *work)return ; } -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, -ENOTCONN );
This emphasizes that this is a redundant parameter.
Quoted 12 lines
drivers/nvme/host/tcp.c +1
-0}static void nvme_tcp_teardown_ctrl (struct nvme_ctrl *ctrl,bool shutdown)@@ -2229,6 +2238,7 @@ static void nvme_reset_ctrl_work(struct work_struct *work) {struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, reset_work); +int ret;nvme_stop_ctrl (ctrl);nvme_tcp_teardown_ctrl (ctrl, false);
drivers/nvme/host/tcp.c static void nvme_reset_ctrl_work(struct work_struct *work) · line 2255
Quoted 7 lines
drivers/nvme/host/tcp.c +2
-1@@ -2240,14 +2250,15 @@ static void nvme_reset_ctrl_work(struct work_struct *work)return ; } -if (nvme_tcp_setup_ctrl (ctrl, false)) + ret =nvme_tcp_setup_ctrl (ctrl, false); +if (ret) goto out_fail;
Similar error handling here.
status = nvme_tcp_setup_ctrl(ctrl, false);
if (status) {
++ctrl->nr_reconnects;
if (status > 0 && (status & NVME_SC_DNR)) {
dev_info(ctrl->device,
"Removing controller...\n");
nvme_delete_ctrl(ctrl);
} else {
nvme_tcp_reconnect_or_remove(ctrl);
}
}Quoted 10 lines
drivers/nvme/host/tcp.c +1
-1return ; out_fail: ++ctrl->nr_reconnects; -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, ret); }static void nvme_tcp_free_ctrl (struct nvme_ctrl *nctrl)
Re: [PATCH 1/2] nvme-tcp: short-circuit connect retries
Quoted 70 lines · Sagi Grimberg
Quoted 30 lines
When a reconnect attempt fails with a non-retryable status (eg when the subsystem has been unprovisioned) there hardly is any reason to retry the reconnect attempt.
So pass the actual error status tonvme_tcp_reconnect_or_remove()and short-circuit retries if the DNR bit is set.Signed-off-by Hannes Reinecke <[email protected]>---
drivers/nvme/host/tcp.c| 25 ++++++++++++++++++-------1 file changed, 18 insertions(+), 7 deletions(-)drivers/nvme/host/tcp.c +3
-1@@ -2065,8 +2065,10 @@ static void nvme_tcp_teardown_io_queues(structnvme_ctrl *ctrl, nvme_tcp_destroy_io_queues(ctrl, remove); } -static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl) +static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl,int status) { + bool recon = true; + /* If we are resetting/deleting then do nothing */ if (ctrl->state != NVME_CTRL_CONNECTING) {WARN_ON_ONCE(ctrl->state ==
NVME_CTRL_NEW||drivers/nvme/host/tcp.c static void nvme_tcp_reconnect_or_remove(struct · line 2083
Quoted 9 lines
drivers/nvme/host/tcp.c +5
-1@@ -2074,7 +2076,12 @@ static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) return; } - if (nvmf_should_reconnect(ctrl)) { + if (status > 0 && (status & NVME_SC_DNR)) { + dev_info(ctrl->device, "reconnect failure %d\n", status); + recon = false; + } +Why should we call this if we need to remove for sure? I suggest we just change the call-site.
Quoted 5 lines
drivers/nvme/host/tcp.c +1
-0+ if (recon && nvmf_should_reconnect(ctrl)) { dev_info(ctrl->device, "Reconnecting in %d seconds...\n", ctrl->opts->reconnect_delay); queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,drivers/nvme/host/tcp.c static void · line 2177
Quoted 10 lines
drivers/nvme/host/tcp.c +3
-1@@ -2162,10 +2169,12 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), struct nvme_tcp_ctrl, connect_work); struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; + int ret; ++ctrl->nr_reconnects; - if (nvme_tcp_setup_ctrl(ctrl, false)) + ret = nvme_tcp_setup_ctrl(ctrl, false); + if (ret) goto requeue;status =
nvme_tcp_setup_ctrl(ctrl, false); if (!status) {
dev_info(ctrl->device,
"Successfully reconnected (%d attempt)\n",ctrl->nr_reconnects);ctrl->nr_reconnects= 0;
} else {
dev_info(ctrl->device,
"Failed reconnect attempt %d (%d)\n",ctrl->nr_reconnects, status); if (status > 0 && (status &NVME_SC_DNR)) {
Quoted 9 lines
dev_info(ctrl->device,
"Removing controller...\n");
nvme_delete_ctrl(ctrl);
} else {
nvme_tcp_reconnect_or_remove(ctrl);
}
}
}
Good point.
Quoted 10 lines
Quoted 10 lines
drivers/nvme/host/tcp.c +0
-0dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",@@ -2178,7 +2187,7 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)requeue:
dev_info(ctrl->device, "Failed reconnect attempt %d\n", ctrl->nr_reconnects);
-nvme_tcp_reconnect_or_remove(ctrl);
+nvme_tcp_reconnect_or_remove(ctrl, ret);
}
static voidnvme_tcp_error_recovery_work(struct work_struct *work)
Quoted 9 lines
drivers/nvme/host/tcp.c static void nvme_tcp_error_recovery_work(struct · line 2215
Quoted 6 lines
drivers/nvme/host/tcp.c +1
-1@@ -2203,7 +2212,7 @@ static void nvme_tcp_error_recovery_work(struct work_struct *work) return; } - nvme_tcp_reconnect_or_remove(ctrl); + nvme_tcp_reconnect_or_remove(ctrl, -ENOTCONN);This emphasizes that this is a redundant parameter.
Ok, will be reworking it.
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), Geschäftsführer: Ivo Totev, Andrew Myers, Andrew McDonald, Martje Boudien Moerman