nvme-fabrics: short-circuit connect retries
v2 cover nvme-fabrics: short-circuit connect retries
I've picked up Hannes' DNR patches. In short the make the transports behave the same way when the DNR bit set on a re-connect attempt. We had a discussion this topic in the past and if I got this right we all agreed is that the host should honor the DNR bit on a connect attempt [1]
The nvme/045 test case (authentication tests) in blktests is a good test case for this after extending it slightly. TCP and RDMA try to reconnect with an invalid key over and over again, while loop and FC stop after the first fail.
[1] https://lore.kernel.org/linux-nvme/[email protected]/
changes:
v2:
- refresh/rebase on current head
- extended blktests (nvme/045) to cover this case
(see separate post)
v1:
- initial version
- https://lore.kernel.org/linux-nvme/[email protected]/
Hannes Reinecke (2):
nvme-tcp: short-circuit reconnect retries
nvme-rdma: short-circuit reconnect retries
drivers/nvme/host/rdma.c | 22 +++++++++++++++------- drivers/nvme/host/tcp.c | 21 +++++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-)
Signature · 1 line
2.44.0
PATCH v2 1/2 nvme-tcp: short-circuit reconnect retries
From: Hannes Reinecke <[email protected]>
Returning an nvme status from nvme_tcp_setup_ctrl() indicates that the association was established and we have received a status from the controller; consequently we should honour the DNR bit. If not any future reconnect attempts will just return the same error, so we can short-circuit the reconnect attempts and fail the connection directly.
--- drivers/nvme/host/tcp.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)
drivers/nvme/host/tcp.c +13 -8
@@ -2149,9 +2149,10 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,nvme_tcp_destroy_io_queues (ctrl, remove); } -static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl) +static void nvme_tcp_reconnect_or_remove (struct nvme_ctrl *ctrl,int status) {enum nvme_ctrl_state state =nvme_ctrl_state (ctrl); +bool recon = status >0 && (status &NVME_SC_DNR ) ? false : true;/* If we are resetting/deleting then do nothing */ if (state !=NVME_CTRL_CONNECTING ) {
@@ -2159,13 +2160,13 @@ static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)return ; } -if (nvmf_should_reconnect (ctrl)) { +if (recon &&nvmf_should_reconnect (ctrl)) {dev_info (ctrl->device,"Reconnecting in %d seconds...\n" , ctrl->opts->reconnect_delay);queue_delayed_work (nvme_wq, &to_tcp_ctrl (ctrl)->connect_work, ctrl->opts->reconnect_delay *HZ ); }else { -dev_info (ctrl->device,"Removing controller...\n" ); +dev_info (ctrl->device,"Removing controller (%d)...\n" , status);nvme_delete_ctrl (ctrl); } }
@@ -2246,10 +2247,12 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)struct nvme_tcp_ctrl *tcp_ctrl =container_of (to_delayed_work (work), struct nvme_tcp_ctrl, connect_work);struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; +int ret; ++ctrl->nr_reconnects; -if (nvme_tcp_setup_ctrl (ctrl, false)) + ret =nvme_tcp_setup_ctrl (ctrl, false); +if (ret) goto requeue; dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
@@ -2262,7 +2265,7 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) requeue:dev_info (ctrl->device,"Failed reconnect attempt %d\n" , ctrl->nr_reconnects); -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, ret); }static void nvme_tcp_error_recovery_work (struct work_struct *work)
@@ -2289,7 +2292,7 @@ static void nvme_tcp_error_recovery_work(struct work_struct *work)return ; } -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, -ENOTCONN ); }static void nvme_tcp_teardown_ctrl (struct nvme_ctrl *ctrl,bool shutdown)
@@ -2309,6 +2312,7 @@ static void nvme_reset_ctrl_work(struct work_struct *work) {struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, reset_work); +int ret;nvme_stop_ctrl (ctrl);nvme_tcp_teardown_ctrl (ctrl, false);
@@ -2322,14 +2326,15 @@ static void nvme_reset_ctrl_work(struct work_struct *work)return ; } -if (nvme_tcp_setup_ctrl (ctrl, false)) + ret =nvme_tcp_setup_ctrl (ctrl, false); +if (ret) goto out_fail;return ; out_fail: ++ctrl->nr_reconnects; -nvme_tcp_reconnect_or_remove (ctrl); +nvme_tcp_reconnect_or_remove (ctrl, ret); }static void nvme_tcp_stop_ctrl (struct nvme_ctrl *ctrl)
Signature · 1 line
2.44.0
Christoph Hellwig
maintainerRe: [PATCH v2 1/2] nvme-tcp: short-circuit reconnect retries
drivers/nvme/host/tcp.c static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, · line 2155
Quoted 1 line
drivers/nvme/host/tcp.c +1
-0+bool recon = status >0 && (status &NVME_SC_DNR ) ? false : true;
This is screaming for a (well-documented) helper.
PATCH v2 2/2 nvme-rdma: short-circuit reconnect retries
In reply to Daniel WagnerFrom: Hannes Reinecke <[email protected]>
Returning an nvme status from nvme_rdma_setup_ctrl() indicates that the association was established and we have received a status from the controller; consequently we should honour the DNR bit. If not any future reconnect attempts will just return the same error, so we can short-circuit the reconnect attempts and fail the connection directly.
--- drivers/nvme/host/rdma.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-)
drivers/nvme/host/rdma.c +15 -7
@@ -982,9 +982,11 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)kfree (ctrl); } -static void nvme_rdma_reconnect_or_remove (struct nvme_rdma_ctrl *ctrl) +static void nvme_rdma_reconnect_or_remove (struct nvme_rdma_ctrl *ctrl, +int status) {enum nvme_ctrl_state state =nvme_ctrl_state (&ctrl->ctrl); +bool recon = status >0 && (status &NVME_SC_DNR ) ? false : true;/* If we are resetting/deleting then do nothing */ if (state !=NVME_CTRL_CONNECTING ) {
@@ -992,12 +994,14 @@ static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)return ; } -if (nvmf_should_reconnect (&ctrl->ctrl)) { +if (recon &&nvmf_should_reconnect (&ctrl->ctrl)) {dev_info (ctrl->ctrl.device,"Reconnecting in %d seconds...\n" , ctrl->ctrl.opts->reconnect_delay);queue_delayed_work (nvme_wq, &ctrl->reconnect_work, ctrl->ctrl.opts->reconnect_delay *HZ ); }else { +dev_info (ctrl->ctrl.device,"Removing controller (%d)...\n" , + status);nvme_delete_ctrl (&ctrl->ctrl); } }
@@ -1098,10 +1102,12 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) {struct nvme_rdma_ctrl *ctrl =container_of (to_delayed_work (work), struct nvme_rdma_ctrl, reconnect_work); +int ret; ++ctrl->ctrl.nr_reconnects; -if (nvme_rdma_setup_ctrl (ctrl, false)) + ret =nvme_rdma_setup_ctrl (ctrl, false); +if (ret) goto requeue; dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
@@ -1114,7 +1120,7 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) requeue:dev_info (ctrl->ctrl.device,"Failed reconnect attempt %d\n" , ctrl->ctrl.nr_reconnects); -nvme_rdma_reconnect_or_remove (ctrl); +nvme_rdma_reconnect_or_remove (ctrl, ret); }static void nvme_rdma_error_recovery_work (struct work_struct *work)
@@ -1139,7 +1145,7 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)return ; } -nvme_rdma_reconnect_or_remove (ctrl); +nvme_rdma_reconnect_or_remove (ctrl, -ENOTCONN ); }static void nvme_rdma_error_recovery (struct nvme_rdma_ctrl *ctrl)
@@ -2163,6 +2169,7 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work) {struct nvme_rdma_ctrl *ctrl = container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work); +int ret;nvme_stop_ctrl (&ctrl->ctrl);nvme_rdma_shutdown_ctrl (ctrl, false);
@@ -2173,14 +2180,15 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)return ; } -if (nvme_rdma_setup_ctrl (ctrl, false)) + ret =nvme_rdma_setup_ctrl (ctrl, false); +if (ret) goto out_fail;return ; out_fail: ++ctrl->ctrl.nr_reconnects; -nvme_rdma_reconnect_or_remove (ctrl); +nvme_rdma_reconnect_or_remove (ctrl, ret); }static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
Signature · 1 line
2.44.0
Keith Busch
maintainerQuoted 4 lines · Daniel Wagner
I've picked up Hannes' DNR patches. In short the make the transports behave the same way when the DNR bit set on a re-connect attempt. We had a discussion this topic in the past and if I got this right we all agreed is that the host should honor the DNR bit on a connect attempt [1]
These look good to me. I think, though, if you're submitting a patch on behalf of another, you should append your 'Signed-off-by' tag after the author's.
Quoted 9 lines · Keith Busch
Quoted 4 lines · Daniel Wagner
I've picked up Hannes' DNR patches. In short the make the transports behave the same way when the DNR bit set on a re-connect attempt. We had a discussion this topic in the past and if I got this right we all agreed is that the host should honor the DNR bit on a connect attempt [1]
These look good to me. I think, though, if you're submitting a patch on behalf of another, you should append your 'Signed-off-by' tag after the author's.
Will do. Yesterday was not my best git day for a while...