nvme: various bugs fix & code cleanup
v2 cover nvme: various bugs fix & code cleanup
From: "Chunguang.xu" <[email protected]>
Here try to fix the hang issue of nvme-rdma, memory leak issue of nvme-tcp, cleanup the code of nvme-tcp.
Chunguang.xu (5):
nvme-tcp: fix the memleak while create new ctrl failed
nvme-rdma: unquiesce admin_q before destroy it
nvme-tcp: no need to quiesec admin_q in nvme_tcp_teardown_io_queues()
nvme-tcp: simplify nvme_tcp_configure_admin_queue()
nvme-tcp: remove nvme_tcp_destroy_io_queues()
drivers/nvme/host/rdma.c | 8 +------ drivers/nvme/host/tcp.c | 49 ++++++++++++++++------------------------ 2 files changed, 20 insertions(+), 37 deletions(-)
Signature · 1 line
2.25.1
PATCH v2 1/5 nvme-tcp: fix the memleak while create new ctrl failed
From: "Chunguang.xu" <[email protected]>
Now while we create new ctrl failed, we have not free the tagset occupied by admin_q, here try to fix it.
--- drivers/nvme/host/tcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/nvme/host/tcp.c +1 -1
@@ -2278,7 +2278,7 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) } destroy_admin:nvme_stop_keep_alive (ctrl); -nvme_tcp_teardown_admin_queue (ctrl, false); +nvme_tcp_teardown_admin_queue (ctrl, new);return ret; }
Signature · 1 line
2.25.1
Re: [PATCH v2 1/5] nvme-tcp: fix the memleak while create new ctrl failed
Looks good:
PATCH v2 2/5 nvme-rdma: unquiesce admin_q before destroy it
In reply to brookxu.cnFrom: "Chunguang.xu" <[email protected]>
Kernel will hang on destroy admin_q while we create ctrl failed, such as following calltrace:
PID: 23644 TASK: ff2d52b40f439fc0 CPU: 2 COMMAND: "nvme"
#0 [ff61d23de260fb78] __schedule at ffffffff8323bc15
#1 [ff61d23de260fc08] schedule at ffffffff8323c014
#2 [ff61d23de260fc28] blk_mq_freeze_queue_wait at ffffffff82a3dba1
#3 [ff61d23de260fc78] blk_freeze_queue at ffffffff82a4113a
#4 [ff61d23de260fc90] blk_cleanup_queue at ffffffff82a33006
#5 [ff61d23de260fcb0] nvme_rdma_destroy_admin_queue at ffffffffc12686ce [nvme_rdma] #6 [ff61d23de260fcc8] nvme_rdma_setup_ctrl at ffffffffc1268ced [nvme_rdma] #7 [ff61d23de260fd28] nvme_rdma_create_ctrl at ffffffffc126919b [nvme_rdma] #8 [ff61d23de260fd68] nvmf_dev_write at ffffffffc024f362 [nvme_fabrics] #9 [ff61d23de260fe38] vfs_write at ffffffff827d5f25
RIP: 00007fda7891d574 RSP: 00007ffe2ef06958 RFLAGS: 00000202
RAX: ffffffffffffffda RBX: 000055e8122a4d90 RCX: 00007fda7891d574
RDX: 000000000000012b RSI: 000055e8122a4d90 RDI: 0000000000000004
RBP: 00007ffe2ef079c0 R8: 000000000000012b R9: 000055e8122a4d90
R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000004
R13: 000055e8122923c0 R14: 000000000000012b R15: 00007fda78a54500
ORIG_RAX: 0000000000000001 CS: 0033 SS: 002bThis due to we have quiesced admi_q before cancel requests, but forgot to unquiesce before destroy it, as a result we fail to drain the pending requests, and hang on blk_mq_freeze_queue_wait() forever. Here try to reuse nvme_rdma_teardown_admin_queue() to fix this issue and simplify the code.
--- drivers/nvme/host/rdma.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
drivers/nvme/host/rdma.c +1 -7
@@ -1091,13 +1091,7 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new) } destroy_admin:nvme_stop_keep_alive (&ctrl->ctrl); -nvme_quiesce_admin_queue (&ctrl->ctrl); -blk_sync_queue (ctrl->ctrl.admin_q); -nvme_rdma_stop_queue (&ctrl->queues[0 ]); -nvme_cancel_admin_tagset (&ctrl->ctrl); -if (new) -nvme_remove_admin_tag_set (&ctrl->ctrl); -nvme_rdma_destroy_admin_queue (ctrl); +nvme_rdma_teardown_admin_queue (ctrl, new);return ret; }
Signature · 1 line
2.25.1
Re: [PATCH v2 2/5] nvme-rdma: unquiesce admin_q before destroy it
Looks good:
PATCH v2 3/5 nvme-tcp: no need to quiesec admin_q in nvme_tcp_teardown_io_queues()
In reply to brookxu.cnFrom: "Chunguang.xu" <[email protected]>
As we quiesec admin_q in nvme_tcp_teardown_admin_queue(), so we should no need to quiesec it in nvme_tcp_reaardown_io_queues(), make things simple.
--- drivers/nvme/host/tcp.c | 1 - 1 file changed, 1 deletion(-)
drivers/nvme/host/tcp.c +0 -1
@@ -2178,7 +2178,6 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, {if (ctrl->queue_count <=1 )return ; -nvme_quiesce_admin_queue (ctrl);nvme_quiesce_io_queues (ctrl);nvme_sync_io_queues (ctrl);nvme_tcp_stop_io_queues (ctrl);
Signature · 1 line
2.25.1
Re: [PATCH v2 3/5] nvme-tcp: no need to quiesec admin_q in nvme_tcp_teardown_io_queues()
Looks good:
PATCH v2 4/5 nvme-tcp: simplify nvme_tcp_configure_admin_queue()
In reply to brookxu.cnFrom: "Chunguang.xu" <[email protected]>
As nvme_tcp_configure_admin_queue() is the only one caller of nvme_tcp_destroy_admin_queue(), so we can merge nvme_tcp_configure_admin_queue() into nvme_tcp_destroy_admin_queue() to simplify the code.
--- drivers/nvme/host/tcp.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-)
drivers/nvme/host/tcp.c +4 -10
@@ -2101,14 +2101,6 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)return ret; } -static void nvme_tcp_destroy_admin_queue (struct nvme_ctrl *ctrl,bool remove) -{ -nvme_tcp_stop_queue (ctrl,0 ); -if (remove) -nvme_remove_admin_tag_set (ctrl); -nvme_tcp_free_admin_queue (ctrl); -} -static int nvme_tcp_configure_admin_queue (struct nvme_ctrl *ctrl,bool new) {int error;
@@ -2163,9 +2155,11 @@ static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,blk_sync_queue (ctrl->admin_q);nvme_tcp_stop_queue (ctrl,0 );nvme_cancel_admin_tagset (ctrl); -if (remove) +if (remove) {nvme_unquiesce_admin_queue (ctrl); -nvme_tcp_destroy_admin_queue (ctrl, remove); +nvme_remove_admin_tag_set (ctrl); + } +nvme_tcp_free_admin_queue (ctrl);if (ctrl->tls_pskid) {dev_dbg (ctrl->device,"Wipe negotiated TLS_PSK %08x\n" , ctrl->tls_pskid);
Signature · 1 line
2.25.1
Re: [PATCH v2 4/5] nvme-tcp: simplify nvme_tcp_configure_admin_queue()
Quoted 5 lines · brookxu.cn
From: "Chunguang.xu" <[email protected]>
As
nvme_tcp_configure_admin_queue()is the only one caller ofnvme_tcp_destroy_admin_queue(), so we can mergenvme_tcp_configure_admin_queue()intonvme_tcp_destroy_admin_queue()to simplify the code.
Need a little fixing for the line length here in the commit message, but otherwise looks good:
PATCH v2 5/5 nvme-tcp: remove nvme_tcp_destroy_io_queues()
In reply to brookxu.cnFrom: "Chunguang.xu" <[email protected]>
Now when destroying the IO queue we call nvme_tcp_stop_io_queues() twice, nvme_tcp_destroy_io_queues() has an unnecessary call. Here we try to remove nvme_tcp_destroy_io_queues() and merge it into nvme_tcp_teardown_io_queues(), simplify the code and align with nvme-rdma, make it easy to maintaince.
--- drivers/nvme/host/tcp.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-)
drivers/nvme/host/tcp.c +14 -18
@@ -2024,14 +2024,6 @@ static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)return __nvme_tcp_alloc_io_queues (ctrl); } -static void nvme_tcp_destroy_io_queues (struct nvme_ctrl *ctrl,bool remove) -{ -nvme_tcp_stop_io_queues (ctrl); -if (remove) -nvme_remove_io_tag_set (ctrl); -nvme_tcp_free_io_queues (ctrl); -} -static int nvme_tcp_configure_io_queues (struct nvme_ctrl *ctrl,bool new) {int ret, nr_queues;
@@ -2170,15 +2162,17 @@ static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,static void nvme_tcp_teardown_io_queues (struct nvme_ctrl *ctrl,bool remove) { -if (ctrl->queue_count <=1 ) -return ; -nvme_quiesce_io_queues (ctrl); -nvme_sync_io_queues (ctrl); -nvme_tcp_stop_io_queues (ctrl); -nvme_cancel_tagset (ctrl); -if (remove) -nvme_unquiesce_io_queues (ctrl); -nvme_tcp_destroy_io_queues (ctrl, remove); +if (ctrl->queue_count >1 ) { +nvme_quiesce_io_queues (ctrl); +nvme_sync_io_queues (ctrl); +nvme_tcp_stop_io_queues (ctrl); +nvme_cancel_tagset (ctrl); +if (remove) { +nvme_unquiesce_io_queues (ctrl); +nvme_remove_io_tag_set (ctrl); + } +nvme_tcp_free_io_queues (ctrl); + } } static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl,
@@ -2267,7 +2261,9 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)nvme_sync_io_queues (ctrl);nvme_tcp_stop_io_queues (ctrl);nvme_cancel_tagset (ctrl); -nvme_tcp_destroy_io_queues (ctrl, new); +if (new) +nvme_remove_io_tag_set (ctrl); +nvme_tcp_free_io_queues (ctrl); } destroy_admin:nvme_stop_keep_alive (ctrl);
Signature · 1 line
2.25.1
Christoph Hellwig
maintainerRe: [PATCH v2 5/5] nvme-tcp: remove nvme_tcp_destroy_io_queues()
Quoted 6 lines · brookxu.cn
From: "Chunguang.xu" <[email protected]>
Now when destroying the IO queue we call
nvme_tcp_stop_io_queues()twice,nvme_tcp_destroy_io_queues()has an unnecessary call. Here we try to removenvme_tcp_destroy_io_queues()and merge it intonvme_tcp_teardown_io_queues(), simplify the code and align with nvme-rdma, make it easy to maintaince.
Please split the reorganization from the fix.
Also can you add Fixes tag to the various bug fix patches?