nvme-fc: fix module unloads while lports still pending
Subsystems (2)
PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending
Current code allows the module to be unloaded even if there are pending data structures, such as localports and controllers on the localports, that have yet to hit their reference counting to remove them.
Fix by having exit entrypoint explicitly delete every controller, which in turn will remove references on the remoteports and localports causing them to be deleted as well. The exit entrypoint, after initiating the deletes, will wait for the last localport to be deleted before continuing.
---
V2: replaced self-module references with code that actually deletes
the outstanding structures.
--- drivers/nvme/host/fc.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-)
drivers/nvme/host/fc.c +53 -2
@@ -204,6 +204,9 @@ static DEFINE_IDA(nvme_fc_ctrl_cnt);static struct workqueue_struct *nvme_fc_wq; +static bool nvme_fc_waiting_to_unload; +struct completion nvme_fc_unload_proceed; +/* * These items are short-term. They will eventually be moved into * a generic FC class. See comments in module init.
@@ -229,6 +232,8 @@ nvme_fc_free_lport(struct kref *ref)/* remove from transport list */ spin_lock_irqsave (&nvme_fc_lock, flags);list_del (&lport->port_list); +if (nvme_fc_waiting_to_unload &&list_empty (&nvme_fc_lport_list)) +complete (&nvme_fc_unload_proceed);spin_unlock_irqrestore (&nvme_fc_lock, flags);ida_simple_remove (&nvme_fc_local_port_cnt, lport->localport.port_num);
@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref)put_device (lport->dev);kfree (lport); + } static void
@@ -3444,6 +3450,8 @@ static int __init nvme_fc_init_module(void)if (ret) goto out_destroy_device; + nvme_fc_waiting_to_unload = false; +return 0 ; out_destroy_device:
@@ -3456,11 +3464,54 @@ static int __init nvme_fc_init_module(void)return ret; } +static void +nvme_fc_delete_controllers (struct nvme_fc_rport *rport) +{ +struct nvme_fc_ctrl *ctrl; + +spin_lock (&rport->lock); +list_for_each_entry (ctrl, &rport->ctrl_list, ctrl_list) { +dev_warn (ctrl->ctrl.device, +"NVME-FC{%d}: transport unloading: deleting ctrl\n" , + ctrl->cnum); +nvme_delete_ctrl (&ctrl->ctrl); + } +spin_unlock (&rport->lock); +} + +static void +nvme_fc_cleanup_for_unload (void ) +{ +struct nvme_fc_lport *lport; +struct nvme_fc_rport *rport; +unsigned long flags; + +spin_lock_irqsave (&nvme_fc_lock, flags); +list_for_each_entry (lport, &nvme_fc_lport_list, port_list) { +list_for_each_entry (rport, &lport->endp_list, endp_list) { +nvme_fc_delete_controllers (rport); + } + } +spin_unlock_irqrestore (&nvme_fc_lock, flags); + +pr_info ("%s: waiting for ctlr deletes\n" , __func__); +wait_for_completion (&nvme_fc_unload_proceed); +pr_info ("%s: ctrl deletes complete\n" , __func__); +} + static void __exit nvme_fc_exit_module(void) { -/* sanity check - all lports should be removed */ +unsigned long flags; +bool need_cleanup = false; + +spin_lock_irqsave (&nvme_fc_lock, flags); + nvme_fc_waiting_to_unload = true; +init_completion (&nvme_fc_unload_proceed);if (!list_empty (&nvme_fc_lport_list)) -pr_warn ("%s: localport list not empty\n" , __func__); + need_cleanup = true; +spin_unlock_irqrestore (&nvme_fc_lock, flags); +if (need_cleanup) +nvme_fc_cleanup_for_unload ();nvmf_unregister_transport (&nvme_fc_transport);
Signature · 1 line
2.13.7
PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending
drivers/nvme/host/fc.c nvme_fc_free_lport(struct kref *ref) · line 246
Quoted 6 lines · James Smart
drivers/nvme/host/fc.c +1
-0@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref)put_device (lport->dev);kfree (lport); + }
Spurious whitespace?
Quoted 25 lines
drivers/nvme/host/fc.c +22
-1+static void +nvme_fc_cleanup_for_unload (void ) +{ +struct nvme_fc_lport *lport; +struct nvme_fc_rport *rport; +unsigned long flags; + +spin_lock_irqsave (&nvme_fc_lock, flags); +list_for_each_entry (lport, &nvme_fc_lport_list, port_list) { +list_for_each_entry (rport, &lport->endp_list, endp_list) { +nvme_fc_delete_controllers (rport); + } + } +spin_unlock_irqrestore (&nvme_fc_lock, flags); + +pr_info ("%s: waiting for ctlr deletes\n" , __func__); +wait_for_completion (&nvme_fc_unload_proceed); +pr_info ("%s: ctrl deletes complete\n" , __func__); +} + static void __exit nvme_fc_exit_module(void) { -/* sanity check - all lports should be removed */ +unsigned long flags; +bool need_cleanup = false;
drivers/nvme/host/fc.c static int __init nvme_fc_init_module(void) · line 3514
Quoted 10 lines
drivers/nvme/host/fc.c +8
-1+ +spin_lock_irqsave (&nvme_fc_lock, flags); + nvme_fc_waiting_to_unload = true; +init_completion (&nvme_fc_unload_proceed);if (!list_empty (&nvme_fc_lport_list)) -pr_warn ("%s: localport list not empty\n" , __func__); + need_cleanup = true; +spin_unlock_irqrestore (&nvme_fc_lock, flags); +if (need_cleanup) +nvme_fc_cleanup_for_unload ();
Can the nvme_fc_lport_list list become empty after nvme_fc_exit_module() has unlocked nvme_fc_lock and before nvme_fc_cleanup_for_unload() is called? If so, shouldn't nvme_fc_cleanup_for_unload() be called unconditionally and shouldn't the wait_for_completion(&nvme_fc_unload_proceed) call depend on whether or not nvme_fc_cleanup_for_unload() has deleted any controllers?
Thanks,
Bart.
PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending
Quoted 8 lines · Bart Van Assche
Review nvme_fc_free_lport(struct kref *ref) · line 245
Quoted 5 lines · James Smart
(quoted diff) +1
-0@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref) ????? put_device(lport->dev); ? ????? kfree(lport); + ? }Spurious whitespace?
yes
Quoted 35 lines
Quoted 35 lines
(quoted diff) +20
-0+static void +nvme_fc_cleanup_for_unload(void) +{ +??? struct nvme_fc_lport *lport; +??? struct nvme_fc_rport *rport; +??? unsigned long flags; + +??? spin_lock_irqsave(&nvme_fc_lock, flags); +??? list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { +??????? list_for_each_entry(rport, &lport->endp_list, endp_list) { +??????????? nvme_fc_delete_controllers(rport); +??????? } +??? } +??? spin_unlock_irqrestore(&nvme_fc_lock, flags); + +??? pr_info("%s: waiting for ctlr deletes\n", __func__); +??? wait_for_completion(&nvme_fc_unload_proceed); +??? pr_info("%s: ctrl deletes complete\n", __func__); +} +? static void
__exitnvme_fc_exit_module(void)
? {
-??? /* sanity check - all lports should be removed */ +??? unsigned long flags;
+??? boolneed_cleanup= false;
+
+???spin_lock_irqsave(&nvme_fc_lock, flags);
+???nvme_fc_waiting_to_unload= true;
+???init_completion(&nvme_fc_unload_proceed); ????? if (!list_empty(&nvme_fc_lport_list)) -???????pr_warn("%s: localport list not empty\n", __func__); +???????need_cleanup= true;
+???spin_unlock_irqrestore(&nvme_fc_lock, flags); +??? if (need_cleanup)
Quoted 8 lines
Quoted 1 line
+???????
nvme_fc_cleanup_for_unload();Can the
nvme_fc_lport_listlist become empty afternvme_fc_exit_module()has unlockednvme_fc_lockand beforenvme_fc_cleanup_for_unload()is called? If so, shouldn'tnvme_fc_cleanup_for_unload()be called unconditionally and shouldn't thewait_for_completion(&nvme_fc_unload_proceed)call depend on whether or notnvme_fc_cleanup_for_unload()has deleted any controllers?
It's possible which is why I init'd the completion while under the lock.?? Therefore, the removal that would make it empty, which is also under lock, will trigger the completion. In this scenario, the cleanup routine would be an empty list, fall into the wait for completion, which immediately succeeds.
-- james
PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending
Quoted 58 lines · James Smart
Quoted 8 lines · Bart Van Assche
Review nvme_fc_free_lport(struct kref *ref) · line 245
Quoted 5 lines · James Smart
(quoted diff) +1
-0@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref) ????? put_device(lport->dev); ? ????? kfree(lport); + ? }Spurious whitespace?
yes
Quoted 35 lines
Quoted 35 lines
(quoted diff) +20
-0+static void +nvme_fc_cleanup_for_unload(void) +{ +??? struct nvme_fc_lport *lport; +??? struct nvme_fc_rport *rport; +??? unsigned long flags; + +??? spin_lock_irqsave(&nvme_fc_lock, flags); +??? list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { +??????? list_for_each_entry(rport, &lport->endp_list, endp_list) { +??????????? nvme_fc_delete_controllers(rport); +??????? } +??? } +??? spin_unlock_irqrestore(&nvme_fc_lock, flags); + +??? pr_info("%s: waiting for ctlr deletes\n", __func__); +??? wait_for_completion(&nvme_fc_unload_proceed); +??? pr_info("%s: ctrl deletes complete\n", __func__); +} +? static void
__exitnvme_fc_exit_module(void)
? {
-??? /* sanity check - all lports should be removed */ +??? unsigned long flags;
+??? boolneed_cleanup= false;
+
+???spin_lock_irqsave(&nvme_fc_lock, flags);
+???nvme_fc_waiting_to_unload= true;
+???init_completion(&nvme_fc_unload_proceed); ????? if (!list_empty(&nvme_fc_lport_list)) -???????pr_warn("%s: localport list not empty\n", __func__); +???????need_cleanup= true;
+???spin_unlock_irqrestore(&nvme_fc_lock, flags); +??? if (need_cleanup)Quoted 8 lines
Quoted 1 line
+???????
nvme_fc_cleanup_for_unload();Can the
nvme_fc_lport_listlist become empty afternvme_fc_exit_module()has unlockednvme_fc_lockand beforenvme_fc_cleanup_for_unload()is called? If so, shouldn'tnvme_fc_cleanup_for_unload()be called unconditionally and shouldn't thewait_for_completion(&nvme_fc_unload_proceed)call depend on whether or notnvme_fc_cleanup_for_unload()has deleted any controllers?
Quoted 5 lines
It's possible which is why I init'd the completion while under the lock.?? Therefore, the removal that would make it empty, which is also under lock, will trigger the completion. In this scenario, the cleanup routine would be an empty list, fall into the wait for completion, which immediately succeeds.
Hi James,
Have you considered to use DECLARE_COMPLETION()?
Thanks,
Bart.