nvme-multipath: fix leak on try_module_get failure
PATCH 1/2 nvme-multipath: fix leak on try_module_get failure
From: Keith Busch <[email protected]>
We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.
--- drivers/nvme/host/multipath.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
drivers/nvme/host/multipath.c +5 -7
@@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)if (!list_empty (&head->list)) goto out; -if (head->delayed_removal_secs) { -/* -* Ensure that no one could remove this module while the head -* remove work is pending. -*/ -if (!try_module_get (THIS_MODULE )) - goto out; +/* +* Ensure that no one could remove this module while the head +* remove work is pending. +*/ +if (head->delayed_removal_secs &&try_module_get (THIS_MODULE )) {mod_delayed_work (nvme_wq, &head->remove_work, head->delayed_removal_secs *HZ ); }else {
Signature · 1 line
2.47.3
PATCH 2/2 nvme: fix unmatched id's under delayed path deletion
From: Keith Busch <[email protected]>
The NVMe controller is allowed to reuse an NSID for a new namespace after deleting the previous namespace that had been using it. The delayed removal may have the stale namespace head in the subsystem list pending the timer, which would cause the scan to falsely report an ID mismatch error for the new namespace. Flush the pending removal work and retry to resolve this.
--- drivers/nvme/host/core.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/nvme/host/core.c +18 -0
@@ -3966,6 +3966,7 @@ static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,static int nvme_init_ns_head (struct nvme_ns *ns,struct nvme_ns_info *info) { +bool retry =IS_ENABLED (CONFIG_NVME_MULTIPATH );struct nvme_ctrl *ctrl = ns->ctrl;struct nvme_ns_head *head =NULL ;int ret;
@@ -4008,6 +4009,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) ctrl->quirks |=NVME_QUIRK_BOGUS_NID ; } +again:mutex_lock (&ctrl->subsys->lock); head =nvme_find_ns_head (ctrl, info->nsid);if (!head) {
@@ -4033,6 +4035,22 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) goto out_put_ns_head; }if (!nvme_ns_ids_equal (&head->ids, &info->ids)) { +/* +* A newly created namespace can reuse an NSID that was +* previously deleted. If the head has no active paths, +* it is pending delayed removal and still occupying +* this NSID in the subsystem list. Flush the removal +* work to clear the stale head and retry. +*/ +if (retry &&list_empty (&head->list)) { +mutex_unlock (&ctrl->subsys->lock); +flush_delayed_work (&head->remove_work); +nvme_put_ns_head (head); + retry = false; + goto again; + } + +WARN_ONCE (list_empty (&head->list));dev_err (ctrl->device,"IDs don't match for shared namespace %d\n" , info->nsid);
Signature · 1 line
2.47.3
Keith Busch
maintainerRe: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) · line 4053
Quoted 2 lines · Keith Busch
drivers/nvme/host/core.c +2
-0+ +WARN_ONCE (list_empty (&head->list));
Ugh, I know this should be WARN_ON_ONCE()...
Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
In reply to Keith BuschQuoted 43 lines · Keith Busch
From: Keith Busch <[email protected]>
The NVMe controller is allowed to reuse an NSID for a new namespace after deleting the previous namespace that had been using it. The delayed removal may have the stale namespace head in the subsystem list pending the timer, which would cause the scan to falsely report an ID mismatch error for the new namespace. Flush the pending removal work and retry to resolve this.
Signed-off-by Keith Busch <[email protected]>--- drivers/nvme/host/core.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/nvme/host/core.c +8
-0@@ -3966,6 +3966,7 @@ static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,static int nvme_init_ns_head (struct nvme_ns *ns,struct nvme_ns_info *info) { +bool retry =IS_ENABLED (CONFIG_NVME_MULTIPATH );struct nvme_ctrl *ctrl = ns->ctrl;struct nvme_ns_head *head =NULL ;int ret;@@ -4008,6 +4009,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) ctrl->quirks |=NVME_QUIRK_BOGUS_NID ; } +again:mutex_lock (&ctrl->subsys->lock); head =nvme_find_ns_head (ctrl, info->nsid);if (!head) {@@ -4033,6 +4035,22 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) goto out_put_ns_head; }if (!nvme_ns_ids_equal (&head->ids, &info->ids)) { +/* +* A newly created namespace can reuse an NSID that was +* previously deleted. If the head has no active paths, +* it is pending delayed removal and still occupying +* this NSID in the subsystem list. Flush the removal +* work to clear the stale head and retry.
drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) · line 4053
Quoted 10 lines
drivers/nvme/host/core.c +10
-0+*/ +if (retry &&list_empty (&head->list)) { +mutex_unlock (&ctrl->subsys->lock); +flush_delayed_work (&head->remove_work); +nvme_put_ns_head (head); + retry = false; + goto again; + } + +WARN_ONCE (list_empty (&head->list));
We need to replace WARN_ONCE with WARN_ON_ONCE (as you already mentioned in another thread), so with that change applied, this looks good to me:
Christoph Hellwig
maintainerRe: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
In reply to Keith BuschQuoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
The NVMe controller is allowed to reuse an NSID for a new namespace after deleting the previous namespace that had been using it. The delayed removal may have the stale namespace head in the subsystem list pending the timer, which
Overlong lines.
Quoted 14 lines
drivers/nvme/host/core.c +2
-0+bool retry =IS_ENABLED (CONFIG_NVME_MULTIPATH );struct nvme_ctrl *ctrl = ns->ctrl;struct nvme_ns_head *head =NULL ;int ret;@@ -4008,6 +4009,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) ctrl->quirks |=NVME_QUIRK_BOGUS_NID ; } +again:mutex_lock (&ctrl->subsys->lock); head =nvme_find_ns_head (ctrl, info->nsid);if (!head) {@@ -4033,6 +4035,22 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) goto out_put_ns_head;
drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) · line 4045
Quoted 10 lines
drivers/nvme/host/core.c +8
-0}if (!nvme_ns_ids_equal (&head->ids, &info->ids)) { +/* +* A newly created namespace can reuse an NSID that was +* previously deleted. If the head has no active paths, +* it is pending delayed removal and still occupying +* this NSID in the subsystem list. Flush the removal +* work to clear the stale head and retry. +*/ +if (retry &&list_empty (&head->list)) {
I find the retry logic a bit odd and different from other places do in similar areas. What I'd expected is either a "nr_retries" or "did_retry" variable initialized to 0/false, then checked here to be not set (plus the IS_ENABLED() for multipath) and incremented/set below.
But independent of that, the actual logic looks fine.
Keith Busch
maintainerRe: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
Quoted 7 lines · Christoph Hellwig
I find the retry logic a bit odd and different from other places do in similar areas. What I'd expected is either a "
nr_retries" or "did_retry" variable initialized to 0/false, then checked here to be not set (plus theIS_ENABLED()for multipath) and incremented/set below.But independent of that, the actual logic looks fine.
I was able to test this, and it does work when we're specifically blocking on the delayed removal. But there's a different race this doesn't handle: controller A's scan_work may depend on controller B's scan_work to finish first to remove a final reference on the deleted namespace when A is trying to add a newly created namespace that recycled the NSID.
This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the scan_work synthesize a NVME_AER_NOTICE_NS_CHANGED event for every controller in the subsystem, then re-kick their scan work if the scan_work removed anything.
Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
Quoted 16 lines · Keith Busch
Quoted 7 lines · Christoph Hellwig
I find the retry logic a bit odd and different from other places do in similar areas. What I'd expected is either a "
nr_retries" or "did_retry" variable initialized to 0/false, then checked here to be not set (plus theIS_ENABLED()for multipath) and incremented/set below.But independent of that, the actual logic looks fine.
I was able to test this, and it does work when we're specifically blocking on the delayed removal. But there's a different race this doesn't handle: controller A's
scan_workmay depend on controller B'sscan_workto finish first to remove a final reference on the deleted namespace when A is trying to add a newly created namespace that recycled the NSID.
Quoted 4 lines
This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the
scan_worksynthesize aNVME_AER_NOTICE_NS_CHANGEDevent for every controller in the subsystem, then re-kick their scan work if thescan_workremoved anything.
So does your disk when reuse NSID, changes ns ids such as NGUID/UUID/EUI64?
Thanks,
--Nilay
Keith Busch
maintainerRe: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
Quoted 7 lines · Nilay Shroff
Quoted 4 lines · Keith Busch
This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the
scan_worksynthesize aNVME_AER_NOTICE_NS_CHANGEDevent for every controller in the subsystem, then re-kick their scan work if thescan_workremoved anything.So does your disk when reuse NSID, changes ns ids such as NGUID/UUID/EUI64?
Yes. The NSID is put back into the available pool when it's deleted, but the UID associated with it is newly generated upon creation.
Christoph Hellwig
maintainerRe: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion
Quoted 11 lines · Keith Busch
Quoted 7 lines · Nilay Shroff
Quoted 4 lines · Keith Busch
This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the
scan_worksynthesize aNVME_AER_NOTICE_NS_CHANGEDevent for every controller in the subsystem, then re-kick their scan work if thescan_workremoved anything.So does your disk when reuse NSID, changes ns ids such as NGUID/UUID/EUI64?
Yes. The NSID is put back into the available pool when it's deleted, but the UID associated with it is newly generated upon creation.
Yeah, that is one of the two allowed nvme behaviors, and there's a bit to detect if the persistent ids can be reused or not.
Quoted 6 lines · Keith Busch
From: Keith Busch <[email protected]>
We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.
Signed-off-by Keith Busch <[email protected]>
Looks good to me:
Quoted 7 lines · Keith Busch
From: Keith Busch<[email protected]>
We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.
Signed-off-by Keith Busch<[email protected]>---
Feel free to add:
Quoted 4 lines · Keith Busch
From: Keith Busch <[email protected]>
We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.
Looks good:
But this should probably have a Fixes tag?
Quoted 23 lines · Keith Busch
From: Keith Busch <[email protected]>
We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.
Signed-off-by Keith Busch <[email protected]>--- drivers/nvme/host/multipath.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)drivers/nvme/host/multipath.c +0
-5@@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)if (!list_empty (&head->list)) goto out; -if (head->delayed_removal_secs) { -/* -* Ensure that no one could remove this module while the head -* remove work is pending. -*/
drivers/nvme/host/multipath.c void nvme_mpath_remove_disk(struct nvme_ns_head *head) · line 1320
Quoted 10 lines
drivers/nvme/host/multipath.c +5
-2-if (!try_module_get (THIS_MODULE )) - goto out; +/* +* Ensure that no one could remove this module while the head +* remove work is pending. +*/ +if (head->delayed_removal_secs &&try_module_get (THIS_MODULE )) {mod_delayed_work (nvme_wq, &head->remove_work, head->delayed_removal_secs *HZ ); }else {
Are we also missing the module_put() for when we cancel the timer? AFAICS, we only put the module for after the disk is removed.
Maybe we need something like this:
Subject: [PATCH] nvme-multipath: put module reference when delayed removal timer is canceled
The delayed removal timer is canceled when a NS appears. However, we do not put the module reference grabbed in nvme_mpath_remove_disk(), so fix that.
drivers/nvme/host/core.c +2 -1
@@ -4083,7 +4083,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)mutex_unlock (&ctrl->subsys->lock);#ifdef CONFIG_NVME_MULTIPATH -cancel_delayed_work (&head->remove_work); +if (cancel_delayed_work_sync (&head->remove_work)) +module_put (THIS_MODULE );#endif return 0 ;
Keith Busch
maintainerQuoted 2 lines · John Garry
Are we also missing the
module_put()for when we cancel the timer? AFAICS, we only put the module for after the disk is removed.
Yes, I believe you are correct.
Quoted 15 lines
Maybe we need something like this:
Subject: [PATCH] nvme-multipath: put module reference when delayed removal timer is canceled
The delayed removal timer is canceled when a NS appears. However, we do not put the module reference grabbed in
nvme_mpath_remove_disk(), so fix that.Signed-off-by John Garry <[email protected]>drivers/nvme/host/core.c +0
-0
drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, · line 4089
Quoted 8 lines
drivers/nvme/host/core.c +2
-1@@ -4083,7 +4083,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) mutex_unlock(&ctrl->subsys->lock);#ifdef CONFIG_NVME_MULTIPATH -cancel_delayed_work (&head->remove_work); +if (cancel_delayed_work_sync (&head->remove_work)) +module_put (THIS_MODULE );#endif
I don't think we need the _sync() variant here, but the check and module_put() look good.
Quoted 11 lines · Keith Busch
drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, · line 4089
Quoted 9 lines
drivers/nvme/host/core.c +2
-1@@ -4083,7 +4083,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) mutex_unlock(&ctrl->subsys->lock);#ifdef CONFIG_NVME_MULTIPATH -cancel_delayed_work (&head->remove_work); +if (cancel_delayed_work_sync (&head->remove_work)) +module_put (THIS_MODULE );#endif I don't think we need the
_sync()variant here, but the check andmodule_put()look good.
Yeah, I don't think that the _sync is required either.
cheers