detect duplicate cntlids v2
Christoph Hellwig
maintainerHi all,
this series detects duplicate cntlids in a subsystems. It is based on a patch from Hannes.
Changes since v1:
- add a lockdep check for a held lock
- improve printk format
Christoph Hellwig
maintainerPATCH 1/2 nvme: change locking for the per-subsystem controller list
Life becomes a lot simpler if we just use the global nvme_subsystems_lock to protect this list. Given that it is only accessed during controller probing and removal that isn't a scalability problem either.
--- drivers/nvme/host/core.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-)
drivers/nvme/host/core.c +14 -18
@@ -2346,13 +2346,13 @@ static int nvme_active_ctrls(struct nvme_subsystem *subsys)int count =0 ;struct nvme_ctrl *ctrl; -mutex_lock (&subsys->lock); +lockdep_assert_held (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD ) count++; } -mutex_unlock (&subsys->lock);return count; }
@@ -2394,6 +2394,9 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)mutex_lock (&nvme_subsystems_lock); found =__nvme_find_get_subsystem (subsys->subnqn);if (found) { +__nvme_release_subsystem (subsys); + subsys = found; +/* * Verify that the subsystem actually supports multiple * controllers, else bail out.
@@ -2402,14 +2405,10 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {dev_err (ctrl->device,"ignoring ctrl due to duplicate subnqn (%s).\n" , - found->subnqn); -nvme_put_subsystem (found); + subsys->subnqn); ret = -EINVAL ; - goto out_unlock; + goto out_put_subsystem; } - -__nvme_release_subsystem (subsys); - subsys = found; }else { ret =device_add (&subsys->dev);if (ret) {
@@ -2421,23 +2420,20 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)list_add_tail (&subsys->entry, &nvme_subsystems); } - ctrl->subsys = subsys; -mutex_unlock (&nvme_subsystems_lock); -if (sysfs_create_link (&subsys->dev.kobj, &ctrl->device->kobj,dev_name (ctrl->device))) {dev_err (ctrl->device,"failed to create sysfs link from subsystem.\n" ); -/* the transport driver will eventually put the subsystem */ -return -EINVAL ; + goto out_put_subsystem; } -mutex_lock (&subsys->lock); + ctrl->subsys = subsys;list_add_tail (&ctrl->subsys_entry, &subsys->ctrls); -mutex_unlock (&subsys->lock); - +mutex_unlock (&nvme_subsystems_lock);return 0 ; +out_put_subsystem: +nvme_put_subsystem (subsys); out_unlock:mutex_unlock (&nvme_subsystems_lock);put_device (&subsys->dev);
@@ -3695,10 +3691,10 @@ static void nvme_free_ctrl(struct device *dev)__free_page (ctrl->discard_page);if (subsys) { -mutex_lock (&subsys->lock); +mutex_lock (&nvme_subsystems_lock);list_del (&ctrl->subsys_entry); -mutex_unlock (&subsys->lock);sysfs_remove_link (&subsys->dev.kobj,dev_name (ctrl->device)); +mutex_unlock (&nvme_subsystems_lock); } ctrl->ops->free_ctrl (ctrl);
Signature · 1 line
2.20.1
PATCH 1/2 nvme: change locking for the per-subsystem controller list
Quoted 10 lines · Christoph Hellwig
Life becomes a lot simpler if we just use the global
nvme_subsystems_lockto protect this list. Given that it is only accessed during controller probing and removal that isn't a scalability problem either.Signed-off-by Christoph Hellwig <hch at lst.de>--- drivers/nvme/host/core.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-)
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Teamlead Storage & Networking hare at suse.de +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg GF: Felix Imend?rffer, Mary Higgins, Sri Rasiah HRB 21284 (AG N?rnberg)
PATCH 1/2 nvme: change locking for the per-subsystem controller list
In reply to Christoph HellwigQuoted 100 lines · Christoph Hellwig
Life becomes a lot simpler if we just use the global
nvme_subsystems_lockto protect this list. Given that it is only accessed during controller probing and removal that isn't a scalability problem either.Signed-off-by Christoph Hellwig <hch at lst.de>--- drivers/nvme/host/core.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-)drivers/nvme/host/core.c +14
-18@@ -2346,13 +2346,13 @@ static int nvme_active_ctrls(struct nvme_subsystem *subsys)int count =0 ;struct nvme_ctrl *ctrl; -mutex_lock (&subsys->lock); +lockdep_assert_held (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD ) count++; } -mutex_unlock (&subsys->lock);return count; }@@ -2394,6 +2394,9 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)mutex_lock (&nvme_subsystems_lock); found =__nvme_find_get_subsystem (subsys->subnqn);if (found) { +__nvme_release_subsystem (subsys); + subsys = found; +/* * Verify that the subsystem actually supports multiple * controllers, else bail out. @@ -2402,14 +2405,10 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {dev_err (ctrl->device,"ignoring ctrl due to duplicate subnqn (%s).\n" , - found->subnqn); -nvme_put_subsystem (found); + subsys->subnqn); ret = -EINVAL ; - goto out_unlock; + goto out_put_subsystem; } - -__nvme_release_subsystem (subsys); - subsys = found; }else { ret =device_add (&subsys->dev);if (ret) {@@ -2421,23 +2420,20 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)list_add_tail (&subsys->entry, &nvme_subsystems); } - ctrl->subsys = subsys; -mutex_unlock (&nvme_subsystems_lock); -if (sysfs_create_link (&subsys->dev.kobj, &ctrl->device->kobj,dev_name (ctrl->device))) {dev_err (ctrl->device,"failed to create sysfs link from subsystem.\n" ); -/* the transport driver will eventually put the subsystem */ -return -EINVAL ; + goto out_put_subsystem; } -mutex_lock (&subsys->lock); + ctrl->subsys = subsys;list_add_tail (&ctrl->subsys_entry, &subsys->ctrls); -mutex_unlock (&subsys->lock); - +mutex_unlock (&nvme_subsystems_lock);return 0 ; +out_put_subsystem: +nvme_put_subsystem (subsys); out_unlock:mutex_unlock (&nvme_subsystems_lock);put_device (&subsys->dev);@@ -3695,10 +3691,10 @@ static void nvme_free_ctrl(struct device *dev)__free_page (ctrl->discard_page);if (subsys) { -mutex_lock (&subsys->lock); +mutex_lock (&nvme_subsystems_lock);list_del (&ctrl->subsys_entry); -mutex_unlock (&subsys->lock);sysfs_remove_link (&subsys->dev.kobj,dev_name (ctrl->device)); +mutex_unlock (&nvme_subsystems_lock); } ctrl->ops->free_ctrl (ctrl);
Christoph Hellwig
maintainerPATCH 2/2 nvme: validate cntlid during controller initialisation
In reply to Christoph HellwigThe CNTLID value is required to be unique, and we do rely on this for correct operation. So reject any controller for which a non-unique CNTLID has been detected.
Based on a patch from Hannes Reinecke.
--- drivers/nvme/host/core.c | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-)
drivers/nvme/host/core.c +24 -17
@@ -2341,20 +2341,35 @@ static const struct attribute_group *nvme_subsys_attrs_groups[] = { NULL, }; -static int nvme_active_ctrls (struct nvme_subsystem *subsys) +static bool nvme_validate_cntlid (struct nvme_subsystem *subsys, +struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id) { -int count =0 ; -struct nvme_ctrl *ctrl; +struct nvme_ctrl *tmp;lockdep_assert_held (&nvme_subsystems_lock); -list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { -if (ctrl->state !=NVME_CTRL_DELETING && - ctrl->state !=NVME_CTRL_DEAD ) - count++; +list_for_each_entry (tmp, &subsys->ctrls, subsys_entry) { +if (ctrl->state ==NVME_CTRL_DELETING || + ctrl->state ==NVME_CTRL_DEAD ) +continue ; + +if (tmp->cntlid == ctrl->cntlid) { +dev_err (ctrl->device, +"Duplicate cntlid %u with %s, rejecting\n" , + ctrl->cntlid,dev_name (tmp->device)); +return false; + } + +if ((id->cmic & (1 <<1 )) || + (ctrl->opts && ctrl->opts->discovery_nqn)) +continue ; + +dev_err (ctrl->device, +"Subsystem does not support multiple controllers\n" ); +return false; } -return count; +return true; }static int nvme_init_subsystem (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id)
@@ -2397,15 +2412,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)__nvme_release_subsystem (subsys); subsys = found; -/* -* Verify that the subsystem actually supports multiple -* controllers, else bail out. -*/ -if (!(ctrl->opts && ctrl->opts->discovery_nqn) && -nvme_active_ctrls (found) && !(id->cmic & (1 <<1 ))) { -dev_err (ctrl->device, -"ignoring ctrl due to duplicate subnqn (%s).\n" , - subsys->subnqn); +if (!nvme_validate_cntlid (subsys, ctrl, id)) { ret = -EINVAL ; goto out_put_subsystem; }
Signature · 1 line
2.20.1
PATCH 2/2 nvme: validate cntlid during controller initialisation
Quoted 11 lines · Christoph Hellwig
The CNTLID value is required to be unique, and we do rely on this for correct operation. So reject any controller for which a non-unique CNTLID has been detected.
Based on a patch from Hannes Reinecke.
Signed-off-by Christoph Hellwig <hch at lst.de>--- drivers/nvme/host/core.c | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-)
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Teamlead Storage & Networking hare at suse.de +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg GF: Felix Imend?rffer, Mary Higgins, Sri Rasiah HRB 21284 (AG N?rnberg)
PATCH 2/2 nvme: validate cntlid during controller initialisation
In reply to Christoph HellwigQuoted 77 lines · Christoph Hellwig
The CNTLID value is required to be unique, and we do rely on this for correct operation. So reject any controller for which a non-unique CNTLID has been detected.
Based on a patch from Hannes Reinecke.
Signed-off-by Christoph Hellwig <hch at lst.de>--- drivers/nvme/host/core.c | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-)drivers/nvme/host/core.c +24
-17@@ -2341,20 +2341,35 @@ static const struct attribute_group *nvme_subsys_attrs_groups[] = { NULL, }; -static int nvme_active_ctrls (struct nvme_subsystem *subsys) +static bool nvme_validate_cntlid (struct nvme_subsystem *subsys, +struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id) { -int count =0 ; -struct nvme_ctrl *ctrl; +struct nvme_ctrl *tmp;lockdep_assert_held (&nvme_subsystems_lock); -list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { -if (ctrl->state !=NVME_CTRL_DELETING && - ctrl->state !=NVME_CTRL_DEAD ) - count++; +list_for_each_entry (tmp, &subsys->ctrls, subsys_entry) { +if (ctrl->state ==NVME_CTRL_DELETING || + ctrl->state ==NVME_CTRL_DEAD ) +continue ; + +if (tmp->cntlid == ctrl->cntlid) { +dev_err (ctrl->device, +"Duplicate cntlid %u with %s, rejecting\n" , + ctrl->cntlid,dev_name (tmp->device)); +return false; + } + +if ((id->cmic & (1 <<1 )) || + (ctrl->opts && ctrl->opts->discovery_nqn)) +continue ; + +dev_err (ctrl->device, +"Subsystem does not support multiple controllers\n" ); +return false; } -return count; +return true; }static int nvme_init_subsystem (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id)@@ -2397,15 +2412,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)__nvme_release_subsystem (subsys); subsys = found; -/* -* Verify that the subsystem actually supports multiple -* controllers, else bail out. -*/ -if (!(ctrl->opts && ctrl->opts->discovery_nqn) && -nvme_active_ctrls (found) && !(id->cmic & (1 <<1 ))) { -dev_err (ctrl->device, -"ignoring ctrl due to duplicate subnqn (%s).\n" , - subsys->subnqn); +if (!nvme_validate_cntlid (subsys, ctrl, id)) { ret = -EINVAL ; goto out_put_subsystem; }
PATCH 2/2 nvme: validate cntlid during controller initialisation
In reply to Christoph HellwigQuoted 38 lines · Christoph Hellwig
The CNTLID value is required to be unique, and we do rely on this for correct operation. So reject any controller for which a non-unique CNTLID has been detected.
Based on a patch from Hannes Reinecke.
Signed-off-by Christoph Hellwig <hch at lst.de>--- drivers/nvme/host/core.c | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-)drivers/nvme/host/core.c +8
-7@@ -2341,20 +2341,35 @@ static const struct attribute_group *nvme_subsys_attrs_groups[] = { NULL, }; -static int nvme_active_ctrls (struct nvme_subsystem *subsys) +static bool nvme_validate_cntlid (struct nvme_subsystem *subsys, +struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id) { -int count =0 ; -struct nvme_ctrl *ctrl; +struct nvme_ctrl *tmp;lockdep_assert_held (&nvme_subsystems_lock); -list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { -if (ctrl->state !=NVME_CTRL_DELETING && - ctrl->state !=NVME_CTRL_DEAD ) - count++; +list_for_each_entry (tmp, &subsys->ctrls, subsys_entry) { +if (ctrl->state ==NVME_CTRL_DELETING || + ctrl->state ==NVME_CTRL_DEAD ) +continue ; +
drivers/nvme/host/core.c static const struct attribute_group *nvme_subsys_attrs_groups[] = { · line 2365
Quoted 10 lines
drivers/nvme/host/core.c +10
-0+if (tmp->cntlid == ctrl->cntlid) { +dev_err (ctrl->device, +"Duplicate cntlid %u with %s, rejecting\n" , + ctrl->cntlid,dev_name (tmp->device)); +return false; + } + +if ((id->cmic & (1 <<1 )) || + (ctrl->opts && ctrl->opts->discovery_nqn)) +continue ;
Replace whitespace with tab
Quoted 11 lines
drivers/nvme/host/core.c +5
-1+ +dev_err (ctrl->device, +"Subsystem does not support multiple controllers\n" ); +return false; } -return count; +return true; }static int nvme_init_subsystem (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id)
Re: [PATCH 2/2] nvme: validate cntlid during controller initialisation
In reply to Christoph HellwigQuoted 37 lines · Christoph Hellwig
The CNTLID value is required to be unique, and we do rely on this for correct operation. So reject any controller for which a non-unique CNTLID has been detected.
Based on a patch from Hannes Reinecke.
Signed-off-by Christoph Hellwig <[email protected]>--- drivers/nvme/host/core.c | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-)drivers/nvme/host/core.c +7
-7@@ -2341,20 +2341,35 @@ static const struct attribute_group *nvme_subsys_attrs_groups[] = { NULL, }; -static int nvme_active_ctrls (struct nvme_subsystem *subsys) +static bool nvme_validate_cntlid (struct nvme_subsystem *subsys, +struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id) { -int count =0 ; -struct nvme_ctrl *ctrl; +struct nvme_ctrl *tmp;lockdep_assert_held (&nvme_subsystems_lock); -list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { -if (ctrl->state !=NVME_CTRL_DELETING && - ctrl->state !=NVME_CTRL_DEAD ) - count++; +list_for_each_entry (tmp, &subsys->ctrls, subsys_entry) { +if (ctrl->state ==NVME_CTRL_DELETING || + ctrl->state ==NVME_CTRL_DEAD ) +continue ;
drivers/nvme/host/core.c static const struct attribute_group *nvme_subsys_attrs_groups[] = { · line 2364
Quoted 10 lines
drivers/nvme/host/core.c +10
-0+ +if (tmp->cntlid == ctrl->cntlid) { +dev_err (ctrl->device, +"Duplicate cntlid %u with %s, rejecting\n" , + ctrl->cntlid,dev_name (tmp->device)); +return false; + } +if ((id->cmic & (1 <<1 )) || + (ctrl->opts && ctrl->opts->discovery_nqn)) +continue ;
Do we have a problem here if the subsystem is a discovery controller nqn ?
Assume we have a connection to a discovery controller in place (it has been created, is in the midst of reading the discovery log; or a long-lived discovery controller) and a new one is being made. The new one could be on completely different transport or different host/subsystem ports:
It looks like __nvme_find_get_subsystem() when called with the discovery nqn will match the subsystem for the existing discovery controller, which then forces the call to nvme_validate_cntlid(), which will find a live controller and it's very possible that controller id can be different. The requirement for sameness does not apply to the discovery controller - so it shouldn't be returning failure in this case. Agree ?
What's the best patch to avoid this ?
- change __nvme_find_get_subsystem() so that if discovery nqn, it treats it as a separate subsystem ?
- change nvme_validate_cntlid() such that "(ctrl->opts && ctrl->opts->discovery_nqn)" is added to the ctrl state checks above this test ?
-- james
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRe: [PATCH 2/2] nvme: validate cntlid during controller initialisation
Quoted 52 lines · Christoph Hellwig
Quoted 35 lines
The CNTLID value is required to be unique, and we do rely on this for correct operation. So reject any controller for which a non-unique CNTLID has been detected.
Based on a patch from Hannes Reinecke.
Signed-off-by Christoph Hellwig <[email protected]>---
drivers/nvme/host/core.c| 41 +++++++++++++++++++++++-----------------1 file changed, 24 insertions(+), 17 deletions(-)drivers/nvme/host/core.c +7
-7@@ -2341,20 +2341,35 @@ static const struct attribute_group *nvme_subsys_attrs_groups[] = { NULL, }; -static int nvme_active_ctrls (struct nvme_subsystem *subsys) +static bool nvme_validate_cntlid (struct nvme_subsystem *subsys, + struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) { - int count = 0; - struct nvme_ctrl *ctrl; + struct nvme_ctrl *tmp; lockdep_assert_held(&nvme_subsystems_lock); - list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { - if (ctrl->state != NVME_CTRL_DELETING && - ctrl->state != NVME_CTRL_DEAD) - count++; + list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) { + if (ctrl->state == NVME_CTRL_DELETING || + ctrl->state == NVME_CTRL_DEAD) + continue;drivers/nvme/host/core.c static const struct attribute_group · line 2362
Quoted 10 lines
drivers/nvme/host/core.c +10
-0+ + if (tmp->cntlid == ctrl->cntlid) { + dev_err(ctrl->device, + "Duplicate cntlid %u with %s, rejecting\n", + ctrl->cntlid, dev_name(tmp->device)); + return false; + } + if ((id->cmic & (1 << 1)) || + (ctrl->opts && ctrl->opts->discovery_nqn)) + continue;Do we have a problem here if the subsystem is a discovery controller nqn ?
Assume we have a connection to a discovery controller in place (it has been created, is in the midst of reading the discovery log; or a long-lived discovery controller) and a new one is being made. The new one could be on completely different transport or different host/subsystem ports:
Quoted 7 lines
It looks like
__nvme_find_get_subsystem()when called with the discovery nqn will match the subsystem for the existing discovery controller, which then forces the call tonvme_validate_cntlid(), which will find a live controller and it's very possible that controller id can be different. The requirement for sameness does not apply to the discovery controller - so it shouldn't be returning failure in this case. Agree ?What's the best patch to avoid this ?
- change__nvme_find_get_subsystem()so that if discovery nqn, it treats it as a separate subsystem ?
- changenvme_validate_cntlid()such that "(ctrl->opts&&ctrl->opts->discovery_nqn)" is added to the ctrl state checks above this test ?
Yes, you're right, we should allow this for the discovery subsystem.
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme