PCI: hv: Fix a bug on removing child devices on the bus
Subsystems (4)
PATCH 1/1 PCI: hv: Fix a bug on removing child devices on the bus
From: Long Li <[email protected]>
In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.
This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +12 -3
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;/* * After the host sends the RESCIND_CHANNEL message, it doesn't
@@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); -list_for_each_entry_safe (hpdev, tmp, &hbus->children, list_entry) { +list_for_each_entry_safe (hpdev, tmp, &hbus->children, list_entry) +list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); } ret =hv_send_resources_released (hdev);
Signature · 1 line
2.25.1
Wei Liu
maintainerQuoted 33 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the device")Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +0
-0
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1
-0@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;
This can be moved to where it is needed -- the if(!keep_dev) branch -- to limit its scope.
Quoted 11 lines
drivers/pci/controller/pci-hyperv.c +3
-1/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244
Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +8
-1spin_lock_irqsave (&hbus->device_list_lock, flags); -list_for_each_entry_safe (hpdev, tmp, &hbus->children, list_entry) { +list_for_each_entry_safe (hpdev, tmp, &hbus->children, list_entry) +list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);
list_for_each_entry_safe can also be used here, right?
Wei.
Quoted 14 lines
drivers/pci/controller/pci-hyperv.c +0
-1list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); } ret =hv_send_resources_released (hdev);Signature · 1 line
2.25.1
Quoted 74 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 35 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing thedevice")
Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +0
-0@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1
-0struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;This can be moved to where it is needed -- the
if(!keep_dev)branch -- to limit its scope.drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, · line 3225
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +0
-1/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3
-2return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); - list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry) {drivers/pci/controller/pci-hyperv.c
Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1
-0+ list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244
Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7
-0+list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);
list_for_each_entry_safecan also be used here, right?
Quoted 1 line
Wei.
I will address your comments.
Long
Quoted 16 lines
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 5 lines
drivers/pci/controller/pci-hyperv.c +0
-0list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0
-2put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); } ret =hv_send_resources_released (hdev); --2.25.1
From: Long Li <[email protected]> Sent: Tuesday, August 24, 2021 10:28 AM
Quoted 75 lines
Quoted 75 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 35 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing thedevice")
Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +0
-0@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1
-0struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;This can be moved to where it is needed -- the
if(!keep_dev)branch -- to limit its scope.drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, · line 3225
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +0
-1/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3
-2return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); - list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry) {drivers/pci/controller/pci-hyperv.c
Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1
-0+ list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244
Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7
-0+list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);
list_for_each_entry_safecan also be used here, right?
Quoted 5 lines
Quoted 1 line
Wei.
I will address your comments.
Long
I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin lock.
Or do I have it backwards?
Michael
Quoted 17 lines
Quoted 17 lines
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 6 lines
drivers/pci/controller/pci-hyperv.c +0
-0list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0
-2put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); } ret =hv_send_resources_released (hdev); --2.25.1
Quoted 98 lines
Subject: RE: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
From: Long Li <[email protected]> Sent: Tuesday, August 24, 2021 10:28 AM
Quoted 5 lines
Quoted 5 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected]
Quoted 75 lines · wrote:
Quoted 75 lines
Quoted 36 lines
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removingthe
device")Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +0
-0drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device · line 3220
Quoted 1 line
drivers/pci/controller/pci-hyperv.c +0
-0@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1
-0struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;This can be moved to where it is needed -- the
if(!keep_dev)branch -- to limit its scope.Quoted 6 lines
drivers/pci/controller/pci-hyperv.c +0
-0/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@
-3229,9 +3230,18 @@ static int hv_pci_bus_exit(structhv_device*hdev, bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3
-2return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); - list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry) {drivers/pci/controller/pci-hyperv.c
Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1
-0+ list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244
Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7
-0+list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);
list_for_each_entry_safecan also be used here, right?Quoted 5 lines
Quoted 1 line
Wei.
I will address your comments.
Long
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spin lock.
Quoted 3 lines
Or do I have it backwards?
Michael
I think we need list_for_each_entry_safe() because we delete the list elements while going through them:
Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element. This
* macro allows for the deletion of a list element while looping through the
* list.
*
* See list_for_each_entry for more details.
*/
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device · line 3249
Quoted 8 lines
drivers/pci/controller/pci-hyperv.c +0
-0list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0
-1put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); }ret =
hv_send_resources_released(hdev);
--
2.25.1
From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM
Quoted 14 lines
Quoted 11 lines
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spin lock.Or do I have it backwards?
Michael
I think we need
list_for_each_entry_safe()because we delete the list elements while going through them:
Quoted 9 lines
Here is the comment on
list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element. This
* macro allows for the deletion of a list element while looping through the
* list.
*
* Seelist_for_each_entryfor more details.
*/
Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.
Interestingly, pci-hyperv.c has another case of looping through this list and removing items where the _safe version is not used. See pci_devices_present_work() where the missing children are moved to a list on the stack.
Michael
Wei Liu
maintainerQuoted 26 lines · Michael Kelley
From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM
Quoted 14 lines
Quoted 11 lines
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spin lock.Or do I have it backwards?
Michael
I think we need
list_for_each_entry_safe()because we delete the list elements while going through them:Quoted 9 lines
Here is the comment on
list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element. This
* macro allows for the deletion of a list element while looping through the
* list.
*
* Seelist_for_each_entryfor more details.
*/
Quoted 7 lines
Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.Interestingly,
pci-hyperv.chas another case of looping through this list and removing items where the_safeversion is not used. Seepci_devices_present_work()where the missing children are moved to a list on the stack.
That can be converted too, I think.
The original code is not wrong per-se. It is just not as concise as using list_for_each_entry_safe.
Wei.
Quoted 2 lines
Michael
Quoted 43 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
Quoted 11 lines · Michael Kelley
From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM
Quoted 8 lines
Quoted 8 lines
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spinlock.
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
Or do I have it backwards?
Michael
I think we need
list_for_each_entry_safe()because we delete the listelements while going through them:
Quoted 13 lines
Quoted 12 lines
Here is the comment on
list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element.
This
* macro allows for the deletion of a list element while looping
through the
* list.
*
* Seelist_for_each_entryfor more details.
*/Quoted 7 lines
Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.Interestingly,
pci-hyperv.chas another case of looping through this list and removing items where the_safeversion is not used. Seepci_devices_present_work()where the missing children are moved to a list on the stack.
Quoted 6 lines
That can be converted too, I think.
The original code is not wrong per-se. It is just not as concise as using
list_for_each_entry_safe.Wei.
I assume we are talking about the following code in pci_devices_present_work():
list_for_each_entry(hpdev, &hbus->children, list_entry) {
if (hpdev->reported_missing) {
found = true;
put_pcichild(hpdev);
list_move_tail(&hpdev->list_entry, &removed);
break;
}
}This code is correct as there is a "break" after a list entry is removed from the list. So there is no need to use the _safe version. This code can be converted to use the _safe version.
Wei Liu
maintainerQuoted 61 lines · Long Li
Quoted 43 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
Quoted 11 lines · Michael Kelley
From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM
Quoted 8 lines
Quoted 8 lines
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spinlock.
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
Or do I have it backwards?
Michael
I think we need
list_for_each_entry_safe()because we delete the listelements while going through them:
Quoted 13 lines
Quoted 12 lines
Here is the comment on
list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element.
This
* macro allows for the deletion of a list element while looping
through the
* list.
*
* Seelist_for_each_entryfor more details.
*/Quoted 7 lines
Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.Interestingly,
pci-hyperv.chas another case of looping through this list and removing items where the_safeversion is not used. Seepci_devices_present_work()where the missing children are moved to a list on the stack.Quoted 6 lines
That can be converted too, I think.
The original code is not wrong per-se. It is just not as concise as using
list_for_each_entry_safe.Wei.
I assume we are talking about the following code in
pci_devices_present_work():list_for_each_entry(hpdev, &hbus->children, list_entry) { if (hpdev->reported_missing) { found = true; put_pcichild(hpdev); list_move_tail(&hpdev->list_entry, &removed); break; } }
Quoted 3 lines
This code is correct as there is a "break" after a list entry is removed from the list. So there is no need to use the
_safeversion. This code can be converted to use the_safeversion.
After this block there is another block like
while (!list_empty(removed)) {
... list_del(...)
}
I assumed Michael was referring to that block. :-)
Wei.
Quoted 77 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
Quoted 47 lines · Long Li
Quoted 47 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
Quoted 13 lines · Michael Kelley
From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021
1:25 PMQuoted 9 lines
Quoted 9 lines
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spinlock.
Quoted 7 lines
Quoted 7 lines
Quoted 4 lines
Or do I have it backwards?
Michael
I think we need
list_for_each_entry_safe()because we delete the listelements while going through them:
Quoted 20 lines
Quoted 12 lines
Here is the comment on
list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element.
This
* macro allows for the deletion of a list element while looping
through the
* list.
*
* Seelist_for_each_entryfor more details.
*/Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.Interestingly,
pci-hyperv.chas another case of looping through this list and removing items where the_safeversion is not used. Seepci_devices_present_work()where the missing children are moved to a list on the stack.Quoted 8 lines
Quoted 6 lines
That can be converted too, I think.
The original code is not wrong per-se. It is just not as concise as using
list_for_each_entry_safe.Wei.
I assume we are talking about the following code in
pci_devices_present_work():Quoted 10 lines
list_for_each_entry(hpdev, &hbus->children, list_entry) { if (hpdev->reported_missing) { found = true; put_pcichild(hpdev); list_move_tail(&hpdev->list_entry, &removed); break; } }Quoted 3 lines
This code is correct as there is a "break" after a list entry is removed from the list. So there is no need to use the
_safeversion. This code can be converted to use the_safeversion.After this block there is another block like
while (!list_empty(removed)) { ...
Quoted 7 lines
list_del(...)}I assumed Michael was referring to that block. :-)
Wei.
This block is also correct. We don't have a bug here but there is a better way to code it.
Long
Rob Herring
reviewerQuoted 91 lines · Michael Kelley
From: Long Li <[email protected]> Sent: Tuesday, August 24, 2021 10:28 AM
Quoted 73 lines
Quoted 73 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 35 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing thedevice")
Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +0
-0@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1
-0struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;This can be moved to where it is needed -- the
if(!keep_dev)branch -- to limit its scope.drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, · line 3225
Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +0
-1/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3
-2return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); - list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry) {Quoted 1 line
+ list_for_each_entry_safe(hpdev, tmp,
&hbus->children,
list_entry)Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7
-0+list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);Quoted 7 lines
Quoted 3 lines
list_for_each_entry_safecan also be used here, right?Wei.
I will address your comments.
Long
I thought
list_for_each_entry_safe()is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), thenlist_for_each_entry()is the simpler implementation. The original implementation didn't need to use the_safeversion because of the spin lock.
Quoted 1 line
Or do I have it backwards?
"_safe" only means "safe against removal of list entry" as the kerneldoc says. But that means removal within the loop iteration, not any writer. A lock is needed in either case if there's another writer.
Don't ask me about the RCU variant though...
Rob
"Fix a bug ..." is not a very useful subject line. It doesn't say anything about what the patch *does*. It doesn't hint at a locking change.
Quoted 4 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.
It's unfortunate that slots are not better integrated into the PCI core. I'm sorry your driver even has to worry about this.
Quoted 7 lines
This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the device")
Quoted 12 lines
Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>
A lore link to Dan's report would be useful here.
Quoted 49 lines
Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +12
-3@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); -list_for_each_entry_safe (hpdev, tmp, &hbus->children, list_entry) { +list_for_each_entry_safe (hpdev, tmp, &hbus->children, list_entry) +list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); } ret =hv_send_resources_released (hdev);Signature · 1 line
2.25.1
Quoted 34 lines
Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus
"Fix a bug ..." is not a very useful subject line. It doesn't say anything about what the patch *does*. It doesn't hint at a locking change.
Quoted 4 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:
From: Long Li <[email protected]>
In
hv_pci_bus_exit, the code is holding a spinlock while callingpci_destroy_slot(), which takes a mutex.It's unfortunate that slots are not better integrated into the PCI core. I'm sorry your driver even has to worry about this.
Quoted 8 lines
This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.
Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing thedevice")
Quoted 12 lines
Cc "K. Y. Srinivasan" <[email protected]>Cc Haiyang Zhang <[email protected]>Cc Stephen Hemminger <[email protected]>Cc Wei Liu <[email protected]>Cc Dexuan Cui <[email protected]>Cc Lorenzo Pieralisi <[email protected]>Cc Rob Herring <[email protected]>Cc "Krzysztof Wilczyński" <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Michael Kelley <[email protected]>Cc Dan Carpenter <[email protected]>Reported-by Dan Carpenter <[email protected]>
Quoted 1 line
A lore link to Dan's report would be useful here.
I will address your comments and send v2.
Long
Quoted 56 lines
drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 12 lines
Signed-off-by Long Li <[email protected]>--- drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)drivers/pci/controller/pci-hyperv.c +0
-0@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
Quoted 8 lines
drivers/pci/controller/pci-hyperv.c +1
-1struct hv_pci_dev *hpdev, *tmp;unsigned long flags;int ret; +struct list_head removed;/* * After the host sends the RESCIND_CHANNEL message, it doesn't @@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool
keep_devs)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3
-2return 0 ;if (!keep_devs) { -/* Delete any children which might still exist. */ +INIT_LIST_HEAD (&removed); + +/* Move all present children to the list on stack */ spin_lock_irqsave (&hbus->device_list_lock, flags); - list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry) {drivers/pci/controller/pci-hyperv.c
Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1
-0+ list_for_each_entry_safe(hpdev, tmp, &hbus->children,
list_entry)drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,
Quoted 11 lines
drivers/pci/controller/pci-hyperv.c +7
-0+list_move_tail (&hpdev->list_entry, &removed); +spin_unlock_irqrestore (&hbus->device_list_lock, flags); + +/* Remove all children in the list */ +while (!list_empty (&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry);list_del (&hpdev->list_entry);if (hpdev->pci_slot)pci_destroy_slot (hpdev->pci_slot);@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,bool
keep_devs)Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0
-2put_pcichild (hpdev);put_pcichild (hpdev); } -spin_unlock_irqrestore (&hbus->device_list_lock, flags); } ret =hv_send_resources_released (hdev); --2.25.1