Skip to contentSkip to conversation

PCI: hv: Fix a bug on removing child devices on the bus

PATCH In review [email protected] 6 participants 13 messages Aug 24, 2021 – Aug 26, 2021

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 PCI: hv: Fix a bug on removing child devices on the bus

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the device")
Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +12 -3
index a53bd8728d0d..d4f3cce18957 100644
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;
 
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't
@@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry) {
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry)
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 
 	ret = hv_send_resources_released(hdev);
Signature · 1 line
2.25.1
maintainer
#1 lore raw
Quoted 33 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the device")
Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +0 -0
index a53bd8728d0d..d4f3cce18957 100644

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1 -0
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;

This can be moved to where it is needed -- the if(!keep_dev) branch -- to limit its scope.

Quoted 11 lines
drivers/pci/controller/pci-hyperv.c +3 -1
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't
@@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244

Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +8 -1
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry) {
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry)
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);

list_for_each_entry_safe can also be used here, right?

Wei.

Quoted 14 lines
drivers/pci/controller/pci-hyperv.c +0 -1
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 
 	ret = hv_send_resources_released(hdev);
Signature · 1 line
2.25.1
#2 lore raw
Quoted 74 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 35 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the

device")

Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +0 -0
index a53bd8728d0d..d4f3cce18957 100644
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1 -0
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;

This can be moved to where it is needed -- the if(!keep_dev) branch -- to limit its scope.

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, · line 3225

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +0 -1
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't @@
-3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool

keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)

Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3 -2
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry) {

drivers/pci/controller/pci-hyperv.c

Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1 -0
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244

Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7 -0
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);

list_for_each_entry_safe can also be used here, right?

Quoted 1 line

Wei.

I will address your comments.

Long

Quoted 16 lines

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 5 lines
drivers/pci/controller/pci-hyperv.c +0 -0
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0 -2
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 
 	ret = hv_send_resources_released(hdev);
--

2.25.1

#3 lore raw

From: Long Li <[email protected]> Sent: Tuesday, August 24, 2021 10:28 AM

Quoted 75 lines
Quoted 75 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 35 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the

device")

Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +0 -0
index a53bd8728d0d..d4f3cce18957 100644
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1 -0
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;

This can be moved to where it is needed -- the if(!keep_dev) branch -- to limit its scope.

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, · line 3225

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +0 -1
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't @@
-3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool

keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)

Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3 -2
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry) {

drivers/pci/controller/pci-hyperv.c

Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1 -0
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244

Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7 -0
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);

list_for_each_entry_safe can also be used here, right?

Quoted 5 lines
Quoted 1 line

Wei.

I will address your comments.

Long

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin lock.

Or do I have it backwards?

Michael

Quoted 17 lines
Quoted 17 lines

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 6 lines
drivers/pci/controller/pci-hyperv.c +0 -0
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0 -2
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 
 	ret = hv_send_resources_released(hdev);
--

2.25.1

#4 lore raw
Quoted 98 lines

Subject: RE: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

From: Long Li <[email protected]> Sent: Tuesday, August 24, 2021 10:28 AM

Quoted 5 lines
Quoted 5 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected]

Quoted 75 lines · wrote:
Quoted 75 lines
Quoted 36 lines

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing

the
device")

Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +0 -0
index a53bd8728d0d..d4f3cce18957 100644

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device · line 3220

Quoted 1 line
drivers/pci/controller/pci-hyperv.c +0 -0
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device
 *hdev,

bool keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1 -0
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;

This can be moved to where it is needed -- the if(!keep_dev) branch -- to limit its scope.

Quoted 6 lines
drivers/pci/controller/pci-hyperv.c +0 -0
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't

@@
-3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool

keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)

Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3 -2
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry) {

drivers/pci/controller/pci-hyperv.c

Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1 -0
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3244

Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7 -0
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);

list_for_each_entry_safe can also be used here, right?

Quoted 5 lines
Quoted 1 line

Wei.

I will address your comments.

Long

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then
list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin lock.

Quoted 3 lines

Or do I have it backwards?

Michael

I think we need list_for_each_entry_safe() because we delete the list elements while going through them:

Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element. This
* macro allows for the deletion of a list element while looping through the
* list.
*
* See list_for_each_entry for more details.
*/

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device · line 3249

Quoted 8 lines
drivers/pci/controller/pci-hyperv.c +0 -0
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device
 *hdev,

bool keep_devs)

Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0 -1
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 

ret = hv_send_resources_released(hdev);
--
2.25.1

#5 lore raw

From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM

Quoted 14 lines
Quoted 11 lines

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then
list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin lock.

Or do I have it backwards?

Michael

I think we need list_for_each_entry_safe() because we delete the list elements while going through them:

Quoted 9 lines

Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element. This
* macro allows for the deletion of a list element while looping through the
* list.
*
* See list_for_each_entry for more details.
*/

Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.

Interestingly, pci-hyperv.c has another case of looping through this list and removing items where the _safe version is not used. See pci_devices_present_work() where the missing children are moved to a list on the stack.

Michael

maintainer
#6 lore raw
Quoted 26 lines · Michael Kelley

From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM

Quoted 14 lines
Quoted 11 lines

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then
list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin lock.

Or do I have it backwards?

Michael

I think we need list_for_each_entry_safe() because we delete the list elements while going through them:

Quoted 9 lines

Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element. This
* macro allows for the deletion of a list element while looping through the
* list.
*
* See list_for_each_entry for more details.
*/

Quoted 7 lines

Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.

Interestingly, pci-hyperv.c has another case of looping through this list and removing items where the _safe version is not used. See pci_devices_present_work() where the missing children are moved to a list on the stack.

That can be converted too, I think.

The original code is not wrong per-se. It is just not as concise as using list_for_each_entry_safe.

Wei.

Quoted 2 lines

Michael

#7 lore raw
Quoted 43 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

Quoted 11 lines · Michael Kelley

From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM

Quoted 8 lines
Quoted 8 lines

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then
list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin

lock.

Quoted 6 lines
Quoted 6 lines
Quoted 4 lines

Or do I have it backwards?

Michael

I think we need list_for_each_entry_safe() because we delete the list

elements while going through them:

Quoted 13 lines
Quoted 12 lines

Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element.
This
* macro allows for the deletion of a list element while looping
through the
* list.
*
* See list_for_each_entry for more details.
*/

Quoted 7 lines

Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.

Interestingly, pci-hyperv.c has another case of looping through this list and removing items where the _safe version is not used. See pci_devices_present_work() where the missing children are moved to a list on the stack.

Quoted 6 lines

That can be converted too, I think.

The original code is not wrong per-se. It is just not as concise as using list_for_each_entry_safe.

Wei.

I assume we are talking about the following code in pci_devices_present_work():

                list_for_each_entry(hpdev, &hbus->children, list_entry) {
                        if (hpdev->reported_missing) {
                                found = true;
                                put_pcichild(hpdev);
                                list_move_tail(&hpdev->list_entry, &removed);
                                break;
                        }
                }

This code is correct as there is a "break" after a list entry is removed from the list. So there is no need to use the _safe version. This code can be converted to use the _safe version.

maintainer
#8 lore raw
Quoted 61 lines · Long Li
Quoted 43 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

Quoted 11 lines · Michael Kelley

From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021 1:25 PM

Quoted 8 lines
Quoted 8 lines

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then
list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin

lock.

Quoted 6 lines
Quoted 6 lines
Quoted 4 lines

Or do I have it backwards?

Michael

I think we need list_for_each_entry_safe() because we delete the list

elements while going through them:

Quoted 13 lines
Quoted 12 lines

Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element.
This
* macro allows for the deletion of a list element while looping
through the
* list.
*
* See list_for_each_entry for more details.
*/

Quoted 7 lines

Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.

Interestingly, pci-hyperv.c has another case of looping through this list and removing items where the _safe version is not used. See pci_devices_present_work() where the missing children are moved to a list on the stack.

Quoted 6 lines

That can be converted too, I think.

The original code is not wrong per-se. It is just not as concise as using list_for_each_entry_safe.

Wei.

I assume we are talking about the following code in pci_devices_present_work():

                list_for_each_entry(hpdev, &hbus->children, list_entry) {
                        if (hpdev->reported_missing) {
                                found = true;
                                put_pcichild(hpdev);
                                list_move_tail(&hpdev->list_entry, &removed);
                                break;
                        }
                }
Quoted 3 lines

This code is correct as there is a "break" after a list entry is removed from the list. So there is no need to use the _safe version. This code can be converted to use the _safe version.

After this block there is another block like

  while (!list_empty(removed)) {
	...

list_del(...)

  }

I assumed Michael was referring to that block. :-)

Wei.

#9 lore raw
Quoted 77 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

Quoted 47 lines · Long Li
Quoted 47 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

Quoted 13 lines · Michael Kelley

From: Long Li <[email protected]> Sent: Wednesday, August 25, 2021
1:25 PM

Quoted 9 lines
Quoted 9 lines

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then
list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin

lock.

Quoted 7 lines
Quoted 7 lines
Quoted 4 lines

Or do I have it backwards?

Michael

I think we need list_for_each_entry_safe() because we delete the list

elements while going through them:

Quoted 20 lines
Quoted 12 lines

Here is the comment on list_for_each_entry_safe():
/**
* Loop through the list, keeping a backup pointer to the element.
This
* macro allows for the deletion of a list element while looping
through the
* list.
*
* See list_for_each_entry for more details.
*/

Got it. Thanks (and to Rob Herring). I read that comment but
with the wrong assumptions and didn't understand it correctly.

Interestingly, pci-hyperv.c has another case of looping through this list and removing items where the _safe version is not used. See pci_devices_present_work() where the missing children are moved to a list on the stack.

Quoted 8 lines
Quoted 6 lines

That can be converted too, I think.

The original code is not wrong per-se. It is just not as concise as using list_for_each_entry_safe.

Wei.

I assume we are talking about the following code in

pci_devices_present_work():

Quoted 10 lines
                list_for_each_entry(hpdev, &hbus->children, list_entry) {
                        if (hpdev->reported_missing) {
                                found = true;
                                put_pcichild(hpdev);
                                list_move_tail(&hpdev->list_entry, &removed);
                                break;
                        }
                }
Quoted 3 lines

This code is correct as there is a "break" after a list entry is removed from the list. So there is no need to use the _safe version. This code can be converted to use the _safe version.

After this block there is another block like

  while (!list_empty(removed)) {
	...
Quoted 7 lines

list_del(...)

  }

I assumed Michael was referring to that block. :-)

Wei.

This block is also correct. We don't have a bug here but there is a better way to code it.

Long

reviewer
#10 lore raw
In reply to Michael Kelley
Quoted 91 lines · Michael Kelley

From: Long Li <[email protected]> Sent: Tuesday, August 24, 2021 10:28 AM

Quoted 73 lines
Quoted 73 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 35 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the

device")

Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +0 -0
index a53bd8728d0d..d4f3cce18957 100644
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) · line 3223

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +1 -0
  struct hv_pci_dev *hpdev, *tmp;
  unsigned long flags;
  int ret;
+ struct list_head removed;

This can be moved to where it is needed -- the if(!keep_dev) branch -- to limit its scope.

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, · line 3225

Quoted 4 lines
drivers/pci/controller/pci-hyperv.c +0 -1
  /*
   * After the host sends the RESCIND_CHANNEL message, it doesn't @@
-3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool

keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)

Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3 -2
          return 0;
 
  if (!keep_devs) {
-         /* Delete any children which might still exist. */
+         INIT_LIST_HEAD(&removed);
+
+         /* Move all present children to the list on stack */
          spin_lock_irqsave(&hbus->device_list_lock, flags);
-         list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry) {

Quoted 1 line

+ list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry)

Quoted 7 lines
drivers/pci/controller/pci-hyperv.c +7 -0
+                 list_move_tail(&hpdev->list_entry, &removed);
+         spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+         /* Remove all children in the list */
+         while (!list_empty(&removed)) {
+                 hpdev = list_first_entry(&removed, struct hv_pci_dev,
+                                          list_entry);
Quoted 7 lines
Quoted 3 lines

list_for_each_entry_safe can also be used here, right?

Wei.

I will address your comments.

Long

I thought list_for_each_entry_safe() is for use when list manipulation is *not* protected by a lock and you want to safely walk the list even if an entry gets removed. If the list is protected by a lock or not subject to contention (as is the case here), then list_for_each_entry() is the simpler implementation. The original implementation didn't need to use the _safe version because of the spin lock.

Quoted 1 line

Or do I have it backwards?

"_safe" only means "safe against removal of list entry" as the kerneldoc says. But that means removal within the loop iteration, not any writer. A lock is needed in either case if there's another writer.

Don't ask me about the RCU variant though...

Rob

#11 lore raw
In reply to [email protected]

"Fix a bug ..." is not a very useful subject line. It doesn't say anything about what the patch *does*. It doesn't hint at a locking change.

Quoted 4 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

It's unfortunate that slots are not better integrated into the PCI core. I'm sorry your driver even has to worry about this.

Quoted 7 lines

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the device")
Quoted 12 lines
Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>

A lore link to Dan's report would be useful here.

Quoted 49 lines
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +12 -3
index a53bd8728d0d..d4f3cce18957 100644
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;
 
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't
@@ -3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry) {
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry)
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 
 	ret = hv_send_resources_released(hdev);
Signature · 1 line
2.25.1
#12 lore raw
Quoted 34 lines

Subject: Re: [PATCH] PCI: hv: Fix a bug on removing child devices on the bus

"Fix a bug ..." is not a very useful subject line. It doesn't say anything about what the patch *does*. It doesn't hint at a locking change.

Quoted 4 lines · On Tue, Aug 24, 2021 at 12:20:20AM -0700, [email protected] wrote:

From: Long Li <[email protected]>

In hv_pci_bus_exit, the code is holding a spinlock while calling pci_destroy_slot(), which takes a mutex.

It's unfortunate that slots are not better integrated into the PCI core. I'm sorry your driver even has to worry about this.

Quoted 8 lines

This is not safe for spinlock. Fix this by moving the children to be deleted to a list on the stack, and removing them after spinlock is released.

Fixes 94d22763207a ("PCI: hv: Fix a race condition when removing the

device")

Quoted 12 lines
Cc "K. Y. Srinivasan" <[email protected]>
Cc Haiyang Zhang <[email protected]>
Cc Stephen Hemminger <[email protected]>
Cc Wei Liu <[email protected]>
Cc Dexuan Cui <[email protected]>
Cc Lorenzo Pieralisi <[email protected]>
Cc Rob Herring <[email protected]>
Cc "Krzysztof Wilczyński" <[email protected]>
Cc Bjorn Helgaas <[email protected]>
Cc Michael Kelley <[email protected]>
Cc Dan Carpenter <[email protected]>
Quoted 1 line

A lore link to Dan's report would be useful here.

I will address your comments and send v2.

Long

Quoted 56 lines

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 12 lines
---
 drivers/pci/controller/pci-hyperv.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
drivers/pci/controller/pci-hyperv.c +0 -0
index a53bd8728d0d..d4f3cce18957 100644
@@ -3220,6 +3220,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)

Quoted 8 lines
drivers/pci/controller/pci-hyperv.c +1 -1
 	struct hv_pci_dev *hpdev, *tmp;
 	unsigned long flags;
 	int ret;
+	struct list_head removed;
 
 	/*
 	 * After the host sends the RESCIND_CHANNEL message, it doesn't @@
-3229,9 +3230,18 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool

keep_devs)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)

Quoted 9 lines
drivers/pci/controller/pci-hyperv.c +3 -2
 		return 0;
 
 	if (!keep_devs) {
-		/* Delete any children which might still exist. */
+		INIT_LIST_HEAD(&removed);
+
+		/* Move all present children to the list on stack */
 		spin_lock_irqsave(&hbus->device_list_lock, flags);
-		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry) {

drivers/pci/controller/pci-hyperv.c

Quoted 1 line
drivers/pci/controller/pci-hyperv.c +1 -0
+		list_for_each_entry_safe(hpdev, tmp, &hbus->children,

list_entry)

drivers/pci/controller/pci-hyperv.c static int hv_pci_bus_exit(struct hv_device *hdev,

Quoted 11 lines
drivers/pci/controller/pci-hyperv.c +7 -0
+			list_move_tail(&hpdev->list_entry, &removed);
+		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+		/* Remove all children in the list */
+		while (!list_empty(&removed)) {
+			hpdev = list_first_entry(&removed, struct hv_pci_dev,
+						 list_entry);
 			list_del(&hpdev->list_entry);
 			if (hpdev->pci_slot)
 				pci_destroy_slot(hpdev->pci_slot);
@@ -3239,7 +3249,6 @@ static int hv_pci_bus_exit(struct hv_device *hdev,

bool keep_devs)

Quoted 10 lines
drivers/pci/controller/pci-hyperv.c +0 -2
 			put_pcichild(hpdev);
 			put_pcichild(hpdev);
 		}
-		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
 	}
 
 	ret = hv_send_resources_released(hdev);
--

2.25.1