Skip to contentSkip to conversation

pci: pciehp update the slot bridge res to get big range for pcie devices

PATCH In review Yinghai Lu 8 participants 129 messages Oct 21, 2009 – Dec 17, 2009 8 messages not in the archive

Subsystems (2)
lore
Share & tools
lore

Yinghai Lu

author
#0 lore raw

PATCH 1/1 pci: pciehp update the slot bridge res to get big range for pcie devices

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW. for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_is_enabled must be removed in pci_setup_bridge(), otherwise update res is not updated to bridge BAR. and we are safe to do that, because only have one bus_size and assigned resources are called.

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++----
 drivers/pci/setup-bus.c          |   70 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 90 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +22 -7
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,28 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
-		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
+	pci_bus_size_bridges(parent);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
 	pci_bus_add_devices(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
+		pci_configure_slot(dev);
+		pci_dev_put(dev);
+	}
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +67 -3

88 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
#1 lore raw

* Yinghai Lu <[email protected]>:

drivers/pci/setup-bus.c static void pci_setup_bridge(struct pci_ · line 186

Quoted 9 lines
drivers/pci/setup-bus.c +0 -3
@@ -144,9 +182,6 @@ static void pci_setup_bridge(struct pci_
 	u32 l, bu, lu, io_upper16;
 	int pref_mem64;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
 		 pci_domain_nr(bus), bus->number);

Have you tested this with the hotplug facilities I added a while back?

Basically, what happens when you offline a bridge using

	/sys/bus/pci/devices/.../remove

And then re-add it using:

/sys/bus/pci/rescan

Please try this both for bridges that are claimed by pciehp as well as bridges that are not claimed.

Thanks.

/ac

author
#2 lore raw

PATCH 1/1 pci: pciehp update the slot bridge res to get big range for pcie devices - v2

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes.

---
 drivers/pci/hotplug/pciehp_pci.c |   30 ++++++++++++----
 drivers/pci/setup-bus.c          |   73 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 94 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +23 -7
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,29 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
-		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
 	pci_bus_add_devices(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
+		pci_configure_slot(dev);
+		pci_dev_put(dev);
+	}
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +70 -3

103 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
#3 lore raw
In reply to Yinghai Lu
Quoted 9 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW. for example pcie devices need 256M, but the bridge only get preallocated 2M...

Though I have not looked at the patch deeply yet (sorry), I have some questions and concerns about this change. Please correct me if my understanding is not correct.

- Your patch doesn't seems to have the code to free resources. If we need to expand the resource range, don't we need to free preallocated resource before allocating the new one?

- Your patch seems to update BARs for bridge itself. I think it would break the bridge's driver (port service driver) that if it controls the device's capability by using IO/Mem, though I don't know if such driver or capabilities exists now.

And one comment about pci_configure_slot(). We need to program hotplug parameters before the device start working. So we need to call pci_configure_slot() before calling pci_bus_add_devices().

Thanks,
Kenji Kaneshige

Quoted 185 lines

pci_is_enabled must be removed in pci_setup_bridge(), otherwise update res is not updated to bridge BAR. and we are safe to do that, because only have one bus_size and assigned resources are called.

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++----
 drivers/pci/setup-bus.c          |   70 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 90 insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +22 -7
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,28 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
-		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
+	pci_bus_size_bridges(parent);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
 	pci_bus_add_devices(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
+		pci_configure_slot(dev);
+		pci_dev_put(dev);
+	}
+
 	return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +67 -3
@@ -27,6 +27,44 @@
 #include <linux/slab.h>
 #include "pci.h"
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev)
+{
+	struct resource *res;
+	struct resource_list head, *list, *tmp;
+	int idx;
+	u16 class = dev->class >> 8;
+
+	head.next = NULL;
+
+	/* Don't touch classless devices or host bridges or ioapics.  */
+	if (class == PCI_CLASS_NOT_DEFINED ||
+	    class == PCI_CLASS_BRIDGE_HOST)
+		return;
+
+	/* Don't touch ioapic devices already enabled by firmware */
+	if (class == PCI_CLASS_SYSTEM_PIC) {
+		u16 command;
+		pci_read_config_word(dev, PCI_COMMAND, &command);
+		if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+			return;
+	}
+
+	pdev_sort_resources(dev, &head);
+
+	for (list = head.next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			res->start = 0;
+			res->end = 0;
+			res->flags = 0;
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
+
 static void pbus_assign_resources_sorted(const struct pci_bus *bus)
 {
 	struct pci_dev *dev;
@@ -144,9 +182,6 @@ static void pci_setup_bridge(struct pci_
 	u32 l, bu, lu, io_upper16;
 	int pref_mem64;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
 		 pci_domain_nr(bus), bus->number);
 
@@ -550,6 +585,35 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
+{
+	struct pci_bus *b;
+
+	pdev_assign_resources_sorted((struct pci_dev *)bridge);
+
+	b = bridge->subordinate;
+	if (!b)
+		return;
+
+	pci_bus_assign_resources(b);
+
+	switch (bridge->class >> 8) {
+	case PCI_CLASS_BRIDGE_PCI:
+		pci_setup_bridge(b);
+		break;
+
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		pci_setup_cardbus(b);
+		break;
+
+	default:
+		dev_info(&bridge->dev, "not setting up bridge for bus "
+			 "%04x:%02x\n", pci_domain_nr(b), b->number);
+		break;
+	}
+}
+EXPORT_SYMBOL(pci_bridge_assign_resources);
+
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
 	struct pci_bus *b;

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +1 -1
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 
--

To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

author
#4 lore raw
Quoted 15 lines · Kenji Kaneshige
Quoted 6 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call
assign resources several times, when there are several bridges under the slot bridge.

Quoted 7 lines

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

Quoted 7 lines

Though I have not looked at the patch deeply yet (sorry), I have some questions and concerns about this change. Please correct me if my understanding is not correct.

- Your patch doesn't seems to have the code to free resources. If we need to expand the resource range, don't we need to free preallocated resource before allocating the new one?

that is done with pci_bus_size_bridges ==> pbus_size_io/pbus_size_mem ==> find_free_bus_resource ==> release_resource.

Quoted 5 lines

- Your patch seems to update BARs for bridge itself. I think it would break the bridge's driver (port service driver) that if it controls the device's capability by using IO/Mem, though I don't know if such driver or capabilities exists now.

port service driver will be AER and pciehotplug.
it seems those driver are not use those BAR... those BAR are supposed for the devices under the pcie bridge.

Quoted 5 lines

And one comment about pci_configure_slot(). We need to program hotplug parameters before the device start working. So we need to call pci_configure_slot() before calling pci_bus_add_devices().

Thanks, will correct that.

YH

YH

#5 lore raw
Quoted 19 lines · Yinghai Lu
Quoted 18 lines · Kenji Kaneshige
Quoted 6 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call
assign resources several times, when there are several bridges under the slot bridge.

Quoted 7 lines

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

Though I have not looked at the patch deeply yet (sorry), I have some questions and concerns about this change. Please correct me if my understanding is not correct.

Quoted 6 lines
Quoted 3 lines

- Your patch doesn't seems to have the code to free resources. If we need to expand the resource range, don't we need to free preallocated resource before allocating the new one?

that is done with pci_bus_size_bridges ==> pbus_size_io/pbus_size_mem ==> find_free_bus_resource ==> release_resource.

I didn't noticed that find_free_bus_resource() was changed to call release_resource() recently...

By the way, does this (release_resource() by find_bus_resource()) change the resource assignment by BIOS also for bridges other than the ports with hotplug slot (switch upstreamport, for example)?

Quoted 9 lines
Quoted 4 lines

- Your patch seems to update BARs for bridge itself. I think it would break the bridge's driver (port service driver) that if it controls the device's capability by using IO/Mem, though I don't know if such driver or capabilities exists now.

port service driver will be AER and pciehotplug.
it seems those driver are not use those BAR... those BAR are supposed for the devices under the pcie bridge.

I understand that there are only two port service drivers (AER and PCIe hotplug) and both doesn't use BAR. But I still have a concern that changing bridge's BARs might cause problems in the future. In my understanding, what you need is expanding IO/Mem base and limit of root or switch downstream ports. If so, I think we should only touch IO/Mem base/limit, and should not touch bridge's BARs.

Thanks,
Kenji Kaneshige

author
#6 lore raw
Quoted 29 lines · Kenji Kaneshige
Quoted 20 lines · Yinghai Lu
Quoted 19 lines · Kenji Kaneshige
Quoted 7 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge()
and at last do them all together one time including slot bridge, to avoid to call
assign resources several times, when there are several bridges under the slot bridge.

Quoted 7 lines

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

Though I have not looked at the patch deeply yet (sorry), I have some questions and concerns about this change. Please correct me if my understanding is not correct.

Quoted 7 lines
Quoted 3 lines

- Your patch doesn't seems to have the code to free resources. If we need to expand the resource range, don't we need to free preallocated resource before allocating the new one?

that is done with pci_bus_size_bridges ==> pbus_size_io/pbus_size_mem ==> find_free_bus_resource ==> release_resource.

Quoted 6 lines

I didn't noticed that find_free_bus_resource() was changed to call release_resource() recently...

By the way, does this (release_resource() by find_bus_resource()) change the resource assignment by BIOS also for bridges other than the ports with hotplug slot (switch upstreamport, for example)?

yes.

BIOS preallocate small range for the bridge, and leave the BAR for the device under that bridge uninitialized.

Quoted 11 lines
Quoted 10 lines
Quoted 5 lines

- Your patch seems to update BARs for bridge itself. I think it would break the bridge's driver (port service driver) that if it controls the device's capability by using IO/Mem, though I don't know if such driver or capabilities exists now.

port service driver will be AER and pciehotplug.
it seems those driver are not use those BAR... those BAR are supposed for the devices under the pcie bridge.

Quoted 6 lines

I understand that there are only two port service drivers (AER and PCIe hotplug) and both doesn't use BAR. But I still have a concern that changing bridge's BARs might cause problems in the future. In my understanding, what you need is expanding IO/Mem base and limit of root or switch downstream ports. If so, I think we should only touch IO/Mem base/limit, and should not touch bridge's BARs.

those bridge BAR are for devices under that bridge. the port device is not supposed to use them.

if we don't touch the bridge's BAR, the hw will not forward the io for those devices under it.

YH

#7 lore raw
Quoted 31 lines · Yinghai Lu
Quoted 30 lines · Kenji Kaneshige
Quoted 20 lines · Yinghai Lu
Quoted 19 lines · Kenji Kaneshige
Quoted 7 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge()
and at last do them all together one time including slot bridge, to avoid to call
assign resources several times, when there are several bridges under the slot bridge.

Quoted 7 lines

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

Though I have not looked at the patch deeply yet (sorry), I have some questions and concerns about this change. Please correct me if my understanding is not correct.

Quoted 6 lines
Quoted 3 lines

- Your patch doesn't seems to have the code to free resources. If we need to expand the resource range, don't we need to free preallocated resource before allocating the new one?

that is done with pci_bus_size_bridges ==> pbus_size_io/pbus_size_mem ==> find_free_bus_resource ==> release_resource.

I didn't noticed that find_free_bus_resource() was changed to call release_resource() recently...

Quoted 8 lines
Quoted 3 lines

By the way, does this (release_resource() by find_bus_resource()) change the resource assignment by BIOS also for bridges other than the ports with hotplug slot (switch upstreamport, for example)?

yes.

BIOS preallocate small range for the bridge, and leave the BAR for the device under that bridge uninitialized.

Does this happen at the boot time regardless of hot-plug?

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines

- Your patch seems to update BARs for bridge itself. I think it would break the bridge's driver (port service driver) that if it controls the device's capability by using IO/Mem, though I don't know if such driver or capabilities exists now.

port service driver will be AER and pciehotplug.
it seems those driver are not use those BAR... those BAR are supposed for the devices under the pcie bridge.

Quoted 9 lines
Quoted 6 lines

I understand that there are only two port service drivers (AER and PCIe hotplug) and both doesn't use BAR. But I still have a concern that changing bridge's BARs might cause problems in the future. In my understanding, what you need is expanding IO/Mem base and limit of root or switch downstream ports. If so, I think we should only touch IO/Mem base/limit, and should not touch bridge's BARs.

those bridge BAR are for devices under that bridge. the port device is not supposed to use them.

Do you mean you touch only BARs of the devices under the bridge?

Quoted 2 lines

if we don't touch the bridge's BAR, the hw will not forward the io for those devices under it.

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

Thanks,
Kenji Kaneshige

author
#8 lore raw
Quoted 36 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 33 lines · Kenji Kaneshige
Quoted 20 lines · Yinghai Lu
Quoted 19 lines · Kenji Kaneshige
Quoted 7 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge()
and at last do them all together one time including slot bridge, to avoid to call
assign resources several times, when there are several bridges under the slot bridge.

Quoted 7 lines

need to introduce pci_bridge_assign_resources there.

handle the case the first bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

Though I have not looked at the patch deeply yet (sorry), I have some questions and concerns about this change. Please correct me if my understanding is not correct.

Quoted 6 lines
Quoted 3 lines

- Your patch doesn't seems to have the code to free resources. If we need to expand the resource range, don't we need to free preallocated resource before allocating the new one?

that is done with pci_bus_size_bridges ==> pbus_size_io/pbus_size_mem ==> find_free_bus_resource ==> release_resource.

I didn't noticed that find_free_bus_resource() was changed to call release_resource() recently...

By the way, does this (release_resource() by find_bus_resource()) change the resource assignment by BIOS also for bridges other than the ports with hotplug slot (switch upstreamport, for example)?

Quoted 7 lines
Quoted 5 lines

yes.

BIOS preallocate small range for the bridge, and leave the BAR for the device under that bridge uninitialized.

Does this happen at the boot time regardless of hot-plug?

yes

Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
Quoted 10 lines
Quoted 6 lines

- Your patch seems to update BARs for bridge itself. I think it would break the bridge's driver (port service driver) that if it controls the device's capability by using IO/Mem, though I don't know if such driver or capabilities exists now.

port service driver will be AER and pciehotplug.
it seems those driver are not use those BAR... those BAR are supposed for the devices under the pcie bridge.

I understand that there are only two port service drivers (AER and PCIe hotplug) and both doesn't use BAR. But I still have a concern that changing bridge's BARs might cause problems in the future. In my understanding, what you need is expanding IO/Mem base and limit of root or switch downstream ports. If so, I think we should only touch IO/Mem base/limit, and should not touch bridge's BARs.

Quoted 5 lines
Quoted 3 lines

those bridge BAR are for devices under that bridge. the port device is not supposed to use them.

Do you mean you touch only BARs of the devices under the bridge?

no. the BAR of 0x1c, 0x20, and 0x28

Quoted 5 lines
Quoted 4 lines

if we don't touch the bridge's BAR, the hw will not forward the io for those devices under it.

Quoted 7 lines

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used...
if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

YH

author
#9 lore raw
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used...
if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if

pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{       
        int i;
                
        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                
                r = &dev->resource[i];
                        
                if (r->flags & IORESOURCE_PCI_FIXED)
                        continue;
        
                if (!(r->flags) || r->parent)
                        continue;

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thanks

Yinghai Lu

#10 lore raw
Quoted 39 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 7 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used...
if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if

pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{       
        int i;
                
        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                
                r = &dev->resource[i];
                        
                if (r->flags & IORESOURCE_PCI_FIXED)
                        continue;
        
                if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Thanks,
Kenji Kaneshige

author
#11 lore raw
Quoted 72 lines · Kenji Kaneshige
Quoted 36 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Quoted 7 lines

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

YH

#12 lore raw
Quoted 81 lines · Yinghai Lu
Quoted 72 lines · Kenji Kaneshige
Quoted 36 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Quoted 7 lines

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

When you do, can you please add printks showing the changes you're making to bridge resources? I think these events are infrequent, and knowing about the changes is vital to debugging problems.

Bjorn

author
#13 lore raw
Quoted 7 lines · Bjorn Helgaas
Quoted 2 lines · Yinghai Lu
Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

When you do, can you please add printks showing the changes you're making to bridge resources? I think these events are infrequent, and knowing about the changes is vital to debugging problems.

sure.

[   91.762205] pci 0000:00:07.0: resource 15 [mem 0x91800000-0x91ffffff 64bit pref] released
[   91.771715] pci 0000:00:07.0: PCI bridge, secondary bus 0000:0e
[   91.783297] pci 0000:00:07.0:   IO window: 0x2000-0x2fff
[   91.791564] pci 0000:00:07.0:   MEM window: 0x90000000-0x917fffff
[   91.806376] pci 0000:00:07.0:   PREFETCH window: disabled
[   91.811537] pci 0000:40:07.0: resource 15 [mem 0xb1800000-0xb1ffffff 64bit pref] released
[   91.826760] pci 0000:40:07.0: PCI bridge, secondary bus 0000:50
[   91.841288] pci 0000:40:07.0:   IO window: 0x7000-0x7fff
[   91.847605] pci 0000:40:07.0:   MEM window: 0xb0000000-0xb17fffff
[   91.861572] pci 0000:40:07.0:   PREFETCH window: disabled
[   91.868426] pci 0000:80:07.0: resource 15 [mem 0xd1800000-0xd1ffffff 64bit pref] released
[   91.884552] pci 0000:80:07.0: PCI bridge, secondary bus 0000:90
[   91.892898] pci 0000:80:07.0:   IO window: 0xa000-0xafff
[   91.905952] pci 0000:80:07.0:   MEM window: 0xd0000000-0xd17fffff
[   91.912304] pci 0000:80:07.0:   PREFETCH window: disabled
[   91.925909] pci 0000:c0:07.0: resource 15 [mem 0xf1800000-0xf1ffffff 64bit pref] released
[   91.942305] pci 0000:c0:07.0: PCI bridge, secondary bus 0000:d0
[   91.950273] pci 0000:c0:07.0:   IO window: 0xf000-0xffff
[   91.961687] pci 0000:c0:07.0:   MEM window: 0xf0000000-0xf17fffff
[   91.969438] pci 0000:c0:07.0:   PREFETCH window: disabled

YH

#14 lore raw
In reply to Yinghai Lu
Quoted 81 lines · Yinghai Lu
Quoted 72 lines · Kenji Kaneshige
Quoted 36 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Quoted 7 lines

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

Today I make plugging in a hotplug bridge work by having the firmware reserve at one level and having the kernel reserve at the next level.

Windows handles the issue in theory by performing some kind of hot-unplugging of drivers that already have assigned resources and then replugging them. Which allows a full renumbering of busses. We don't have the infrastructure to do that safely today.

Eric

author
#15 lore raw
Quoted 80 lines · Eric W. Biederman
Quoted 78 lines · Yinghai Lu
Quoted 77 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 5 lines
Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

no.

you need to plug device in those slots and then insert it into a leaf hotplug slot.

Quoted 8 lines

Today I make plugging in a hotplug bridge work by having the firmware reserve at one level and having the kernel reserve at the next level.

Windows handles the issue in theory by performing some kind of hot-unplugging of drivers that already have assigned resources and then replugging them. Which allows a full renumbering of busses. We don't have the infrastructure to do that safely today.

that will take some drivers offline at first ?

YH

#16 lore raw
Quoted 83 lines · Yinghai Lu
Quoted 82 lines · Eric W. Biederman
Quoted 79 lines · Yinghai Lu
Quoted 70 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Quoted 7 lines

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Quoted 7 lines
Quoted 3 lines

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

no.

you need to plug device in those slots and then insert it into a leaf hotplug slot.

Scenario.

I insert a bridge with pci hotplug slots into a leaf hotplug slot. Which adds more leave hotplug slots.

Since the bridge itself is no longer a leaf slot it's resources will not get reassigned.

Then I will have no resources to assign to the leaves?

Quoted 9 lines
Quoted 7 lines

Today I make plugging in a hotplug bridge work by having the firmware reserve at one level and having the kernel reserve at the next level.

Windows handles the issue in theory by performing some kind of hot-unplugging of drivers that already have assigned resources and then replugging them. Which allows a full renumbering of busses. We don't have the infrastructure to do that safely today.

that will take some drivers offline at first ?

I believe windows only does that for drivers that support being temporarily disconnected from their hardware.

Eric

author
#17 lore raw
Quoted 93 lines · Eric W. Biederman
Quoted 81 lines · Yinghai Lu
Quoted 80 lines · Eric W. Biederman
Quoted 78 lines · Yinghai Lu
Quoted 77 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

no.

you need to plug device in those slots and then insert it into a leaf hotplug slot.

Scenario.

Quoted 7 lines

I insert a bridge with pci hotplug slots into a leaf hotplug slot. Which adds more leave hotplug slots.

Since the bridge itself is no longer a leaf slot it's resources will not get reassigned.

Then I will have no resources to assign to the leaves?

so we still have your min_size code there.

in your case:
you need plug all card in your slots on that daughter card at first, and then insert the daughter card to leaf slot in the MB.

my setup is :

system got 4 io chains. and will get slot:

00:03.0  00:05.0 00:07.0 00:09.0
40:03.0  40:05.0 40:07.0 40:09.0
80:03.0  80:05.0 80:07.0 80:09.0
c0:03.0  c0:05.0 c0:07.0 c0:09.0

those are hanged on peer root buses directly. but bios assign to them every one get 8M, if user plug one card need 256M, then it will not work.

with those two patches, could clear the resource assigned by BIOS, and get resource as needed. ( with mmio 64 bit )

YH

#18 lore raw
Quoted 100 lines · Yinghai Lu
Quoted 99 lines · Eric W. Biederman
Quoted 81 lines · Yinghai Lu
Quoted 80 lines · Eric W. Biederman
Quoted 78 lines · Yinghai Lu
Quoted 77 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

no.

you need to plug device in those slots and then insert it into a leaf hotplug slot.

Scenario.

I insert a bridge with pci hotplug slots into a leaf hotplug slot. Which adds more leave hotplug slots.

Since the bridge itself is no longer a leaf slot it's resources will not get reassigned.

Quoted 7 lines
Quoted 1 line

Then I will have no resources to assign to the leaves?

so we still have your min_size code there.

in your case: you need plug all card in your slots on that daughter card at first, and then insert the daughter card to leaf slot in the MB.

Operationally that is an impossibility. I would not have multiple layers of hotplug if I only needed a single layer.

Which means your patch would cause a regression in my setup.

Quoted 8 lines

my setup is :

system got 4 io chains. and will get slot:

00:03.0  00:05.0 00:07.0 00:09.0
40:03.0  40:05.0 40:07.0 40:09.0
80:03.0  80:05.0 80:07.0 80:09.0
c0:03.0  c0:05.0 c0:07.0 c0:09.0
Quoted 6 lines

those are hanged on peer root buses directly. but bios assign to them every one get 8M, if user plug one card need 256M, then it will not work.

with those two patches, could clear the resource assigned by BIOS, and get resource as needed. ( with mmio 64 bit )

Hmm.

Could you avoid reallocating resources until a pci device is plugged in that has problems?

A lot of root bridges have important configuration registers that are not in standard locations. Which means in general we can not reprogram root bridges successfully from linux. At least not without code that knows the root bridge magic.

You can almost solve your problem by simply saying: pci=hpmemsize=256M. Which works except that allocating 4G of pci memory isn't very likely to work.

One of the suggestions when I made my patch was to have a per port option instead of a global minimum. That is an option for your case. But it is not as elegant.

The truly elegant approach is to make certain the hibernate in the drivers can handle bars being changed under them, hibernate everything that needs renumbering and then bring them back.

Personally I think you should walk over to whomever did your firmware and tell them they goofed.

Eric

author
#19 lore raw
Quoted 104 lines · Eric W. Biederman
Quoted 102 lines · Yinghai Lu
Quoted 92 lines · Eric W. Biederman
Quoted 81 lines · Yinghai Lu
Quoted 80 lines · Eric W. Biederman
Quoted 78 lines · Yinghai Lu
Quoted 77 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

no.

you need to plug device in those slots and then insert it into a leaf hotplug slot.

Scenario.

Quoted 7 lines

I insert a bridge with pci hotplug slots into a leaf hotplug slot. Which adds more leave hotplug slots.

Since the bridge itself is no longer a leaf slot it's resources will not get reassigned.

Then I will have no resources to assign to the leaves?

so we still have your min_size code there.

Quoted 8 lines
Quoted 3 lines

in your case: you need plug all card in your slots on that daughter card at first, and then insert the daughter card to leaf slot in the MB.

Operationally that is an impossibility. I would not have multiple layers of hotplug if I only needed a single layer.

Which means your patch would cause a regression in my setup.

ok, may need to compare new range size and old range size before clear it.

Quoted 18 lines
Quoted 9 lines

my setup is :

system got 4 io chains. and will get slot:

00:03.0  00:05.0 00:07.0 00:09.0
40:03.0  40:05.0 40:07.0 40:09.0
80:03.0  80:05.0 80:07.0 80:09.0
c0:03.0  c0:05.0 c0:07.0 c0:09.0
Quoted 6 lines

those are hanged on peer root buses directly. but bios assign to them every one get 8M, if user plug one card need 256M, then it will not work.

with those two patches, could clear the resource assigned by BIOS, and get resource as needed. ( with mmio 64 bit )

Hmm.

Quoted 7 lines

Could you avoid reallocating resources until a pci device is plugged in that has problems?

A lot of root bridges have important configuration registers that are not in standard locations. Which means in general we can not reprogram root bridges successfully from linux. At least not without code that knows the root bridge magic.

no one change that

Quoted 9 lines

You can almost solve your problem by simply saying: pci=hpmemsize=256M. Which works except that allocating 4G of pci memory isn't very likely to work.

One of the suggestions when I made my patch was to have a per port option instead of a global minimum. That is an option for your case. But it is not as elegant.

Quoted 6 lines

The truly elegant approach is to make certain the hibernate in the drivers can handle bars being changed under them, hibernate everything that needs renumbering and then bring them back.

Personally I think you should walk over to whomever did your firmware and tell them they goofed.

they said it IS Linux problem. because other os is ok.

YH

author
#20 lore raw
Quoted 108 lines · Yinghai Lu
Quoted 107 lines · Eric W. Biederman
Quoted 105 lines · Yinghai Lu
Quoted 92 lines · Eric W. Biederman
Quoted 81 lines · Yinghai Lu
Quoted 80 lines · Eric W. Biederman
Quoted 78 lines · Yinghai Lu
Quoted 77 lines · Kenji Kaneshige
Quoted 35 lines · Yinghai Lu
Quoted 5 lines · Yinghai Lu
Quoted 4 lines · Kenji Kaneshige

I understand you need to touch I/O base/limit and Mem base/limit. But I don't understand why you also need to update bridge's BARs. Could you please explain a little more about it?

Quoted 8 lines
Quoted 3 lines

Just in case, my terminology "bridge's BARs" is Base Address Register 0 (offset 0x10) and Base Address Register 1 (offset 0x14) in the (type 1) configuration space header of the bridge.

i mean 0x1c, 0x20, 0x28

did not notice that bridge device's 0x10, 0x14 are used... if port service need to use 0x10, 0x14, and the device is enabled, we should touch 0x10, and 0x14.

after check the code, if
pci_bridge_assign_resources ==> pdev_assign_resources_sorted ==> pdev_sort_resources

will not touch 0x10 and 0x14, if those resource is claimed by port service.

/* Sort resources by alignment */
void pdev_sort_resources(struct pci_dev *dev, struct resource_list *head)

{               int i;
                        for (i = 0; i < PCI_NUM_RESOURCES; i++) {
                struct resource *r;
                struct resource_list *list, *tmp;
                resource_size_t r_align;
                                r = &dev->resource[i];
                                        if (r->flags &
IORESOURCE_PCI_FIXED)
                        continue;
                        if (!(r->flags) || r->parent)
                        continue;
Quoted 4 lines

r->parent != NULL, will make it skip those two.

So -v3 should be safe.

Thank you for the clarification.

But I still don't understand the whole picture of your set of changes. Let me ask some questions.

In my understanding of your set of changes, if there is a PCIe switch with some hot-plug slots and all of those slots are empty, I/O and Memory resources assigned by BIOS are all released at the boot time. For example, suppose the following case.

               bridge(A)
                  |
       -----------------------
       |                     |
    bridge(B)             bridge(C)
       |                     |
     slot(1)               slot(2)
     (empty)               (empty)

bridge(A): P2P bridge for switch upstream port bridge(B): P2P bridge for switch downstream port bridge(C): P2P bridge for switch downstream port

In the above example, I/O and Mem resource assigned to bridge(A), bridge(B) and bridge(C) are all released at the boot time. Correct?

Then, when a adapter card is hot-added to slot(1), I/O and Mem resources enough for enabling the hot-added adapter card is assigned to bridge(A), bridge(B) and the adapter card. Correct?

Then, when an another adpater card is hot-added to slot(2), we need to assign enough resource to bridge(C) and the new card. But bridge(A) doesn't have enough resource for bridge(C) and the new card. In addition, all bridge(A) and bridge(B) and the adapter card on slot(1) are already working. How do you assign resource to bridge(C) and the card on slot(2)?

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

thanks, will update the patches to only handle leaf bridge, and don't touch min_size etc.

Tell me what is your expected behavior if I plug a bridge with hotplug slots into a leaf hotplug slot? Will you assign me enough resources so that I can plug in additional devices?

no.

you need to plug device in those slots and then insert it into a leaf hotplug slot.

Scenario.

Quoted 7 lines

I insert a bridge with pci hotplug slots into a leaf hotplug slot. Which adds more leave hotplug slots.

Since the bridge itself is no longer a leaf slot it's resources will not get reassigned.

Then I will have no resources to assign to the leaves?

so we still have your min_size code there.

in your case: you need plug all card in your slots on that daughter card at first, and then insert the daughter card to leaf slot in the MB.

Quoted 6 lines
Quoted 4 lines

Operationally that is an impossibility. I would not have multiple layers of hotplug if I only needed a single layer.

Which means your patch would cause a regression in my setup.

ok, may need to compare new range size and old range size before clear it.

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?
http://patchwork.kernel.org/patch/56344/
http://patchwork.kernel.org/patch/56343/

Thanks

Yinghai Lu

author
#21 lore raw
Quoted 10 lines · Yinghai Lu
Quoted 3 lines
Quoted 2 lines

Which means your patch would cause a regression in my setup.

ok, may need to compare new range size and old range size before clear it.

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 8 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?
http://patchwork.kernel.org/patch/56344/
http://patchwork.kernel.org/patch/56343/

on top Jesse today's PCI tree.

YH

#22 lore raw
In reply to Yinghai Lu
Quoted 7 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

I don't see how throwing away the work that the firmware has done in preallocation is something that we can afford to do in general if what the firmware has done works for us.

Eric

author
#23 lore raw
Quoted 7 lines · Eric W. Biederman
Quoted 6 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 8 lines
Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

which kernel version?

Quoted 3 lines

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

one or two ht chains?

do you still have lspci -tv with it?

Quoted 7 lines

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

that mean kernel doesn't get peer root bus res probed properly

YH

#24 lore raw
Quoted 15 lines · Yinghai Lu
Quoted 14 lines · Eric W. Biederman
Quoted 12 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

Quoted 7 lines
Quoted 1 line

I have a much simpler case I will break, as I tried something similar by accident.

which kernel version?

Quoted 3 lines

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

one or two ht chains?

One chain.

Quoted 11 lines

do you still have lspci -tv with it?

Quoted 7 lines

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

that mean kernel doesn't get peer root bus res probed properly

How do you do that without having drivers for the peer root bus?

Eric

author
#25 lore raw
Quoted 30 lines · Eric W. Biederman
Quoted 13 lines · Yinghai Lu
Quoted 12 lines · Eric W. Biederman
Quoted 11 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines
Quoted 2 lines
Quoted 1 line

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

which kernel version?

Quoted 2 lines

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

one or two ht chains?

One chain.

Quoted 6 lines

do you still have lspci -tv with it?

Quoted 4 lines

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

Quoted 5 lines
Quoted 3 lines
Quoted 2 lines

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

that mean kernel doesn't get peer root bus res probed properly

How do you do that without having drivers for the peer root bus?

we have amd_bus.c to handle amd k8 system with two chains. but one chain is skipped. (wonder if need to reenable that for one chain k8 system)

another intel_bus.c is on the way to 2.6.33.

when use_crs is used, those info from pci conf space is not used but just print out for check if _CRS is right or not.

YH

#26 lore raw
Quoted 31 lines · Yinghai Lu
Quoted 30 lines · Eric W. Biederman
Quoted 13 lines · Yinghai Lu
Quoted 12 lines · Eric W. Biederman
Quoted 11 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines
Quoted 2 lines
Quoted 1 line

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

which kernel version?

Quoted 2 lines

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

one or two ht chains?

One chain.

Quoted 6 lines

do you still have lspci -tv with it?

Quoted 4 lines

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

that mean kernel doesn't get peer root bus res probed properly

How do you do that without having drivers for the peer root bus?

we have amd_bus.c to handle amd k8 system with two chains. but one chain is skipped. (wonder if need to reenable that for one chain k8 system)

I was running a 32bit kernel so this didn't kick in. That might have helped. At least as far as recognizing the resources weren't properly routed. If we don't setup the infrastructure so that we can reprogram those resources I'm not certain how much good it will do in general.

Quoted 3 lines

another intel_bus.c is on the way to 2.6.33.

when use_crs is used, those info from pci conf space is not used but just print out for check if _CRS is right or not.

If enough space is routed and we get accurate information I am certain that is fine. I am still worried about the change in policy though.

Only rerouting things when there is a need gives us the best chance of working everywhere. Freeing unused resources on hotplug ports before we plug in a device scares me, because we do something that should but doesn't we reallocate them. If there is simply not enough room we can do something different.

Eric

author
#27 lore raw
Quoted 9 lines

If enough space is routed and we get accurate information I am certain that is fine. I am still worried about the change in policy though.

Only rerouting things when there is a need gives us the best chance of working everywhere. Freeing unused resources on hotplug ports before we plug in a device scares me, because we do something that should but doesn't we reallocate them. If there is simply not enough room we can do something different.

sure.

will send out another version, only release those res that don't have big range to support children.

YH

author
#28 lore raw

PATCH 1/2 pci: release that leaf bridge' resource that is not big -v8

In reply to Eric W. Biederman

only for index < 3

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big enough
need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.

---
 drivers/pci/setup-bus.c |  258 ++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 239 insertions(+), 19 deletions(-)
drivers/pci/setup-bus.c +239 -19
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -27,11 +27,54 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+
+static void add_to_failed_list(struct resource_list *head, struct pci_dev *dev,
+				 struct resource *res)
+{
+	struct resource_list *list = head;
+	struct resource_list *ln = list->next;
+	struct resource_list *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list *head)
+{
+	struct resource_list *list, *tmp;
+	struct resource *res;
+	/*
+	 * Try to release leaf bridge's resources that there is no child
+	 * under it
+	 */
+	for (list = head->next; list;) {
+		res = list->res;
+		res->start = 0;
+		res->end = 0;
+		res->flags = 0;
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
 	struct resource_list head, *list, *tmp;
+	resource_size_t size;
 	int idx;
 
 	head.next = NULL;
@@ -57,10 +100,22 @@ static void pbus_assign_resources_sorted
 	for (list = head.next; list;) {
 		res = list->res;
 		idx = res - &list->dev->resource[0];
+		/* save the size at first */
+		size = resource_size(res);
 		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
+			if (fail_head && !list->dev->subordinate) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				res->start = 0;
+				res->end   = size - 1;
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
 		}
 		tmp = list;
 		list = list->next;
@@ -137,18 +192,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
-		 pci_domain_nr(bus), bus->number);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	pcibios_resource_to_bus(bridge, &region, bus->resource[0]);
@@ -175,7 +224,12 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	pcibios_resource_to_bus(bridge, &region, bus->resource[1]);
@@ -191,6 +245,13 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  MEM window: disabled\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -226,10 +287,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
+		 pci_domain_nr(bus), bus->number);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
@@ -541,19 +629,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -571,15 +660,105 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type)
+{
+	struct pci_bus *b;
+
+	/* If the bus is cardbus, do nothing */
+	if (bus->self && (bus->self->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
+		return;
+
+	/* We only release the resources under the leaf bridge */
+	if (list_empty(&bus->children)) {
+		if (!pci_is_root_bus(bus))
+			pci_bridge_release_unused_res(bus, type);
+		return;
+	}
+
+	/* Scan child buses if the bus is not leaf */
+	list_for_each_entry(b, &bus->children, node)
+		pci_bus_release_unused_bridge_res(b, type);
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);
@@ -607,6 +786,12 @@ void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	bool second_tried = false;
+	struct resource_list head, *list, *tmp;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+again:
+	head.next = NULL;
 
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
@@ -615,7 +800,42 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+
+	if (second_tried) {
+		/* still fail, don't want to try more */
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		bus = list->dev->bus;
+		/* don't need to play with root bus */
+		if (!pci_is_root_bus(bus))
+			pci_bus_release_unused_bridge_res(bus,
+					 list->res->flags & type_mask);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
author
#29 lore raw

PATCH 2/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v8

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big enough

---
 drivers/pci/hotplug/pciehp_pci.c |   23 ++++--
 drivers/pci/setup-bus.c          |  133 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 147 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +129 -4

160 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -760,6 +760,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
#30 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v8

In reply to Yinghai Lu

Hi Yinghai,

Sorry for the delayed response.
I'll try your patches on my machine soon (maybe in the few days).

Thanks,
Kenji Kaneshige

Quoted 392 lines · Yinghai Lu

only for index < 3

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big enough
need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.

---
 drivers/pci/setup-bus.c |  258 ++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 239 insertions(+), 19 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +239 -20
@@ -27,11 +27,54 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+
+static void add_to_failed_list(struct resource_list *head, struct pci_dev *dev,
+				 struct resource *res)
+{
+	struct resource_list *list = head;
+	struct resource_list *ln = list->next;
+	struct resource_list *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list *head)
+{
+	struct resource_list *list, *tmp;
+	struct resource *res;
+	/*
+	 * Try to release leaf bridge's resources that there is no child
+	 * under it
+	 */
+	for (list = head->next; list;) {
+		res = list->res;
+		res->start = 0;
+		res->end = 0;
+		res->flags = 0;
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
 	struct resource_list head, *list, *tmp;
+	resource_size_t size;
 	int idx;
 
 	head.next = NULL;
@@ -57,10 +100,22 @@ static void pbus_assign_resources_sorted
 	for (list = head.next; list;) {
 		res = list->res;
 		idx = res - &list->dev->resource[0];
+		/* save the size at first */
+		size = resource_size(res);
 		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
+			if (fail_head && !list->dev->subordinate) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				res->start = 0;
+				res->end   = size - 1;
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
 		}
 		tmp = list;
 		list = list->next;
@@ -137,18 +192,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
-		 pci_domain_nr(bus), bus->number);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	pcibios_resource_to_bus(bridge, &region, bus->resource[0]);
@@ -175,7 +224,12 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	pcibios_resource_to_bus(bridge, &region, bus->resource[1]);
@@ -191,6 +245,13 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  MEM window: disabled\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -226,10 +287,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
+		 pci_domain_nr(bus), bus->number);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
@@ -541,19 +629,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -571,15 +660,105 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type)
+{
+	struct pci_bus *b;
+
+	/* If the bus is cardbus, do nothing */
+	if (bus->self && (bus->self->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
+		return;
+
+	/* We only release the resources under the leaf bridge */
+	if (list_empty(&bus->children)) {
+		if (!pci_is_root_bus(bus))
+			pci_bridge_release_unused_res(bus, type);
+		return;
+	}
+
+	/* Scan child buses if the bus is not leaf */
+	list_for_each_entry(b, &bus->children, node)
+		pci_bus_release_unused_bridge_res(b, type);
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);
@@ -607,6 +786,12 @@ void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	bool second_tried = false;
+	struct resource_list head, *list, *tmp;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+again:
+	head.next = NULL;
 
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
@@ -615,7 +800,42 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+
+	if (second_tried) {
+		/* still fail, don't want to try more */
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		bus = list->dev->bus;
+		/* don't need to play with root bus */
+		if (!pci_is_root_bus(bus))
+			pci_bus_release_unused_bridge_res(bus,
+					 list->res->flags & type_mask);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
--

To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#31 lore raw
In reply to Eric W. Biederman
Quoted 20 lines · Eric W. Biederman
Quoted 7 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

Quoted 6 lines

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

Do you mean you had to have firmware program a hot-added device, or just that firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it?

Firmware normally supplies ACPI _CRS information that tells us how it programmed the host bridge windows. On x86, Linux normally ignores that and just assumes a range based on memory size. If we paid attention to it (as with "pci=use_crs"), it's likely that we could do a better job of doing this setup.

Or, of course, we could add a Linux driver that knows about "the hidden AMD bar registers." But I think that should be a last resort, for when firmware supplied incorrect _CRS information.

Quoted 5 lines

I don't see how throwing away the work that the firmware has done in preallocation is something that we can afford to do in general if what the firmware has done works for us.

Eric

#32 lore raw
Quoted 25 lines · Bjorn Helgaas
Quoted 24 lines · Eric W. Biederman
Quoted 7 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

Quoted 6 lines
Quoted 2 lines

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

Do you mean you had to have firmware program a hot-added device, or just that firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it?

Firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it.

Quoted 9 lines

Firmware normally supplies ACPI _CRS information that tells us how it programmed the host bridge windows. On x86, Linux normally ignores that and just assumes a range based on memory size. If we paid attention to it (as with "pci=use_crs"), it's likely that we could do a better job of doing this setup.

Or, of course, we could add a Linux driver that knows about "the hidden AMD bar registers." But I think that should be a last resort, for when firmware supplied incorrect _CRS information.

In this case there was no ACPI, and even if there was correct _CRS information it would have said only those addresses routed to bars/apertures on the root bridge was routed to the MCP55. So while it looked like we had gobs of unallocated space we could use. In practice we did not.

Eric

#33 lore raw
Quoted 43 lines · Eric W. Biederman
Quoted 25 lines · Bjorn Helgaas
Quoted 24 lines · Eric W. Biederman
Quoted 7 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

Quoted 6 lines
Quoted 2 lines

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

Do you mean you had to have firmware program a hot-added device, or just that firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it?

Firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it.

Quoted 6 lines

Firmware normally supplies ACPI _CRS information that tells us how it programmed the host bridge windows. On x86, Linux normally ignores that and just assumes a range based on memory size. If we paid attention to it (as with "pci=use_crs"), it's likely that we could do a better job of doing this setup.

Quoted 8 lines
Quoted 3 lines

Or, of course, we could add a Linux driver that knows about "the hidden AMD bar registers." But I think that should be a last resort, for when firmware supplied incorrect _CRS information.

In this case there was no ACPI, and even if there was correct _CRS information it would have said only those addresses routed to bars/apertures on the root bridge was routed to the MCP55. So while it looked like we had gobs of unallocated space we could use. In practice we did not.

I know this is a hypothetical case since you don't have ACPI, but I'm curious about this.

I assume the magic AMD BARs only affect the host bridge, and that the downstream root ports look like standard PCI-to-PCI bridges. If that's the case, and if we have correct descriptions of the host bridge apertures, Linux should theoretically be able to do as well as firmware.

But you seem to be suggesting that even with a correct host bridge description, there's space that *looks* available but is not. I don't understand how this can be.

Bjorn

#34 lore raw
Quoted 56 lines · Bjorn Helgaas
Quoted 43 lines · Eric W. Biederman
Quoted 25 lines · Bjorn Helgaas
Quoted 24 lines · Eric W. Biederman
Quoted 7 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

Quoted 6 lines
Quoted 2 lines

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

Do you mean you had to have firmware program a hot-added device, or just that firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it?

Firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it.

Quoted 6 lines

Firmware normally supplies ACPI _CRS information that tells us how it programmed the host bridge windows. On x86, Linux normally ignores that and just assumes a range based on memory size. If we paid attention to it (as with "pci=use_crs"), it's likely that we could do a better job of doing this setup.

Quoted 8 lines
Quoted 3 lines

Or, of course, we could add a Linux driver that knows about "the hidden AMD bar registers." But I think that should be a last resort, for when firmware supplied incorrect _CRS information.

In this case there was no ACPI, and even if there was correct _CRS information it would have said only those addresses routed to bars/apertures on the root bridge was routed to the MCP55. So while it looked like we had gobs of unallocated space we could use. In practice we did not.

I know this is a hypothetical case since you don't have ACPI, but I'm curious about this.

Quoted 8 lines

I assume the magic AMD BARs only affect the host bridge, and that the downstream root ports look like standard PCI-to-PCI bridges. If that's the case, and if we have correct descriptions of the host bridge apertures, Linux should theoretically be able to do as well as firmware.

But you seem to be suggesting that even with a correct host bridge description, there's space that *looks* available but is not. I don't understand how this can be.

What I meant was simply that not all of the non-memory space was routed down the hypertransport chain to the mcp55. If you have an accurate description of that you should be fine.

Eric

#35 lore raw
Quoted 63 lines · Eric W. Biederman
Quoted 61 lines · Bjorn Helgaas
Quoted 43 lines · Eric W. Biederman
Quoted 25 lines · Bjorn Helgaas
Quoted 24 lines · Eric W. Biederman
Quoted 7 lines · Yinghai Lu

after closing look up the code, it looks it will not break your setup.

1. before the patches:
a. when master card is inserted, all bridge in that card will get assigned with min_size b. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

Quoted 6 lines

2. after the patches: v5
a. booted up, all leaf bridge mmio get clearred.
b. when master card is inserted, all bridge in that card will get assigned with min_size, and master bridge will be sum of them c. when new cards is inserted to those slots in master card, will get assigned in the bridge size.

can you check those two patches in your setup to verify it?

I have a much simpler case I will break, as I tried something similar by accident.

AMD cpu MCP55 with one pcie port setup as hotplug.
The system only has 2GB of RAM. So plenty of space for pcie devices.

If the firmware assigns nothing and linux at boot time assigns the pci mmio space:
Reads from the bar of the hotplugged device work
Writes to the bar of the hotplugged device, cause further writes to go to lala land.

Quoted 6 lines
Quoted 2 lines

So I had to have the firmware make the assignment, because only it knows the details of the hidden AMD bar registers for each hypertransport chain etc.

Do you mean you had to have firmware program a hot-added device, or just that firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it?

Firmware had to program the apertures of the root port that was present at boot, even though it had no devices below it.

Quoted 6 lines

Firmware normally supplies ACPI _CRS information that tells us how it programmed the host bridge windows. On x86, Linux normally ignores that and just assumes a range based on memory size. If we paid attention to it (as with "pci=use_crs"), it's likely that we could do a better job of doing this setup.

Quoted 8 lines
Quoted 3 lines

Or, of course, we could add a Linux driver that knows about "the hidden AMD bar registers." But I think that should be a last resort, for when firmware supplied incorrect _CRS information.

In this case there was no ACPI, and even if there was correct _CRS information it would have said only those addresses routed to bars/apertures on the root bridge was routed to the MCP55. So while it looked like we had gobs of unallocated space we could use. In practice we did not.

I know this is a hypothetical case since you don't have ACPI, but I'm curious about this.

I assume the magic AMD BARs only affect the host bridge, and that the downstream root ports look like standard PCI-to-PCI bridges. If that's the case, and if we have correct descriptions of the host bridge apertures, Linux should theoretically be able to do as well as firmware.

Quoted 7 lines
Quoted 3 lines

But you seem to be suggesting that even with a correct host bridge description, there's space that *looks* available but is not. I don't understand how this can be.

What I meant was simply that not all of the non-memory space was routed down the hypertransport chain to the mcp55. If you have an accurate description of that you should be fine.

OK, yep, that makes perfect sense. That's a good example of why I believe we should start paying attention to the root bridge _CRS, because that's exactly what it would tell us.

Bjorn

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#37 lore raw

PATCH 2/2 pci: only release that resource index is less than 3 -v5

after

| commit 308cf8e13f42f476dfd6552aeff58fdc0788e566
|
|    PCI: get larger bridge ranges when space is available

found one of resource of peer root bus (0x00) get released from root resource. later one hotplug device can not get big range anymore. other peer root buses is ok.

it turns out it is from transparent path.

those resources will be used for pci bridge BAR updated. so need to limit it to 3.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big range for pcie devices - v4
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.

---
 arch/x86/pci/i386.c              |    6 ++
 drivers/pci/hotplug/pciehp_pci.c |    2 
 drivers/pci/setup-bus.c          |   81 ++++++++++++++++++++++++++++++++++++++-
 include/linux/pci.h              |    2 
 4 files changed, 90 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +80 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -319,6 +319,42 @@ static void pci_bridge_check_ranges(stru
 	}
 }
 
+void pci_bridge_release_not_used_res(struct pci_bus *bus)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if (r->flags & type_mask) {
+			if (!r->parent)
+				continue;
+			/*
+			 * if there is no child under that, we should release
+			 * and use it.
+			 */
+			if (!r->child && !release_resource(r)) {
+				dev_info(&dev->dev,
+					 "resource %d %pRt released\n",
+					 idx, r);
+				r->flags = 0;
+				changed = true;
+			}
+		}
+	}
+
+	if (changed)
+		pci_setup_bridge(bus);
+}
+EXPORT_SYMBOL(pci_bridge_release_not_used_res);
+
 /* Helper function for sizing routines: find first available
    bus resource of a given type. Note: we intentionally skip
    the bus resources which have already been assigned (that is,
@@ -576,6 +612,48 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+
+/* only release those resources that is on leaf bridge */
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			pci_bus_release_bridges_not_used_res(b);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if (is_leaf_bridge)
+			pci_bridge_release_not_used_res(bus);
+		break;
+	}
+}
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
 {
 	struct pci_bus *b;
@@ -644,7 +722,8 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);
drivers/pci/hotplug/pciehp_pci.c +2 -0
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -98,6 +98,7 @@ int pciehp_configure_device(struct slot
 		pci_dev_put(dev);
 	}
 
+	pci_bridge_release_not_used_res(parent);
 	pci_bus_size_bridges(parent);
 	pci_clear_master(bridge);
 	pci_bridge_assign_resources(bridge);
@@ -171,6 +172,7 @@ int pciehp_unconfigure_device(struct slo
 		}
 		pci_dev_put(temp);
 	}
+	pci_bridge_release_not_used_res(parent);
 
 	return rc;
 }
include/linux/pci.h +2 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -759,6 +759,8 @@ int pci_vpd_truncate(struct pci_dev *dev
 void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
+void pci_bus_release_bridges_not_used_res(struct pci_bus *bus);
+void pci_bridge_release_not_used_res(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
 void pdev_enable_device(struct pci_dev *);
arch/x86/pci/i386.c +6 -0
Index: linux-2.6/arch/x86/pci/i386.c
===================================================================
@@ -194,6 +194,7 @@ static void __init pcibios_allocate_reso
 static int __init pcibios_assign_resources(void)
 {
 	struct pci_dev *dev = NULL;
+	struct pci_bus *bus;
 	struct resource *r;
 
 	if (!(pci_probe & PCI_ASSIGN_ROMS)) {
@@ -213,6 +214,11 @@ static int __init pcibios_assign_resourc
 		}
 	}
 
+	/* Try to release bridge resources, that there is not child under it */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_bus_release_bridges_not_used_res(bus);
+	}
+
 	pci_assign_unassigned_resources();
 
 	return 0;
#38 lore raw

Re: [PATCH 2/2] pci: only release that resource index is less than 3 -v5

Quoted 45 lines · Yinghai Lu

after

| commit 308cf8e13f42f476dfd6552aeff58fdc0788e566
|
|    PCI: get larger bridge ranges when space is available

found one of resource of peer root bus (0x00) get released from root resource. later one hotplug device can not get big range anymore. other peer root buses is ok.

it turns out it is from transparent path.

those resources will be used for pci bridge BAR updated. so need to limit it to 3.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big range for pcie devices - v4
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.

---
 arch/x86/pci/i386.c              |    6 ++
 drivers/pci/hotplug/pciehp_pci.c |    2 
 drivers/pci/setup-bus.c          |   81 ++++++++++++++++++++++++++++++++++++++-
 include/linux/pci.h              |    2 
 4 files changed, 90 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +3 -0
@@ -319,6 +319,42 @@ static void pci_bridge_check_ranges(stru
 	}
 }
 
+void pci_bridge_release_not_used_res(struct pci_bus *bus)
+{
+	int idx;

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 334

Quoted 10 lines
drivers/pci/setup-bus.c +10 -0
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {

If this function is only for normal PCI bridge, we need additional check at the head of this function.

	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
		return;

if not, 3 should be 4 instead. And we can do as follows

       for (i = PCI_BRIDGE_RESOURCE; i <= PCI_BRIDGE_RESOURCE_END; i++) {

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 336

Quoted 2 lines
drivers/pci/setup-bus.c +2 -0
+		r = &dev->resource[idx];
+		if (r->flags & type_mask) {

How about

		if (!(r->flags & type_mask))
			continue;

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 342

Quoted 6 lines
drivers/pci/setup-bus.c +6 -0
+			if (!r->parent)
+				continue;
+			/*
+			 * if there is no child under that, we should release
+			 * and use it.
+			 */

I think this comment should be updated because whether "we should release and use it" is decided by the caller of this function.

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 349

Quoted 7 lines
drivers/pci/setup-bus.c +7 -0
+			if (!r->child && !release_resource(r)) {
+				dev_info(&dev->dev,
+					 "resource %d %pRt released\n",
+					 idx, r);
+				r->flags = 0;
+				changed = true;
+			}

How about

		if (r->child)
			continue;

		if (!release_resource(r)) {
			...
		}

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 354

Quoted 5 lines
drivers/pci/setup-bus.c +5 -0
+		}
+	}
+
+	if (changed)
+		pci_setup_bridge(bus);

The pci_setup_bridge() is called even if some resources are used by children. For example, mem resource is used by children, but prefetchable mem resource is not used by children. In this case, I think mem resource is released and pci_setup_bridge() is called while prefetcable mem resource is being used. I'm worrying that it might cause something wrong.

Quoted 40 lines
drivers/pci/setup-bus.c +33 -0
+}
+EXPORT_SYMBOL(pci_bridge_release_not_used_res);
+
 /* Helper function for sizing routines: find first available
    bus resource of a given type. Note: we intentionally skip
    the bus resources which have already been assigned (that is,
@@ -576,6 +612,48 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+
+/* only release those resources that is on leaf bridge */
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			pci_bus_release_bridges_not_used_res(b);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {

drivers/pci/setup-bus.c void __ref pci_bus_size_bridges(struct p · line 654

Quoted 10 lines
drivers/pci/setup-bus.c +10 -0
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if (is_leaf_bridge)
+			pci_bridge_release_not_used_res(bus);
+		break;
+	}
+}

How about like below. This might need to be called with "pci_bus_sem" held.

void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)

{
       struct pci_bus *b;

       /* If the bus is cardbus, do nothing */
       if (bus->self && (bus->self->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
               return;

       /* We only release the resources under the leaf bridge */
       if (list_empty(&bus->children)) {
               if (!pci_is_root_bus(bus))
                       pci_bridge_release_not_used_res(bus);
               return;
       }

       /* Scan child buses if the bus is not leaf */
       list_for_each_entry(b, &bus->children, node)
		pci_bus_release_bridges_not_used_res(b);
}
Quoted 20 lines
drivers/pci/setup-bus.c +4 -1
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
 {
 	struct pci_bus *b;
@@ -644,7 +722,8 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c int pciehp_configure_device(struct slot · line 101

Quoted 4 lines
drivers/pci/hotplug/pciehp_pci.c +1 -0
@@ -98,6 +98,7 @@ int pciehp_configure_device(struct slot
 		pci_dev_put(dev);
 	}
 
+	pci_bridge_release_not_used_res(parent);

I guess this is for the slots that are empty at the boot time on non x86 systems. And it only works at the first hot-add. Correct? How about doing this at the pciehp's slot initialization.

drivers/pci/hotplug/pciehp_pci.c int pciehp_unconfigure_device(struct slo · line 175

Quoted 8 lines
drivers/pci/hotplug/pciehp_pci.c +1 -0
 	pci_bus_size_bridges(parent);
 	pci_clear_master(bridge);
 	pci_bridge_assign_resources(bridge);
@@ -171,6 +172,7 @@ int pciehp_unconfigure_device(struct slo
 		}
 		pci_dev_put(temp);
 	}
+	pci_bridge_release_not_used_res(parent);

How about call pci_bus_release_bridges_not_used_res() instead and make pci_bridge_release_not_used_res() static function.

Quoted 29 lines
drivers/pci/hotplug/pciehp_pci.c +0 -0
 	return rc;
 }

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +2 -0
@@ -759,6 +759,8 @@ int pci_vpd_truncate(struct pci_dev *dev
 void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
+void pci_bus_release_bridges_not_used_res(struct pci_bus *bus);
+void pci_bridge_release_not_used_res(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
 void pdev_enable_device(struct pci_dev *);

Index: linux-2.6/arch/x86/pci/i386.c
===================================================================

arch/x86/pci/i386.c +1 -0
@@ -194,6 +194,7 @@ static void __init pcibios_allocate_reso
 static int __init pcibios_assign_resources(void)
 {
 	struct pci_dev *dev = NULL;
+	struct pci_bus *bus;
 	struct resource *r;
 
 	if (!(pci_probe & PCI_ASSIGN_ROMS)) {

arch/x86/pci/i386.c static int __init pcibios_assign_resourc · line 221

Quoted 8 lines
arch/x86/pci/i386.c +5 -0
@@ -213,6 +214,11 @@ static int __init pcibios_assign_resourc
 		}
 	}
 
+	/* Try to release bridge resources, that there is not child under it */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_bus_release_bridges_not_used_res(bus);
+	}
+

If this is only for PCIe hotplug, I don't think we need it here (as you're doing for non x86 platforms). If not, I think you should do it in the another patch.

Thanks,
Kenji Kaneshige

author
#39 lore raw

Re: [PATCH 2/2] pci: only release that resource index is less than 3 -v5

Quoted 62 lines · Kenji Kaneshige
Quoted 47 lines · Yinghai Lu

after

| commit 308cf8e13f42f476dfd6552aeff58fdc0788e566
|
|    PCI: get larger bridge ranges when space is available

found one of resource of peer root bus (0x00) get released from root resource. later one hotplug device can not get big range anymore. other peer root buses is ok.

it turns out it is from transparent path.

those resources will be used for pci bridge BAR updated. so need to limit it to 3.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big range
for pcie devices - v4
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.

---
 arch/x86/pci/i386.c              |    6 ++
 drivers/pci/hotplug/pciehp_pci.c |    2
 drivers/pci/setup-bus.c          |   81

++++++++++++++++++++++++++++++++++++++-
include/linux/pci.h | 2 4 files changed, 90
insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +3 -0
@@ -319,6 +319,42 @@ static void pci_bridge_check_ranges(stru
     }
 }
 
+void pci_bridge_release_not_used_res(struct pci_bus *bus)
+{
+    int idx;

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 334

Quoted 10 lines
drivers/pci/setup-bus.c +10 -0
+    bool changed = false;
+    struct pci_dev *dev;
+    struct resource *r;
+    unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+                  IORESOURCE_PREFETCH;
+
+    /* for pci bridges res only */
+    dev = bus->self;
+    for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+        idx++) {

If this function is only for normal PCI bridge, we need additional check at the head of this function.

Quoted 6 lines
    if ((dev->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
        return;

if not, 3 should be 4 instead. And we can do as follows

       for (i = PCI_BRIDGE_RESOURCE; i <= PCI_BRIDGE_RESOURCE_END; i++) {

don't want to mess up with pci cardbus yet. can not access related stuff.

Quoted 8 lines

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 336

Quoted 3 lines
drivers/pci/setup-bus.c +2 -0
+        r = &dev->resource[idx];
+        if (r->flags & type_mask) {

How about

        if (!(r->flags & type_mask))
            continue;

ok

Quoted 10 lines

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 342

Quoted 6 lines
drivers/pci/setup-bus.c +6 -0
+            if (!r->parent)
+                continue;
+            /*
+             * if there is no child under that, we should release
+             * and use it.
+             */

I think this comment should be updated because whether "we should release and use it" is decided by the caller of this function.

ok

Quoted 8 lines

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 349

Quoted 7 lines
drivers/pci/setup-bus.c +7 -0
+            if (!r->child && !release_resource(r)) {
+                dev_info(&dev->dev,
+                     "resource %d %pRt released\n",
+                     idx, r);
+                r->flags = 0;
+                changed = true;
+            }
Quoted 8 lines

How about

        if (r->child)
            continue;

        if (!release_resource(r)) {
            ...
        }

ok

Quoted 7 lines

drivers/pci/setup-bus.c static void pci_bridge_check_ranges(stru · line 354

Quoted 5 lines
drivers/pci/setup-bus.c +5 -0
+        }
+    }
+
+    if (changed)
+        pci_setup_bridge(bus);
Quoted 6 lines

The pci_setup_bridge() is called even if some resources are used by children. For example, mem resource is used by children, but prefetchable mem resource is not used by children. In this case, I think mem resource is released and pci_setup_bridge() is called while prefetcable mem resource is being used. I'm worrying that it might cause something wrong.

should be ok
case 1: before assigned unassigned resource. no driver etc are loaded yet case 2: before config pcie slot bridge, no device under it yet.

Quoted 66 lines
Quoted 41 lines
drivers/pci/setup-bus.c +33 -0
+}
+EXPORT_SYMBOL(pci_bridge_release_not_used_res);
+
 /* Helper function for sizing routines: find first available
    bus resource of a given type. Note: we intentionally skip
    the bus resources which have already been assigned (that is,
@@ -576,6 +612,48 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+
+/* only release those resources that is on leaf bridge */
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)
+{
+    struct pci_dev *dev;
+    bool is_leaf_bridge = true;
+
+    list_for_each_entry(dev, &bus->devices, bus_list) {
+        struct pci_bus *b = dev->subordinate;
+        if (!b)
+            continue;
+
+        switch (dev->class >> 8) {
+        case PCI_CLASS_BRIDGE_CARDBUS:
+            is_leaf_bridge = false;
+            break;
+
+        case PCI_CLASS_BRIDGE_PCI:
+        default:
+            is_leaf_bridge = false;
+            pci_bus_release_bridges_not_used_res(b);
+            break;
+        }
+    }
+
+    /* The root bus? */
+    if (!bus->self)
+        return;
+
+    switch (bus->self->class >> 8) {

drivers/pci/setup-bus.c void __ref pci_bus_size_bridges(struct p · line 654

Quoted 10 lines
drivers/pci/setup-bus.c +10 -0
+    case PCI_CLASS_BRIDGE_CARDBUS:
+        break;
+
+    case PCI_CLASS_BRIDGE_PCI:
+    default:
+        if (is_leaf_bridge)
+            pci_bridge_release_not_used_res(bus);
+        break;
+    }
+}

How about like below. This might need to be called with "pci_bus_sem" held.

void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)

{
       struct pci_bus *b;

       /* If the bus is cardbus, do nothing */
       if (bus->self && (bus->self->class >> 8) ==
PCI_CLASS_BRIDGE_CARDBUS)
               return;

       /* We only release the resources under the leaf bridge */
       if (list_empty(&bus->children)) {
               if (!pci_is_root_bus(bus))
Quoted 9 lines
                       pci_bridge_release_not_used_res(bus);
               return;
       }

       /* Scan child buses if the bus is not leaf */
       list_for_each_entry(b, &bus->children, node)
        pci_bus_release_bridges_not_used_res(b);
}

ok

Quoted 23 lines
Quoted 23 lines
drivers/pci/setup-bus.c +4 -1
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
 {
     struct pci_bus *b;
@@ -644,7 +722,8 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+        if (!res || !res->end || !res->flags)
                         continue;
 
         dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +0 -0
@@ -98,6 +98,7 @@ int pciehp_configure_device(struct slot
         pci_dev_put(dev);
     }
 
Quoted 5 lines

drivers/pci/hotplug/pciehp_pci.c int pciehp_configure_device(struct slot · line 101

Quoted 1 line
drivers/pci/hotplug/pciehp_pci.c +1 -0
+    pci_bridge_release_not_used_res(parent);

I guess this is for the slots that are empty at the boot time on non x86 systems. And it only works at the first hot-add. Correct? How about doing this at the pciehp's slot initialization.

this one could be removed.

Quoted 14 lines

drivers/pci/hotplug/pciehp_pci.c int pciehp_unconfigure_device(struct slo · line 175

Quoted 10 lines
drivers/pci/hotplug/pciehp_pci.c +1 -0
     pci_bus_size_bridges(parent);
     pci_clear_master(bridge);
     pci_bridge_assign_resources(bridge);
@@ -171,6 +172,7 @@ int pciehp_unconfigure_device(struct slo
         }
         pci_dev_put(temp);
     }
+    pci_bridge_release_not_used_res(parent);

How about call pci_bus_release_bridges_not_used_res() instead and make pci_bridge_release_not_used_res() static function.

ok

Quoted 32 lines
Quoted 32 lines
drivers/pci/hotplug/pciehp_pci.c +0 -0
     return rc;
 }

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +2 -0
@@ -759,6 +759,8 @@ int pci_vpd_truncate(struct pci_dev *dev
 void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
+void pci_bus_release_bridges_not_used_res(struct pci_bus *bus);
+void pci_bridge_release_not_used_res(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
 void pdev_enable_device(struct pci_dev *);

Index: linux-2.6/arch/x86/pci/i386.c
===================================================================

arch/x86/pci/i386.c +1 -0
@@ -194,6 +194,7 @@ static void __init pcibios_allocate_reso
 static int __init pcibios_assign_resources(void)
 {
     struct pci_dev *dev = NULL;
+    struct pci_bus *bus;
     struct resource *r;
 
     if (!(pci_probe & PCI_ASSIGN_ROMS)) {
@@ -213,6 +214,11 @@ static int __init pcibios_assign_resourc
         }
     }
 
Quoted 10 lines

arch/x86/pci/i386.c static int __init pcibios_assign_resourc · line 221

Quoted 6 lines
arch/x86/pci/i386.c +1 -0
+    /* Try to release bridge resources, that there is not child under

it */

arch/x86/pci/i386.c +4 -0
+    list_for_each_entry(bus, &pci_root_buses, node) {
+        pci_bus_release_bridges_not_used_res(bus);
+    }
+

If this is only for PCIe hotplug, I don't think we need it here (as you're doing for non x86 platforms). If not, I think you should do it in the another patch.

it is needed for the first boot, when pcie card is already plugged in at boot time. need to move back to setup_bus.c

YH

author
#40 lore raw

PATCH 1/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v4

In reply to Eric W. Biederman

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes. -v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices() -v4: move out pci_is_enabled checkout of pci_setup_bridge()

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73 ++++++++++++++++++++++++++++++++++++---
 include/linux/pci.h              |    1 
 3 files changed, 93 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +23 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +69 -4

97 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
#41 lore raw

Re: [PATCH 1/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v4

Quoted 199 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

(quoted diff) +0 -3
-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes.
-v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices()
-v4: move out pci_is_enabled checkout of pci_setup_bridge()
---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73 ++++++++++++++++++++++++++++++++++++---
 include/linux/pci.h              |    1 
 3 files changed, 93 insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +23 -6
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +69 -4
@@ -27,6 +27,44 @@
 #include <linux/slab.h>
 #include "pci.h"
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev)
+{
+	struct resource *res;
+	struct resource_list head, *list, *tmp;
+	int idx;
+	u16 class = dev->class >> 8;
+
+	head.next = NULL;
+
+	/* Don't touch classless devices or host bridges or ioapics.  */
+	if (class == PCI_CLASS_NOT_DEFINED ||
+	    class == PCI_CLASS_BRIDGE_HOST)
+		return;
+
+	/* Don't touch ioapic devices already enabled by firmware */
+	if (class == PCI_CLASS_SYSTEM_PIC) {
+		u16 command;
+		pci_read_config_word(dev, PCI_COMMAND, &command);
+		if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+			return;
+	}
+
+	pdev_sort_resources(dev, &head);
+
+	for (list = head.next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			res->start = 0;
+			res->end = 0;
+			res->flags = 0;
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
+
 static void pbus_assign_resources_sorted(const struct pci_bus *bus)
 {
 	struct pci_dev *dev;
@@ -144,9 +182,6 @@ static void pci_setup_bridge(struct pci_
 	u32 l, bu, lu, io_upper16;
 	int pref_mem64;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
 		 pci_domain_nr(bus), bus->number);
 
@@ -541,6 +576,35 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
+{
+	struct pci_bus *b;
+
+	pdev_assign_resources_sorted((struct pci_dev *)bridge);
+
+	b = bridge->subordinate;
+	if (!b)
+		return;
+
+	pci_bus_assign_resources(b);
+
+	switch (bridge->class >> 8) {
+	case PCI_CLASS_BRIDGE_PCI:
+		pci_setup_bridge(b);
+		break;
+
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		pci_setup_cardbus(b);
+		break;
+
+	default:
+		dev_info(&bridge->dev, "not setting up bridge for bus "
+			 "%04x:%02x\n", pci_domain_nr(b), b->number);
+		break;
+	}
+}
+EXPORT_SYMBOL(pci_bridge_assign_resources);
+
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
 	struct pci_bus *b;
@@ -557,7 +621,8 @@ void __ref pci_bus_assign_resources(cons
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
-			pci_setup_bridge(b);
+			if (!pci_is_enabled(dev))
+				pci_setup_bridge(b);
 			break;
 
 		case PCI_CLASS_BRIDGE_CARDBUS:

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h ssize_t pci_write_vpd(struct pci_dev *de · line 762

Quoted 7 lines
include/linux/pci.h +1 -0
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);

Does this patch work without [PATCH 2/2]? I don't understand who releases resouces? Does find_free_bus_resource() still release resources?

Thanks,
Kenji Kaneshige

author
#42 lore raw

Re: [PATCH 1/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v4

Quoted 210 lines · Kenji Kaneshige
Quoted 209 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to
call assign resources several times, when there are several bridges under the
slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res
from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise
update res is not updated to bridge BAR. that is bridge is enabled already for
port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes.
-v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices()
-v4: move out pci_is_enabled checkout of pci_setup_bridge()

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73

++++++++++++++++++++++++++++++++++++---
include/linux/pci.h | 1 3 files changed, 93
insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +23 -6
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
         busnr = pci_scan_bridge(parent, dev, busnr, pass);
     if (!dev->subordinate)
         return -1;
-    pci_bus_size_bridges(dev->subordinate);
-    pci_bus_assign_resources(parent);
-    pci_enable_bridges(parent);
-    pci_bus_add_devices(parent);
+
     return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
     struct pci_dev *dev;
-    struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+    struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+    struct pci_bus *parent = bridge->subordinate;
     int num, fn;
     struct controller *ctrl = p_slot->ctrl;
+    int retval;
 
     dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
     if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
                 (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
             pciehp_add_bridge(dev);
         }
+        pci_dev_put(dev);
+    }
+
+    pci_bus_size_bridges(parent);
+    pci_clear_master(bridge);
+    pci_bridge_assign_resources(bridge);
+    retval = pci_reenable_device(bridge);
+    pci_set_master(bridge);
+    pci_enable_bridges(parent);
+
+    for (fn = 0; fn < 8; fn++) {
+        dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+        if (!dev)
+            continue;
+        if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+            pci_dev_put(dev);
+            continue;
+        }
         pci_configure_slot(dev);
         pci_dev_put(dev);
     }
 
-    pci_bus_assign_resources(parent);
     pci_bus_add_devices(parent);
+
     return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +69 -4
@@ -27,6 +27,44 @@
 #include <linux/slab.h>
 #include "pci.h"
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev)
+{
+    struct resource *res;
+    struct resource_list head, *list, *tmp;
+    int idx;
+    u16 class = dev->class >> 8;
+
+    head.next = NULL;
+
+    /* Don't touch classless devices or host bridges or ioapics.  */
+    if (class == PCI_CLASS_NOT_DEFINED ||
+        class == PCI_CLASS_BRIDGE_HOST)
+        return;
+
+    /* Don't touch ioapic devices already enabled by firmware */
+    if (class == PCI_CLASS_SYSTEM_PIC) {
+        u16 command;
+        pci_read_config_word(dev, PCI_COMMAND, &command);
+        if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+            return;
+    }
+
+    pdev_sort_resources(dev, &head);
+
+    for (list = head.next; list;) {
+        res = list->res;
+        idx = res - &list->dev->resource[0];
+        if (pci_assign_resource(list->dev, idx)) {
+            res->start = 0;
+            res->end = 0;
+            res->flags = 0;
+        }
+        tmp = list;
+        list = list->next;
+        kfree(tmp);
+    }
+}
+
 static void pbus_assign_resources_sorted(const struct pci_bus *bus)
 {
     struct pci_dev *dev;
@@ -144,9 +182,6 @@ static void pci_setup_bridge(struct pci_
     u32 l, bu, lu, io_upper16;
     int pref_mem64;
 
-    if (pci_is_enabled(bridge))
-        return;
-
     dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
          pci_domain_nr(bus), bus->number);
 
@@ -541,6 +576,35 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
+{
+    struct pci_bus *b;
+
+    pdev_assign_resources_sorted((struct pci_dev *)bridge);
+
+    b = bridge->subordinate;
+    if (!b)
+        return;
+
+    pci_bus_assign_resources(b);
+
+    switch (bridge->class >> 8) {
+    case PCI_CLASS_BRIDGE_PCI:
+        pci_setup_bridge(b);
+        break;
+
+    case PCI_CLASS_BRIDGE_CARDBUS:
+        pci_setup_cardbus(b);
+        break;
+
+    default:
+        dev_info(&bridge->dev, "not setting up bridge for bus "
+             "%04x:%02x\n", pci_domain_nr(b), b->number);
+        break;
+    }
+}
+EXPORT_SYMBOL(pci_bridge_assign_resources);
+
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
     struct pci_bus *b;
@@ -557,7 +621,8 @@ void __ref pci_bus_assign_resources(cons
 
         switch (dev->class >> 8) {
         case PCI_CLASS_BRIDGE_PCI:
-            pci_setup_bridge(b);
+            if (!pci_is_enabled(dev))
+                pci_setup_bridge(b);
             break;
 
         case PCI_CLASS_BRIDGE_CARDBUS:

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +0 -0
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
Quoted 11 lines
Quoted 7 lines
include/linux/pci.h +0 -0
 /* Helper functions for low-level code

(drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
void pci_bus_assign_resources(const struct pci_bus *bus); void pci_bus_size_bridges(struct pci_bus *bus);
int pci_claim_resource(struct pci_dev *, int);

Does this patch work without [PATCH 2/2]? I don't understand who releases resouces? Does find_free_bus_resource() still release resources?

need to work with [2/2].

YH

#43 lore raw

Re: [PATCH 1/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v4

Quoted 221 lines · Yinghai Lu
Quoted 209 lines · Kenji Kaneshige
Quoted 208 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to
call assign resources several times, when there are several bridges under the
slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res
from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise
update res is not updated to bridge BAR. that is bridge is enabled already for
port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes.
-v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices()
-v4: move out pci_is_enabled checkout of pci_setup_bridge()

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73

++++++++++++++++++++++++++++++++++++---
include/linux/pci.h | 1 3 files changed, 93
insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +23 -6
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
         busnr = pci_scan_bridge(parent, dev, busnr, pass);
     if (!dev->subordinate)
         return -1;
-    pci_bus_size_bridges(dev->subordinate);
-    pci_bus_assign_resources(parent);
-    pci_enable_bridges(parent);
-    pci_bus_add_devices(parent);
+
     return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
     struct pci_dev *dev;
-    struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+    struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+    struct pci_bus *parent = bridge->subordinate;
     int num, fn;
     struct controller *ctrl = p_slot->ctrl;
+    int retval;
 
     dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
     if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
                 (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
             pciehp_add_bridge(dev);
         }
+        pci_dev_put(dev);
+    }
+
+    pci_bus_size_bridges(parent);
+    pci_clear_master(bridge);
+    pci_bridge_assign_resources(bridge);
+    retval = pci_reenable_device(bridge);
+    pci_set_master(bridge);
+    pci_enable_bridges(parent);
+
+    for (fn = 0; fn < 8; fn++) {
+        dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+        if (!dev)
+            continue;
+        if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+            pci_dev_put(dev);
+            continue;
+        }
         pci_configure_slot(dev);
         pci_dev_put(dev);
     }
 
-    pci_bus_assign_resources(parent);
     pci_bus_add_devices(parent);
+
     return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +69 -4
@@ -27,6 +27,44 @@
 #include <linux/slab.h>
 #include "pci.h"
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev)
+{
+    struct resource *res;
+    struct resource_list head, *list, *tmp;
+    int idx;
+    u16 class = dev->class >> 8;
+
+    head.next = NULL;
+
+    /* Don't touch classless devices or host bridges or ioapics.  */
+    if (class == PCI_CLASS_NOT_DEFINED ||
+        class == PCI_CLASS_BRIDGE_HOST)
+        return;
+
+    /* Don't touch ioapic devices already enabled by firmware */
+    if (class == PCI_CLASS_SYSTEM_PIC) {
+        u16 command;
+        pci_read_config_word(dev, PCI_COMMAND, &command);
+        if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+            return;
+    }
+
+    pdev_sort_resources(dev, &head);
+
+    for (list = head.next; list;) {
+        res = list->res;
+        idx = res - &list->dev->resource[0];
+        if (pci_assign_resource(list->dev, idx)) {
+            res->start = 0;
+            res->end = 0;
+            res->flags = 0;
+        }
+        tmp = list;
+        list = list->next;
+        kfree(tmp);
+    }
+}
+
 static void pbus_assign_resources_sorted(const struct pci_bus *bus)
 {
     struct pci_dev *dev;
@@ -144,9 +182,6 @@ static void pci_setup_bridge(struct pci_
     u32 l, bu, lu, io_upper16;
     int pref_mem64;
 
-    if (pci_is_enabled(bridge))
-        return;
-
     dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
          pci_domain_nr(bus), bus->number);
 
@@ -541,6 +576,35 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
+{
+    struct pci_bus *b;
+
+    pdev_assign_resources_sorted((struct pci_dev *)bridge);
+
+    b = bridge->subordinate;
+    if (!b)
+        return;
+
+    pci_bus_assign_resources(b);
+
+    switch (bridge->class >> 8) {
+    case PCI_CLASS_BRIDGE_PCI:
+        pci_setup_bridge(b);
+        break;
+
+    case PCI_CLASS_BRIDGE_CARDBUS:
+        pci_setup_cardbus(b);
+        break;
+
+    default:
+        dev_info(&bridge->dev, "not setting up bridge for bus "
+             "%04x:%02x\n", pci_domain_nr(b), b->number);
+        break;
+    }
+}
+EXPORT_SYMBOL(pci_bridge_assign_resources);
+
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
     struct pci_bus *b;
@@ -557,7 +621,8 @@ void __ref pci_bus_assign_resources(cons
 
         switch (dev->class >> 8) {
         case PCI_CLASS_BRIDGE_PCI:
-            pci_setup_bridge(b);
+            if (!pci_is_enabled(dev))
+                pci_setup_bridge(b);
             break;
 
         case PCI_CLASS_BRIDGE_CARDBUS:

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +0 -0
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
Quoted 10 lines
Quoted 7 lines
include/linux/pci.h +0 -0
 /* Helper functions for low-level code

(drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
void pci_bus_assign_resources(const struct pci_bus *bus); void pci_bus_size_bridges(struct pci_bus *bus);
int pci_claim_resource(struct pci_dev *, int);

Does this patch work without [PATCH 2/2]? I don't understand who releases resouces? Does find_free_bus_resource() still release resources?

Quoted 2 lines

need to work with [2/2].

Ok. Could you rearrange the set of patches with the right pieces and with the right order? It's very difficult for me to understand and review the current patches.

By the way, is release_resource() in find_free_bus_resource() already removed?

Thanks,
Kenji Kaneshige

author
#44 lore raw

Re: [PATCH 1/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v4

Quoted 223 lines · Kenji Kaneshige
Quoted 222 lines · Yinghai Lu
Quoted 220 lines · Kenji Kaneshige
Quoted 210 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge()
and at last do them all together one time including slot bridge, to avoid to
call assign resources several times, when there are several bridges under the
slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res
from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise
update res is not updated to bridge BAR. that is bridge is enabled already for
port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes.
-v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices()
-v4: move out pci_is_enabled checkout of pci_setup_bridge()

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73

++++++++++++++++++++++++++++++++++++---
include/linux/pci.h | 1 3 files changed, 93
insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +23 -6
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
         busnr = pci_scan_bridge(parent, dev, busnr, pass);
     if (!dev->subordinate)
         return -1;
-    pci_bus_size_bridges(dev->subordinate);
-    pci_bus_assign_resources(parent);
-    pci_enable_bridges(parent);
-    pci_bus_add_devices(parent);
+
     return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
     struct pci_dev *dev;
-    struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+    struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+    struct pci_bus *parent = bridge->subordinate;
     int num, fn;
     struct controller *ctrl = p_slot->ctrl;
+    int retval;
 
     dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
     if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
                 (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
             pciehp_add_bridge(dev);
         }
+        pci_dev_put(dev);
+    }
+
+    pci_bus_size_bridges(parent);
+    pci_clear_master(bridge);
+    pci_bridge_assign_resources(bridge);
+    retval = pci_reenable_device(bridge);
+    pci_set_master(bridge);
+    pci_enable_bridges(parent);
+
+    for (fn = 0; fn < 8; fn++) {
+        dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+        if (!dev)
+            continue;
+        if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+            pci_dev_put(dev);
+            continue;
+        }
         pci_configure_slot(dev);
         pci_dev_put(dev);
     }
 
-    pci_bus_assign_resources(parent);
     pci_bus_add_devices(parent);
+
     return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +69 -4
@@ -27,6 +27,44 @@
 #include <linux/slab.h>
 #include "pci.h"
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev)
+{
+    struct resource *res;
+    struct resource_list head, *list, *tmp;
+    int idx;
+    u16 class = dev->class >> 8;
+
+    head.next = NULL;
+
+    /* Don't touch classless devices or host bridges or ioapics.  */
+    if (class == PCI_CLASS_NOT_DEFINED ||
+        class == PCI_CLASS_BRIDGE_HOST)
+        return;
+
+    /* Don't touch ioapic devices already enabled by firmware */
+    if (class == PCI_CLASS_SYSTEM_PIC) {
+        u16 command;
+        pci_read_config_word(dev, PCI_COMMAND, &command);
+        if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+            return;
+    }
+
+    pdev_sort_resources(dev, &head);
+
+    for (list = head.next; list;) {
+        res = list->res;
+        idx = res - &list->dev->resource[0];
+        if (pci_assign_resource(list->dev, idx)) {
+            res->start = 0;
+            res->end = 0;
+            res->flags = 0;
+        }
+        tmp = list;
+        list = list->next;
+        kfree(tmp);
+    }
+}
+
 static void pbus_assign_resources_sorted(const struct pci_bus *bus)
 {
     struct pci_dev *dev;
@@ -144,9 +182,6 @@ static void pci_setup_bridge(struct pci_
     u32 l, bu, lu, io_upper16;
     int pref_mem64;
 
-    if (pci_is_enabled(bridge))
-        return;
-
     dev_info(&bridge->dev, "PCI bridge, secondary bus %04x:%02x\n",
          pci_domain_nr(bus), bus->number);
 
@@ -541,6 +576,35 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
+{
+    struct pci_bus *b;
+
+    pdev_assign_resources_sorted((struct pci_dev *)bridge);
+
+    b = bridge->subordinate;
+    if (!b)
+        return;
+
+    pci_bus_assign_resources(b);
+
+    switch (bridge->class >> 8) {
+    case PCI_CLASS_BRIDGE_PCI:
+        pci_setup_bridge(b);
+        break;
+
+    case PCI_CLASS_BRIDGE_CARDBUS:
+        pci_setup_cardbus(b);
+        break;
+
+    default:
+        dev_info(&bridge->dev, "not setting up bridge for bus "
+             "%04x:%02x\n", pci_domain_nr(b), b->number);
+        break;
+    }
+}
+EXPORT_SYMBOL(pci_bridge_assign_resources);
+
 void __ref pci_bus_assign_resources(const struct pci_bus *bus)
 {
     struct pci_bus *b;
@@ -557,7 +621,8 @@ void __ref pci_bus_assign_resources(cons
 
         switch (dev->class >> 8) {
         case PCI_CLASS_BRIDGE_PCI:
-            pci_setup_bridge(b);
+            if (!pci_is_enabled(dev))
+                pci_setup_bridge(b);
             break;
 
         case PCI_CLASS_BRIDGE_CARDBUS:

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +0 -0
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code
Quoted 6 lines

(drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
void pci_bus_assign_resources(const struct pci_bus *bus); void pci_bus_size_bridges(struct pci_bus *bus);
int pci_claim_resource(struct pci_dev *, int);

Does this patch work without [PATCH 2/2]? I don't understand who releases resouces? Does find_free_bus_resource() still release resources?

Quoted 6 lines
Quoted 2 lines

need to work with [2/2].

Ok. Could you rearrange the set of patches with the right pieces and with the right order? It's very difficult for me to understand and review the current patches.

ok

Quoted 3 lines

By the way, is release_resource() in find_free_bus_resource() already removed?

YES. Jesse sent request to Linus to revert that.

YH

author
#45 lore raw

PATCH 1/2 pci: release that leaf bridge' resource index is not used -v6

In reply to Kenji Kaneshige

only for index < 3

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big range for pcie devices - v4
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource

---
 drivers/pci/setup-bus.c |   65 +++++++++++++++++++++++++++++++++++++++++++++++-
 include/linux/pci.h     |    1 
 2 files changed, 65 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +64 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -573,13 +573,68 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void pci_bridge_release_not_used_res(struct pci_bus *bus)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if (!(r->flags & type_mask))
+			continue;
+		if (!r->parent)
+			continue;
+		/* if there is no child under that, we should release it */
+		if (r->child)
+			continue;
+		if (!release_resource(r)) {
+			dev_info(&dev->dev, "resource %d %pRt released\n",
+				 idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed)
+		pci_setup_bridge(bus);
+}
+
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)
+{
+	struct pci_bus *b;
+
+	/* If the bus is cardbus, do nothing */
+	if (bus->self && (bus->self->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
+		return;
+
+	/* We only release the resources under the leaf bridge */
+	if (list_empty(&bus->children)) {
+		if (!pci_is_root_bus(bus))
+			pci_bridge_release_not_used_res(bus);
+		return;
+	}
+
+	/* Scan child buses if the bus is not leaf */
+	list_for_each_entry(b, &bus->children, node)
+		pci_bus_release_bridges_not_used_res(b);
+}
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);
@@ -608,6 +663,14 @@ pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
 
+	/*
+	 * Try to release leaf bridge's resources that there is not child
+	 * under it
+	 */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_bus_release_bridges_not_used_res(bus);
+	}
+
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -758,6 +758,7 @@ int pci_vpd_truncate(struct pci_dev *dev
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
+void pci_bus_release_bridges_not_used_res(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
 void pdev_enable_device(struct pci_dev *);
#46 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource index is not used -v6

Quoted 15 lines · On Thu, 29 Oct 2009 02:52:00 -0700 Yinghai Lu <[email protected]> wrote:

only for index < 3

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big
range for pcie devices - v4 v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res. only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order move back release to pci_assign_unassigned_resource

Quoted 1 line

Starting to look better...

Quoted 12 lines
---
 drivers/pci/setup-bus.c |   65

+++++++++++++++++++++++++++++++++++++++++++++++-
include/linux/pci.h | 1 2 files changed, 65 insertions(+), 1
deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 576

Quoted 4 lines
drivers/pci/setup-bus.c +1 -0
@@ -573,13 +573,68 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void pci_bridge_release_not_used_res(struct pci_bus *bus)

Can we call these functions something else? Maybe pci_bridge_release_unused() with a kdoc comment describing exactly what it does and why?

Quoted 26 lines
drivers/pci/setup-bus.c +19 -0
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES
+ 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if (!(r->flags & type_mask))
+			continue;
+		if (!r->parent)
+			continue;
+		/* if there is no child under that, we should

release it */

drivers/pci/setup-bus.c +4 -0
+		if (r->child)
+			continue;
+		if (!release_resource(r)) {
+			dev_info(&dev->dev, "resource %d %pRt

released\n",

drivers/pci/setup-bus.c +1 -0
+				 idx, r);

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 609

Quoted 10 lines
drivers/pci/setup-bus.c +10 -0
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed)
+		pci_setup_bridge(bus);
+}
+
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)

Likewise, maybe pci_bus_release_unused_bridge_res or something, with comments again.

Quoted 40 lines
drivers/pci/setup-bus.c +5 -0
+{
+	struct pci_bus *b;
+
+	/* If the bus is cardbus, do nothing */
+	if (bus->self && (bus->self->class >> 8) ==

PCI_CLASS_BRIDGE_CARDBUS)

drivers/pci/setup-bus.c +17 -1
+		return;
+
+	/* We only release the resources under the leaf bridge */
+	if (list_empty(&bus->children)) {
+		if (!pci_is_root_bus(bus))
+			pci_bridge_release_not_used_res(bus);
+		return;
+	}
+
+	/* Scan child buses if the bus is not leaf */
+	list_for_each_entry(b, &bus->children, node)
+		pci_bus_release_bridges_not_used_res(b);
+}
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d

%pRt\n", i, res); @@ -608,6 +663,14 @@
pci_assign_unassigned_resources(void) {

drivers/pci/setup-bus.c +2 -0
 	struct pci_bus *bus;
 
+	/*
+	 * Try to release leaf bridge's resources that there is not

child

drivers/pci/setup-bus.c pci_assign_unassigned_resources(void) · line 673

Quoted 6 lines
drivers/pci/setup-bus.c +6 -0
+	 * under it
+	 */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_bus_release_bridges_not_used_res(bus);
+	}
+

Also I wonder if we need to make this a command line option that isn't run by default?

Really, as Eric says, we need a real dynamic allocation system with the ability to move devices around at some point. Any volunteers? :)

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center
author
#47 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource index is not used -v6

Quoted 35 lines · Jesse Barnes
Quoted 14 lines · On Thu, 29 Oct 2009 02:52:00 -0700 Yinghai Lu <[email protected]> wrote:

only for index < 3

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big
range for pcie devices - v4 v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res. only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order move back release to pci_assign_unassigned_resource

Quoted 1 line

Starting to look better...

Quoted 14 lines
---
 drivers/pci/setup-bus.c |   65

+++++++++++++++++++++++++++++++++++++++++++++++-
include/linux/pci.h | 1 2 files changed, 65 insertions(+), 1
deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +0 -0
@@ -573,13 +573,68 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
Quoted 5 lines

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 576

Quoted 1 line
drivers/pci/setup-bus.c +1 -0
+static void pci_bridge_release_not_used_res(struct pci_bus *bus)

Can we call these functions something else? Maybe pci_bridge_release_unused() with a kdoc comment describing exactly what it does and why?

ok.

Quoted 32 lines
Quoted 32 lines
drivers/pci/setup-bus.c +19 -0
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES
+ 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if (!(r->flags & type_mask))
+			continue;
+		if (!r->parent)
+			continue;
+		/* if there is no child under that, we should

release it */

drivers/pci/setup-bus.c +4 -0
+		if (r->child)
+			continue;
+		if (!release_resource(r)) {
+			dev_info(&dev->dev, "resource %d %pRt

released\n",

drivers/pci/setup-bus.c +6 -0
+				 idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
Quoted 8 lines

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 609

Quoted 5 lines
drivers/pci/setup-bus.c +5 -0
+	if (changed)
+		pci_setup_bridge(bus);
+}
+
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)

Likewise, maybe pci_bus_release_unused_bridge_res or something, with comments again.

ok

Quoted 38 lines
Quoted 38 lines
drivers/pci/setup-bus.c +5 -0
+{
+	struct pci_bus *b;
+
+	/* If the bus is cardbus, do nothing */
+	if (bus->self && (bus->self->class >> 8) ==

PCI_CLASS_BRIDGE_CARDBUS)

drivers/pci/setup-bus.c +17 -1
+		return;
+
+	/* We only release the resources under the leaf bridge */
+	if (list_empty(&bus->children)) {
+		if (!pci_is_root_bus(bus))
+			pci_bridge_release_not_used_res(bus);
+		return;
+	}
+
+	/* Scan child buses if the bus is not leaf */
+	list_for_each_entry(b, &bus->children, node)
+		pci_bus_release_bridges_not_used_res(b);
+}
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d

%pRt\n", i, res); @@ -608,6 +663,14 @@
pci_assign_unassigned_resources(void) {
struct pci_bus *bus;

Quoted 12 lines

drivers/pci/setup-bus.c pci_assign_unassigned_resources(void) · line 673

Quoted 9 lines
drivers/pci/setup-bus.c +2 -0
+	/*
+	 * Try to release leaf bridge's resources that there is not

child

drivers/pci/setup-bus.c +6 -0
+	 * under it
+	 */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_bus_release_bridges_not_used_res(bus);
+	}
+

Also I wonder if we need to make this a command line option that isn't run by default?

thinking:
1. try not to touch bridge resource and do the allocation and record the fail device
2. release leaf bridge for those failed device
3. do second around bus_size and allocation.

Quoted 3 lines

Really, as Eric says, we need a real dynamic allocation system with the ability to move devices around at some point. Any volunteers? :)

that will need suspend driver and rescan allocation and wakeup driver...

YH

#48 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource index is not used -v6

Quoted 7 lines · On Thu, 29 Oct 2009 10:10:22 -0700 Yinghai Lu <[email protected]> wrote:
Quoted 2 lines

Also I wonder if we need to make this a command line option that isn't run by default?

thinking:
1. try not to touch bridge resource and do the allocation and record the fail device 2. release leaf bridge for those failed device
3. do second around bus_size and allocation.

Yeah, that would make it more automatic at least, which would be nice.

Quoted 6 lines
Quoted 3 lines

Really, as Eric says, we need a real dynamic allocation system with the ability to move devices around at some point. Any volunteers? :)

that will need suspend driver and rescan allocation and wakeup driver...

Right, it's fairly invasive. But it's the right direction to go long term.

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#50 lore raw

PATCH 2/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v5

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes. -v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices() -v4: move out pci_is_enabled checkout of pci_setup_bridge()
-v5: change the applying sequence.

---
 drivers/pci/hotplug/pciehp_pci.c |   31 +++++++++++++---
 drivers/pci/setup-bus.c          |   73 ++++++++++++++++++++++++++++++++++++---
 include/linux/pci.h              |    1 
 3 files changed, 95 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +25 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
@@ -155,5 +172,7 @@ int pciehp_unconfigure_device(struct slo
 		pci_dev_put(temp);
 	}
 
+	pci_bus_release_bridges_not_used_res(parent);
+
 	return rc;
 }
drivers/pci/setup-bus.c +69 -4

97 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 void pci_bus_release_bridges_not_used_res(struct pci_bus *bus);
#51 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v5

Quoted 17 lines · On Thu, 29 Oct 2009 02:52:25 -0700 Yinghai Lu <[email protected]> wrote:

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

Quoted 7 lines

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes. -v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices() -v4: move out pci_is_enabled checkout of pci_setup_bridge() -v5: change the applying sequence.

Where are we with this patchset? Given the nature of the changes I'll definitely want to get a few Tested-bys in addition to acks from people involved in this thread.

Thanks,

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center
author
#52 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v5

Quoted 25 lines · Jesse Barnes
Quoted 22 lines · On Thu, 29 Oct 2009 02:52:25 -0700 Yinghai Lu <[email protected]> wrote:

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes. -v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices() -v4: move out pci_is_enabled checkout of pci_setup_bridge() -v5: change the applying sequence.

Quoted 5 lines
Quoted 1 line

Where are we with this patchset? Given the nature of the changes I'll definitely want to get a few Tested-bys in addition to acks from people involved in this thread.

already have -v8.

but will refresh them after you push out your tree with several patches from Bjorn.

YH

author
#53 lore raw

PATCH 1/2 pci: release that leaf bridge' resource that is not big -v9

only for index < 3

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big
enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.

---
 drivers/pci/setup-bus.c |  253 ++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 234 insertions(+), 19 deletions(-)
drivers/pci/setup-bus.c +234 -19
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -27,7 +27,49 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+
+static void add_to_failed_list(struct resource_list *head, struct pci_dev *dev,
+				 struct resource *res)
+{
+	struct resource_list *list = head;
+	struct resource_list *ln = list->next;
+	struct resource_list *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list *head)
+{
+	struct resource_list *list, *tmp;
+	struct resource *res;
+	/*
+	 * Try to release leaf bridge's resources that there is no child
+	 * under it
+	 */
+	for (list = head->next; list;) {
+		res = list->res;
+		res->start = 0;
+		res->end = 0;
+		res->flags = 0;
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
@@ -58,9 +100,18 @@ static void pbus_assign_resources_sorted
 		res = list->res;
 		idx = res - &list->dev->resource[0];
 		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
+			if (fail_head && !list->dev->subordinate &&
+			    !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
 		}
 		tmp = list;
 		list = list->next;
@@ -134,19 +185,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct resource *res;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
-		 bus->secondary, bus->subordinate);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	res = bus->resource[0];
@@ -172,7 +216,13 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	res = bus->resource[1];
@@ -187,6 +237,14 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  bridge window [mem disabled]\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -219,10 +277,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
+		 bus->secondary, bus->subordinate);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
@@ -543,19 +628,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -573,15 +659,105 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type)
+{
+	struct pci_bus *b;
+
+	/* If the bus is cardbus, do nothing */
+	if (bus->self && (bus->self->class >> 8) == PCI_CLASS_BRIDGE_CARDBUS)
+		return;
+
+	/* We only release the resources under the leaf bridge */
+	if (list_empty(&bus->children)) {
+		if (!pci_is_root_bus(bus))
+			pci_bridge_release_unused_res(bus, type);
+		return;
+	}
+
+	/* Scan child buses if the bus is not leaf */
+	list_for_each_entry(b, &bus->children, node)
+		pci_bus_release_unused_bridge_res(b, type);
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pR\n", i, res);
@@ -609,6 +785,12 @@ void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	bool second_tried = false;
+	struct resource_list head, *list, *tmp;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+again:
+	head.next = NULL;
 
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
@@ -617,7 +799,40 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+
+	if (second_tried) {
+		/* still fail, don't want to try more */
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		bus = list->dev->bus;
+		pci_bus_release_unused_bridge_res(bus,
+				 list->res->flags & type_mask);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
author
#54 lore raw

PATCH 2/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v9

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++-
 drivers/pci/setup-bus.c          |  129 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 143 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +125 -4

156 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -767,6 +767,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
#55 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v9

* Yinghai Lu <[email protected]>:

Quoted 28 lines

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.

Quoted 1 line

How did you build and test this? I got this during building:

ERROR: "pci_assign_unassigned_bridge_resources" [drivers/pci/hotplug/pciehp.ko] undefined! make[1]: *** [__modpost] Error 1
make: *** [modules] Error 2
make: *** Waiting for unfinished jobs....

This patch fixed it for me:
---

drivers/pci/setup-bus.c +1 -0
index 95fe3f4..2f15473 100644
@@ -962,3 +962,4 @@ again:
 
 	goto again;
 }
+EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
author
#56 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v9

Quoted 44 lines · Alex Chiang

* Yinghai Lu <[email protected]>:

Quoted 27 lines

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.

Quoted 1 line

How did you build and test this? I got this during building:

ERROR: "pci_assign_unassigned_bridge_resources" [drivers/pci/hotplug/pciehp.ko] undefined! make[1]: *** [__modpost] Error 1
make: *** [modules] Error 2
make: *** Waiting for unfinished jobs....

This patch fixed it for me:
---

drivers/pci/setup-bus.c +0 -0
index 95fe3f4..2f15473 100644

drivers/pci/setup-bus.c again: · line 965

Quoted 4 lines
drivers/pci/setup-bus.c +1 -0
@@ -962,3 +962,4 @@ again:
 
 	goto again;
 }
+EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);

thanks.

i built that in

YH

author
#57 lore raw

PATCH 2/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v10

In reply to Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.

Quoted 258 lines · v3: Kenji pointed out that pci_config_slot need to be Yinghai Lu wrote:

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++-
 drivers/pci/setup-bus.c          |  129 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 143 insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +17 -6
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +125 -4
@@ -68,6 +68,52 @@ static void free_failed_list(struct reso
 	head->next = NULL;
 }
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev,
+				 struct resource_list *fail_head)
+{
+	struct resource *res;
+	struct resource_list head, *list, *tmp;
+	int idx;
+	u16 class = dev->class >> 8;
+
+	head.next = NULL;
+
+	/* Don't touch classless devices or host bridges or ioapics.  */
+	if (class == PCI_CLASS_NOT_DEFINED || class == PCI_CLASS_BRIDGE_HOST)
+		return;
+
+	/* Don't touch ioapic devices already enabled by firmware */
+	if (class == PCI_CLASS_SYSTEM_PIC) {
+		u16 command;
+		pci_read_config_word(dev, PCI_COMMAND, &command);
+		if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+			return;
+	}
+
+	pdev_sort_resources(dev, &head);
+
+	for (list = head.next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			if (fail_head && !list->dev->subordinate &&
+			    !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
 static void pbus_assign_resources_sorted(const struct pci_bus *bus,
 					 struct resource_list *fail_head)
 {
@@ -282,9 +328,6 @@ static void __pci_setup_bridge(struct pc
 {
 	struct pci_dev *bridge = bus->self;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
 		 bus->secondary, bus->subordinate);
 
@@ -645,7 +688,8 @@ static void __ref __pci_bus_assign_resou
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
-			pci_setup_bridge(b);
+			if (!pci_is_enabled(dev))
+				pci_setup_bridge(b);
 			break;
 
 		case PCI_CLASS_BRIDGE_CARDBUS:
@@ -666,6 +710,34 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void __ref __pci_bridge_assign_resources(const struct pci_dev *bridge,
+					 struct resource_list *fail_head)
+{
+	struct pci_bus *b;
+
+	pdev_assign_resources_sorted((struct pci_dev *)bridge, fail_head);
+
+	b = bridge->subordinate;
+	if (!b)
+		return;
+
+	__pci_bus_assign_resources(b, fail_head);
+
+	switch (bridge->class >> 8) {
+	case PCI_CLASS_BRIDGE_PCI:
+		pci_setup_bridge(b);
+		break;
+
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		pci_setup_cardbus(b);
+		break;
+
+	default:
+		dev_info(&bridge->dev, "not setting up bridge for bus "
+			 "%04x:%02x\n", pci_domain_nr(b), b->number);
+		break;
+	}
+}
 static void release_children_resource(struct resource *r)
 {
 	struct resource *p;
@@ -841,3 +913,52 @@ enable_and_dump:
 		pci_bus_dump_resources(bus);
 	}
 }
+
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
+{
+	struct pci_bus *bus;
+	struct pci_bus *parent = bridge->subordinate;
+	bool second_tried = false;
+	struct resource_list head, *list, *tmp;
+	int retval;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+again:
+	head.next = NULL;
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	__pci_bridge_assign_resources(bridge, &head);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	/* any device complain? */
+	if (!head.next)
+		return;
+
+	if (second_tried) {
+		/* still fail, don't want to try more */
+		free_failed_list(&head);
+		return;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		bus = list->dev->bus;
+		pci_bus_release_unused_bridge_res(bus,
+				 list->res->flags & type_mask);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+}

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h void pci_bus_assign_resources(const stru · line 773

Quoted 7 lines
include/linux/pci.h +1 -0
@@ -767,6 +767,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);

called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10: alex found need to have export for pci_assign_unassigned_bridge_resources

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++-
 drivers/pci/setup-bus.c          |  130 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 144 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +126 -4

157 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -768,6 +768,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
author
#58 lore raw

PATCH 2/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v10

In reply to Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10: alex found need to have export for pci_assign_unassigned_bridge_resources

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++-
 drivers/pci/setup-bus.c          |  130 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 144 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +126 -4

157 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -768,6 +768,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
#59 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v10

Can I ask which is the latest version?

I think -v10 is the latest. But I could not find -v10 for patch 1/2.

Thanks,
Kenji Kaneshige

Quoted 268 lines · Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10: alex found need to have export for pci_assign_unassigned_bridge_resources

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++-
 drivers/pci/setup-bus.c          |  130 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 144 insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +17 -6
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +126 -4
@@ -68,6 +68,52 @@ static void free_failed_list(struct reso
 	head->next = NULL;
 }
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev,
+				 struct resource_list *fail_head)
+{
+	struct resource *res;
+	struct resource_list head, *list, *tmp;
+	int idx;
+	u16 class = dev->class >> 8;
+
+	head.next = NULL;
+
+	/* Don't touch classless devices or host bridges or ioapics.  */
+	if (class == PCI_CLASS_NOT_DEFINED || class == PCI_CLASS_BRIDGE_HOST)
+		return;
+
+	/* Don't touch ioapic devices already enabled by firmware */
+	if (class == PCI_CLASS_SYSTEM_PIC) {
+		u16 command;
+		pci_read_config_word(dev, PCI_COMMAND, &command);
+		if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+			return;
+	}
+
+	pdev_sort_resources(dev, &head);
+
+	for (list = head.next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			if (fail_head && !list->dev->subordinate &&
+			    !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
 static void pbus_assign_resources_sorted(const struct pci_bus *bus,
 					 struct resource_list *fail_head)
 {
@@ -282,9 +328,6 @@ static void __pci_setup_bridge(struct pc
 {
 	struct pci_dev *bridge = bus->self;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
 		 bus->secondary, bus->subordinate);
 
@@ -645,7 +688,8 @@ static void __ref __pci_bus_assign_resou
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
-			pci_setup_bridge(b);
+			if (!pci_is_enabled(dev))
+				pci_setup_bridge(b);
 			break;
 
 		case PCI_CLASS_BRIDGE_CARDBUS:
@@ -666,6 +710,34 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void __ref __pci_bridge_assign_resources(const struct pci_dev *bridge,
+					 struct resource_list *fail_head)
+{
+	struct pci_bus *b;
+
+	pdev_assign_resources_sorted((struct pci_dev *)bridge, fail_head);
+
+	b = bridge->subordinate;
+	if (!b)
+		return;
+
+	__pci_bus_assign_resources(b, fail_head);
+
+	switch (bridge->class >> 8) {
+	case PCI_CLASS_BRIDGE_PCI:
+		pci_setup_bridge(b);
+		break;
+
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		pci_setup_cardbus(b);
+		break;
+
+	default:
+		dev_info(&bridge->dev, "not setting up bridge for bus "
+			 "%04x:%02x\n", pci_domain_nr(b), b->number);
+		break;
+	}
+}
 static void release_children_resource(struct resource *r)
 {
 	struct resource *p;
@@ -841,3 +913,53 @@ enable_and_dump:
 		pci_bus_dump_resources(bus);
 	}
 }
+
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
+{
+	struct pci_bus *bus;
+	struct pci_bus *parent = bridge->subordinate;
+	bool second_tried = false;
+	struct resource_list head, *list, *tmp;
+	int retval;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+again:
+	head.next = NULL;
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	__pci_bridge_assign_resources(bridge, &head);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	/* any device complain? */
+	if (!head.next)
+		return;
+
+	if (second_tried) {
+		/* still fail, don't want to try more */
+		free_failed_list(&head);
+		return;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		bus = list->dev->bus;
+		pci_bus_release_unused_bridge_res(bus,
+				 list->res->flags & type_mask);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+}
+EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +1 -0
@@ -768,6 +768,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
#61 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v10

Quoted 10 lines · Yinghai Lu
Quoted 3 lines · Kenji Kaneshige

Can I ask which is the latest version?

I think -v10 is the latest. But I could not find -v10 for patch 1/2.

please use

http://patchwork.kernel.org/patch/57814/ 1/2 -v9
http://patchwork.kernel.org/patch/58294/ 2/2 -v10

Regardless of PCIe hotplug, some of PCIe devices (not on hotplug slots) doesn't work with your patches by the following error.

Copyright (c) 2007-2009 Intel Corporation.
igb 0000:07:00.0: PCI INT A -> GSI 16 (level, low) -> IRQ 16 igb 0000:07:00.0: setting latency timer to 64
igb 0000:07:00.0: PCI INT A disabled
igb: probe of 0000:07:00.0 failed with error -5 igb 0000:07:00.1: PCI INT B -> GSI 17 (level, low) -> IRQ 17 igb 0000:07:00.1: setting latency timer to 64
igb 0000:07:00.1: PCI INT B disabled
igb: probe of 0000:07:00.1 failed with error -5 igb 0000:08:00.0: PCI INT A -> GSI 17 (level, low) -> IRQ 17 igb 0000:08:00.0: setting latency timer to 64
igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -5 igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: setting latency timer to 64
igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -5

I'm using Jesse's latest linux-next.
I'm attaching the /proc/iomem outputs. The iomem-default.txt is /proc/iomem output without your patches. The iomem-yinghai.txt is /proc/iomem output with your patches.

Thanks,
Kenji Kaneshige

00000000-0009efff : System RAM
0009f000-0009ffff : ACPI Non-volatile Storage
00100000-79190fff : System RAM
01000000-0141dbe2 : Kernel code
0141dbe3-0183639f : Kernel data
01acb000-027006f7 : Kernel bss
79191000-791c0fff : reserved
791c1000-79207fff : System RAM
79208000-79214fff : ACPI Tables
79215000-79321fff : System RAM
79322000-79322fff : reserved
79323000-7934afff : System RAM
7934b000-7934bfff : ACPI Tables
7934c000-793dffff : System RAM
793e0000-793e1fff : ACPI Tables
793e2000-79401fff : System RAM
79402000-79404fff : ACPI Tables
79405000-79450fff : System RAM
79451000-79451fff : ACPI Tables
79452000-79452fff : ACPI Non-volatile Storage
79453000-799e6fff : System RAM
799e7000-799e9fff : reserved
799ea000-799eefff : System RAM
799ef000-799f1fff : reserved
799f2000-79a30fff : System RAM
79a31000-79a34fff : reserved
79a35000-79a3cfff : System RAM
79a3d000-79a5ffff : reserved
79a60000-79aa1fff : System RAM
79aa2000-7a2b2fff : ACPI Non-volatile Storage
7a2b3000-7a302fff : System RAM
7a303000-7a303fff : reserved
7a304000-7a31dfff : System RAM
7a31e000-7a31efff : ACPI Tables
7a31f000-7a320fff : ACPI Non-volatile Storage
7a321000-7a3a4fff : System RAM
7a3a5000-7a3a7fff : reserved
7a3a8000-7a3a8fff : ACPI Tables
7a3a9000-7a3b4fff : System RAM
7a3b5000-7a3d4fff : reserved
7a3d5000-7a43dfff : System RAM
7a43e000-7a443fff : reserved
7a444000-7a4a1fff : System RAM
7a4a2000-7a4a5fff : reserved
7a4a6000-7a4f3fff : System RAM
7a4f4000-7a4fbfff : reserved
7a4fc000-7a514fff : System RAM
7a515000-7a515fff : ACPI Tables
7a516000-7a553fff : System RAM
7a554000-7a557fff : reserved
7a558000-7a9a9fff : System RAM
7a9aa000-7a9aafff : reserved
7a9ab000-7a9e3fff : System RAM
7a9e4000-7a9e7fff : reserved
7a9e8000-7aad4fff : System RAM
7aad5000-7aad5fff : reserved
7aad6000-7af3efff : System RAM
7af3f000-7af3ffff : reserved
7af40000-7af44fff : System RAM
7af45000-7af45fff : reserved
7af46000-7effffff : System RAM
7f000000-7fffffff : RAM buffer
80000000-8fffffff : PCI MMCONFIG 0 [00-ff]
80000000-8fffffff : reserved
90000000-fbffffff : PCI Bus #00
90000000-90ffffff : PCI Bus 0000:19

    90000000-90ffffff : 0000:19:00.0
      90000000-907fffff : efifb
  91000000-917fffff : PCI Bus 0000:03
    91000000-917fffff : PCI Bus 0000:04
      91000000-917fffff : PCI Bus 0000:0c
        91000000-917fffff : PCI Bus 0000:0d
          91000000-911fffff : PCI Bus 0000:11
          91000000-9103ffff : 0000:11:00.0
          91200000-913fffff : PCI Bus 0000:10
          91400000-915fffff : PCI Bus 0000:0f
          91400000-9141ffff : 0000:0f:00.0
          91420000-9143ffff : 0000:0f:00.1
          91600000-917fffff : PCI Bus 0000:0e
  c6000000-c68fffff : PCI Bus 0000:19
    c6000000-c67fffff : 0000:19:00.0
    c6800000-c6803fff : 0000:19:00.0
    c6810000-c681ffff : 0000:19:00.0
  c6a00000-c79fffff : PCI Bus 0000:03
    c6a00000-c79fffff : PCI Bus 0000:04
      c6a00000-c75fffff : PCI Bus 0000:0c
        c6a00000-c75fffff : PCI Bus 0000:0d
          c6a00000-c6cfffff : PCI Bus 0000:11
          c6c00000-c6c03fff : 0000:11:00.0
          c6c04000-c6c04fff : 0000:11:00.0
          c6e00000-c6ffffff : PCI Bus 0000:10
          c7000000-c72fffff : PCI Bus 0000:0f
          c7200000-c721ffff : 0000:0f:00.1
          c7200000-c721ffff : e1000e
          c7220000-c723ffff : 0000:0f:00.1
          c7220000-c723ffff : e1000e
          c7240000-c725ffff : 0000:0f:00.0
          c7240000-c725ffff : e1000e
          c7260000-c727ffff : 0000:0f:00.0
          c7260000-c727ffff : e1000e
          c7400000-c75fffff : PCI Bus 0000:0e
      c7600000-c79fffff : PCI Bus 0000:05
        c7600000-c79fffff : PCI Bus 0000:06
          c7600000-c76fffff : PCI Bus 0000:0a
          c7600000-c760ffff : 0000:0a:00.0
          c7600000-c760ffff : mpt
          c7610000-c7613fff : 0000:0a:00.0
          c7610000-c7613fff : mpt
          c7700000-c77fffff : PCI Bus 0000:09
          c7700000-c770ffff : 0000:09:00.0
          c7700000-c770ffff : mpt
          c7710000-c7713fff : 0000:09:00.0
          c7710000-c7713fff : mpt
          c7800000-c78fffff : PCI Bus 0000:08
          c7800000-c781ffff : 0000:08:00.1
          c7800000-c781ffff : igb
          c7820000-c783ffff : 0000:08:00.1
          c7820000-c783ffff : igb
          c7840000-c785ffff : 0000:08:00.0
          c7840000-c785ffff : igb
          c7860000-c787ffff : 0000:08:00.0
          c7860000-c787ffff : igb
          c7880000-c7883fff : 0000:08:00.1
          c7880000-c7883fff : igb
          c7884000-c7887fff : 0000:08:00.0
          c7884000-c7887fff : igb
          c78a0000-c78bffff : 0000:08:00.0
          c78c0000-c78dffff : 0000:08:00.1
          c78e0000-c78fffff : 0000:08:00.0
          c7900000-c79fffff : PCI Bus 0000:07
          c7900000-c791ffff : 0000:07:00.1
          c7900000-c791ffff : igb
          c7920000-c793ffff : 0000:07:00.1
          c7920000-c793ffff : igb
          c7940000-c795ffff : 0000:07:00.0
          c7940000-c795ffff : igb
          c7960000-c797ffff : 0000:07:00.0
          c7960000-c797ffff : igb
          c7980000-c7983fff : 0000:07:00.1
          c7980000-c7983fff : igb
          c7984000-c7987fff : 0000:07:00.0
          c7984000-c7987fff : igb
          c79a0000-c79bffff : 0000:07:00.0
          c79c0000-c79dffff : 0000:07:00.1
          c79e0000-c79fffff : 0000:07:00.0
  c7a00000-c7a1ffff : 0000:00:19.0
    c7a00000-c7a1ffff : e1000e
  c7a20000-c7a23fff : 0000:00:16.0
    c7a20000-c7a23fff : ioatdma
  c7a24000-c7a27fff : 0000:00:16.1
    c7a24000-c7a27fff : ioatdma
  c7a28000-c7a2bfff : 0000:00:16.2
    c7a28000-c7a2bfff : ioatdma
  c7a2c000-c7a2ffff : 0000:00:16.3
    c7a2c000-c7a2ffff : ioatdma
  c7a30000-c7a33fff : 0000:00:16.4
    c7a30000-c7a33fff : ioatdma
  c7a34000-c7a37fff : 0000:00:16.5
    c7a34000-c7a37fff : ioatdma
  c7a38000-c7a3bfff : 0000:00:16.6
    c7a38000-c7a3bfff : ioatdma
  c7a3c000-c7a3ffff : 0000:00:16.7
    c7a3c000-c7a3ffff : ioatdma
  c7a40000-c7a40fff : 0000:00:1f.6
  c7a41000-c7a41fff : 0000:00:19.0
    c7a41000-c7a41fff : e1000e
  c7a42000-c7a42fff : 0000:00:13.0
  c7a43000-c7a433ff : 0000:00:1d.7
    c7a43000-c7a433ff : ehci_hcd
  c7a43400-c7a437ff : 0000:00:1a.7
    c7a43400-c7a437ff : ehci_hcd

c7a43800-c7a438ff : 0000:00:1f.3
fec00000-fec00fff : IOAPIC 0
fee00000-fee00fff : Local APIC
100000000-47fffffff : System RAM
fc000000000-fc07fffffff : PCI Bus #00

00000000-0009efff : System RAM
0009f000-0009ffff : ACPI Non-volatile Storage
00100000-79190fff : System RAM
01000000-0141e142 : Kernel code
0141e143-0183639f : Kernel data
01acb000-027006f7 : Kernel bss
79191000-791c0fff : reserved
791c1000-79207fff : System RAM
79208000-79214fff : ACPI Tables
79215000-79321fff : System RAM
79322000-79322fff : reserved
79323000-7934afff : System RAM
7934b000-7934bfff : ACPI Tables
7934c000-793dffff : System RAM
793e0000-793e1fff : ACPI Tables
793e2000-79401fff : System RAM
79402000-79404fff : ACPI Tables
79405000-79450fff : System RAM
79451000-79451fff : ACPI Tables
79452000-79452fff : ACPI Non-volatile Storage
79453000-799e6fff : System RAM
799e7000-799e9fff : reserved
799ea000-799eefff : System RAM
799ef000-799f1fff : reserved
799f2000-79a30fff : System RAM
79a31000-79a34fff : reserved
79a35000-79a3cfff : System RAM
79a3d000-79a5ffff : reserved
79a60000-79aa1fff : System RAM
79aa2000-7a2b2fff : ACPI Non-volatile Storage
7a2b3000-7a302fff : System RAM
7a303000-7a303fff : reserved
7a304000-7a31dfff : System RAM
7a31e000-7a31efff : ACPI Tables
7a31f000-7a320fff : ACPI Non-volatile Storage
7a321000-7a3a4fff : System RAM
7a3a5000-7a3a7fff : reserved
7a3a8000-7a3a8fff : ACPI Tables
7a3a9000-7a3b4fff : System RAM
7a3b5000-7a3d4fff : reserved
7a3d5000-7a43dfff : System RAM
7a43e000-7a443fff : reserved
7a444000-7a4a1fff : System RAM
7a4a2000-7a4a5fff : reserved
7a4a6000-7a4f3fff : System RAM
7a4f4000-7a4fbfff : reserved
7a4fc000-7a514fff : System RAM
7a515000-7a515fff : ACPI Tables
7a516000-7a553fff : System RAM
7a554000-7a557fff : reserved
7a558000-7a9a9fff : System RAM
7a9aa000-7a9aafff : reserved
7a9ab000-7a9e3fff : System RAM
7a9e4000-7a9e7fff : reserved
7a9e8000-7aad4fff : System RAM
7aad5000-7aad5fff : reserved
7aad6000-7af3efff : System RAM
7af3f000-7af3ffff : reserved
7af40000-7af44fff : System RAM
7af45000-7af45fff : reserved
7af46000-7effffff : System RAM
7f000000-7fffffff : RAM buffer
80000000-8fffffff : PCI MMCONFIG 0 [00-ff]
80000000-8fffffff : reserved
90000000-fbffffff : PCI Bus #00
90000000-90ffffff : PCI Bus 0000:19

    90000000-90ffffff : 0000:19:00.0
      90000000-907fffff : efifb
  91000000-917fffff : PCI Bus 0000:03
    91000000-917fffff : PCI Bus 0000:04
      91000000-917fffff : PCI Bus 0000:0c
        91000000-917fffff : PCI Bus 0000:0d
          91000000-911fffff : PCI Bus 0000:11
          91000000-9103ffff : 0000:11:00.0
          91200000-913fffff : PCI Bus 0000:10
          91400000-915fffff : PCI Bus 0000:0f
          91400000-9141ffff : 0000:0f:00.0
          91420000-9143ffff : 0000:0f:00.1
          91600000-917fffff : PCI Bus 0000:0e
  c6000000-c68fffff : PCI Bus 0000:19
    c6000000-c67fffff : 0000:19:00.0
    c6800000-c6803fff : 0000:19:00.0
    c6810000-c681ffff : 0000:19:00.0
  c6a00000-c79fffff : PCI Bus 0000:03
    c6a00000-c79fffff : PCI Bus 0000:04
      c6a00000-c75fffff : PCI Bus 0000:0c
        c6a00000-c75fffff : PCI Bus 0000:0d
          c6a00000-c6cfffff : PCI Bus 0000:11
          c6c00000-c6c03fff : 0000:11:00.0
          c6c00000-c6c03fff : lpfc
          c6c04000-c6c04fff : 0000:11:00.0
          c6c04000-c6c04fff : lpfc
          c6e00000-c6ffffff : PCI Bus 0000:10
          c7000000-c72fffff : PCI Bus 0000:0f
          c7200000-c721ffff : 0000:0f:00.1
          c7200000-c721ffff : e1000e
          c7220000-c723ffff : 0000:0f:00.1
          c7220000-c723ffff : e1000e
          c7240000-c725ffff : 0000:0f:00.0
          c7240000-c725ffff : e1000e
          c7260000-c727ffff : 0000:0f:00.0
          c7260000-c727ffff : e1000e
          c7400000-c75fffff : PCI Bus 0000:0e
      c7600000-c79fffff : PCI Bus 0000:05
        c7600000-c79fffff : PCI Bus 0000:06
          c7600000-c76fffff : PCI Bus 0000:0a
          c7600000-c760ffff : 0000:0a:00.0
          c7600000-c760ffff : mpt
          c7610000-c7613fff : 0000:0a:00.0
          c7610000-c7613fff : mpt
          c7700000-c77fffff : PCI Bus 0000:09
          c7700000-c770ffff : 0000:09:00.0
          c7700000-c770ffff : mpt
          c7710000-c7713fff : 0000:09:00.0
          c7710000-c7713fff : mpt
          c7800000-c79fffff : PCI Bus 0000:09
          c7800000-c79fffff : 0000:09:00.0
  c7a00000-c7a1ffff : 0000:00:19.0
    c7a00000-c7a1ffff : e1000e
  c7a20000-c7a23fff : 0000:00:16.0
    c7a20000-c7a23fff : ioatdma
  c7a24000-c7a27fff : 0000:00:16.1
    c7a24000-c7a27fff : ioatdma
  c7a28000-c7a2bfff : 0000:00:16.2
    c7a28000-c7a2bfff : ioatdma
  c7a2c000-c7a2ffff : 0000:00:16.3
    c7a2c000-c7a2ffff : ioatdma
  c7a30000-c7a33fff : 0000:00:16.4
    c7a30000-c7a33fff : ioatdma
  c7a34000-c7a37fff : 0000:00:16.5
    c7a34000-c7a37fff : ioatdma
  c7a38000-c7a3bfff : 0000:00:16.6
    c7a38000-c7a3bfff : ioatdma
  c7a3c000-c7a3ffff : 0000:00:16.7
    c7a3c000-c7a3ffff : ioatdma
  c7a40000-c7a40fff : 0000:00:1f.6
  c7a41000-c7a41fff : 0000:00:19.0
    c7a41000-c7a41fff : e1000e
  c7a42000-c7a42fff : 0000:00:13.0
  c7a43000-c7a433ff : 0000:00:1d.7
    c7a43000-c7a433ff : ehci_hcd
  c7a43400-c7a437ff : 0000:00:1a.7
    c7a43400-c7a437ff : ehci_hcd

c7a43800-c7a438ff : 0000:00:1f.3
fec00000-fec00fff : IOAPIC 0
fee00000-fee00fff : Local APIC
100000000-47fffffff : System RAM
fc000000000-fc07fffffff : PCI Bus #00

author
#62 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v10

Quoted 33 lines · Kenji Kaneshige
Quoted 10 lines · Yinghai Lu
Quoted 3 lines · Kenji Kaneshige

Can I ask which is the latest version?

I think -v10 is the latest. But I could not find -v10 for patch 1/2.

please use

http://patchwork.kernel.org/patch/57814/ 1/2 -v9
http://patchwork.kernel.org/patch/58294/ 2/2 -v10

Regardless of PCIe hotplug, some of PCIe devices (not on hotplug slots) doesn't work with your patches by the following error.

Copyright (c) 2007-2009 Intel Corporation.
igb 0000:07:00.0: PCI INT A -> GSI 16 (level, low) -> IRQ 16 igb 0000:07:00.0: setting latency timer to 64
igb 0000:07:00.0: PCI INT A disabled
igb: probe of 0000:07:00.0 failed with error -5 igb 0000:07:00.1: PCI INT B -> GSI 17 (level, low) -> IRQ 17 igb 0000:07:00.1: setting latency timer to 64
igb 0000:07:00.1: PCI INT B disabled
igb: probe of 0000:07:00.1 failed with error -5 igb 0000:08:00.0: PCI INT A -> GSI 17 (level, low) -> IRQ 17 igb 0000:08:00.0: setting latency timer to 64
igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -5 igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: setting latency timer to 64
igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -5

Quoted 4 lines

I'm using Jesse's latest linux-next.
I'm attaching the /proc/iomem outputs. The iomem-default.txt is /proc/iomem output without your patches. The iomem-yinghai.txt is /proc/iomem output with your patches.

can you post whole bootlog with pci debug enabled?

Thanks

Yinghai

#63 lore raw

Re: [PATCH 2/2] pci: pciehp update the slot bridge res to get big range for pcie devices - v10

Quoted 32 lines · Yinghai Lu
Quoted 31 lines · Kenji Kaneshige
Quoted 9 lines · Yinghai Lu
Quoted 3 lines · Kenji Kaneshige

Can I ask which is the latest version?

I think -v10 is the latest. But I could not find -v10 for patch 1/2.

please use

http://patchwork.kernel.org/patch/57814/ 1/2 -v9
http://patchwork.kernel.org/patch/58294/ 2/2 -v10

Regardless of PCIe hotplug, some of PCIe devices (not on hotplug slots) doesn't work with your patches by the following error.

Copyright (c) 2007-2009 Intel Corporation.
igb 0000:07:00.0: PCI INT A -> GSI 16 (level, low) -> IRQ 16 igb 0000:07:00.0: setting latency timer to 64
igb 0000:07:00.0: PCI INT A disabled
igb: probe of 0000:07:00.0 failed with error -5 igb 0000:07:00.1: PCI INT B -> GSI 17 (level, low) -> IRQ 17 igb 0000:07:00.1: setting latency timer to 64
igb 0000:07:00.1: PCI INT B disabled
igb: probe of 0000:07:00.1 failed with error -5 igb 0000:08:00.0: PCI INT A -> GSI 17 (level, low) -> IRQ 17 igb 0000:08:00.0: setting latency timer to 64
igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -5 igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: setting latency timer to 64
igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -5

Quoted 7 lines
Quoted 4 lines

I'm using Jesse's latest linux-next.
I'm attaching the /proc/iomem outputs. The iomem-default.txt is /proc/iomem output without your patches. The iomem-yinghai.txt is /proc/iomem output with your patches.

can you post whole bootlog with pci debug enabled?

I'll send it privately.

Thanks,
Kenji Kaneshige

author
#64 lore raw

PATCH 1/2 pci: release that leaf bridge' resource that is not big -v11

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big
enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v11:add pci=try=5, about more try to change more bridge

---
 drivers/pci/pci.c       |    5 
 drivers/pci/pci.h       |    2 
 drivers/pci/setup-bus.c |  304 +++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 292 insertions(+), 19 deletions(-)
drivers/pci/setup-bus.c +285 -19
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -27,7 +27,49 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+
+static void add_to_failed_list(struct resource_list *head, struct pci_dev *dev,
+				 struct resource *res)
+{
+	struct resource_list *list = head;
+	struct resource_list *ln = list->next;
+	struct resource_list *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list *head)
+{
+	struct resource_list *list, *tmp;
+	struct resource *res;
+	/*
+	 * Try to release leaf bridge's resources that there is no child
+	 * under it
+	 */
+	for (list = head->next; list;) {
+		res = list->res;
+		res->start = 0;
+		res->end = 0;
+		res->flags = 0;
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
@@ -58,9 +100,17 @@ static void pbus_assign_resources_sorted
 		res = list->res;
 		idx = res - &list->dev->resource[0];
 		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
+			if (fail_head && !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for next try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
 		}
 		tmp = list;
 		list = list->next;
@@ -134,19 +184,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct resource *res;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
-		 bus->secondary, bus->subordinate);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	res = bus->resource[0];
@@ -172,7 +215,13 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	res = bus->resource[1];
@@ -187,6 +236,14 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  bridge window [mem disabled]\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -219,10 +276,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
+		 bus->secondary, bus->subordinate);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
@@ -543,19 +627,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -573,15 +658,130 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ * check:
+ *    0: only release the bridge and only the bridge is leaf
+ *    1: release all down side bridge for third shoot
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type,
+						    int check_leaf)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			if (!check_leaf)
+				pci_bus_release_unused_bridge_res(b, type,
+							 check_leaf);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if ((check_leaf && is_leaf_bridge) || !check_leaf)
+			pci_bridge_release_unused_res(bus, type);
+		break;
+	}
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pR\n", i, res);
@@ -605,10 +805,25 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
+/*
+ * first try will not touch pci bridge res
+ * second try will clear small leaf bridge res
+ * third try will clear related bridge: some aggressive
+ */
+/* assume we only have 4 level bridges, so only try 5 times */
+int pci_try_num = 5;
 void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	int tried_times = 0;
+	int check_leaf = 1;
+	struct resource_list head, *list, *tmp;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+	unsigned long failed_type;
+again:
+	head.next = NULL;
 
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
@@ -617,7 +832,58 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+	tried_times++;
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+	failed_type = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		failed_type |= flags;
+		list = list->next;
+	}
+	/*
+	 * io port are tight, don't try extra
+	 * or if reach the limit, don't want to try more
+	 */
+	failed_type &= type_mask;
+	if ((failed_type == IORESOURCE_IO) || (tried_times >= pci_try_num)) {
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
+			 tried_times + 1);
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	/* third times and later will not check if it is leaf */
+	if ((tried_times + 1) > 2)
+		check_leaf = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		bus = list->dev->bus;
+		if (list->dev->subordinate)
+			list->res->flags = 0;
+		pci_bus_release_unused_bridge_res(bus, flags & type_mask,
+						  check_leaf);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
drivers/pci/pci.c +5 -0
Index: linux-2.6/drivers/pci/pci.c
===================================================================
@@ -2779,6 +2779,11 @@ static int __init pci_setup(char *str)
 				pci_no_aer();
 			} else if (!strcmp(str, "nodomains")) {
 				pci_no_domains();
+			} else if (!strncmp(str, "try=", 4)) {
+				int try_num = memparse(str + 4, &str);
+
+				if (try_num > 0 && try_num < 10)
+					pci_try_num = try_num;
 			} else if (!strncmp(str, "cbiosize=", 9)) {
 				pci_cardbus_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "cbmemsize=", 10)) {
drivers/pci/pci.h +2 -0
Index: linux-2.6/drivers/pci/pci.h
===================================================================
@@ -203,6 +203,8 @@ static inline int pci_ari_enabled(struct
 	return bus->self && bus->self->ari_enabled;
 }
 
+extern int pci_try_num;
+
 #ifdef CONFIG_PCI_QUIRKS
 extern int pci_is_reassigndev(struct pci_dev *dev);
 resource_size_t pci_specified_resource_alignment(struct pci_dev *dev);
#65 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Hi,

I tried v11 patches. This version seems to fix the problem I reported against previous version.

I have no objection against the idea of resource allocation changes for PCI express hotplug slots.

But I still have concern about changing resource allocation for other than PCI express hotplug slots. For example, some hotplug controller other than PCI express can have multiple slots under the same bus. If some hotplug slots are occupied and the others are empty at the boot time, I think your code try to shrink the bus resources for hotplug slots allocated by BIOS. It would break the hot-add on the empty slots due to the resource allocation failure.

Thanks,
Kenji Kaneshige

Quoted 465 lines · Yinghai Lu

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big
enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v11:add pci=try=5, about more try to change more bridge

---
 drivers/pci/pci.c       |    5 
 drivers/pci/pci.h       |    2 
 drivers/pci/setup-bus.c |  304 +++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 292 insertions(+), 19 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +285 -19
@@ -27,7 +27,49 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+
+static void add_to_failed_list(struct resource_list *head, struct pci_dev *dev,
+				 struct resource *res)
+{
+	struct resource_list *list = head;
+	struct resource_list *ln = list->next;
+	struct resource_list *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list *head)
+{
+	struct resource_list *list, *tmp;
+	struct resource *res;
+	/*
+	 * Try to release leaf bridge's resources that there is no child
+	 * under it
+	 */
+	for (list = head->next; list;) {
+		res = list->res;
+		res->start = 0;
+		res->end = 0;
+		res->flags = 0;
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
@@ -58,9 +100,17 @@ static void pbus_assign_resources_sorted
 		res = list->res;
 		idx = res - &list->dev->resource[0];
 		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
+			if (fail_head && !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for next try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
 		}
 		tmp = list;
 		list = list->next;
@@ -134,19 +184,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct resource *res;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
-		 bus->secondary, bus->subordinate);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	res = bus->resource[0];
@@ -172,7 +215,13 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	res = bus->resource[1];
@@ -187,6 +236,14 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  bridge window [mem disabled]\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -219,10 +276,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
+		 bus->secondary, bus->subordinate);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
@@ -543,19 +627,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -573,15 +658,130 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ * check:
+ *    0: only release the bridge and only the bridge is leaf
+ *    1: release all down side bridge for third shoot
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type,
+						    int check_leaf)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			if (!check_leaf)
+				pci_bus_release_unused_bridge_res(b, type,
+							 check_leaf);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if ((check_leaf && is_leaf_bridge) || !check_leaf)
+			pci_bridge_release_unused_res(bus, type);
+		break;
+	}
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pR\n", i, res);
@@ -605,10 +805,25 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
+/*
+ * first try will not touch pci bridge res
+ * second try will clear small leaf bridge res
+ * third try will clear related bridge: some aggressive
+ */
+/* assume we only have 4 level bridges, so only try 5 times */
+int pci_try_num = 5;
 void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	int tried_times = 0;
+	int check_leaf = 1;
+	struct resource_list head, *list, *tmp;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+	unsigned long failed_type;
+again:
+	head.next = NULL;
 
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
@@ -617,7 +832,58 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+	tried_times++;
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+	failed_type = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		failed_type |= flags;
+		list = list->next;
+	}
+	/*
+	 * io port are tight, don't try extra
+	 * or if reach the limit, don't want to try more
+	 */
+	failed_type &= type_mask;
+	if ((failed_type == IORESOURCE_IO) || (tried_times >= pci_try_num)) {
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
+			 tried_times + 1);
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	/* third times and later will not check if it is leaf */
+	if ((tried_times + 1) > 2)
+		check_leaf = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		bus = list->dev->bus;
+		if (list->dev->subordinate)
+			list->res->flags = 0;
+		pci_bus_release_unused_bridge_res(bus, flags & type_mask,
+						  check_leaf);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 

Index: linux-2.6/drivers/pci/pci.c
===================================================================

drivers/pci/pci.c +5 -0
@@ -2779,6 +2779,11 @@ static int __init pci_setup(char *str)
 				pci_no_aer();
 			} else if (!strcmp(str, "nodomains")) {
 				pci_no_domains();
+			} else if (!strncmp(str, "try=", 4)) {
+				int try_num = memparse(str + 4, &str);
+
+				if (try_num > 0 && try_num < 10)
+					pci_try_num = try_num;
 			} else if (!strncmp(str, "cbiosize=", 9)) {
 				pci_cardbus_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "cbmemsize=", 10)) {

Index: linux-2.6/drivers/pci/pci.h
===================================================================

drivers/pci/pci.h +2 -0
@@ -203,6 +203,8 @@ static inline int pci_ari_enabled(struct
 	return bus->self && bus->self->ari_enabled;
 }
 
+extern int pci_try_num;
+
 #ifdef CONFIG_PCI_QUIRKS
 extern int pci_is_reassigndev(struct pci_dev *dev);
 resource_size_t pci_specified_resource_alignment(struct pci_dev *dev);
author
#66 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 7 lines · Kenji Kaneshige

Hi,

I tried v11 patches. This version seems to fix the problem I reported against previous version.

I have no objection against the idea of resource allocation changes for PCI express hotplug slots.

thanks

Quoted 9 lines

But I still have concern about changing resource allocation for other than PCI express hotplug slots. For example, some hotplug controller other than PCI express can have multiple slots under the same bus. If some hotplug slots are occupied and the others are empty at the boot time, I think your code try to shrink the bus resources for hotplug slots allocated by BIOS. It would break the hot-add on the empty slots due to the resource allocation failure.

no,
it will not touch bridge resources that already assigned by BIOS except some bridge resource is not big enough. and get big one for them.

YH

#67 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 20 lines · Yinghai Lu
Quoted 7 lines · Kenji Kaneshige

Hi,

I tried v11 patches. This version seems to fix the problem I reported against previous version.

I have no objection against the idea of resource allocation changes for PCI express hotplug slots.

thanks

Quoted 8 lines

But I still have concern about changing resource allocation for other than PCI express hotplug slots. For example, some hotplug controller other than PCI express can have multiple slots under the same bus. If some hotplug slots are occupied and the others are empty at the boot time, I think your code try to shrink the bus resources for hotplug slots allocated by BIOS. It would break the hot-add on the empty slots due to the resource allocation failure.

Quoted 4 lines

no,
it will not touch bridge resources that already assigned by BIOS except some bridge resource is not big enough. and get big one for them.

Ok, I understood that if the BIOS assigns enough resources to the bridge, it has no impact.

One question. I thought your patch shrinks the bridge resource to allocate enough resource for sibling bridge. But it actually doesn't. Right?

It would be appreciated if you update the patch description about the problem and how to fix/improbe it.

Thanks,
Kenji Kaneshige

author
#68 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 26 lines · Kenji Kaneshige
Quoted 20 lines · Yinghai Lu
Quoted 7 lines · Kenji Kaneshige

Hi,

I tried v11 patches. This version seems to fix the problem I reported against previous version.

I have no objection against the idea of resource allocation changes for PCI express hotplug slots.

thanks

Quoted 8 lines

But I still have concern about changing resource allocation for other than PCI express hotplug slots. For example, some hotplug controller other than PCI express can have multiple slots under the same bus. If some hotplug slots are occupied and the others are empty at the boot time, I think your code try to shrink the bus resources for hotplug slots allocated by BIOS. It would break the hot-add on the empty slots due to the resource allocation failure.

Quoted 4 lines

no,
it will not touch bridge resources that already assigned by BIOS except some bridge resource is not big enough. and get big one for them.

Quoted 6 lines

Ok, I understood that if the BIOS assigns enough resources to the bridge, it has no impact.

One question. I thought your patch shrinks the bridge resource to allocate enough resource for sibling bridge. But it actually doesn't. Right?

ok, i got it. will change pci_try_num default to 1.

Quoted 3 lines

It would be appreciated if you update the patch description about the problem and how to fix/improbe it.

sure.

YH

author
#69 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

In reply to Kenji Kaneshige
Quoted 26 lines · Kenji Kaneshige
Quoted 20 lines · Yinghai Lu
Quoted 7 lines · Kenji Kaneshige

Hi,

I tried v11 patches. This version seems to fix the problem I reported against previous version.

I have no objection against the idea of resource allocation changes for PCI express hotplug slots.

thanks

Quoted 8 lines

But I still have concern about changing resource allocation for other than PCI express hotplug slots. For example, some hotplug controller other than PCI express can have multiple slots under the same bus. If some hotplug slots are occupied and the others are empty at the boot time, I think your code try to shrink the bus resources for hotplug slots allocated by BIOS. It would break the hot-add on the empty slots due to the resource allocation failure.

Quoted 4 lines

no,
it will not touch bridge resources that already assigned by BIOS except some bridge resource is not big enough. and get big one for them.

Quoted 6 lines

Ok, I understood that if the BIOS assigns enough resources to the bridge, it has no impact.

One question. I thought your patch shrinks the bridge resource to allocate enough resource for sibling bridge. But it actually doesn't. Right?

please check if this one could fix the shrinking bridge resource problem...

[PATCH] pci: don't shrink bridge resources

when we are clearing leaf bridge resource and try to get big one, we could shrink the bridge if there resource under it.

let check with old resource size and make sure we are trying to get big one.

---
 drivers/pci/setup-bus.c |   22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)
drivers/pci/setup-bus.c +12 -10
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -432,7 +432,7 @@ static void pbus_size_io(struct pci_bus
 {
 	struct pci_dev *dev;
 	struct resource *b_res = find_free_bus_resource(bus, IORESOURCE_IO);
-	unsigned long size = 0, size1 = 0;
+	unsigned long size = 0, size1 = 0, old_size;
 
 	if (!b_res)
  		return;
@@ -457,17 +457,18 @@ static void pbus_size_io(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
 /* To be fixed in 2.5: we should have sort of HAVE_ISA
    flag in the struct pci_bus. */
 #if defined(CONFIG_ISA) || defined(CONFIG_EISA)
 	size = (size & 0xff) + ((size & ~0xffUL) << 2);
 #endif
 	size = ALIGN(size + size1, 4096);
+	if (size < old_size)
+		size = old_size;
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return;
 	}
@@ -483,7 +484,7 @@ static int pbus_size_mem(struct pci_bus
 			 unsigned long type, resource_size_t min_size)
 {
 	struct pci_dev *dev;
-	resource_size_t min_align, align, size;
+	resource_size_t min_align, align, size, old_size;
 	resource_size_t aligns[12];	/* Alignments from 1Mb to 2Gb */
 	int order, max_order;
 	struct resource *b_res = find_free_bus_resource(bus, type);
@@ -533,6 +534,11 @@ static int pbus_size_mem(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
+	if (size < old_size)
+		size = old_size;
 
 	align = 0;
 	min_align = 0;
@@ -549,10 +555,6 @@ static int pbus_size_mem(struct pci_bus
 	}
 	size = ALIGN(size, min_align);
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return 1;
 	}
#70 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Hi Yinghai,

I would like to reconfirm what is the problem you're trying to solve before reviewing and testing the additional patch. To be honest, your patch looks more and more complicated and it is becoming difficult for me to review and test it.

By the way, if your problem is that BIOS doesn't assign the resource to the parent bridge (root port or switch downstream port) of PCIe hotplug slots, I guess we can improve it with simpler way. I made a sample patches (no enough testing). Please take a look. Patches are

- [PATCH 1/2] pciehp: remove redundancy in bridge resource allocation
- [PATCH 2/2] pciehp: add support for bridge resource reallocation

Thanks,
Kenji Kaneshige

Quoted 116 lines · Yinghai Lu
Quoted 23 lines · Kenji Kaneshige
Quoted 18 lines · Yinghai Lu
Quoted 7 lines · Kenji Kaneshige

Hi,

I tried v11 patches. This version seems to fix the problem I reported against previous version.

I have no objection against the idea of resource allocation changes for PCI express hotplug slots.

thanks

Quoted 8 lines

But I still have concern about changing resource allocation for other than PCI express hotplug slots. For example, some hotplug controller other than PCI express can have multiple slots under the same bus. If some hotplug slots are occupied and the others are empty at the boot time, I think your code try to shrink the bus resources for hotplug slots allocated by BIOS. It would break the hot-add on the empty slots due to the resource allocation failure.

Quoted 4 lines

no,
it will not touch bridge resources that already assigned by BIOS except some bridge resource is not big enough. and get big one for them.

Quoted 6 lines

Ok, I understood that if the BIOS assigns enough resources to the bridge, it has no impact.

One question. I thought your patch shrinks the bridge resource to allocate enough resource for sibling bridge. But it actually doesn't. Right?

please check if this one could fix the shrinking bridge resource problem...

[PATCH] pci: don't shrink bridge resources

when we are clearing leaf bridge resource and try to get big one, we could shrink the bridge if there resource under it.

let check with old resource size and make sure we are trying to get big one.

---
 drivers/pci/setup-bus.c |   22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +12 -10
@@ -432,7 +432,7 @@ static void pbus_size_io(struct pci_bus
 {
 	struct pci_dev *dev;
 	struct resource *b_res = find_free_bus_resource(bus, IORESOURCE_IO);
-	unsigned long size = 0, size1 = 0;
+	unsigned long size = 0, size1 = 0, old_size;
 
 	if (!b_res)
  		return;
@@ -457,17 +457,18 @@ static void pbus_size_io(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
 /* To be fixed in 2.5: we should have sort of HAVE_ISA
    flag in the struct pci_bus. */
 #if defined(CONFIG_ISA) || defined(CONFIG_EISA)
 	size = (size & 0xff) + ((size & ~0xffUL) << 2);
 #endif
 	size = ALIGN(size + size1, 4096);
+	if (size < old_size)
+		size = old_size;
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return;
 	}
@@ -483,7 +484,7 @@ static int pbus_size_mem(struct pci_bus
 			 unsigned long type, resource_size_t min_size)
 {
 	struct pci_dev *dev;
-	resource_size_t min_align, align, size;
+	resource_size_t min_align, align, size, old_size;
 	resource_size_t aligns[12];	/* Alignments from 1Mb to 2Gb */
 	int order, max_order;
 	struct resource *b_res = find_free_bus_resource(bus, type);
@@ -533,6 +534,11 @@ static int pbus_size_mem(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
+	if (size < old_size)
+		size = old_size;
 
 	align = 0;
 	min_align = 0;
@@ -549,10 +555,6 @@ static int pbus_size_mem(struct pci_bus
 	}
 	size = ALIGN(size, min_align);
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return 1;
 	}
#71 lore raw

PATCH 1/2 pciehp: remove redundancy in bridge resource allocation

Current pciehp driver configures bridge's child bus each time the bridge is found on the hot-added PCIe adapter card. It is redundant and strange because pciehp tryies to add devices on the child bus before adding its parent bridge. It should be done at one time on the slot's parent bus.

---
 drivers/pci/hotplug/pciehp_pci.c |   28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +13 -15
Index: 20091125/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -34,30 +34,26 @@
 #include "../pci.h"
 #include "pciehp.h"
 
-static int __ref pciehp_add_bridge(struct pci_dev *dev)
+static void __ref pciehp_scan_bridge(struct pci_dev *dev)
 {
 	struct pci_bus *parent = dev->bus;
-	int pass, busnr, start = parent->secondary;
-	int end = parent->subordinate;
+	int pass, busnr, start, end;
 
+	start = parent->secondary;
+	end = parent->subordinate;
 	for (busnr = start; busnr <= end; busnr++) {
 		if (!pci_find_bus(pci_domain_nr(parent), busnr))
 			break;
 	}
+
 	if (busnr-- > end) {
 		err("No bus number available for hot-added bridge %s\n",
-				pci_name(dev));
-		return -1;
+		    pci_name(dev));
+		return;
 	}
+
 	for (pass = 0; pass < 2; pass++)
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
-	if (!dev->subordinate)
-		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
-	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
@@ -93,14 +89,16 @@ int pciehp_configure_device(struct slot 
 			continue;
 		}
 		if ((dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) ||
-				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
-			pciehp_add_bridge(dev);
-		}
+		    (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS))
+			pciehp_scan_bridge(dev);
+
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
+	pci_bus_size_bridges(parent);
 	pci_bus_assign_resources(parent);
+	pci_enable_bridges(parent);
 	pci_bus_add_devices(parent);
 	return 0;
 }
author
#72 lore raw

Re: [PATCH 1/2] pciehp: remove redundancy in bridge resource allocation

Quoted 65 lines · Kenji Kaneshige

Current pciehp driver configures bridge's child bus each time the bridge is found on the hot-added PCIe adapter card. It is redundant and strange because pciehp tryies to add devices on the child bus before adding its parent bridge. It should be done at one time on the slot's parent bus.

---
 drivers/pci/hotplug/pciehp_pci.c |   28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

Index: 20091125/drivers/pci/hotplug/pciehp_pci.c
===================================================================

drivers/pci/hotplug/pciehp_pci.c +11 -15
@@ -34,30 +34,26 @@
 #include "../pci.h"
 #include "pciehp.h"
 
-static int __ref pciehp_add_bridge(struct pci_dev *dev)
+static void __ref pciehp_scan_bridge(struct pci_dev *dev)
 {
 	struct pci_bus *parent = dev->bus;
-	int pass, busnr, start = parent->secondary;
-	int end = parent->subordinate;
+	int pass, busnr, start, end;
 
+	start = parent->secondary;
+	end = parent->subordinate;
 	for (busnr = start; busnr <= end; busnr++) {
 		if (!pci_find_bus(pci_domain_nr(parent), busnr))
 			break;
 	}
+
 	if (busnr-- > end) {
 		err("No bus number available for hot-added bridge %s\n",
-				pci_name(dev));
-		return -1;
+		    pci_name(dev));
+		return;
 	}
+
 	for (pass = 0; pass < 2; pass++)
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
-	if (!dev->subordinate)
-		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
-	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
@@ -93,14 +89,16 @@ int pciehp_configure_device(struct slot 
 			continue;
 		}
 		if ((dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) ||
-				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
-			pciehp_add_bridge(dev);
-		}
+		    (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS))
+			pciehp_scan_bridge(dev);
+

drivers/pci/hotplug/pciehp_pci.c int pciehp_configure_device(struct slot · line 104

Quoted 10 lines
drivers/pci/hotplug/pciehp_pci.c +2 -0
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
+	pci_bus_size_bridges(parent);
 	pci_bus_assign_resources(parent);
+	pci_enable_bridges(parent);
 	pci_bus_add_devices(parent);
 	return 0;
 }

you comment on my patch2, that we should move pci_configure_slot later?

YH

#73 lore raw

PATCH 2/2 pciehp: add support for bridge resource reallocation

In reply to Kenji Kaneshige

Some platform BIOS doesn't assign PCI bridge resources (IO base/limit, Mem base/limit and Prefechable Mem base/limit) to root port or switch downstream port with hotplug slots. On such platform, PCIe hotplug doesn't work due to the resource allocation failure. This patch is to improbe this situation.

With this patch, pciehp driver try to assign PCI bridge resources to parent bridge (root port or switch downstream port) of the slot in addition to assign resources to the hot-added device.

With this patch, pciehp driver try to release PCI bridge resources so that we can reassign those resources to another root port on the same PCI bus segment, or to another downstream port on the same swith.

This feature is enabled when 'pciehp_realloc' option is specified.

---
 drivers/pci/hotplug/pciehp.h      |    1 
 drivers/pci/hotplug/pciehp_core.c |    9 ++++
 drivers/pci/hotplug/pciehp_pci.c  |    9 +++-
 drivers/pci/setup-bus.c           |   84 +++++++++++++++++++++++++++++---------
 include/linux/pci.h               |    2 
 5 files changed, 85 insertions(+), 20 deletions(-)
drivers/pci/setup-bus.c +65 -19
Index: 20091125/drivers/pci/setup-bus.c
===================================================================
@@ -27,13 +27,31 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+static void pbus_assign_resources_list(struct resource_list *head)
 {
-	struct pci_dev *dev;
 	struct resource *res;
-	struct resource_list head, *list, *tmp;
+	struct resource_list *list, *tmp;
 	int idx;
 
+	for (list = head->next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			res->start = 0;
+			res->end = 0;
+			res->flags = 0;
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+{
+	struct pci_dev *dev;
+	struct resource_list head;
+
 	head.next = NULL;
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		u16 class = dev->class >> 8;
@@ -54,18 +72,7 @@ static void pbus_assign_resources_sorted
 		pdev_sort_resources(dev, &head);
 	}
 
-	for (list = head.next; list;) {
-		res = list->res;
-		idx = res - &list->dev->resource[0];
-		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
-		}
-		tmp = list;
-		list = list->next;
-		kfree(tmp);
-	}
+	pbus_assign_resources_list(&head);
 }
 
 void pci_setup_cardbus(struct pci_bus *bus)
@@ -142,9 +149,6 @@ static void pci_setup_bridge(struct pci_
 	u32 l, bu, lu, io_upper16;
 	int pref_mem64;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
 		 bus->secondary, bus->subordinate);
 
@@ -559,7 +563,8 @@ void __ref pci_bus_assign_resources(cons
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
-			pci_setup_bridge(b);
+			if (!pci_is_enabled(dev))
+				pci_setup_bridge(b);
 			break;
 
 		case PCI_CLASS_BRIDGE_CARDBUS:
@@ -575,6 +580,47 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+void __ref pci_bridge_assign_resources(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource_list head;
+
+	if (pci_is_root_bus(bus) ||
+	    (bridge->class >> 8) != PCI_CLASS_BRIDGE_PCI)
+		return;
+
+	head.next = NULL;
+	pdev_sort_resources(bridge, &head);
+	pbus_assign_resources_list(&head);
+	pci_bus_assign_resources(bus);
+	pci_setup_bridge(bus);
+}
+EXPORT_SYMBOL(pci_bridge_assign_resources);
+
+void __ref pci_bridge_release_window(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	int i;
+
+	if (pci_is_root_bus(bus) ||
+	    (bridge->class >> 8) != PCI_CLASS_BRIDGE_PCI)
+		return;
+
+	for (i = 0; i < 3; i++)
+		if (bus->resource[i]->child)
+			return;
+
+	for (i = 0; i < 3; i++)
+		bus->resource[i]->flags = 0;
+
+	pci_setup_bridge(bus);
+
+	for (i = 0; i < 3; i++)
+		if (bus->resource[i]->parent)
+			release_resource(bus->resource[i]);
+}
+EXPORT_SYMBOL(pci_bridge_release_window);
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
drivers/pci/hotplug/pciehp_core.c +9 -0
Index: 20091125/drivers/pci/hotplug/pciehp_core.c
===================================================================
@@ -41,6 +41,7 @@ int pciehp_debug;
 int pciehp_poll_mode;
 int pciehp_poll_time;
 int pciehp_force;
+int pciehp_realloc;
 struct workqueue_struct *pciehp_wq;
 
 #define DRIVER_VERSION	"0.4"
@@ -55,10 +56,13 @@ module_param(pciehp_debug, bool, 0644);
 module_param(pciehp_poll_mode, bool, 0644);
 module_param(pciehp_poll_time, int, 0644);
 module_param(pciehp_force, bool, 0644);
+module_param(pciehp_realloc, bool, 0644);
+
 MODULE_PARM_DESC(pciehp_debug, "Debugging mode enabled or not");
 MODULE_PARM_DESC(pciehp_poll_mode, "Using polling mechanism for hot-plug events or not");
 MODULE_PARM_DESC(pciehp_poll_time, "Polling mechanism frequency, in seconds");
 MODULE_PARM_DESC(pciehp_force, "Force pciehp, even if _OSC and OSHP are missing");
+MODULE_PARM_DESC(pciehp_realloc, "Realloc resources for slot's parent bridge");
 
 #define PCIE_MODULE_NAME "pciehp"
 
@@ -293,10 +297,15 @@ static int pciehp_probe(struct pcie_devi
 	pciehp_get_power_status(slot, &poweron);
 	if (occupied && pciehp_force)
 		pciehp_enable_slot(slot);
+
 	/* If empty slot's power status is on, turn power off */
 	if (!occupied && poweron && POWER_CTRL(ctrl))
 		pciehp_power_off_slot(slot);
 
+	/* If no device is running on the slot, release bridge's I/O window */
+	if (pciehp_realloc && !poweron)
+		pci_bridge_release_window(dev->port->subordinate);
+
 	return 0;
 
 err_out_free_ctrl_slot:
drivers/pci/hotplug/pciehp_pci.c +8 -1
Index: 20091125/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -97,7 +97,10 @@ int pciehp_configure_device(struct slot 
 	}
 
 	pci_bus_size_bridges(parent);
-	pci_bus_assign_resources(parent);
+	if (pciehp_realloc)
+		pci_bridge_assign_resources(parent);
+	else
+		pci_bus_assign_resources(parent);
 	pci_enable_bridges(parent);
 	pci_bus_add_devices(parent);
 	return 0;
@@ -153,5 +156,9 @@ int pciehp_unconfigure_device(struct slo
 		pci_dev_put(temp);
 	}
 
+	/* Release I/O window of the slots's parent bridge */
+	if (pciehp_realloc)
+		pci_bridge_release_window(parent);
+
 	return rc;
 }
include/linux/pci.h +2 -0
Index: 20091125/include/linux/pci.h
===================================================================
@@ -764,6 +764,8 @@ int pci_vpd_truncate(struct pci_dev *dev
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
 void pci_bus_assign_resources(const struct pci_bus *bus);
+void pci_bridge_assign_resources(struct pci_bus *bus);
+void pci_bridge_release_window(struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
drivers/pci/hotplug/pciehp.h +1 -0
Index: 20091125/drivers/pci/hotplug/pciehp.h
===================================================================
@@ -43,6 +43,7 @@ extern int pciehp_poll_mode;
 extern int pciehp_poll_time;
 extern int pciehp_debug;
 extern int pciehp_force;
+extern int pciehp_realloc;
 extern struct workqueue_struct *pciehp_wq;
 
 #define dbg(format, arg...)						\
author
#74 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

In reply to Kenji Kaneshige
Quoted 6 lines · Kenji Kaneshige

Hi Yinghai,

I would like to reconfirm what is the problem you're trying to solve before reviewing and testing the additional patch. To be honest, your patch looks more and more complicated and it is becoming difficult for me to review and test it.

the real problem:
1. boot time:

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

so patch1 is trying to
a. pci assign unassign and record the failed device resource. b. clear the BIOS assigned resource of the parent bridge of fail device c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.

2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M) but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource.

so patch2 is trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.

so it will keep original assigned resource by BIOS if possible.

and you have tested patch1 and patch2 in V11, but said patch1 may have shrinking resource problem. the patch3 is addressing the patch1 that could shrinking resource for non-pcie hotplug bridge...

Quoted 8 lines

By the way, if your problem is that BIOS doesn't assign the resource to the parent bridge (root port or switch downstream port) of PCIe hotplug slots, I guess we can improve it with simpler way. I made a sample patches (no enough testing). Please take a look. Patches are

- [PATCH 1/2] pciehp: remove redundancy in bridge resource allocation
- [PATCH 2/2] pciehp: add support for bridge resource reallocation

like some earlier version of patch2 (release it them all at first) but have pciehp_realloc parameter. could be useful in some case when current patch2 (try and increase) could use up all space. i like to have that after patch2.

YH

#75 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 28 lines · Yinghai Lu
Quoted 6 lines · Kenji Kaneshige

Hi Yinghai,

I would like to reconfirm what is the problem you're trying to solve before reviewing and testing the additional patch. To be honest, your patch looks more and more complicated and it is becoming difficult for me to review and test it.

the real problem:
1. boot time:

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

so patch1 is trying to a. pci assign unassign and record the failed device resource. b. clear the BIOS assigned resource of the parent bridge of fail device c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.

2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M) but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource.

so patch2 is trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.

Quoted 5 lines

so it will keep original assigned resource by BIOS if possible.

and you have tested patch1 and patch2 in V11, but said patch1 may have shrinking resource problem. the patch3 is addressing the patch1 that could shrinking resource for non-pcie hotplug bridge...

Thank you for the explanation. The patch3 seems to solve my concern.

Your patch only touch the leaf bridge at the 2nd try of resource assignment. IIRC, this behavior is to prevent from shrinking bridge resources. Am I correct? I'm not sure but I think we don't need this behavior because now that we have another mechanism to prevent from shrinking bridge resource.

Thanks,
Kenji Kaneshige

Quoted 16 lines
Quoted 8 lines

By the way, if your problem is that BIOS doesn't assign the resource to the parent bridge (root port or switch downstream port) of PCIe hotplug slots, I guess we can improve it with simpler way. I made a sample patches (no enough testing). Please take a look. Patches are

- [PATCH 1/2] pciehp: remove redundancy in bridge resource allocation
- [PATCH 2/2] pciehp: add support for bridge resource reallocation

like some earlier version of patch2 (release it them all at first) but have pciehp_realloc parameter. could be useful in some case when current patch2 (try and increase) could use up all space. i like to have that after patch2.

YH

author
#76 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 28 lines · Kenji Kaneshige
Quoted 17 lines · Yinghai Lu
Quoted 6 lines · Kenji Kaneshige

Hi Yinghai,

I would like to reconfirm what is the problem you're trying to solve before reviewing and testing the additional patch. To be honest, your patch looks more and more complicated and it is becoming difficult for me to review and test it.

the real problem:
1. boot time:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.
so patch1 is trying to a. pci assign unassign and record the failed device resource.
b. clear the BIOS assigned resource of the parent bridge of fail device
c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.
2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M)

Quoted 9 lines

but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource. so patch2 is trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.
so it will keep original assigned resource by BIOS if possible.
and you have tested patch1 and patch2 in V11, but said patch1 may have shrinking resource problem.
the patch3 is addressing the patch1 that could shrinking resource for non-pcie hotplug bridge...

Quoted 8 lines

Thank you for the explanation. The patch3 seems to solve my concern.

Your patch only touch the leaf bridge at the 2nd try of resource assignment. IIRC, this behavior is to prevent from shrinking bridge resources. Am I correct? I'm not sure but I think we don't need this behavior because now that we have another mechanism to prevent from shrinking bridge resource.

Quoted 1 line
Quoted 1 line

Third patch will only try to increase the bridge res

Try num still default to 5 , it could help other case

Can you send me whole bootlog ?

Thanks

Yinghai

#77 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 43 lines · Yinghai
Quoted 28 lines · Kenji Kaneshige
Quoted 17 lines · Yinghai Lu
Quoted 6 lines · Kenji Kaneshige

Hi Yinghai,

I would like to reconfirm what is the problem you're trying to solve before reviewing and testing the additional patch. To be honest, your patch looks more and more complicated and it is becoming difficult for me to review and test it.

the real problem:
1. boot time:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.
so patch1 is trying to a. pci assign unassign and record the failed device resource.
b. clear the BIOS assigned resource of the parent bridge of fail device
c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.
2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M)

Quoted 9 lines

but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource. so patch2 is trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.
so it will keep original assigned resource by BIOS if possible.
and you have tested patch1 and patch2 in V11, but said patch1 may have shrinking resource problem.
the patch3 is addressing the patch1 that could shrinking resource for non-pcie hotplug bridge...

Quoted 8 lines

Thank you for the explanation. The patch3 seems to solve my concern.

Your patch only touch the leaf bridge at the 2nd try of resource assignment. IIRC, this behavior is to prevent from shrinking bridge resources. Am I correct? I'm not sure but I think we don't need this behavior because now that we have another mechanism to prevent from shrinking bridge resource.

Quoted 1 line
Quoted 1 line
Quoted 6 lines

Third patch will only try to increase the bridge res

Try num still default to 5 , it could help other case

Can you send me whole bootlog ?

Bad news...

My system doesn't boot (hangup) with your latest set of patches. Fusion MPT SAS driver initialization failed on some devices. Please see below

...
Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18 mptsas 0000:09:00.0: BAR 1: can't reserve [mem 0x00510000-0x00513fff 64bit]
mptbase: ioc0: ERROR - pci_request_selected_regions() with MEM failed mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19 mptsas 0000:0a:00.0: BAR 1: can't reserve [mem 0x00610000-0x00613fff 64bit]
mptbase: ioc1: ERROR - pci_request_selected_regions() with MEM failed
...

This problem disappear when I revert the patch 6/9, and my system can boot. But I found igb driver initialization failed on some devices even in this case. Please see below

...
igb 0000:08:00.0: BAR 0: can't reserve [mem 0x00100000-0x0011ffff] igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -16
igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: BAR 0: can't reserve [mem 0x00140000-0x0015ffff] igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -16
...

I'll send the whole boot log in both cases privately later.

Thanks,
Kenji Kaneshige

author
#78 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 23 lines
Quoted 1 line

Bad news...

My system doesn't boot (hangup) with your latest set of patches. Fusion MPT SAS driver initialization failed on some devices. Please see below

...
Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18 mptsas 0000:09:00.0: BAR 1: can't reserve [mem 0x00510000-0x00513fff 64bit]
mptbase: ioc0: ERROR - pci_request_selected_regions() with MEM failed mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19 mptsas 0000:0a:00.0: BAR 1: can't reserve [mem 0x00610000-0x00613fff 64bit]
mptbase: ioc1: ERROR - pci_request_selected_regions() with MEM failed
...

This problem disappear when I revert the patch 6/9, and my system can boot. But I found igb driver initialization failed on some devices even in this case. Please see below

Quoted 9 lines

...
igb 0000:08:00.0: BAR 0: can't reserve [mem 0x00100000-0x0011ffff] igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -16
igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: BAR 0: can't reserve [mem 0x00140000-0x0015ffff] igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -16
...

that looks werid, without patch 6/9 should only have pci bridge res shrink problem with your last test?

maybe we need keep that feature default to be disabled.

please try

---
 drivers/pci/setup-bus.c |    3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/pci/setup-bus.c +1 -2
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -918,8 +918,7 @@ static void pci_bus_dump_resources(struc
  * second try will clear small leaf bridge res
  * third try will clear related bridge: some aggressive
  */
-/* assume we only have 4 level bridges, so only try 5 times */
-int pci_try_num = 5;
+int pci_try_num = 1;
 void __init
 pci_assign_unassigned_resources(void)
 {
#79 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 31 lines · Yinghai Lu
Quoted 21 lines

Bad news...

My system doesn't boot (hangup) with your latest set of patches. Fusion MPT SAS driver initialization failed on some devices. Please see below

...
Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18 mptsas 0000:09:00.0: BAR 1: can't reserve [mem 0x00510000-0x00513fff 64bit]
mptbase: ioc0: ERROR - pci_request_selected_regions() with MEM failed mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19 mptsas 0000:0a:00.0: BAR 1: can't reserve [mem 0x00610000-0x00613fff 64bit]
mptbase: ioc1: ERROR - pci_request_selected_regions() with MEM failed
...

This problem disappear when I revert the patch 6/9, and my system can boot. But I found igb driver initialization failed on some devices even in this case. Please see below

Quoted 9 lines

...
igb 0000:08:00.0: BAR 0: can't reserve [mem 0x00100000-0x0011ffff] igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -16
igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: BAR 0: can't reserve [mem 0x00140000-0x0015ffff] igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -16
...

Quoted 2 lines

that looks werid, without patch 6/9 should only have pci bridge res shrink problem with your last test?

I don't know why. I might have overlooked it at the previous test.

Quoted 1 line

maybe we need keep that feature default to be disabled.

Will do it next week.

Thanks,
Kenji Kaneshige

Quoted 23 lines

please try

---
 drivers/pci/setup-bus.c |    3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +1 -2
@@ -918,8 +918,7 @@ static void pci_bus_dump_resources(struc
  * second try will clear small leaf bridge res
  * third try will clear related bridge: some aggressive
  */
-/* assume we only have 4 level bridges, so only try 5 times */
-int pci_try_num = 5;
+int pci_try_num = 1;
 void __init
 pci_assign_unassigned_resources(void)
 {
author
#80 lore raw

Re: [PATCH 1/2] pci: release that leaf bridge' resource that is not big -v11

Quoted 58 lines · Kenji Kaneshige
Quoted 33 lines · Yinghai Lu
Quoted 23 lines

Bad news...

My system doesn't boot (hangup) with your latest set of patches. Fusion MPT SAS driver initialization failed on some devices. Please see below

...
Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18 mptsas 0000:09:00.0: BAR 1: can't reserve [mem 0x00510000-0x00513fff 64bit]
mptbase: ioc0: ERROR - pci_request_selected_regions() with MEM failed mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19 mptsas 0000:0a:00.0: BAR 1: can't reserve [mem 0x00610000-0x00613fff 64bit]
mptbase: ioc1: ERROR - pci_request_selected_regions() with MEM failed
...

This problem disappear when I revert the patch 6/9, and my system can boot. But I found igb driver initialization failed on some devices even in this case. Please see below

Quoted 9 lines

...
igb 0000:08:00.0: BAR 0: can't reserve [mem 0x00100000-0x0011ffff] igb 0000:08:00.0: PCI INT A disabled
igb: probe of 0000:08:00.0 failed with error -16
igb 0000:08:00.1: PCI INT B -> GSI 18 (level, low) -> IRQ 18 igb 0000:08:00.1: BAR 0: can't reserve [mem 0x00140000-0x0015ffff] igb 0000:08:00.1: PCI INT B disabled
igb: probe of 0000:08:00.1 failed with error -16
...

Quoted 3 lines

that looks werid, without patch 6/9 should only have pci bridge res shrink problem with your last test?

I don't know why. I might have overlooked it at the previous test.

Quoted 1 line

maybe we need keep that feature default to be disabled.

Will do it next week.

Thanks,
Kenji Kaneshige

Quoted 7 lines

please try

---
 drivers/pci/setup-bus.c |    3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
Quoted 17 lines
Quoted 7 lines

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c static void pci_bus_dump_resources(struc · line 924

Quoted 9 lines
drivers/pci/setup-bus.c +1 -2
@@ -918,8 +918,7 @@ static void pci_bus_dump_resources(struc
  * second try will clear small leaf bridge res
  * third try will clear related bridge: some aggressive
  */
-/* assume we only have 4 level bridges, so only try 5 times */
-int pci_try_num = 5;
+int pci_try_num = 1;
 void __init
 pci_assign_unassigned_resources(void)
 {

don't try it. found the cause.

even change pci_try_num = 1 will not help.

...

will have another patch.

Thanks

Yinghai

author
#81 lore raw

cover pci: update pci bridge resource to get more big range for devices under it - v12

In reply to Kenji Kaneshige

split the two patches in to 9 for easy review...

please check

1. boot time:

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

so patches (first 6) are trying to
a. pci assign unassign and record the failed device resource. b. clear the BIOS assigned resource of the parent bridge of fail device c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big
enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v11:add pci=try=5, about more try to change more bridge
v12:not shrink pci bridge resource

2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M) but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource.

so patches (last 3) are trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10:alex found need to have export for pci_assign_unassigned_bridge_resources
v11: pass check_leaf with pci_bus_release_unused_bridge_res

Thanks

Yinghai

A message that is not in the archive (sent off-list or not yet received): 4B0D88A4.5050904@kerne! l.org

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#84 lore raw

PATCH 1/9 pci: separate pci_setup_bridge to small functions

prepare to use those small functions according to resource type later

---
 drivers/pci/setup-bus.c |   54 +++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 10 deletions(-)
drivers/pci/setup-bus.c +44 -10
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -134,19 +134,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct resource *res;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
-		 bus->secondary, bus->subordinate);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	res = bus->resource[0];
@@ -172,7 +165,13 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	res = bus->resource[1];
@@ -187,6 +186,14 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  bridge window [mem disabled]\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -219,10 +226,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
+		 bus->secondary, bus->subordinate);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
author
#85 lore raw

PATCH 2/9 pci: add pci_bridge_release_unused_res and pci_bus_release_unused_bridge_res

In reply to a message not in the archive

so later we could use it to release small resource before pci assign unassign res

---
 drivers/pci/setup-bus.c |  112 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 111 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +111 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -609,13 +609,123 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ * check:
+ *    0: only release the bridge and only the bridge is leaf
+ *    1: release all down side bridge for third shoot
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type,
+						    int check_leaf)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			if (!check_leaf)
+				pci_bus_release_unused_bridge_res(b, type,
+							 check_leaf);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if ((check_leaf && is_leaf_bridge) || !check_leaf)
+			pci_bridge_release_unused_res(bus, type);
+		break;
+	}
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pR\n", i, res);
author
#86 lore raw

PATCH 3/9 pci: don't dump it when bus resource flags is not set

In reply to a message not in the archive

mean it is not used, so skip it.

---
 drivers/pci/setup-bus.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/pci/setup-bus.c +1 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -725,7 +725,7 @@ static void pci_bus_dump_res(struct pci_
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
 
-		if (!res || !res->end)
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pR\n", i, res);
author
#87 lore raw

PATCH 4/9 pci: add failed_list to record failed one for pci_bus_assign_resources

In reply to a message not in the archive

so later we can do sth with those failed one

---
 drivers/pci/setup-bus.c |   70 +++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 63 insertions(+), 7 deletions(-)
drivers/pci/setup-bus.c +63 -7
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -27,7 +27,49 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+
+static void add_to_failed_list(struct resource_list *head, struct pci_dev *dev,
+				 struct resource *res)
+{
+	struct resource_list *list = head;
+	struct resource_list *ln = list->next;
+	struct resource_list *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list *head)
+{
+	struct resource_list *list, *tmp;
+	struct resource *res;
+	/*
+	 * Try to release leaf bridge's resources that there is no child
+	 * under it
+	 */
+	for (list = head->next; list;) {
+		res = list->res;
+		res->start = 0;
+		res->end = 0;
+		res->flags = 0;
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
@@ -58,9 +100,17 @@ static void pbus_assign_resources_sorted
 		res = list->res;
 		idx = res - &list->dev->resource[0];
 		if (pci_assign_resource(list->dev, idx)) {
-			res->start = 0;
-			res->end = 0;
-			res->flags = 0;
+			if (fail_head && !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for next try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
 		}
 		tmp = list;
 		list = list->next;
@@ -577,19 +627,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -607,6 +658,11 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
 static void release_children_resource(struct resource *r)
author
#88 lore raw

PATCH 5/9 pci: update leaf bridge res to get more big range in pci assign unssign

In reply to a message not in the archive

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

1. pci assign unassign and record the failed device resource.
2. clear the BIOS assigned resource of the parent bridge of fail device
3. go back and call pci assign unsigned
4. if it still fail, will go up more bridges. and clear and try again.

use pci_try_num to control back track bridge levels.

---
 drivers/pci/pci.c       |    5 +++
 drivers/pci/pci.h       |    2 +
 drivers/pci/setup-bus.c |   68 +++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +67 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -805,10 +805,25 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
+/*
+ * first try will not touch pci bridge res
+ * second try will clear small leaf bridge res
+ * third try will clear related bridge: some aggressive
+ */
+/* assume we only have 4 level bridges, so only try 5 times */
+int pci_try_num = 5;
 void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	int tried_times = 0;
+	int check_leaf = 1;
+	struct resource_list head, *list, *tmp;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+	unsigned long failed_type;
+again:
+	head.next = NULL;
 
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
@@ -817,7 +832,58 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+	tried_times++;
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+	failed_type = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		failed_type |= flags;
+		list = list->next;
+	}
+	/*
+	 * io port are tight, don't try extra
+	 * or if reach the limit, don't want to try more
+	 */
+	failed_type &= type_mask;
+	if ((failed_type == IORESOURCE_IO) || (tried_times >= pci_try_num)) {
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
+			 tried_times + 1);
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	/* third times and later will not check if it is leaf */
+	if ((tried_times + 1) > 2)
+		check_leaf = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		bus = list->dev->bus;
+		if (list->dev->subordinate)
+			list->res->flags = 0;
+		pci_bus_release_unused_bridge_res(bus, flags & type_mask,
+						  check_leaf);
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
drivers/pci/pci.c +5 -0
Index: linux-2.6/drivers/pci/pci.c
===================================================================
@@ -2779,6 +2779,11 @@ static int __init pci_setup(char *str)
 				pci_no_aer();
 			} else if (!strcmp(str, "nodomains")) {
 				pci_no_domains();
+			} else if (!strncmp(str, "try=", 4)) {
+				int try_num = memparse(str + 4, &str);
+
+				if (try_num > 0 && try_num < 10)
+					pci_try_num = try_num;
 			} else if (!strncmp(str, "cbiosize=", 9)) {
 				pci_cardbus_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "cbmemsize=", 10)) {
drivers/pci/pci.h +2 -0
Index: linux-2.6/drivers/pci/pci.h
===================================================================
@@ -203,6 +203,8 @@ static inline int pci_ari_enabled(struct
 	return bus->self && bus->self->ari_enabled;
 }
 
+extern int pci_try_num;
+
 #ifdef CONFIG_PCI_QUIRKS
 extern int pci_is_reassigndev(struct pci_dev *dev);
 resource_size_t pci_specified_resource_alignment(struct pci_dev *dev);
author
#89 lore raw

PATCH 6/9 pci: don't shrink bridge resources

In reply to a message not in the archive

when we are clearing leaf bridge resource and try to get big one, we could shrink the bridge if there is no resource under it.

let check with old resource size and make sure we are trying to get big one.

---
 drivers/pci/setup-bus.c |   22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)
drivers/pci/setup-bus.c +12 -10
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -390,7 +390,7 @@ static void pbus_size_io(struct pci_bus
 {
 	struct pci_dev *dev;
 	struct resource *b_res = find_free_bus_resource(bus, IORESOURCE_IO);
-	unsigned long size = 0, size1 = 0;
+	unsigned long size = 0, size1 = 0, old_size;
 
 	if (!b_res)
  		return;
@@ -415,17 +415,18 @@ static void pbus_size_io(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
 /* To be fixed in 2.5: we should have sort of HAVE_ISA
    flag in the struct pci_bus. */
 #if defined(CONFIG_ISA) || defined(CONFIG_EISA)
 	size = (size & 0xff) + ((size & ~0xffUL) << 2);
 #endif
 	size = ALIGN(size + size1, 4096);
+	if (size < old_size)
+		size = old_size;
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return;
 	}
@@ -441,7 +442,7 @@ static int pbus_size_mem(struct pci_bus
 			 unsigned long type, resource_size_t min_size)
 {
 	struct pci_dev *dev;
-	resource_size_t min_align, align, size;
+	resource_size_t min_align, align, size, old_size;
 	resource_size_t aligns[12];	/* Alignments from 1Mb to 2Gb */
 	int order, max_order;
 	struct resource *b_res = find_free_bus_resource(bus, type);
@@ -491,6 +492,11 @@ static int pbus_size_mem(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
+	if (size < old_size)
+		size = old_size;
 
 	align = 0;
 	min_align = 0;
@@ -507,10 +513,6 @@ static int pbus_size_mem(struct pci_bus
 	}
 	size = ALIGN(size, min_align);
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return 1;
 	}
author
#90 lore raw

PATCH 7/9 pci: introduce pci_assign_unassigned_bridge_resources

In reply to a message not in the archive

for pciehp to use it later

pci_setup_bridge() will not check enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

---
 drivers/pci/setup-bus.c |   94 +++++++++++++++++++++++++++++++++++++++++++++---
 include/linux/pci.h     |    1 
 2 files changed, 91 insertions(+), 4 deletions(-)
drivers/pci/setup-bus.c +90 -4
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -68,6 +68,51 @@ static void free_failed_list(struct reso
 	head->next = NULL;
 }
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev,
+				 struct resource_list *fail_head)
+{
+	struct resource *res;
+	struct resource_list head, *list, *tmp;
+	int idx;
+	u16 class = dev->class >> 8;
+
+	head.next = NULL;
+
+	/* Don't touch classless devices or host bridges or ioapics.  */
+	if (class == PCI_CLASS_NOT_DEFINED || class == PCI_CLASS_BRIDGE_HOST)
+		return;
+
+	/* Don't touch ioapic devices already enabled by firmware */
+	if (class == PCI_CLASS_SYSTEM_PIC) {
+		u16 command;
+		pci_read_config_word(dev, PCI_COMMAND, &command);
+		if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+			return;
+	}
+
+	pdev_sort_resources(dev, &head);
+
+	for (list = head.next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			if (fail_head && !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			} else {
+				res->start = 0;
+				res->end = 0;
+				res->flags = 0;
+			}
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
 static void pbus_assign_resources_sorted(const struct pci_bus *bus,
 					 struct resource_list *fail_head)
 {
@@ -281,9 +326,6 @@ static void __pci_setup_bridge(struct pc
 {
 	struct pci_dev *bridge = bus->self;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
 		 bus->secondary, bus->subordinate);
 
@@ -646,7 +688,8 @@ static void __ref __pci_bus_assign_resou
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
-			pci_setup_bridge(b);
+			if (!pci_is_enabled(dev))
+				pci_setup_bridge(b);
 			break;
 
 		case PCI_CLASS_BRIDGE_CARDBUS:
@@ -667,6 +710,34 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void __ref __pci_bridge_assign_resources(const struct pci_dev *bridge,
+					 struct resource_list *fail_head)
+{
+	struct pci_bus *b;
+
+	pdev_assign_resources_sorted((struct pci_dev *)bridge, fail_head);
+
+	b = bridge->subordinate;
+	if (!b)
+		return;
+
+	__pci_bus_assign_resources(b, fail_head);
+
+	switch (bridge->class >> 8) {
+	case PCI_CLASS_BRIDGE_PCI:
+		pci_setup_bridge(b);
+		break;
+
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		pci_setup_cardbus(b);
+		break;
+
+	default:
+		dev_info(&bridge->dev, "not setting up bridge for bus "
+			 "%04x:%02x\n", pci_domain_nr(b), b->number);
+		break;
+	}
+}
 static void release_children_resource(struct resource *r)
 {
 	struct resource *p;
@@ -894,3 +965,18 @@ enable_and_dump:
 		pci_bus_dump_resources(bus);
 	}
 }
+
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
+{
+	struct pci_bus *bus;
+	struct pci_bus *parent = bridge->subordinate;
+	int retval;
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	__pci_bridge_assign_resources(bridge, NULL);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+}
+EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -768,6 +768,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
author
#91 lore raw

PATCH 8/9 pci: pciehp clean flow in pciehp_configure_device

In reply to a message not in the archive

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.
use pci_assign_nassigned_bridge_resources

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
author
#92 lore raw

PATCH 9/9 pci: pciehp second try to get big range for pcie devices

In reply to a message not in the archive

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

---
 drivers/pci/setup-bus.c |   41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +40 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -970,13 +970,52 @@ void pci_assign_unassigned_bridge_resour
 {
 	struct pci_bus *bus;
 	struct pci_bus *parent = bridge->subordinate;
+	bool second_tried = false;
+	struct resource_list head, *list, *tmp;
 	int retval;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+again:
+	head.next = NULL;
 
 	pci_bus_size_bridges(parent);
 	pci_clear_master(bridge);
-	__pci_bridge_assign_resources(bridge, NULL);
+	__pci_bridge_assign_resources(bridge, &head);
 	retval = pci_reenable_device(bridge);
 	pci_set_master(bridge);
 	pci_enable_bridges(parent);
+
+	/* any device complain? */
+	if (!head.next)
+		return;
+
+	if (second_tried) {
+		/* still fail, don't want to try more */
+		free_failed_list(&head);
+		return;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		unsigned long flags = list->res->flags;
+
+		bus = list->dev->bus;
+		if (list->dev->subordinate)
+			list->res->flags = 0;
+		pci_bus_release_unused_bridge_res(bus, flags & type_mask, 0);
+
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	goto again;
 }
 EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
author
#93 lore raw

cover pci: update pci bridge resource to get more big range for devices under it - v13

In reply to Kenji Kaneshige

split the two patches in to 9 for easy review...

please check

1. boot time:

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

so patches (first 6) are trying to
a. pci assign unassign and record the failed device resource. b. clear the BIOS assigned resource of the parent bridge of fail device c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big
enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v11:add pci=try=5, about more try to change more bridge
v12:not shrink pci bridge resource

2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M) but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource.

so patches (last 3) are trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10:alex found need to have export for pci_assign_unassigned_bridge_resources
v11: pass check_leaf with pci_bus_release_unused_bridge_res

-v13: change resource_list to resource_list_x, to save size and flags aside, otherwise grandchild res will
get confused with son's res as could be used

Thanks

Yinghai

#95 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

In reply to Yinghai Lu
Quoted 56 lines · Yinghai Lu

split the two patches in to 9 for easy review...

please check

1. boot time:

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

so patches (first 6) are trying to a. pci assign unassign and record the failed device resource. b. clear the BIOS assigned resource of the parent bridge of fail device
c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order
move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big
enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first.
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v11:add pci=try=5, about more try to change more bridge
v12:not shrink pci bridge resource

2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M) but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource.

so patches (last 3) are trying to
a. assign resource to devices with that slot. and record fail devices
b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10:alex found need to have export for pci_assign_unassigned_bridge_resources
v11: pass check_leaf with pci_bus_release_unused_bridge_res

Quoted 3 lines

-v13: change resource_list to resource_list_x, to save size and flags aside, otherwise grandchild res will
get confused with son's res as could be used

My system doesn't boot (hangup) with the following error messages from Fusion MPT SAS driver. Those are different from the one captured with the previous version of your patches.

Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18
mptbase: ioc0: Initiating bringup
mptbase: ioc0: WARNING - Unexpected doorbell active!
mptbase: ioc0: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc0: WARNING - ResetHistory bit failed to clear!
mptbase: ioc0: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc0: WARNING - NOT READY WARNING!
mptbase: ioc0: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:09:00.0 failed with error -1
mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
mptbase: ioc1: Initiating bringup
mptbase: ioc1: WARNING - Unexpected doorbell active!
mptbase: ioc1: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc1: WARNING - ResetHistory bit failed to clear!
mptbase: ioc1: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc1: WARNING - NOT READY WARNING!
mptbase: ioc1: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:0a:00.0 failed with error -1

I'll send the whole boot log privately.

Thanks,
Kenji Kaneshige

author
#96 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

Quoted 26 lines · Kenji Kaneshige

My system doesn't boot (hangup) with the following error messages from Fusion MPT SAS driver. Those are different from the one captured with the previous version of your patches.

Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18
mptbase: ioc0: Initiating bringup
mptbase: ioc0: WARNING - Unexpected doorbell active!
mptbase: ioc0: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc0: WARNING - ResetHistory bit failed to clear!
mptbase: ioc0: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc0: WARNING - NOT READY WARNING!
mptbase: ioc0: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:09:00.0 failed with error -1
mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
mptbase: ioc1: Initiating bringup
mptbase: ioc1: WARNING - Unexpected doorbell active!
mptbase: ioc1: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc1: WARNING - ResetHistory bit failed to clear!
mptbase: ioc1: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc1: WARNING - NOT READY WARNING!
mptbase: ioc1: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:0a:00.0 failed with error -1

Quoted 1 line

I'll send the whole boot log privately.

thanks, at the same time please try "debug pci=try=1"

Yinghai

#97 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

Quoted 27 lines · Yinghai Lu
Quoted 26 lines · Kenji Kaneshige

My system doesn't boot (hangup) with the following error messages from Fusion MPT SAS driver. Those are different from the one captured with the previous version of your patches.

Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18
mptbase: ioc0: Initiating bringup
mptbase: ioc0: WARNING - Unexpected doorbell active!
mptbase: ioc0: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc0: WARNING - ResetHistory bit failed to clear!
mptbase: ioc0: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc0: WARNING - NOT READY WARNING!
mptbase: ioc0: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:09:00.0 failed with error -1
mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
mptbase: ioc1: Initiating bringup
mptbase: ioc1: WARNING - Unexpected doorbell active!
mptbase: ioc1: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc1: WARNING - ResetHistory bit failed to clear!
mptbase: ioc1: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc1: WARNING - NOT READY WARNING!
mptbase: ioc1: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:0a:00.0 failed with error -1

Quoted 4 lines
Quoted 1 line

I'll send the whole boot log privately.

thanks, at the same time please try "debug pci=try=1"

I confirmed system boot up without errors by "pci=try=1". I'll capture the bootlog with "debug" option and send it to you soon.

Thanks,
Kenji Kaneshige

author
#98 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

Quoted 30 lines · Kenji Kaneshige
Quoted 29 lines · Yinghai Lu
Quoted 28 lines · Kenji Kaneshige

My system doesn't boot (hangup) with the following error messages from Fusion MPT SAS driver. Those are different from the one captured with the previous version of your patches.

Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18
mptbase: ioc0: Initiating bringup
mptbase: ioc0: WARNING - Unexpected doorbell active!
mptbase: ioc0: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc0: WARNING - ResetHistory bit failed to clear!
mptbase: ioc0: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc0: WARNING - NOT READY WARNING!
mptbase: ioc0: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:09:00.0 failed with error -1 mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
mptbase: ioc1: Initiating bringup
mptbase: ioc1: WARNING - Unexpected doorbell active!
mptbase: ioc1: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc1: WARNING - ResetHistory bit failed to clear!
mptbase: ioc1: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc1: WARNING - NOT READY WARNING!
mptbase: ioc1: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:0a:00.0 failed with error -1

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

I'll send the whole boot log privately.

thanks, at the same time please try "debug pci=try=1"

I confirmed system boot up without errors by "pci=try=1". I'll capture the bootlog with "debug" option and send it to you soon.

thanks.

please send one only have "debug" only.

Yinghai

author
#99 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

In reply to Kenji Kaneshige
Quoted 30 lines · Kenji Kaneshige
Quoted 29 lines · Yinghai Lu
Quoted 28 lines · Kenji Kaneshige

My system doesn't boot (hangup) with the following error messages from Fusion MPT SAS driver. Those are different from the one captured with the previous version of your patches.

Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18
mptbase: ioc0: Initiating bringup
mptbase: ioc0: WARNING - Unexpected doorbell active!
mptbase: ioc0: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc0: WARNING - ResetHistory bit failed to clear!
mptbase: ioc0: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc0: WARNING - NOT READY WARNING!
mptbase: ioc0: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:09:00.0 failed with error -1 mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
mptbase: ioc1: Initiating bringup
mptbase: ioc1: WARNING - Unexpected doorbell active!
mptbase: ioc1: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc1: WARNING - ResetHistory bit failed to clear!
mptbase: ioc1: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc1: WARNING - NOT READY WARNING!
mptbase: ioc1: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:0a:00.0 failed with error -1

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

I'll send the whole boot log privately.

thanks, at the same time please try "debug pci=try=1"

I confirmed system boot up without errors by "pci=try=1". I'll capture the bootlog with "debug" option and send it to you soon.

Please also try "debug pci=try=7"
it should work too.

Thanks

Yinghai

#100 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

Quoted 32 lines · Yinghai Lu
Quoted 31 lines · Kenji Kaneshige
Quoted 30 lines · Yinghai Lu
Quoted 29 lines · Kenji Kaneshige

My system doesn't boot (hangup) with the following error messages from Fusion MPT SAS driver. Those are different from the one captured with the previous version of your patches.

Fusion MPT SAS Host driver 3.04.12
mptsas 0000:09:00.0: PCI INT A -> GSI 18 (level, low) -> IRQ 18
mptbase: ioc0: Initiating bringup
mptbase: ioc0: WARNING - Unexpected doorbell active!
mptbase: ioc0: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc0: WARNING - ResetHistory bit failed to clear!
mptbase: ioc0: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc0: WARNING - NOT READY WARNING!
mptbase: ioc0: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:09:00.0 failed with error -1 mptsas 0000:0a:00.0: PCI INT A -> GSI 19 (level, low) -> IRQ 19
mptbase: ioc1: Initiating bringup
mptbase: ioc1: WARNING - Unexpected doorbell active!
mptbase: ioc1: ERROR - Failed to come READY after reset! IocState=f0000000
mptbase: ioc1: WARNING - ResetHistory bit failed to clear!
mptbase: ioc1: ERROR - Diagnostic reset FAILED! (ffffffffh)
mptbase: ioc1: WARNING - NOT READY WARNING!
mptbase: ioc1: ERROR - didn't initialize properly! (-1)
mptsas: probe of 0000:0a:00.0 failed with error -1

I'll send the whole boot log privately.

Quoted 9 lines
Quoted 5 lines
Quoted 2 lines

thanks, at the same time please try "debug pci=try=1"

I confirmed system boot up without errors by "pci=try=1". I'll capture the bootlog with "debug" option and send it to you soon.

Please also try "debug pci=try=7"
it should work too.

I confirmed it works.
I will send the whole boot log soon.

Thanks,
Kenji Kaneshige

#101 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

In reply to Yinghai Lu
Quoted 62 lines · On Fri, 27 Nov 2009 23:34:30 -0800 Yinghai Lu <[email protected]> wrote:

split the two patches in to 9 for easy review...

please check

1. boot time:

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

so patches (first 6) are trying to
a. pci assign unassign and record the failed device resource. b. clear the BIOS assigned resource of the parent bridge of fail device c. go back and call pci assign unsigned
d. if it still fail, will go up more bridges. and clear and try again.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res. only clear release those res for x86.
v4: Bjorn want to release use dev instead of bus.
v5: Kenji pointed out it will have problem with several level bridge.
so let only handle leaf bridge.
v6: address Kenji's request (new pci_bus_release...). and change applying order move back release to pci_assign_unassigned_resource
v7: change functions name pci_bus_release_unused_bridge_res according to Jesse v8: address Eric's concern, only overwrite leaf bridge resource that is not big enough need to do it in two steps, and first step recore the failed res, and don't touch bridge res that programmed by firmware. second step will try to release bridge resource that is too small at first. v9: refresh to be applied after bjorn's patch, and remove trick about save size and restore resource second try. v11:add pci=try=5, about more try to change more bridge v12:not shrink pci bridge resource

2. hotplug:
BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to every bridge. (8M) but when insert one card that big resource, the card can not get resource. because kernel will not touch the bridge resource.

so patches (last 3) are trying to
a. assign resource to devices with that slot. and record fail devices b. if there is some failed, will clear sepcifically io port of bridge, or mmio of bridge, or mmio pref of bridge. c. try to assign the parent bridge of the slot.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save size and restore resource second try.
v10:alex found need to have export for pci_assign_unassigned_bridge_resources v11: pass check_leaf with pci_bus_release_unused_bridge_res

Quoted 3 lines

-v13: change resource_list to resource_list_x, to save size and flags aside, otherwise grandchild res will get confused with son's res as could be used

Patches 4-6 make me the most nervous, since they have the potential of really changing our resource allocation (hopefully for the better) on many platforms.

Linus, can you check them out and see if you're ok with the direction?

Patches 7-9 seem like they've recieved some review from Alex and Kenji-san, but I don't see acks or reviewed-bys on them.

Alex and Kenji-san, are you ok with them assuming the previous patches or something like them go upstream?

Thanks,

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center
#102 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

Hi Yinghai,

* Jesse Barnes <[email protected]>:

Quoted 4 lines
Quoted 4 lines

-v13: change resource_list to resource_list_x, to save size and flags aside, otherwise grandchild res will get confused with son's res as could be used

One difficulty that I have in reviewing is the multiple patch revisions buried deep within a thread.

Blame my small brain for not being able to keep track of all the revision-as-mail-responses in one coherent body of work.

Quoted 11 lines

Patches 4-6 make me the most nervous, since they have the potential of really changing our resource allocation (hopefully for the better) on many platforms.

Linus, can you check them out and see if you're ok with the direction?

Patches 7-9 seem like they've recieved some review from Alex and Kenji-san, but I don't see acks or reviewed-bys on them.

Alex and Kenji-san, are you ok with them assuming the previous patches or something like them go upstream?

Can you please repost your next revision (after taking Jesse's review comments) in a new thread?

Thank you.
/ac

author
#103 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

Quoted 19 lines · Alex Chiang

Hi Yinghai,

* Jesse Barnes <[email protected]>:

Quoted 3 lines
Quoted 3 lines

-v13: change resource_list to resource_list_x, to save size and flags aside, otherwise grandchild res will get confused with son's res as could be used

One difficulty that I have in reviewing is the multiple patch revisions buried deep within a thread.

Blame my small brain for not being able to keep track of all the revision-as-mail-responses in one coherent body of work.

Quoted 6 lines

Patches 4-6 make me the most nervous, since they have the potential of really changing our resource allocation (hopefully for the better) on many platforms.

Linus, can you check them out and see if you're ok with the direction?

Quoted 8 lines
Quoted 5 lines

Patches 7-9 seem like they've recieved some review from Alex and Kenji-san, but I don't see acks or reviewed-bys on them.

Alex and Kenji-san, are you ok with them assuming the previous patches or something like them go upstream?

Can you please repost your next revision (after taking Jesse's review comments) in a new thread?

ok.

YH

author
#104 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

In reply to Alex Chiang
Quoted 19 lines · Alex Chiang

Hi Yinghai,

* Jesse Barnes <[email protected]>:

Quoted 3 lines
Quoted 3 lines

-v13: change resource_list to resource_list_x, to save size and flags aside, otherwise grandchild res will get confused with son's res as could be used

One difficulty that I have in reviewing is the multiple patch revisions buried deep within a thread.

Blame my small brain for not being able to keep track of all the revision-as-mail-responses in one coherent body of work.

Quoted 6 lines

Patches 4-6 make me the most nervous, since they have the potential of really changing our resource allocation (hopefully for the better) on many platforms.

Linus, can you check them out and see if you're ok with the direction?

Quoted 9 lines
Quoted 5 lines

Patches 7-9 seem like they've recieved some review from Alex and Kenji-san, but I don't see acks or reviewed-bys on them.

Alex and Kenji-san, are you ok with them assuming the previous patches or something like them go upstream?

Can you please repost your next revision (after taking Jesse's review comments) in a new thread?

can you check

git://git.kernel.org/pub/scm/linux/kernel/git/yinghai/linux-2.6-yinghai.git >> master

http://git.kernel.org/?p=linux/kernel/git/yinghai/linux-2.6-yinghai.git;a=shortlog;h=refs/heads/master

i rebased them to linus tree 12-12-2009.

Thanks

Yinghai

#105 lore raw

Re: [PATCH 0/9] pci: update pci bridge resource to get more big range for devices under it - v13

* Yinghai Lu <[email protected]>:

Quoted 13 lines
Quoted 11 lines · Alex Chiang

* Jesse Barnes <[email protected]>:

Quoted 6 lines

Patches 7-9 seem like they've recieved some review from Alex and Kenji-san, but I don't see acks or reviewed-bys on them.

Alex and Kenji-san, are you ok with them assuming the previous patches or something like them go upstream?

Can you please repost your next revision (after taking Jesse's review comments) in a new thread?

Quoted 8 lines

can you check

git://git.kernel.org/pub/scm/linux/kernel/git/yinghai/linux-2.6-yinghai.git

Quoted 1 line
Quoted 1 line

master

http://git.kernel.org/?p=linux/kernel/git/yinghai/linux-2.6-yinghai.git;a=shortlog;h=refs/heads/master

i rebased them to linus tree 12-12-2009.

Well, yes, I can read those, but now I have to try and guess which patches are 7-9 that Jesse asked me to review.

One reason that it's hard to review your patches is because there are many revisions, buried deep within threads. It's not easy for reviewers to a) keep track of all them or b) keep track of how they inter-relate.

I might be following one subthread, see N revisions for patch x/y as replies in there, keep track of the changes mentally, and then keep all that state in my mind as I navigate around in the thread to try and review patch x+1/y.

Then, trying to figure out how x, x+1, x+2 are all related into one coherent change is difficult.

In the future, for PCI patch series, when you have to make revisions, please repost as a new thread every time. This will make life much easier for reviewers.

Thanks,
/ac

A message that is not in the archive (sent off-list or not yet received): 4B10D084.8070608@kerne! l.org

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#108 lore raw

PATCH 1/9 pci: separate pci_setup_bridge to small functions

prepare to use those small functions according to resource type later

---
 drivers/pci/setup-bus.c |   54 +++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 10 deletions(-)
drivers/pci/setup-bus.c +44 -10
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -134,19 +134,12 @@ EXPORT_SYMBOL(pci_setup_cardbus);
    config space writes, so it's quite possible that an I/O window of
    the bridge will have some undesirable address (e.g. 0) after the
    first write. Ditto 64-bit prefetchable MMIO.  */
-static void pci_setup_bridge(struct pci_bus *bus)
+static void pci_setup_bridge_io(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
 	struct resource *res;
 	struct pci_bus_region region;
-	u32 l, bu, lu, io_upper16;
-	int pref_mem64;
-
-	if (pci_is_enabled(bridge))
-		return;
-
-	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
-		 bus->secondary, bus->subordinate);
+	u32 l, io_upper16;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus. */
 	res = bus->resource[0];
@@ -172,7 +165,13 @@ static void pci_setup_bridge(struct pci_
 	pci_write_config_dword(bridge, PCI_IO_BASE, l);
 	/* Update upper 16 bits of I/O base/limit. */
 	pci_write_config_dword(bridge, PCI_IO_BASE_UPPER16, io_upper16);
-
+}
+static void pci_setup_bridge_mmio(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l;
 	/* Set up the top and bottom of the PCI Memory segment
 	   for this bus. */
 	res = bus->resource[1];
@@ -187,6 +186,14 @@ static void pci_setup_bridge(struct pci_
 		dev_info(&bridge->dev, "  bridge window [mem disabled]\n");
 	}
 	pci_write_config_dword(bridge, PCI_MEMORY_BASE, l);
+}
+static void pci_setup_bridge_mmio_pref(struct pci_bus *bus)
+{
+	struct pci_dev *bridge = bus->self;
+	struct resource *res;
+	struct pci_bus_region region;
+	u32 l, bu, lu;
+	int pref_mem64;
 
 	/* Clear out the upper 32 bits of PREF limit.
 	   If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
@@ -219,10 +226,37 @@ static void pci_setup_bridge(struct pci_
 		pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, bu);
 		pci_write_config_dword(bridge, PCI_PREF_LIMIT_UPPER32, lu);
 	}
+}
+static void __pci_setup_bridge(struct pci_bus *bus, unsigned long type)
+{
+	struct pci_dev *bridge = bus->self;
+
+	if (pci_is_enabled(bridge))
+		return;
+
+	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
+		 bus->secondary, bus->subordinate);
+
+	if (type & IORESOURCE_IO)
+		pci_setup_bridge_io(bus);
+
+	if (type & IORESOURCE_MEM)
+		pci_setup_bridge_mmio(bus);
+
+	if (type & IORESOURCE_PREFETCH)
+		pci_setup_bridge_mmio_pref(bus);
 
 	pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, bus->bridge_ctl);
 }
 
+static void pci_setup_bridge(struct pci_bus *bus)
+{
+	unsigned long type = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	__pci_setup_bridge(bus, type);
+}
+
 /* Check whether the bridge supports optional I/O and
    prefetchable memory ranges. If not, the respective
    base/limit registers must be read-only and read as 0. */
#109 lore raw

Re: [PATCH 1/9] pci: separate pci_setup_bridge to small functions

Quoted 9 lines · On Fri, 27 Nov 2009 23:34:55 -0800 Yinghai Lu <[email protected]> wrote:

prepare to use those small functions according to resource type later

---
 drivers/pci/setup-bus.c |   54

+++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 10 deletions(-)

Seems like a reasonable cleanup, anyone have objections?

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center
#110 lore raw

Re: [PATCH 1/9] pci: separate pci_setup_bridge to small functions

Quoted 5 lines · Jesse Barnes

On Fri, 27 Nov 2009 23:34:55 -0800

Quoted 2 lines · Yinghai Lu

prepare to use those small functions according to resource type later

Quoted 8 lines
Quoted 6 lines
---
 drivers/pci/setup-bus.c |   54

+++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 10 deletions(-)

Seems like a reasonable cleanup, anyone have objections?

There are some newlines missing between some functions, e.g. before pci_setup_bridge_mmio().

Eike

author
#111 lore raw

PATCH 2/9 pci: add pci_bridge_release_unused_res and pci_bus_release_unused_bridge_res

In reply to a message not in the archive

so later we could use it to release small resource before pci assign unassign res

---
 drivers/pci/setup-bus.c |  114 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 113 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +112 -0
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -609,6 +609,118 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should release them
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx, r);
+			/* keep the old size */
+			r->end = resource_size(r) - 1;
+			r->start = 0;
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ * check:
+ *    0: only release the bridge and only the bridge is leaf
+ *    1: release all down side bridge for third shoot
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus *bus,
+						    unsigned long type,
+						    int check_leaf)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			if (!check_leaf)
+				pci_bus_release_unused_bridge_res(b, type,
+							 check_leaf);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if ((check_leaf && is_leaf_bridge) || !check_leaf)
+			pci_bridge_release_unused_res(bus, type);
+		break;
+	}
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {
         int i;
#112 lore raw

Re: [PATCH 2/9] pci: add pci_bridge_release_unused_res and pci_bus_release_unused_bridge_res

Quoted 16 lines · On Fri, 27 Nov 2009 23:35:14 -0800 Yinghai Lu <[email protected]> wrote:

so later we could use it to release small resource before pci assign unassign res

---
 drivers/pci/setup-bus.c |  114

+++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 612

Quoted 4 lines
drivers/pci/setup-bus.c +1 -0
@@ -609,6 +609,118 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void release_children_resource(struct resource *r)

release_child_resources?

Quoted 7 lines
drivers/pci/setup-bus.c +7 -0
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 628

Quoted 10 lines
drivers/pci/setup-bus.c +9 -0
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource
 %pRt\n", p);
+		/* need to restore size, and keep flags */
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}

Also seems like it should go into resource.c instead?

Quoted 93 lines
drivers/pci/setup-bus.c +23 -0
+
+static void pci_bridge_release_unused_res(struct pci_bus *bus,
+					  unsigned long type)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES
+ 3;
+	    idx++) {
+		r = &dev->resource[idx];
+		if ((r->flags & type_mask) != type)
+			continue;
+		if (!r->parent)
+			continue;
+		/*
+		 * if there are children under that, we should

release them

drivers/pci/setup-bus.c +6 -0
+		 *  all
+		 */
+		release_children_resource(r);
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pRt released\n", idx,

r);

drivers/pci/setup-bus.c +25 -0
+			/* keep the old size */
+			r->end = resource_size(r) - 1;
+			r->start = 0;
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed) {
+		if (type & IORESOURCE_PREFETCH) {
+			/* avoiding touch the one without PREF */
+			type = IORESOURCE_PREFETCH;
+		}
+		__pci_setup_bridge(bus, type);
+	}
+}
+
+/*
+ * try to release pci bridge resources that is from leaf bridge,
+ * so we can allocate big new one later
+ * check:
+ *    0: only release the bridge and only the bridge is leaf
+ *    1: release all down side bridge for third shoot
+ */
+static void __ref pci_bus_release_unused_bridge_res(struct pci_bus

*bus,

drivers/pci/setup-bus.c +1 -0
+						    unsigned long

type,

drivers/pci/setup-bus.c +20 -0
+						    int check_leaf)
+{
+	struct pci_dev *dev;
+	bool is_leaf_bridge = true;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			is_leaf_bridge = false;
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			is_leaf_bridge = false;
+			if (!check_leaf)
+				pci_bus_release_unused_bridge_res(b,

type,

drivers/pci/setup-bus.c +13 -0
+							 check_leaf);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 725

Quoted 10 lines
drivers/pci/setup-bus.c +8 -0
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		if ((check_leaf && is_leaf_bridge) || !check_leaf)
+			pci_bridge_release_unused_res(bus, type);
+		break;
+	}
+}
+
 static void pci_bus_dump_res(struct pci_bus *bus)
 {

The idea here seems good, of allowing reallocation of bridge windows when the space is insufficient. Would be good if Linus could check out this series though.

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center
author
#113 lore raw

Re: [PATCH 2/9] pci: add pci_bridge_release_unused_res and pci_bus_release_unused_bridge_res

Quoted 21 lines · Jesse Barnes
Quoted 18 lines · On Fri, 27 Nov 2009 23:35:14 -0800 Yinghai Lu <[email protected]> wrote:

so later we could use it to release small resource before pci assign unassign res

---
 drivers/pci/setup-bus.c |  114

+++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +0 -0
@@ -609,6 +609,118 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
Quoted 3 lines

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 612

Quoted 1 line
drivers/pci/setup-bus.c +1 -0
+static void release_children_resource(struct resource *r)

release_child_resources?

ok

Quoted 12 lines
Quoted 12 lines
drivers/pci/setup-bus.c +10 -0
+{
+	struct resource *p;
+	resource_size_t size;
+
+	p = r->child;
+	while (p) {
+		release_children_resource(p);
+		release_resource(p);
+		printk(KERN_DEBUG "PCI: release child resource
 %pRt\n", p);
+		/* need to restore size, and keep flags */
Quoted 8 lines

drivers/pci/setup-bus.c void __ref pci_bus_assign_resources(cons · line 628

Quoted 6 lines
drivers/pci/setup-bus.c +6 -0
+		size = resource_size(p);
+		p->start = 0;
+		p->end = size - 1;
+		p = r->child;
+	}
+}

Also seems like it should go into resource.c instead?

only one user, may move that later...

YH

author
#114 lore raw

PATCH 3/9 pci: don't dump it when bus resource flags is not used

In reply to a message not in the archive

mean it is not used, so skip it.

---
 drivers/pci/setup-bus.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +2 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -727,7 +727,8 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pR\n", i, res);
#115 lore raw

Re: [PATCH 3/9] pci: don't dump it when bus resource flags is not used

Quoted 23 lines · On Fri, 27 Nov 2009 23:35:28 -0800 Yinghai Lu <[email protected]> wrote:

mean it is not used, so skip it.

---
 drivers/pci/setup-bus.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +2 -1
@@ -727,7 +727,8 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d
Quoted 1 line

%pR\n", i, res);

Seems fine apart from whitespace (looks like the if (...) isn't tabbed in properly?).

Signature · 1 line
Jesse Barnes, Intel Open Source Technology Center
author
#116 lore raw

Re: [PATCH 3/9] pci: don't dump it when bus resource flags is not used

Quoted 24 lines · Jesse Barnes
Quoted 21 lines · On Fri, 27 Nov 2009 23:35:28 -0800 Yinghai Lu <[email protected]> wrote:

mean it is not used, so skip it.

---
 drivers/pci/setup-bus.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +2 -1
@@ -727,7 +727,8 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+
+		if (!res || !res->end || !res->flags)
                         continue;
 
Quoted 6 lines
Quoted 2 lines
drivers/pci/setup-bus.c +0 -0
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d

%pR\n", i, res);

Seems fine apart from whitespace (looks like the if (...) isn't tabbed in properly?).

old one have space instead of tab

author
#117 lore raw

PATCH 4/9 pci: add failed_list to record failed one for pci_bus_assign_resources -v2

In reply to a message not in the archive

so later we can do sth with those failed one

-v2: store start, end, flags aside. so could keep res cleared when assign
failed. and make following assignment of its children do not go wild

---
 drivers/pci/setup-bus.c |   66 +++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 62 insertions(+), 4 deletions(-)
drivers/pci/setup-bus.c +62 -4
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -27,7 +27,52 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-static void pbus_assign_resources_sorted(const struct pci_bus *bus)
+struct resource_list_x {
+	struct resource_list_x *next;
+	struct resource *res;
+	struct pci_dev *dev;
+	resource_size_t start;
+	resource_size_t end;
+	unsigned long flags;
+};
+
+static void add_to_failed_list(struct resource_list_x *head,
+				 struct pci_dev *dev, struct resource *res)
+{
+	struct resource_list_x *list = head;
+	struct resource_list_x *ln = list->next;
+	struct resource_list_x *tmp;
+
+	tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+	if (!tmp) {
+		pr_warning("add_to_failed_list: kmalloc() failed!\n");
+		return;
+	}
+
+	tmp->next = ln;
+	tmp->res = res;
+	tmp->dev = dev;
+	tmp->start = res->start;
+	tmp->end = res->end;
+	tmp->flags = res->flags;
+	list->next = tmp;
+}
+
+static void free_failed_list(struct resource_list_x *head)
+{
+	struct resource_list_x *list, *tmp;
+
+	for (list = head->next; list;) {
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+
+	head->next = NULL;
+}
+
+static void pbus_assign_resources_sorted(const struct pci_bus *bus,
+					 struct resource_list_x *fail_head)
 {
 	struct pci_dev *dev;
 	struct resource *res;
@@ -58,6 +103,13 @@ static void pbus_assign_resources_sorted
 		res = list->res;
 		idx = res - &list->dev->resource[0];
 		if (pci_assign_resource(list->dev, idx)) {
+			if (fail_head && !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for next try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			}
 			res->start = 0;
 			res->end = 0;
 			res->flags = 0;
@@ -577,19 +629,20 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
-void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+static void __ref __pci_bus_assign_resources(const struct pci_bus *bus,
+					 struct resource_list_x *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus);
+	pbus_assign_resources_sorted(bus, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		b = dev->subordinate;
 		if (!b)
 			continue;
 
-		pci_bus_assign_resources(b);
+		__pci_bus_assign_resources(b, fail_head);
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
@@ -607,6 +660,11 @@ void __ref pci_bus_assign_resources(cons
 		}
 	}
 }
+
+void __ref pci_bus_assign_resources(const struct pci_bus *bus)
+{
+	__pci_bus_assign_resources(bus, NULL);
+}
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
 static void release_children_resource(struct resource *r)
author
#118 lore raw

PATCH 5/9 pci: update leaf bridge res to get more big range in pci assign unssign -v2

In reply to a message not in the archive

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

1. pci assign unassign and record the failed device resource.
2. clear the BIOS assigned resource of the parent bridge of fail device
3. go back and call pci assign unsigned
4. if it still fail, will go up more bridges. and clear and try again.

use pci_try_num to control back track bridge levels.

-v2: update it with resource_list_x

---
 drivers/pci/pci.c       |    5 +++
 drivers/pci/pci.h       |    2 +
 drivers/pci/setup-bus.c |   79 +++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 85 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +78 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -810,11 +810,27 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
+/*
+ * first try will not touch pci bridge res
+ * second try will clear small leaf bridge res
+ * third try will clear related bridge: some aggressive
+ */
+/* assume we only have 4 level bridges, so only try 5 times */
+int pci_try_num = 5;
 void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	int tried_times = 0;
+	int check_leaf = 1;
+	struct resource_list_x head, *list;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+	unsigned long failed_type;
 
+	head.next = NULL;
+
+again:
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
@@ -822,7 +838,68 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+	tried_times++;
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+	failed_type = 0;
+	for (list = head.next; list;) {
+		unsigned long flags = list->flags;
+
+		failed_type |= flags;
+		list = list->next;
+	}
+	/*
+	 * io port are tight, don't try extra
+	 * or if reach the limit, don't want to try more
+	 */
+	failed_type &= type_mask;
+	if ((failed_type == IORESOURCE_IO) || (tried_times >= pci_try_num)) {
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
+			 tried_times + 1);
+
+	/* third times and later will not check if it is leaf */
+	if ((tried_times + 1) > 2)
+		check_leaf = 0;
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		unsigned long flags = list->flags;
+
+		bus = list->dev->bus;
+		pci_bus_release_unused_bridge_res(bus, flags & type_mask,
+						  check_leaf);
+		list = list->next;
+	}
+	/* retore size and flags */
+	for (list = head.next; list;) {
+		struct resource *res = list->res;
+
+		res->start = list->start;
+		res->end = list->end;
+		res->flags = list->flags;
+		if (list->dev->subordinate)
+			res->flags = 0;
+
+		list = list->next;
+	}
+	free_failed_list(&head);
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
drivers/pci/pci.c +5 -0
Index: linux-2.6/drivers/pci/pci.c
===================================================================
@@ -2777,6 +2777,11 @@ static int __init pci_setup(char *str)
 				pci_no_aer();
 			} else if (!strcmp(str, "nodomains")) {
 				pci_no_domains();
+			} else if (!strncmp(str, "try=", 4)) {
+				int try_num = memparse(str + 4, &str);
+
+				if (try_num > 0 && try_num < 10)
+					pci_try_num = try_num;
 			} else if (!strncmp(str, "cbiosize=", 9)) {
 				pci_cardbus_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "cbmemsize=", 10)) {
drivers/pci/pci.h +2 -0
Index: linux-2.6/drivers/pci/pci.h
===================================================================
@@ -203,6 +203,8 @@ static inline int pci_ari_enabled(struct
 	return bus->self && bus->self->ari_enabled;
 }
 
+extern int pci_try_num;
+
 #ifdef CONFIG_PCI_QUIRKS
 extern int pci_is_reassigndev(struct pci_dev *dev);
 resource_size_t pci_specified_resource_alignment(struct pci_dev *dev);
author
#119 lore raw

PATCH 5/9 pci: update leaf bridge res to get more big range in pci assign unssign -v3

BIOS separate IO range between several IOHs, and on some slots, BIOS assign the resource to the bridge, but stop assigning resource to the device under that bridge, because the device need big resource.

1. pci assign unassign and record the failed device resource.
2. clear the BIOS assigned resource of the parent bridge of fail device
3. go back and call pci assign unsigned
4. if it still fail, will go up more bridges. and clear and try again.

use pci_try_num to control back track bridge levels.

-v2: update it with resource_list_x
-v3: make pci_try_num default to 1. and when pci_try_num is set to more than 1
will check it with max_depth, and adjust that to make sure it is bigger enough

---
 drivers/pci/pci.c       |    5 ++
 drivers/pci/pci.h       |    2 
 drivers/pci/setup-bus.c |  117 +++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 123 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +116 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -810,11 +810,69 @@ static void pci_bus_dump_resources(struc
 	}
 }
 
+static int __init pci_bus_get_depth(struct pci_bus *bus)
+{
+	int depth = 0;
+	struct pci_dev *dev;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		int ret;
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		ret = pci_bus_get_depth(b);
+		if (ret + 1 > depth)
+			depth = ret + 1;
+	}
+
+	return depth;
+}
+static int __init pci_get_max_depth(void)
+{
+	int depth = 0;
+	struct pci_bus *bus;
+
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		int ret;
+
+		ret = pci_bus_get_depth(bus);
+		if (ret > depth)
+			depth = ret;
+	}
+
+	return depth;
+}
+
+/*
+ * first try will not touch pci bridge res
+ * second try will clear small leaf bridge res
+ * third try will clear related bridge: some aggressive
+ */
+int pci_try_num = 1;
 void __init
 pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *bus;
+	int tried_times = 0;
+	int check_leaf = 1;
+	struct resource_list_x head, *list;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+	unsigned long failed_type;
+	int max_depth = pci_get_max_depth();
 
+	head.next = NULL;
+
+	if (pci_try_num > 1) {
+		if (max_depth + 1 > pci_try_num)
+			pci_try_num = max_depth + 1;
+	}
+
+	printk(KERN_DEBUG "PCI: max depth: %d pci_try_num: %d\n",
+		 max_depth, pci_try_num);
+
+again:
 	/* Depth first, calculate sizes and alignments of all
 	   subordinate buses. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
@@ -822,7 +880,64 @@ pci_assign_unassigned_resources(void)
 	}
 	/* Depth last, allocate resources and update the hardware. */
 	list_for_each_entry(bus, &pci_root_buses, node) {
-		pci_bus_assign_resources(bus);
+		__pci_bus_assign_resources(bus, &head);
+	}
+	tried_times++;
+
+	/* any device complain? */
+	if (!head.next)
+		goto enable_and_dump;
+	failed_type = 0;
+	for (list = head.next; list;) {
+		failed_type |= list->flags;
+		list = list->next;
+	}
+	/*
+	 * io port are tight, don't try extra
+	 * or if reach the limit, don't want to try more
+	 */
+	failed_type &= type_mask;
+	if ((failed_type == IORESOURCE_IO) || (tried_times >= pci_try_num)) {
+		free_failed_list(&head);
+		goto enable_and_dump;
+	}
+
+	printk(KERN_DEBUG "PCI: No. %d try to assign unassigned res\n",
+			 tried_times + 1);
+
+	/* third times and later will not check if it is leaf */
+	if ((tried_times + 1) > 2)
+		check_leaf = 0;
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		bus = list->dev->bus;
+		pci_bus_release_unused_bridge_res(bus, list->flags & type_mask,
+						  check_leaf);
+		list = list->next;
+	}
+	/* retore size and flags */
+	for (list = head.next; list;) {
+		struct resource *res = list->res;
+
+		res->start = list->start;
+		res->end = list->end;
+		res->flags = list->flags;
+		if (list->dev->subordinate)
+			res->flags = 0;
+
+		list = list->next;
+	}
+	free_failed_list(&head);
+
+	goto again;
+
+enable_and_dump:
+	/* Depth last, update the hardware. */
+	list_for_each_entry(bus, &pci_root_buses, node) {
 		pci_enable_bridges(bus);
 	}
 
drivers/pci/pci.c +5 -0
Index: linux-2.6/drivers/pci/pci.c
===================================================================
@@ -2777,6 +2777,11 @@ static int __init pci_setup(char *str)
 				pci_no_aer();
 			} else if (!strcmp(str, "nodomains")) {
 				pci_no_domains();
+			} else if (!strncmp(str, "try=", 4)) {
+				int try_num = memparse(str + 4, &str);
+
+				if (try_num > 0 && try_num < 16)
+					pci_try_num = try_num;
 			} else if (!strncmp(str, "cbiosize=", 9)) {
 				pci_cardbus_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "cbmemsize=", 10)) {
drivers/pci/pci.h +2 -0
Index: linux-2.6/drivers/pci/pci.h
===================================================================
@@ -203,6 +203,8 @@ static inline int pci_ari_enabled(struct
 	return bus->self && bus->self->ari_enabled;
 }
 
+extern int pci_try_num;
+
 #ifdef CONFIG_PCI_QUIRKS
 extern int pci_is_reassigndev(struct pci_dev *dev);
 resource_size_t pci_specified_resource_alignment(struct pci_dev *dev);
author
#120 lore raw

PATCH 6/9 pci: don't shrink bridge resources

In reply to a message not in the archive

when we are clearing leaf bridge resource and try to get big one, we could shrink the bridge if there is no resource under it.

let check with old resource size and make sure we are trying to get big one.

---
 drivers/pci/setup-bus.c |   22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)
drivers/pci/setup-bus.c +12 -10
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -392,7 +392,7 @@ static void pbus_size_io(struct pci_bus
 {
 	struct pci_dev *dev;
 	struct resource *b_res = find_free_bus_resource(bus, IORESOURCE_IO);
-	unsigned long size = 0, size1 = 0;
+	unsigned long size = 0, size1 = 0, old_size;
 
 	if (!b_res)
  		return;
@@ -417,17 +417,18 @@ static void pbus_size_io(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
 /* To be fixed in 2.5: we should have sort of HAVE_ISA
    flag in the struct pci_bus. */
 #if defined(CONFIG_ISA) || defined(CONFIG_EISA)
 	size = (size & 0xff) + ((size & ~0xffUL) << 2);
 #endif
 	size = ALIGN(size + size1, 4096);
+	if (size < old_size)
+		size = old_size;
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return;
 	}
@@ -443,7 +444,7 @@ static int pbus_size_mem(struct pci_bus
 			 unsigned long type, resource_size_t min_size)
 {
 	struct pci_dev *dev;
-	resource_size_t min_align, align, size;
+	resource_size_t min_align, align, size, old_size;
 	resource_size_t aligns[12];	/* Alignments from 1Mb to 2Gb */
 	int order, max_order;
 	struct resource *b_res = find_free_bus_resource(bus, type);
@@ -493,6 +494,11 @@ static int pbus_size_mem(struct pci_bus
 	}
 	if (size < min_size)
 		size = min_size;
+	old_size = resource_size(b_res);
+	if (old_size == 1)
+		old_size = 0;
+	if (size < old_size)
+		size = old_size;
 
 	align = 0;
 	min_align = 0;
@@ -509,10 +515,6 @@ static int pbus_size_mem(struct pci_bus
 	}
 	size = ALIGN(size, min_align);
 	if (!size) {
-		if (b_res->start || b_res->end)
-			dev_info(&bus->self->dev, "disabling bridge window "
-				 "%pR to [bus %02x-%02x] (unused)\n", b_res,
-				 bus->secondary, bus->subordinate);
 		b_res->flags = 0;
 		return 1;
 	}
author
#121 lore raw

PATCH 7/9 pci: introduce pci_assign_unassigned_bridge_resources -v2

In reply to a message not in the archive

for pciehp to use it later

pci_setup_bridge() will not check enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: update it with resource_list_x

---
 drivers/pci/setup-bus.c |   93 +++++++++++++++++++++++++++++++++++++++++++++---
 include/linux/pci.h     |    1 
 2 files changed, 90 insertions(+), 4 deletions(-)
drivers/pci/setup-bus.c +89 -4
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -71,6 +71,50 @@ static void free_failed_list(struct reso
 	head->next = NULL;
 }
 
+static void pdev_assign_resources_sorted(struct pci_dev *dev,
+				 struct resource_list_x *fail_head)
+{
+	struct resource *res;
+	struct resource_list head, *list, *tmp;
+	int idx;
+	u16 class = dev->class >> 8;
+
+	head.next = NULL;
+
+	/* Don't touch classless devices or host bridges or ioapics.  */
+	if (class == PCI_CLASS_NOT_DEFINED || class == PCI_CLASS_BRIDGE_HOST)
+		return;
+
+	/* Don't touch ioapic devices already enabled by firmware */
+	if (class == PCI_CLASS_SYSTEM_PIC) {
+		u16 command;
+		pci_read_config_word(dev, PCI_COMMAND, &command);
+		if (command & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY))
+			return;
+	}
+
+	pdev_sort_resources(dev, &head);
+
+	for (list = head.next; list;) {
+		res = list->res;
+		idx = res - &list->dev->resource[0];
+		if (pci_assign_resource(list->dev, idx)) {
+			if (fail_head && !pci_is_root_bus(list->dev->bus)) {
+				/*
+				 * device need to keep flags and size
+				 * for second try
+				 */
+				add_to_failed_list(fail_head, list->dev, res);
+			}
+			res->start = 0;
+			res->end = 0;
+			res->flags = 0;
+		}
+		tmp = list;
+		list = list->next;
+		kfree(tmp);
+	}
+}
 static void pbus_assign_resources_sorted(const struct pci_bus *bus,
 					 struct resource_list_x *fail_head)
 {
@@ -283,9 +327,6 @@ static void __pci_setup_bridge(struct pc
 {
 	struct pci_dev *bridge = bus->self;
 
-	if (pci_is_enabled(bridge))
-		return;
-
 	dev_info(&bridge->dev, "PCI bridge to [bus %02x-%02x]\n",
 		 bus->secondary, bus->subordinate);
 
@@ -648,7 +689,8 @@ static void __ref __pci_bus_assign_resou
 
 		switch (dev->class >> 8) {
 		case PCI_CLASS_BRIDGE_PCI:
-			pci_setup_bridge(b);
+			if (!pci_is_enabled(dev))
+				pci_setup_bridge(b);
 			break;
 
 		case PCI_CLASS_BRIDGE_CARDBUS:
@@ -669,6 +711,34 @@ void __ref pci_bus_assign_resources(cons
 }
 EXPORT_SYMBOL(pci_bus_assign_resources);
 
+static void __ref __pci_bridge_assign_resources(const struct pci_dev *bridge,
+					 struct resource_list_x *fail_head)
+{
+	struct pci_bus *b;
+
+	pdev_assign_resources_sorted((struct pci_dev *)bridge, fail_head);
+
+	b = bridge->subordinate;
+	if (!b)
+		return;
+
+	__pci_bus_assign_resources(b, fail_head);
+
+	switch (bridge->class >> 8) {
+	case PCI_CLASS_BRIDGE_PCI:
+		pci_setup_bridge(b);
+		break;
+
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		pci_setup_cardbus(b);
+		break;
+
+	default:
+		dev_info(&bridge->dev, "not setting up bridge for bus "
+			 "%04x:%02x\n", pci_domain_nr(b), b->number);
+		break;
+	}
+}
 static void release_children_resource(struct resource *r)
 {
 	struct resource *p;
@@ -910,3 +980,18 @@ enable_and_dump:
 		pci_bus_dump_resources(bus);
 	}
 }
+
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
+{
+	struct pci_bus *bus;
+	struct pci_bus *parent = bridge->subordinate;
+	int retval;
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	__pci_bridge_assign_resources(bridge, NULL);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+}
+EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -768,6 +768,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
author
#122 lore raw

PATCH 8/9 pci: pciehp clean flow in pciehp_configure_device

In reply to a message not in the archive

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.
use pci_assign_nassigned_bridge_resources

---
 drivers/pci/hotplug/pciehp_pci.c |   23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
author
#123 lore raw

PATCH 9/9 pci: pciehp second try to get big range for pcie devices -v2

In reply to a message not in the archive

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

-v2: use resource_list_x

---
 drivers/pci/setup-bus.c |   49 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +48 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -985,13 +985,60 @@ void pci_assign_unassigned_bridge_resour
 {
 	struct pci_bus *bus;
 	struct pci_bus *parent = bridge->subordinate;
+	bool second_tried = false;
+	struct resource_list_x head, *list;
 	int retval;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
 
+	head.next = NULL;
+
+again:
 	pci_bus_size_bridges(parent);
 	pci_clear_master(bridge);
-	__pci_bridge_assign_resources(bridge, NULL);
+	__pci_bridge_assign_resources(bridge, &head);
 	retval = pci_reenable_device(bridge);
 	pci_set_master(bridge);
 	pci_enable_bridges(parent);
+
+	/* any device complain? */
+	if (!head.next)
+		return;
+
+	if (second_tried) {
+		/* still fail, don't need to try more */
+		free_failed_list(&head);
+		return;
+	}
+
+	second_tried = true;
+	printk(KERN_DEBUG "PCI: second try to assign unassigned res\n");
+
+	/*
+	 * Try to release leaf bridge's resources that doesn't fit resource of
+	 * child device under that bridge
+	 */
+	for (list = head.next; list;) {
+		unsigned long flags = list->flags;
+
+		bus = list->dev->bus;
+		pci_bus_release_unused_bridge_res(bus, flags & type_mask, 0);
+		list = list->next;
+	}
+	/* retore size and flags */
+	for (list = head.next; list;) {
+		struct resource *res = list->res;
+
+		res->start = list->start;
+		res->end = list->end;
+		res->flags = list->flags;
+		if (list->dev->subordinate)
+			res->flags = 0;
+
+		list = list->next;
+	}
+	free_failed_list(&head);
+
+	goto again;
 }
 EXPORT_SYMBOL_GPL(pci_assign_unassigned_bridge_resources);
author
#124 lore raw

PATCH 1/2 pci: pci_bridge_release_res

need to apply this after 9/9 patchset

prepare for pciehp_realloc

it will clear the resource size for bridge

---
 drivers/pci/setup-bus.c |   42 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/pci.h     |    1 +
 2 files changed, 43 insertions(+)
drivers/pci/setup-bus.c +42 -0
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -739,6 +739,48 @@ static void __ref __pci_bridge_assign_re
 		break;
 	}
 }
+
+void pci_bridge_release_res(struct pci_bus *bus)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	if ((dev->class >> 8) != PCI_CLASS_BRIDGE_PCI)
+		return;
+
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	     idx++) {
+		r = &dev->resource[idx];
+		if (!r->parent)
+			continue;
+
+		/* if there are children under that, we should not release it */
+		if (r->child)
+			continue;
+
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pR released\n", idx, r);
+			/* old size is not kept */
+			r->start = 0;
+			r->end   = 0;
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed)
+		pci_setup_bridge(bus);
+}
+
 static void release_children_resource(struct resource *r)
 {
 	struct resource *p;
include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -767,6 +767,7 @@ int pci_vpd_truncate(struct pci_dev *dev
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
+void pci_bridge_release_res(struct pci_bus *bus);
 void pci_assign_unassigned_resources(void);
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
#125 lore raw

Re: [PATCH 1/2] pci: pci_bridge_release_res

Quoted 53 lines · Yinghai Lu

need to apply this after 9/9 patchset

prepare for pciehp_realloc

it will clear the resource size for bridge

---
 drivers/pci/setup-bus.c |   42 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/pci.h     |    1 +
 2 files changed, 43 insertions(+)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +32 -0
@@ -739,6 +739,48 @@ static void __ref __pci_bridge_assign_re
 		break;
 	}
 }
+
+void pci_bridge_release_res(struct pci_bus *bus)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	if ((dev->class >> 8) != PCI_CLASS_BRIDGE_PCI)
+		return;
+
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	     idx++) {
+		r = &dev->resource[idx];
+		if (!r->parent)
+			continue;
+
+		/* if there are children under that, we should not release it */
+		if (r->child)
+			continue;
+
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pR released\n", idx, r);
+			/* old size is not kept */
+			r->start = 0;

drivers/pci/setup-bus.c static void __ref __pci_bridge_assign_re · line 783

Quoted 10 lines
drivers/pci/setup-bus.c +10 -0
+			r->end   = 0;
+			r->flags = 0;
+			changed = true;
+		}
+	}
+
+	if (changed)
+		pci_setup_bridge(bus);
+}
+

Need to add EXPORT_SYMBOL(pci_bridge_release_res); here so the hotplug kernel module will find this function.

Quoted 19 lines
drivers/pci/setup-bus.c +0 -0
 static void release_children_resource(struct resource *r)
 {
 	struct resource *p;

Index: linux-2.6/include/linux/pci.h
===================================================================

include/linux/pci.h +1 -1
@@ -767,6 +767,7 @@ int pci_vpd_truncate(struct pci_dev *dev
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
+void pci_bridge_release_res(struct pci_bus *bus);
 void pci_assign_unassigned_resources(void);
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
--

To unsubscribe from this list: send the line "unsubscribe linux-pci" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

author
#126 lore raw

Re: [PATCH 1/2] pci: pci_bridge_release_res

Quoted 60 lines · Patrick Keller
Quoted 58 lines · Yinghai Lu

need to apply this after 9/9 patchset

prepare for pciehp_realloc

it will clear the resource size for bridge

---
 drivers/pci/setup-bus.c |   42 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/pci.h     |    1 +
 2 files changed, 43 insertions(+)

Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================

drivers/pci/setup-bus.c +37 -0
@@ -739,6 +739,48 @@ static void __ref __pci_bridge_assign_re
 		break;
 	}
 }
+
+void pci_bridge_release_res(struct pci_bus *bus)
+{
+	int idx;
+	bool changed = false;
+	struct pci_dev *dev;
+	struct resource *r;
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	/* for pci bridges res only */
+	dev = bus->self;
+	if ((dev->class >> 8) != PCI_CLASS_BRIDGE_PCI)
+		return;
+
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_BRIDGE_RESOURCES + 3;
+	     idx++) {
+		r = &dev->resource[idx];
+		if (!r->parent)
+			continue;
+
+		/* if there are children under that, we should not release it */
+		if (r->child)
+			continue;
+
+		if (!release_resource(r)) {
+			dev_printk(KERN_DEBUG, &dev->dev,
+				 "resource %d %pR released\n", idx, r);
+			/* old size is not kept */
+			r->start = 0;
+			r->end   = 0;
+			r->flags = 0;
+			changed = true;
+		}
+	}
Quoted 8 lines

drivers/pci/setup-bus.c static void __ref __pci_bridge_assign_re · line 783

Quoted 5 lines
drivers/pci/setup-bus.c +5 -0
+
+	if (changed)
+		pci_setup_bridge(bus);
+}
+

Need to add EXPORT_SYMBOL(pci_bridge_release_res); here so the hotplug kernel module will find this function.

thanks, will update that in next round.

YH

author
#127 lore raw

PATCH 2/2 pciehp: add support for bridge resource reallocation -v2

In reply to Yinghai Lu

From: Kenji Kaneshige <[email protected]>

With this patch, pciehp driver try to clear PCI bridge resources to parent bridge (root port or switch downstream port) of the slot

so we can shrink pci bridge resource for those port

This feature is enabled when 'pciehp_realloc' option is specified.

-v2: make it could be appiled after Yinghai patchset that touch pci bridge resource
also remove poweron check, because pci_bridge_release_res will check child at first need to apply this one after 9/9 patchset

---
 drivers/pci/hotplug/pciehp.h      |    1 +
 drivers/pci/hotplug/pciehp_core.c |    7 +++++++
 drivers/pci/hotplug/pciehp_pci.c  |    4 ++++
 3 files changed, 12 insertions(+)
drivers/pci/hotplug/pciehp.h +1 -0
Index: linux-2.6/drivers/pci/hotplug/pciehp.h
===================================================================
@@ -43,6 +43,7 @@ extern int pciehp_poll_mode;
 extern int pciehp_poll_time;
 extern int pciehp_debug;
 extern int pciehp_force;
+extern int pciehp_realloc;
 extern struct workqueue_struct *pciehp_wq;
 
 #define dbg(format, arg...)						\
drivers/pci/hotplug/pciehp_core.c +7 -0
Index: linux-2.6/drivers/pci/hotplug/pciehp_core.c
===================================================================
@@ -41,6 +41,7 @@ int pciehp_debug;
 int pciehp_poll_mode;
 int pciehp_poll_time;
 int pciehp_force;
+int pciehp_realloc;
 struct workqueue_struct *pciehp_wq;
 
 #define DRIVER_VERSION	"0.4"
@@ -55,10 +56,12 @@ module_param(pciehp_debug, bool, 0644);
 module_param(pciehp_poll_mode, bool, 0644);
 module_param(pciehp_poll_time, int, 0644);
 module_param(pciehp_force, bool, 0644);
+module_param(pciehp_realloc, bool, 0644);
 MODULE_PARM_DESC(pciehp_debug, "Debugging mode enabled or not");
 MODULE_PARM_DESC(pciehp_poll_mode, "Using polling mechanism for hot-plug events or not");
 MODULE_PARM_DESC(pciehp_poll_time, "Polling mechanism frequency, in seconds");
 MODULE_PARM_DESC(pciehp_force, "Force pciehp, even if _OSC and OSHP are missing");
+MODULE_PARM_DESC(pciehp_realloc, "Realloc resources for slot's parent bridge");
 
 #define PCIE_MODULE_NAME "pciehp"
 
@@ -297,6 +300,10 @@ static int pciehp_probe(struct pcie_devi
 	if (!occupied && poweron && POWER_CTRL(ctrl))
 		pciehp_power_off_slot(slot);
 
+	/* Release I/O window of the slots's parent bridge */
+	if (pciehp_realloc)
+		pci_bridge_release_res(dev->port->subordinate);
+
 	return 0;
 
 err_out_free_ctrl_slot:
drivers/pci/hotplug/pciehp_pci.c +4 -0
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -166,5 +166,9 @@ int pciehp_unconfigure_device(struct slo
 		pci_dev_put(temp);
 	}
 
+	/* Release I/O window of the slots's parent bridge */
+	if (pciehp_realloc)
+		pci_bridge_release_res(parent);
+
 	return rc;
 }
author
#128 lore raw

PATCH 2/2 pci: pciehp update the slot bridge res to get big range for pcie devices - v11

In reply to Kenji Kaneshige

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

v2: address Alex's concern about pci remove/rescan feature about
pci_setup_bridge changes.
v3: Kenji pointed out that pci_config_slot need to be called before
pci_bus_add_devices()
v4: move out pci_is_enabled checkout of pci_setup_bridge()
v5: change the applying sequence.
v6: change the functions name according to Jesse
v8: address Eric's concern, only overwrite leaf bridge resource that is not
big enough
v9: refresh to be applied after bjorn's patch, and remove trick about save
size and restore resource second try.
v10:alex found need to have export for pci_assign_unassigned_bridge_resources
v11: pass check_leaf with pci_bus_release_unused_bridge_res

---
 drivers/pci/hotplug/pciehp_pci.c |   23 ++++--
 drivers/pci/setup-bus.c          |  133 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 147 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +17 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,17 +53,15 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
 
@@ -96,12 +94,25 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_assign_unassigned_bridge_resources(bridge);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +129 -4

160 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -768,6 +768,7 @@ void pci_bus_assign_resources(const stru
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
+void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
 void pdev_enable_device(struct pci_dev *);
 void pdev_sort_resources(struct pci_dev *, struct resource_list *);
 int pci_enable_resources(struct pci_dev *, int mask);
author
#129 lore raw

PATCH 1/1 pci: pciehp update the slot bridge res to get big range for pcie devices - v3

In reply to Kenji Kaneshige

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes. -v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices()

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73 +++++++++++++++++++++++++++++++++++++--
 include/linux/pci.h              |    1 
 3 files changed, 94 insertions(+), 9 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +23 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +70 -3

103 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#131 lore raw

PATCH 2/4 pci: revert "get larger bridge ranges when space is available"

after

| commit 308cf8e13f42f476dfd6552aeff58fdc0788e566
|
|    PCI: get larger bridge ranges when space is available

found one of resource of peer root bus (0x00) get released from root resource. later one hotplug device can not get big range anymore. other peer root buses is ok.

it turns out it is from transparent path.

those resources will be used for pci bridge BAR updated. so need to limit it to 3.

but revert the commit at first.

---
 drivers/pci/setup-bus.c |   13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)
drivers/pci/setup-bus.c +2 -11
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -334,17 +334,8 @@ static struct resource *find_free_bus_re
 		r = bus->resource[i];
 		if (r == &ioport_resource || r == &iomem_resource)
 			continue;
-		if (r && (r->flags & type_mask) == type) {
-			if (!r->parent)
-				return r;
-			/*
-			 * if there is no child under that, we should release
-			 * and use it. don't need to reset it, pbus_size_* will
-			 * set it again
-			 */
-			if (!r->child && !release_resource(r))
-				return r;
-		}
+		if (r && (r->flags & type_mask) == type && !r->parent)
+			return r;
 	}
 	return NULL;
 }
author
#132 lore raw

PATCH 3/4 pci: only release that resource index is less than 3 -v3

In reply to a message not in the archive

after

| commit 308cf8e13f42f476dfd6552aeff58fdc0788e566
|
|    PCI: get larger bridge ranges when space is available

found one of resource of peer root bus (0x00) get released from root resource. later one hotplug device can not get big range anymore. other peer root buses is ok.

it turns out it is from transparent path.

those resources will be used for pci bridge BAR updated. so need to limit it to 3.

v2: Jesse doesn't like it is in find_free_bus_resource...
try to move out of pci_bus_size_bridges loop.
need to apply after:
[PATCH] pci: pciehp update the slot bridge res to get big range for pcie devices - v4
v3: add pci_setup_bridge calling after pci_bridge_release_not_used_res.
only clear release those res for x86.

---
 arch/x86/pci/i386.c              |    6 +++
 drivers/pci/hotplug/pciehp_pci.c |    1 
 drivers/pci/setup-bus.c          |   69 ++++++++++++++++++++++++++++++++++++++-
 include/linux/pci.h              |    2 +
 4 files changed, 77 insertions(+), 1 deletion(-)
drivers/pci/setup-bus.c +68 -1
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -319,6 +319,37 @@ static void pci_bridge_check_ranges(stru
 	}
 }
 
+void pci_bridge_release_not_used_res(struct pci_bus *bus)
+{
+	int i;
+	struct resource *r;
+	unsigned long type_mask = IORESOURCE_IO | IORESOURCE_MEM |
+				  IORESOURCE_PREFETCH;
+
+	/* for pci bridges res only */
+	for (i = 0; i < 3; i++) {
+		r = bus->resource[i];
+		if (r && (r->flags & type_mask)) {
+			if (!r->parent)
+				continue;
+			/*
+			 * if there is no child under that, we should release
+			 * and use it. don't need to reset it, pbus_size_* will
+			 * set it again
+			 */
+			if (!r->child && !release_resource(r)) {
+				r->flags = 0;
+				dev_printk(KERN_DEBUG, &bus->dev,
+					   "resource %d %pRt released\n",
+					   i, r);
+			}
+		}
+	}
+
+	pci_setup_bridge(bus);
+}
+EXPORT_SYMBOL(pci_bridge_release_not_used_res);
+
 /* Helper function for sizing routines: find first available
    bus resource of a given type. Note: we intentionally skip
    the bus resources which have already been assigned (that is,
@@ -576,6 +607,42 @@ void __ref pci_bus_size_bridges(struct p
 }
 EXPORT_SYMBOL(pci_bus_size_bridges);
 
+void __ref pci_bus_release_bridges_not_used_res(struct pci_bus *bus)
+{
+	struct pci_dev *dev;
+
+	list_for_each_entry(dev, &bus->devices, bus_list) {
+		struct pci_bus *b = dev->subordinate;
+		if (!b)
+			continue;
+
+		switch (dev->class >> 8) {
+		case PCI_CLASS_BRIDGE_CARDBUS:
+			break;
+
+		case PCI_CLASS_BRIDGE_PCI:
+		default:
+			pci_bus_release_bridges_not_used_res(b);
+			break;
+		}
+	}
+
+	/* The root bus? */
+	if (!bus->self)
+		return;
+
+	switch (bus->self->class >> 8) {
+	case PCI_CLASS_BRIDGE_CARDBUS:
+		break;
+
+	case PCI_CLASS_BRIDGE_PCI:
+	default:
+		pci_bridge_release_not_used_res(bus);
+		break;
+	}
+}
+EXPORT_SYMBOL(pci_bus_release_bridges_not_used_res);
+
 void __ref pci_bridge_assign_resources(const struct pci_dev *bridge)
 {
 	struct pci_bus *b;
@@ -644,7 +711,7 @@ static void pci_bus_dump_res(struct pci_
 
         for (i = 0; i < PCI_BUS_NUM_RESOURCES; i++) {
                 struct resource *res = bus->resource[i];
-                if (!res || !res->end)
+                if (!res || !res->end || !res->flags)
                         continue;
 
 		dev_printk(KERN_DEBUG, &bus->dev, "resource %d %pRt\n", i, res);
drivers/pci/hotplug/pciehp_pci.c +1 -0
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -171,6 +171,7 @@ int pciehp_unconfigure_device(struct slo
 		}
 		pci_dev_put(temp);
 	}
+	pci_bridge_release_not_used_res(parent);
 
 	return rc;
 }
include/linux/pci.h +2 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -759,6 +759,8 @@ int pci_vpd_truncate(struct pci_dev *dev
 void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
+void pci_bus_release_bridges_not_used_res(struct pci_bus *bus);
+void pci_bridge_release_not_used_res(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);
 void pci_assign_unassigned_resources(void);
 void pdev_enable_device(struct pci_dev *);
arch/x86/pci/i386.c +6 -0
Index: linux-2.6/arch/x86/pci/i386.c
===================================================================
@@ -194,6 +194,7 @@ static void __init pcibios_allocate_reso
 static int __init pcibios_assign_resources(void)
 {
 	struct pci_dev *dev = NULL;
+	struct pci_bus *bus;
 	struct resource *r;
 
 	if (!(pci_probe & PCI_ASSIGN_ROMS)) {
@@ -213,6 +214,11 @@ static int __init pcibios_assign_resourc
 		}
 	}
 
+	/* Try to release bridge resources, that there is not child under it */
+	list_for_each_entry(bus, &pci_root_buses, node) {
+		pci_bus_release_bridges_not_used_res(bus);
+	}
+
 	pci_assign_unassigned_resources();
 
 	return 0;
author
#133 lore raw

PATCH 4/4 pci: remove min_size for hotplug bridge

In reply to a message not in the archive

revert

| commit 28760489a3f1e136c5ae8581c0fa8f63511f2f4c
| Author: Eric W. Biederman <[email protected]>
| Date:   Wed Sep 9 14:09:24 2009 -0700
|
|    PCI: pcie: Ensure hotplug ports have a minimum number of resources

we don't need this trick anymore. the bridge BAR will be updated automatically

---
 drivers/pci/pci.c       |   10 ----------
 drivers/pci/probe.c     |   18 ------------------
 drivers/pci/setup-bus.c |   22 +++++-----------------
 include/linux/pci.h     |    4 ----
 4 files changed, 5 insertions(+), 49 deletions(-)
drivers/pci/pci.c +0 -10
Index: linux-2.6/drivers/pci/pci.c
===================================================================
@@ -41,12 +41,6 @@ int pci_domains_supported = 1;
 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
 
-#define DEFAULT_HOTPLUG_IO_SIZE		(256)
-#define DEFAULT_HOTPLUG_MEM_SIZE	(2*1024*1024)
-/* pci=hpmemsize=nnM,hpiosize=nn can override this */
-unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
-unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE;
-
 /*
  * The default CLS is used if arch didn't set CLS explicitly and not
  * all pci devices agree on the same value.  Arch can override either
@@ -2787,10 +2781,6 @@ static int __init pci_setup(char *str)
 							strlen(str + 19));
 			} else if (!strncmp(str, "ecrc=", 5)) {
 				pcie_ecrc_get_policy(str + 5);
-			} else if (!strncmp(str, "hpiosize=", 9)) {
-				pci_hotplug_io_size = memparse(str + 9, &str);
-			} else if (!strncmp(str, "hpmemsize=", 10)) {
-				pci_hotplug_mem_size = memparse(str + 10, &str);
 			} else {
 				printk(KERN_ERR "PCI: Unknown option `%s'\n",
 						str);
drivers/pci/probe.c +0 -18
Index: linux-2.6/drivers/pci/probe.c
===================================================================
@@ -689,23 +689,6 @@ static void set_pcie_port_type(struct pc
 	pdev->pcie_type = (reg16 & PCI_EXP_FLAGS_TYPE) >> 4;
 }
 
-static void set_pcie_hotplug_bridge(struct pci_dev *pdev)
-{
-	int pos;
-	u16 reg16;
-	u32 reg32;
-
-	pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
-	if (!pos)
-		return;
-	pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, &reg16);
-	if (!(reg16 & PCI_EXP_FLAGS_SLOT))
-		return;
-	pci_read_config_dword(pdev, pos + PCI_EXP_SLTCAP, &reg32);
-	if (reg32 & PCI_EXP_SLTCAP_HPC)
-		pdev->is_hotplug_bridge = 1;
-}
-
 #define LEGACY_IO_RESOURCE	(IORESOURCE_IO | IORESOURCE_PCI_FIXED)
 
 /**
@@ -813,7 +796,6 @@ int pci_setup_device(struct pci_dev *dev
 		pci_read_irq(dev);
 		dev->transparent = ((dev->class & 0xff) == 1);
 		pci_read_bases(dev, 2, PCI_ROM_ADDRESS1);
-		set_pcie_hotplug_bridge(dev);
 		break;
 
 	case PCI_HEADER_TYPE_CARDBUS:		    /* CardBus bridge header */
drivers/pci/setup-bus.c +5 -17
Index: linux-2.6/drivers/pci/setup-bus.c
===================================================================
@@ -375,7 +375,7 @@ static struct resource *find_free_bus_re
    since these windows have 4K granularity and the IO ranges
    of non-bridge PCI devices are limited to 256 bytes.
    We must be careful with the ISA aliasing though. */
-static void pbus_size_io(struct pci_bus *bus, resource_size_t min_size)
+static void pbus_size_io(struct pci_bus *bus)
 {
 	struct pci_dev *dev;
 	struct resource *b_res = find_free_bus_resource(bus, IORESOURCE_IO);
@@ -402,8 +402,6 @@ static void pbus_size_io(struct pci_bus
 				size1 += r_size;
 		}
 	}
-	if (size < min_size)
-		size = min_size;
 /* To be fixed in 2.5: we should have sort of HAVE_ISA
    flag in the struct pci_bus. */
 #if defined(CONFIG_ISA) || defined(CONFIG_EISA)
@@ -422,8 +420,7 @@ static void pbus_size_io(struct pci_bus
 
 /* Calculate the size of the bus and minimal alignment which
    guarantees that all child resources fit in this size. */
-static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
-			 unsigned long type, resource_size_t min_size)
+static int pbus_size_mem(struct pci_bus *bus, unsigned long mask, unsigned long type)
 {
 	struct pci_dev *dev;
 	resource_size_t min_align, align, size;
@@ -473,8 +470,6 @@ static int pbus_size_mem(struct pci_bus
 			mem64_mask &= r->flags & IORESOURCE_MEM_64;
 		}
 	}
-	if (size < min_size)
-		size = min_size;
 
 	align = 0;
 	min_align = 0;
@@ -554,7 +549,6 @@ void __ref pci_bus_size_bridges(struct p
 {
 	struct pci_dev *dev;
 	unsigned long mask, prefmask;
-	resource_size_t min_mem_size = 0, min_io_size = 0;
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		struct pci_bus *b = dev->subordinate;
@@ -584,12 +578,8 @@ void __ref pci_bus_size_bridges(struct p
 
 	case PCI_CLASS_BRIDGE_PCI:
 		pci_bridge_check_ranges(bus);
-		if (bus->self->is_hotplug_bridge) {
-			min_io_size  = pci_hotplug_io_size;
-			min_mem_size = pci_hotplug_mem_size;
-		}
 	default:
-		pbus_size_io(bus, min_io_size);
+		pbus_size_io(bus);
 		/* If the bridge supports prefetchable range, size it
 		   separately. If it doesn't, or its prefetchable window
 		   has already been allocated by arch code, try
@@ -597,11 +587,9 @@ void __ref pci_bus_size_bridges(struct p
 		   resources. */
 		mask = IORESOURCE_MEM;
 		prefmask = IORESOURCE_MEM | IORESOURCE_PREFETCH;
-		if (pbus_size_mem(bus, prefmask, prefmask, min_mem_size))
+		if (pbus_size_mem(bus, prefmask, prefmask))
 			mask = prefmask; /* Success, size non-prefetch only. */
-		else
-			min_mem_size += min_mem_size;
-		pbus_size_mem(bus, mask, IORESOURCE_MEM, min_mem_size);
+		pbus_size_mem(bus, mask, IORESOURCE_MEM);
 		break;
 	}
 }
include/linux/pci.h +0 -4
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -279,7 +279,6 @@ struct pci_dev {
 	unsigned int	is_physfn:1;
 	unsigned int	is_virtfn:1;
 	unsigned int	reset_fn:1;
-	unsigned int    is_hotplug_bridge:1;
 	pci_dev_flags_t dev_flags;
 	atomic_t	enable_cnt;	/* pci_enable_device has been called */
 
@@ -1256,9 +1255,6 @@ extern unsigned long pci_cardbus_mem_siz
 extern u8 pci_dfl_cache_line_size;
 extern u8 pci_cache_line_size;
 
-extern unsigned long pci_hotplug_io_size;
-extern unsigned long pci_hotplug_mem_size;
-
 int pcibios_add_platform_entries(struct pci_dev *dev);
 void pcibios_disable_device(struct pci_dev *dev);
 int pcibios_set_pcie_reset_state(struct pci_dev *dev,
#134 lore raw

Re: [PATCH 4/4] pci: remove min_size for hotplug bridge

Quoted 9 lines · Yinghai Lu

revert

| commit 28760489a3f1e136c5ae8581c0fa8f63511f2f4c
| Author: Eric W. Biederman <[email protected]>
| Date:   Wed Sep 9 14:09:24 2009 -0700
|
|    PCI: pcie: Ensure hotplug ports have a minimum number of resources

we don't need this trick anymore. the bridge BAR will be updated automatically

How so? We can't reassign resources after they have been allocated as someone might be using them. If the firmware doesn't reserve the resource for a hotplug we need to do it ourselves.

Eric

author
#135 lore raw

PATCH 1/4 pci: pciehp update the slot bridge res to get big range for pcie devices - v4

In reply to Yinghai Lu

move out bus_size_bridges and assign resources out of pciehp_add_bridge() and at last do them all together one time including slot bridge, to avoid to call assign resources several times, when there are several bridges under the slot bridge.

need to introduce pci_bridge_assign_resources there.

handle the case the slot bridge that doesn't get pre-allocated big enough res from FW.
for example pcie devices need 256M, but the bridge only get preallocated 2M...

pci_setup_bridge() will take extra check_enabled for the slot bridge, otherwise update res is not updated to bridge BAR. that is bridge is enabled already for port service.

-v2: address Alex's concern about pci remove/rescan feature about pci_setup_bridge changes. -v3: Kenji pointed out that pci_config_slot need to be called before pci_bus_add_devices() -v4: move out pci_is_enabled checkout of pci_setup_bridge()

---
 drivers/pci/hotplug/pciehp_pci.c |   29 ++++++++++++---
 drivers/pci/setup-bus.c          |   73 ++++++++++++++++++++++++++++++++++++---
 include/linux/pci.h              |    1 
 3 files changed, 93 insertions(+), 10 deletions(-)
drivers/pci/hotplug/pciehp_pci.c +23 -6
Index: linux-2.6/drivers/pci/hotplug/pciehp_pci.c
===================================================================
@@ -53,19 +53,18 @@ static int __ref pciehp_add_bridge(struc
 		busnr = pci_scan_bridge(parent, dev, busnr, pass);
 	if (!dev->subordinate)
 		return -1;
-	pci_bus_size_bridges(dev->subordinate);
-	pci_bus_assign_resources(parent);
-	pci_enable_bridges(parent);
-	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
 int pciehp_configure_device(struct slot *p_slot)
 {
 	struct pci_dev *dev;
-	struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
+	struct pci_dev *bridge = p_slot->ctrl->pcie->port;
+	struct pci_bus *parent = bridge->subordinate;
 	int num, fn;
 	struct controller *ctrl = p_slot->ctrl;
+	int retval;
 
 	dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
 	if (dev) {
@@ -96,12 +95,30 @@ int pciehp_configure_device(struct slot
 				(dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)) {
 			pciehp_add_bridge(dev);
 		}
+		pci_dev_put(dev);
+	}
+
+	pci_bus_size_bridges(parent);
+	pci_clear_master(bridge);
+	pci_bridge_assign_resources(bridge);
+	retval = pci_reenable_device(bridge);
+	pci_set_master(bridge);
+	pci_enable_bridges(parent);
+
+	for (fn = 0; fn < 8; fn++) {
+		dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
+		if (!dev)
+			continue;
+		if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+			pci_dev_put(dev);
+			continue;
+		}
 		pci_configure_slot(dev);
 		pci_dev_put(dev);
 	}
 
-	pci_bus_assign_resources(parent);
 	pci_bus_add_devices(parent);
+
 	return 0;
 }
 
drivers/pci/setup-bus.c +69 -4

97 lines not on this page · raw message

include/linux/pci.h +1 -0
Index: linux-2.6/include/linux/pci.h
===================================================================
@@ -756,6 +756,7 @@ ssize_t pci_write_vpd(struct pci_dev *de
 int pci_vpd_truncate(struct pci_dev *dev, size_t size);
 
 /* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
+void pci_bridge_assign_resources(const struct pci_dev *bridge);
 void pci_bus_assign_resources(const struct pci_bus *bus);
 void pci_bus_size_bridges(struct pci_bus *bus);
 int pci_claim_resource(struct pci_dev *, int);