Skip to contentSkip to conversation

powerpc/pnv/pci: Fix incorrect PE reservation attempt on some 64-bit BARs

PATCH Superseded Benjamin Herrenschmidt 2 participants 3 messages Jun 22, 2016 PowerPC

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 3/3 powerpc/pnv/pci: Fix incorrect PE reservation attempt on some 64-bit BARs

The generic allocation code may sometimes decide to assign a prefetchable 64-bit BAR to the M32 window. In fact it may also decide to allocate a 64-bit non-prefetchable BAR to the M64 one ! So using the resource flags as a test to decide which window was used for PE allocation is just wrong and leads to insane PE numbers.

Instead, compare the addresses to figure it out.

---

This is a pretty nasty bug, I'd like to have Gavin ack it first but then we should push it back to distros. I don't know yet *why* the generic code is eager to put my BARs into 32-bit space but that's irrelevant here, it's allowed to do that and we should do the right thing anyway.

 arch/powerpc/platforms/powernv/pci-ioda.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)
arch/powerpc/platforms/powernv/pci-ioda.c +17 -10
index c6396b6..0321ba3 100644
@@ -110,10 +110,16 @@ static int __init iommu_setup(char *str)
 }
 early_param("iommu", iommu_setup);
 
-static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
+static inline bool pnv_pci_is_mem_pref_64(struct pnv_phb *phb, struct resource *r)
 {
-	return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
-		(IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
+	/* WARNING: We cannot rely on the resource flags. The Linux PCI
+	 * allocation code sometimes decides to put a 64-bit prefetchable
+	 * BAR in the 32-bit window, so we have to compare the addresses.
+	 *
+	 * For simplicity we only test resource start.
+	 */
+	return (r->start >= phb->ioda.m64_base &&
+		r->start < (phb->ioda.m64_base + phb->ioda.m64_size));
 }
 
 static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
@@ -230,7 +236,7 @@ static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
 	sgsz = phb->ioda.m64_segsize;
 	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
 		r = &pdev->resource[i];
-		if (!r->parent || !pnv_pci_is_mem_pref_64(r->flags))
+		if (!r->parent || !pnv_pci_is_mem_pref_64(phb, r))
 			continue;
 
 		start = _ALIGN_DOWN(r->start - base, sgsz);
@@ -3059,7 +3065,7 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
 		res = &pdev->resource[i + PCI_IOV_RESOURCES];
 		if (!res->flags || res->parent)
 			continue;
-		if (!pnv_pci_is_mem_pref_64(res->flags)) {
+		if (!pnv_pci_is_mem_pref_64(phb, res)) {
 			dev_warn(&pdev->dev, "Don't support SR-IOV with"
 					" non M64 VF BAR%d: %pR. \n",
 				 i, res);
@@ -3154,8 +3160,7 @@ static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
 			region.start += phb->ioda.io_segsize;
 			index++;
 		}
-	} else if ((res->flags & IORESOURCE_MEM) &&
-		   !pnv_pci_is_mem_pref_64(res->flags)) {
+	} else if ((res->flags & IORESOURCE_MEM) && !pnv_pci_is_mem_pref_64(phb, res)) {
 		region.start = res->start -
 			       phb->hose->mem_offset[0] -
 			       phb->ioda.m32_pci_base;
@@ -3314,9 +3319,11 @@ static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
 		bridge = bridge->bus->self;
 	}
 
-	/* We fail back to M32 if M64 isn't supported */
-	if (phb->ioda.m64_segsize &&
-	    pnv_pci_is_mem_pref_64(type))
+	/* We fail back to M32 if M64 isn't supported. We enforce the M64
+	 * alignment for any 64-bit resource, PCIe doesn't care and
+	 * bridges only do 64-bit prefetchable anyway
+	 */
+	if (phb->ioda.m64_segsize && (type & IORESOURCE_MEM_64))
 		return phb->ioda.m64_segsize;
 	if (type & IORESOURCE_MEM)
 		return phb->ioda.m32_segsize;
Acked-by on v1 3/3
#1 lore raw
Quoted 10 lines · Benjamin Herrenschmidt

The generic allocation code may sometimes decide to assign a prefetchable 64-bit BAR to the M32 window. In fact it may also decide to allocate a 64-bit non-prefetchable BAR to the M64 one ! So using the resource flags as a test to decide which window was used for PE allocation is just wrong and leads to insane PE numbers.

Instead, compare the addresses to figure it out.

Quoted 8 lines

---

This is a pretty nasty bug, I'd like to have Gavin ack it first but then we should push it back to distros. I don't know yet *why* the generic code is eager to put my BARs into 32-bit space but that's irrelevant here, it's allowed to do that and we should do the right thing anyway.

It's likely related the lost 64-bits flag in prefetchable window on root port. The similar issue was observed on CAPI adapter connected to RC directly on Garrison platform, which was fixed by commit d40160f ("PHB3: Emulate root complex pref 64-bits window") in skiboot.

pcibios_init
pcibios_scan_phb
pci_scan_child_bus -> Scan root port
pci_scan_bridge -> Create bus behind root port
pci_scan_child_bus -> Scan devices on bus#1
pcibios_fixup_bus
pci_read_bridge_bases -> Setup bridge windows of root port
pci_read_bridge_mmio_pref

In pci_read_bridge_mmio_pref(), no prefetchable window (64bits+pref) is populated if bit PCI_PREF_RANGE_TYPE_64 (0x1) isn't set on PCI config register (PCI_PREF_MEMORY_BASE, 0x24). During the resource resizing and assigning stage in PCI core, all resources including 64-bits prefetchable resources will be covered by 32-bits bridge window.

Thanks,
Gavin

Quoted 72 lines

arch/powerpc/platforms/powernv/pci-ioda.c | 27 +++++++++++++++++----------

1 file changed, 17 insertions(+), 10 deletions(-)
arch/powerpc/platforms/powernv/pci-ioda.c +17 -10
index c6396b6..0321ba3 100644
@@ -110,10 +110,16 @@ static int __init iommu_setup(char *str)
 }
 early_param("iommu", iommu_setup);
 
-static inline bool pnv_pci_is_mem_pref_64(unsigned long flags)
+static inline bool pnv_pci_is_mem_pref_64(struct pnv_phb *phb, struct resource *r)
 {
-	return ((flags & (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH)) ==
-		(IORESOURCE_MEM_64 | IORESOURCE_PREFETCH));
+	/* WARNING: We cannot rely on the resource flags. The Linux PCI
+	 * allocation code sometimes decides to put a 64-bit prefetchable
+	 * BAR in the 32-bit window, so we have to compare the addresses.
+	 *
+	 * For simplicity we only test resource start.
+	 */
+	return (r->start >= phb->ioda.m64_base &&
+		r->start < (phb->ioda.m64_base + phb->ioda.m64_size));
 }
 
 static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
@@ -230,7 +236,7 @@ static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
 	sgsz = phb->ioda.m64_segsize;
 	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
 		r = &pdev->resource[i];
-		if (!r->parent || !pnv_pci_is_mem_pref_64(r->flags))
+		if (!r->parent || !pnv_pci_is_mem_pref_64(phb, r))
 			continue;
 
 		start = _ALIGN_DOWN(r->start - base, sgsz);
@@ -3059,7 +3065,7 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
 		res = &pdev->resource[i + PCI_IOV_RESOURCES];
 		if (!res->flags || res->parent)
 			continue;
-		if (!pnv_pci_is_mem_pref_64(res->flags)) {
+		if (!pnv_pci_is_mem_pref_64(phb, res)) {
 			dev_warn(&pdev->dev, "Don't support SR-IOV with"
 					" non M64 VF BAR%d: %pR. \n",
 				 i, res);
@@ -3154,8 +3160,7 @@ static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
 			region.start += phb->ioda.io_segsize;
 			index++;
 		}
-	} else if ((res->flags & IORESOURCE_MEM) &&
-		   !pnv_pci_is_mem_pref_64(res->flags)) {
+	} else if ((res->flags & IORESOURCE_MEM) && !pnv_pci_is_mem_pref_64(phb, res)) {
 		region.start = res->start -
 			       phb->hose->mem_offset[0] -
 			       phb->ioda.m32_pci_base;
@@ -3314,9 +3319,11 @@ static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
 		bridge = bridge->bus->self;
 	}
 
-	/* We fail back to M32 if M64 isn't supported */
-	if (phb->ioda.m64_segsize &&
-	    pnv_pci_is_mem_pref_64(type))
+	/* We fail back to M32 if M64 isn't supported. We enforce the M64
+	 * alignment for any 64-bit resource, PCIe doesn't care and
+	 * bridges only do 64-bit prefetchable anyway
+	 */
+	if (phb->ioda.m64_segsize && (type & IORESOURCE_MEM_64))
 		return phb->ioda.m64_segsize;
 	if (type & IORESOURCE_MEM)
 		return phb->ioda.m32_segsize;
#2 lore raw
Quoted 6 lines · Gavin Shan

In pci_read_bridge_mmio_pref(), no prefetchable window (64bits+pref) is populated if bit PCI_PREF_RANGE_TYPE_64 (0x1) isn't set on PCI config register (PCI_PREF_MEMORY_BASE, 0x24). During the resource resizing and assigning stage in PCI core, all resources including 64-bits prefetchable resources will be covered by 32-bits bridge window.

Yeah ok, that's it, thanks !

Cheers,
Ben.