powerpc/pnv/pci: Fix incorrect PE reservation attempt on some 64-bit BARs
PATCH 3/3 powerpc/pnv/pci: Fix incorrect PE reservation attempt on some 64-bit BARs
The generic allocation code may sometimes decide to assign a prefetchable 64-bit BAR to the M32 window. In fact it may also decide to allocate a 64-bit non-prefetchable BAR to the M64 one ! So using the resource flags as a test to decide which window was used for PE allocation is just wrong and leads to insane PE numbers.
Instead, compare the addresses to figure it out.
---
This is a pretty nasty bug, I'd like to have Gavin ack it first but then we should push it back to distros. I don't know yet *why* the generic code is eager to put my BARs into 32-bit space but that's irrelevant here, it's allowed to do that and we should do the right thing anyway.
arch/powerpc/platforms/powernv/pci-ioda.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-)
arch/powerpc/platforms/powernv/pci-ioda.c +17 -10
@@ -110,10 +110,16 @@ static int __init iommu_setup(char *str) }early_param ("iommu" , iommu_setup); -static inline bool pnv_pci_is_mem_pref_64 (unsigned long flags) +static inline bool pnv_pci_is_mem_pref_64 (struct pnv_phb *phb,struct resource *r) { -return ((flags & (IORESOURCE_MEM_64 |IORESOURCE_PREFETCH )) == - (IORESOURCE_MEM_64 |IORESOURCE_PREFETCH )); +/* WARNING: We cannot rely on the resource flags. The Linux PCI +* allocation code sometimes decides to put a 64-bit prefetchable +* BAR in the 32-bit window, so we have to compare the addresses. +* +* For simplicity we only test resource start. +*/ +return (r->start >= phb->ioda.m64_base && + r->start < (phb->ioda.m64_base + phb->ioda.m64_size)); }static struct pnv_ioda_pe *pnv_ioda_init_pe (struct pnv_phb *phb,int pe_no)
@@ -230,7 +236,7 @@ static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev, sgsz = phb->ioda.m64_segsize;for (i =0 ; i <=PCI_ROM_RESOURCE ; i++) { r = &pdev->resource[i]; -if (!r->parent || !pnv_pci_is_mem_pref_64 (r->flags)) +if (!r->parent || !pnv_pci_is_mem_pref_64 (phb, r))continue ; start =_ALIGN_DOWN (r->start - base, sgsz);
@@ -3059,7 +3065,7 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev) res = &pdev->resource[i +PCI_IOV_RESOURCES ];if (!res->flags || res->parent)continue ; -if (!pnv_pci_is_mem_pref_64 (res->flags)) { +if (!pnv_pci_is_mem_pref_64 (phb, res)) {dev_warn (&pdev->dev,"Don't support SR-IOV with" " non M64 VF BAR%d: %pR. \n" , i, res);
@@ -3154,8 +3160,7 @@ static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe, region.start += phb->ioda.io_segsize; index++; } - }else if ((res->flags &IORESOURCE_MEM ) && - !pnv_pci_is_mem_pref_64 (res->flags)) { + }else if ((res->flags &IORESOURCE_MEM ) && !pnv_pci_is_mem_pref_64 (phb, res)) { region.start = res->start - phb->hose->mem_offset[0 ] - phb->ioda.m32_pci_base;
@@ -3314,9 +3319,11 @@ static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus, bridge = bridge->bus->self; } -/* We fail back to M32 if M64 isn't supported */ -if (phb->ioda.m64_segsize && -pnv_pci_is_mem_pref_64 (type)) +/* We fail back to M32 if M64 isn't supported. We enforce the M64 +* alignment for any 64-bit resource, PCIe doesn't care and +* bridges only do 64-bit prefetchable anyway +*/ +if (phb->ioda.m64_segsize && (type &IORESOURCE_MEM_64 ))return phb->ioda.m64_segsize;if (type &IORESOURCE_MEM )return phb->ioda.m32_segsize;
Quoted 10 lines · Benjamin Herrenschmidt
The generic allocation code may sometimes decide to assign a prefetchable 64-bit BAR to the M32 window. In fact it may also decide to allocate a 64-bit non-prefetchable BAR to the M64 one ! So using the resource flags as a test to decide which window was used for PE allocation is just wrong and leads to insane PE numbers.
Instead, compare the addresses to figure it out.
Signed-off-by Benjamin Herrenschmidt <[email protected]>
Quoted 8 lines
---
This is a pretty nasty bug, I'd like to have Gavin ack it first but then we should push it back to distros. I don't know yet *why* the generic code is eager to put my BARs into 32-bit space but that's irrelevant here, it's allowed to do that and we should do the right thing anyway.
It's likely related the lost 64-bits flag in prefetchable window on root port. The similar issue was observed on CAPI adapter connected to RC directly on Garrison platform, which was fixed by commit d40160f ("PHB3: Emulate root complex pref 64-bits window") in skiboot.
pcibios_initpcibios_scan_phbpci_scan_child_bus -> Scan root portpci_scan_bridge -> Create bus behind root portpci_scan_child_bus -> Scan devices on bus#1pcibios_fixup_buspci_read_bridge_bases -> Setup bridge windows of root portpci_read_bridge_mmio_pref
In pci_read_bridge_mmio_pref(), no prefetchable window (64bits+pref) is populated if bit PCI_PREF_RANGE_TYPE_64 (0x1) isn't set on PCI config register (PCI_PREF_MEMORY_BASE, 0x24). During the resource resizing and assigning stage in PCI core, all resources including 64-bits prefetchable resources will be covered by 32-bits bridge window.
Thanks,
Gavin
Quoted 72 lines
arch/powerpc/platforms/powernv/pci-ioda.c| 27 +++++++++++++++++----------1 file changed, 17 insertions(+), 10 deletions(-)arch/powerpc/platforms/powernv/pci-ioda.c +17
-10@@ -110,10 +110,16 @@ static int __init iommu_setup(char *str) }early_param ("iommu" , iommu_setup); -static inline bool pnv_pci_is_mem_pref_64 (unsigned long flags) +static inline bool pnv_pci_is_mem_pref_64 (struct pnv_phb *phb,struct resource *r) { -return ((flags & (IORESOURCE_MEM_64 |IORESOURCE_PREFETCH )) == - (IORESOURCE_MEM_64 |IORESOURCE_PREFETCH )); +/* WARNING: We cannot rely on the resource flags. The Linux PCI +* allocation code sometimes decides to put a 64-bit prefetchable +* BAR in the 32-bit window, so we have to compare the addresses. +* +* For simplicity we only test resource start. +*/ +return (r->start >= phb->ioda.m64_base && + r->start < (phb->ioda.m64_base + phb->ioda.m64_size)); }static struct pnv_ioda_pe *pnv_ioda_init_pe (struct pnv_phb *phb,int pe_no)@@ -230,7 +236,7 @@ static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev, sgsz = phb->ioda.m64_segsize;for (i =0 ; i <=PCI_ROM_RESOURCE ; i++) { r = &pdev->resource[i]; -if (!r->parent || !pnv_pci_is_mem_pref_64 (r->flags)) +if (!r->parent || !pnv_pci_is_mem_pref_64 (phb, r))continue ; start =_ALIGN_DOWN (r->start - base, sgsz);@@ -3059,7 +3065,7 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev) res = &pdev->resource[i +PCI_IOV_RESOURCES ];if (!res->flags || res->parent)continue ; -if (!pnv_pci_is_mem_pref_64 (res->flags)) { +if (!pnv_pci_is_mem_pref_64 (phb, res)) {dev_warn (&pdev->dev,"Don't support SR-IOV with" " non M64 VF BAR%d: %pR. \n" , i, res);@@ -3154,8 +3160,7 @@ static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe, region.start += phb->ioda.io_segsize; index++; } - }else if ((res->flags &IORESOURCE_MEM ) && - !pnv_pci_is_mem_pref_64 (res->flags)) { + }else if ((res->flags &IORESOURCE_MEM ) && !pnv_pci_is_mem_pref_64 (phb, res)) { region.start = res->start - phb->hose->mem_offset[0 ] - phb->ioda.m32_pci_base;@@ -3314,9 +3319,11 @@ static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus, bridge = bridge->bus->self; } -/* We fail back to M32 if M64 isn't supported */ -if (phb->ioda.m64_segsize && -pnv_pci_is_mem_pref_64 (type)) +/* We fail back to M32 if M64 isn't supported. We enforce the M64 +* alignment for any 64-bit resource, PCIe doesn't care and +* bridges only do 64-bit prefetchable anyway +*/ +if (phb->ioda.m64_segsize && (type &IORESOURCE_MEM_64 ))return phb->ioda.m64_segsize;if (type &IORESOURCE_MEM )return phb->ioda.m32_segsize;
Quoted 6 lines · Gavin Shan
In
pci_read_bridge_mmio_pref(), no prefetchable window (64bits+pref) is populated if bitPCI_PREF_RANGE_TYPE_64(0x1) isn't set on PCI config register (PCI_PREF_MEMORY_BASE,0x24). During the resource resizing and assigning stage in PCI core, all resources including 64-bits prefetchable resources will be covered by 32-bits bridge window.
Yeah ok, that's it, thanks !
Cheers,
Ben.