pci: Allow very large resource windows
Quoted 11 lines · Alan
Quoted 5 lines
From: Alan <alan@xxxxxxxxxxxxxxx>
This is needed for some of the Xeon Phi type systems.
Signed-off-by Alan Cox <alan@xxxxxxxxxxxxxxx>I applied this to my pci/resource branch for v3.16. Nikhil posted essentially the same patch a couple years ago, so I added his signed-off-by and adopted his use of
ARRAY_SIZE()to connect the "order > 13" test with the aligns[] declaration.
Well, that's not enough for our 16G BAR ...
I thought Yinghai fixed it permanently last time:
http://permalink.gmane.org/gmane.linux.kernel.pci/29142
Are you still working on this, Yinghai?
Thanks,
Guo Chao
Quoted 7 lines · Guo Chao
Well, that's not enough for our 16G BAR ...
I thought Yinghai fixed it permanently last time:
http://permalink.gmane.org/gmane.linux.kernel.pci/29142Are you still working on this, Yinghai?
Hi Guo,
Please check updated version on your setup.
Thanks
Yinghai
Subject: [PATCH -v3] PCI: Fix bus align checking to support more than 8G
So add mmio64 mask checking, only allow more than 4G when bridge and all child support 64bit res. Still keep old 2G checking for other path.
--- drivers/pci/setup-bus.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-)
drivers/pci/setup-bus.c +38 -2
@@ -928,15 +928,50 @@ static int pbus_size_mem(struct pci_bus {struct pci_dev *dev;resource_size_t min_align, align, size, size0, size1; -resource_size_t aligns[14 ];/* Alignments from 1Mb to 8Gb */ +resource_size_t aligns[44 ];/* Alignments from 1Mb to 2^63 */ int order, max_order;struct resource *b_res =find_free_bus_resource (bus, mask |IORESOURCE_PREFETCH , type);resource_size_t children_add_size =0 ; +unsigned int mem64_mask;if (!b_res)return -ENOSPC ; + mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + +/* check if mem64 support is supported and needed at first */ +list_for_each_entry (dev, &bus->devices, bus_list) { +int i; + +for (i =0 ; i <PCI_NUM_RESOURCES ; i++) { +struct resource *r = &dev->resource[i]; + +if (r->parent || ((r->flags & mask) != type && + (r->flags & mask) != type2 && + (r->flags & mask) != type3)) +continue ; +#ifdef CONFIG_PCI_IOV +/* Skip SRIOV checking, as optional res */ +if (realloc_head && i >=PCI_IOV_RESOURCES && + i <=PCI_IOV_RESOURCE_END ) +continue ; +#endif + mem64_mask &= r->flags &IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + } + } +mem64_mask_check_done: +memset (aligns,0 ,sizeof (aligns)); max_order =0 ; size =0 ;
@@ -973,7 +1008,8 @@ static int pbus_size_mem(struct pci_bus order =__ffs (align) -20 ;if (order <0 ) order =0 ; -if (order >=ARRAY_SIZE (aligns)) { +if (order >=ARRAY_SIZE (aligns) || + (!mem64_mask && order >11 /* 2Gb */ )) {dev_warn (&dev->dev,"disabling BAR %d: %pR " "(bad alignment %#llx)\n" , i, r, (unsigned long long ) align);
Guo Chao
Tested-byQuoted 15 lines · Yinghai Lu
Quoted 7 lines · Guo Chao
Well, that's not enough for our 16G BAR ...
I thought Yinghai fixed it permanently last time:
http://permalink.gmane.org/gmane.linux.kernel.pci/29142Are you still working on this, Yinghai?
Hi Guo,
Please check updated version on your setup.
Thanks
Quoted 9 lines
Yinghai
Subject: [PATCH -v3] PCI: Fix bus align checking to support more than 8G
So add mmio64 mask checking, only allow more than 4G when bridge and all child support 64bit res. Still keep old 2G checking for other path.
Signed-off-by Yinghai Lu <[email protected]>
Thanks
Quoted 70 lines
--- drivers/pci/setup-bus.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-)Index:
linux-2.6/drivers/pci/setup-bus.c
===================================================================drivers/pci/setup-bus.c +38
-2@@ -928,15 +928,50 @@ static int pbus_size_mem(struct pci_bus {struct pci_dev *dev;resource_size_t min_align, align, size, size0, size1; -resource_size_t aligns[14 ];/* Alignments from 1Mb to 8Gb */ +resource_size_t aligns[44 ];/* Alignments from 1Mb to 2^63 */ int order, max_order;struct resource *b_res =find_free_bus_resource (bus, mask |IORESOURCE_PREFETCH , type);resource_size_t children_add_size =0 ; +unsigned int mem64_mask;if (!b_res)return -ENOSPC ; + mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + +/* check if mem64 support is supported and needed at first */ +list_for_each_entry (dev, &bus->devices, bus_list) { +int i; + +for (i =0 ; i <PCI_NUM_RESOURCES ; i++) { +struct resource *r = &dev->resource[i]; + +if (r->parent || ((r->flags & mask) != type && + (r->flags & mask) != type2 && + (r->flags & mask) != type3)) +continue ; +#ifdef CONFIG_PCI_IOV +/* Skip SRIOV checking, as optional res */ +if (realloc_head && i >=PCI_IOV_RESOURCES && + i <=PCI_IOV_RESOURCE_END ) +continue ; +#endif + mem64_mask &= r->flags &IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + } + } +mem64_mask_check_done: +memset (aligns,0 ,sizeof (aligns)); max_order =0 ; size =0 ;@@ -973,7 +1008,8 @@ static int pbus_size_mem(struct pci_bus order =__ffs (align) -20 ;if (order <0 ) order =0 ; -if (order >=ARRAY_SIZE (aligns)) { +if (order >=ARRAY_SIZE (aligns) || + (!mem64_mask && order >11 /* 2Gb */ )) {dev_warn (&dev->dev,"disabling BAR %d: %pR " "(bad alignment %#llx)\n" , i, r, (unsigned long long ) align);
Bjorn Helgaas
maintainerQuoted 70 lines · Yinghai Lu
Quoted 7 lines · Guo Chao
Well, that's not enough for our 16G BAR ...
I thought Yinghai fixed it permanently last time:
http://permalink.gmane.org/gmane.linux.kernel.pci/29142Are you still working on this, Yinghai?
Hi Guo,
Please check updated version on your setup.
Thanks
Yinghai
Subject: [PATCH -v3] PCI: Fix bus align checking to support more than 8G
So add mmio64 mask checking, only allow more than 4G when bridge and all child support 64bit res. Still keep old 2G checking for other path.
Signed-off-by Yinghai Lu <[email protected]>--- drivers/pci/setup-bus.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-)Index:
linux-2.6/drivers/pci/setup-bus.c
===================================================================drivers/pci/setup-bus.c +25
-1@@ -928,15 +928,50 @@ static int pbus_size_mem(struct pci_bus {struct pci_dev *dev;resource_size_t min_align, align, size, size0, size1; -resource_size_t aligns[14 ];/* Alignments from 1Mb to 8Gb */ +resource_size_t aligns[44 ];/* Alignments from 1Mb to 2^63 */ int order, max_order;struct resource *b_res =find_free_bus_resource (bus, mask |IORESOURCE_PREFETCH , type);resource_size_t children_add_size =0 ; +unsigned int mem64_mask;if (!b_res)return -ENOSPC ; + mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + +/* check if mem64 support is supported and needed at first */ +list_for_each_entry (dev, &bus->devices, bus_list) { +int i; + +for (i =0 ; i <PCI_NUM_RESOURCES ; i++) { +struct resource *r = &dev->resource[i]; + +if (r->parent || ((r->flags & mask) != type && + (r->flags & mask) != type2 && + (r->flags & mask) != type3)) +continue ; +#ifdef CONFIG_PCI_IOV +/* Skip SRIOV checking, as optional res */ + if (realloc_head && i >= PCI_IOV_RESOURCES &&
drivers/pci/setup-bus.c static int pbus_size_mem(struct pci_bus · line 973
Quoted 10 lines
drivers/pci/setup-bus.c +10
-0+ i <= PCI_IOV_RESOURCE_END) +continue ; +#endif + mem64_mask &= r->flags &IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + } + } +mem64_mask_check_done:
What happens if we increase the size of aligns[], but omit this loop and the mem64_mask stuff? Is mem64_mask just an optimization, so the code would still work without it? If the existing code works for 8GB bars (without mem64_mask), when does it break and start requiring mem64_mask?
Quoted 13 lines
drivers/pci/setup-bus.c +2
-1memset (aligns,0 ,sizeof (aligns)); max_order =0 ; size =0 ;@@ -973,7 +1008,8 @@ static int pbus_size_mem(struct pci_bus order =__ffs (align) -20 ;if (order <0 ) order =0 ; -if (order >=ARRAY_SIZE (aligns)) { +if (order >=ARRAY_SIZE (aligns) || + (!mem64_mask && order >11 /* 2Gb */ )) {dev_warn (&dev->dev,"disabling BAR %d: %pR " "(bad alignment %#llx)\n" , i, r, (unsigned long long ) align);
Quoted 61 lines · Bjorn Helgaas
Quoted 61 lines
Subject: [PATCH -v3] PCI: Fix bus align checking to support more than 8G
So add mmio64 mask checking, only allow more than 4G when bridge and all child support 64bit res. Still keep old 2G checking for other path.
Signed-off-by Yinghai Lu <[email protected]>--- drivers/pci/setup-bus.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-)Index:
linux-2.6/drivers/pci/setup-bus.c
===================================================================drivers/pci/setup-bus.c +32
-1@@ -928,15 +928,50 @@ static int pbus_size_mem(struct pci_bus {struct pci_dev *dev;resource_size_t min_align, align, size, size0, size1; -resource_size_t aligns[14 ];/* Alignments from 1Mb to 8Gb */ +resource_size_t aligns[44 ];/* Alignments from 1Mb to 2^63 */ int order, max_order;struct resource *b_res =find_free_bus_resource (bus, mask |IORESOURCE_PREFETCH , type);resource_size_t children_add_size =0 ; +unsigned int mem64_mask;if (!b_res)return -ENOSPC ; + mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + +/* check if mem64 support is supported and needed at first */ +list_for_each_entry (dev, &bus->devices, bus_list) { +int i; + +for (i =0 ; i <PCI_NUM_RESOURCES ; i++) { +struct resource *r = &dev->resource[i]; + +if (r->parent || ((r->flags & mask) != type && + (r->flags & mask) != type2 && + (r->flags & mask) != type3)) +continue ; +#ifdef CONFIG_PCI_IOV +/* Skip SRIOV checking, as optional res */ +if (realloc_head && i >=PCI_IOV_RESOURCES && + i <=PCI_IOV_RESOURCE_END ) +continue ; +#endif + mem64_mask &= r->flags &IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done;
Quoted 8 lines
drivers/pci/setup-bus.c static int pbus_size_mem(struct pci_bus · line 973
Quoted 3 lines
drivers/pci/setup-bus.c +3
-0+ } + } +mem64_mask_check_done:What happens if we increase the size of aligns[], but omit this loop and the
mem64_maskstuff? Ismem64_maskjust an optimization, so the code would still work without it? If the existing code works for 8GB bars (withoutmem64_mask), when does it break and start requiringmem64_mask?
Yes, you are right. with current version __pci_bus_size_bridges(), we could get rid of the checking. As child 32bit pref mmio will go underwith bridge 32bit non-pref mmio.
We only need keep
Quoted 5 lines
drivers/pci/setup-bus.c static int pbus_size_mem(struct pci_bus · line 945
Quoted 5 lines
drivers/pci/setup-bus.c +5
-0+ mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ;
Let me know if you want me to send new version.
Thanks
Yinghai
Bjorn Helgaas
maintainerQuoted 71 lines · Yinghai Lu
Quoted 61 lines · Bjorn Helgaas
Quoted 61 lines
Subject: [PATCH -v3] PCI: Fix bus align checking to support more than 8G
So add mmio64 mask checking, only allow more than 4G when bridge and all child support 64bit res. Still keep old 2G checking for other path.
Signed-off-by Yinghai Lu <[email protected]>--- drivers/pci/setup-bus.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-)Index:
linux-2.6/drivers/pci/setup-bus.c
===================================================================drivers/pci/setup-bus.c +32
-1@@ -928,15 +928,50 @@ static int pbus_size_mem(struct pci_bus {struct pci_dev *dev;resource_size_t min_align, align, size, size0, size1; -resource_size_t aligns[14 ];/* Alignments from 1Mb to 8Gb */ +resource_size_t aligns[44 ];/* Alignments from 1Mb to 2^63 */ int order, max_order;struct resource *b_res =find_free_bus_resource (bus, mask |IORESOURCE_PREFETCH , type);resource_size_t children_add_size =0 ; +unsigned int mem64_mask;if (!b_res)return -ENOSPC ; + mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done; + +/* check if mem64 support is supported and needed at first */ +list_for_each_entry (dev, &bus->devices, bus_list) { +int i; + +for (i =0 ; i <PCI_NUM_RESOURCES ; i++) { +struct resource *r = &dev->resource[i]; + +if (r->parent || ((r->flags & mask) != type && + (r->flags & mask) != type2 && + (r->flags & mask) != type3)) +continue ; +#ifdef CONFIG_PCI_IOV +/* Skip SRIOV checking, as optional res */ +if (realloc_head && i >=PCI_IOV_RESOURCES && + i <=PCI_IOV_RESOURCE_END ) +continue ; +#endif + mem64_mask &= r->flags &IORESOURCE_MEM_64 ; + +if (!mem64_mask) + goto mem64_mask_check_done;Quoted 8 lines
drivers/pci/setup-bus.c static int pbus_size_mem(struct pci_bus · line 973
Quoted 3 lines
drivers/pci/setup-bus.c +3
-0+ } + } +mem64_mask_check_done:What happens if we increase the size of aligns[], but omit this loop and the
mem64_maskstuff? Ismem64_maskjust an optimization, so the code would still work without it? If the existing code works for 8GB bars (withoutmem64_mask), when does it break and start requiringmem64_mask?
Quoted 11 lines
Yes, you are right. with current version
__pci_bus_size_bridges(), we could get rid of the checking. As child 32bit pref mmio will go underwith bridge 32bit non-pref mmio.We only need keep
Quoted 5 lines
Quoted 5 lines
drivers/pci/setup-bus.c +5
-0+ mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ;
I think you're fixing two things at once, and they should be split into two separate patches:
1) Change aligns[] size to increase support alignment from 8GB to 2^63
I'm not sure about going all the way to aligns[44]. That array by itself puts 352 bytes in the stack frame (240 of which are added by this patch), which seems excessive. I suspect that supporting BARs up to 64GB or 128GB would be enough for the foreseeable future.
2) Adding mem64_mask
I think the idea is that even if we have a 64-bit window, we can't use anything above 4GB if we only have 32-bit resources. That's true, but I don't think we can enforce that in pbus_size_mem() because we're only figuring out how much space is needed; we have no idea where that space will be allocated.
And I think there are more problems:
- I don't think we handle overflow of "size" correctly. Assume that we have BARs of 2GB, 2GB, and 8GB. If we have 32-bit resources, when we add those up, it will overflow and we'll mistakenly think we only need 8MB.
- We shouldn't set "r->flags = 0". The warning says we're disabling the BAR, but this *doesn't* disable the BAR, and in fact, there is no way to disable a single BAR. What we can do is turn off PCI_COMMAND_MEMORY to disable all the memory BARs for the device. And to do that, we need to keep IORESOURCE_MEM in r->flags so pci_enable_resources() can tell that this is a memory BAR.
Bjorn
Quoted 10 lines · Bjorn Helgaas
Quoted 6 lines
Quoted 6 lines
drivers/pci/setup-bus.c static int pbus_size_mem(struct pci_bus · line 945
Quoted 6 lines
drivers/pci/setup-bus.c +5
-0+ mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ;I think you're fixing two things at once, and they should be split into two separate patches:
Quoted 7 lines
1) Change aligns[] size to increase support alignment from 8GB to 2^63
I'm not sure about going all the way to aligns[44]. That array by itself puts 352 bytes in the stack frame (240 of which are added by this patch), which seems excessive. I suspect that supporting BARs up to 64GB or 128GB would be enough for the foreseeable future.
ok, let's use 128G this time.
Quoted 8 lines
2) Adding
mem64_maskI think the idea is that even if we have a 64-bit window, we can't use anything above 4GB if we only have 32-bit resources. That's true, but I don't think we can enforce that in
pbus_size_mem()because we're only figuring out how much space is needed; we have no idea where that space will be allocated.
with current version of __pci_bus_size_bridges, we separate pref_mem64 with pref_mem32 to calling pbus_size_mem.
so if we have pref 64bit window, we will only size pref 64 bit children under it.
and will only assign pref 64bit mem64 to them late.
If the bridge does not support 64bit pref windows, and child support 64bit pref mmio, then bridge will try to use pref 32 window, in that case, .... could have
size overflow as you state follow.
Quoted 7 lines
And I think there are more problems:
- I don't think we handle overflow of "size" correctly. Assume that we have BARs of 2GB, 2GB, and 8GB. If we have 32-bit resources, when we add those up, it will overflow and we'll mistakenly think we only need 8MB.
in this case we should check the overflow.
Quoted 7 lines
- We shouldn't set "
r->flags= 0". The warning says we're disabling the BAR, but this *doesn't* disable the BAR, and in fact, there is no way to disable a single BAR. What we can do is turn offPCI_COMMAND_MEMORYto disable all the memory BARs for the device. And to do that, we need to keepIORESOURCE_MEMinr->flagssopci_enable_resources()can tell that this is a memory BAR.
when we try to size it, means that bar is not assigned. with r->flags = 0, means
we will ignore it all the way.
Thanks
Yinghai
Bjorn Helgaas
maintainerQuoted 33 lines · Yinghai Lu
Quoted 10 lines · Bjorn Helgaas
Quoted 6 lines
Quoted 6 lines
drivers/pci/setup-bus.c static int pbus_size_mem(struct pci_bus · line 945
Quoted 6 lines
drivers/pci/setup-bus.c +5
-0+ mem64_mask = b_res->flags &IORESOURCE_MEM_64 ; + +/* kernel does not support 64bit res */ +if (sizeof (resource_size_t) ==4 ) + mem64_mask &= ~IORESOURCE_MEM_64 ;I think you're fixing two things at once, and they should be split into two separate patches:
Quoted 7 lines
1) Change aligns[] size to increase support alignment from 8GB to 2^63
I'm not sure about going all the way to aligns[44]. That array by itself puts 352 bytes in the stack frame (240 of which are added by this patch), which seems excessive. I suspect that supporting BARs up to 64GB or 128GB would be enough for the foreseeable future.
ok, let's use 128G this time.
Quoted 8 lines
2) Adding
mem64_maskI think the idea is that even if we have a 64-bit window, we can't use anything above 4GB if we only have 32-bit resources. That's true, but I don't think we can enforce that in
pbus_size_mem()because we're only figuring out how much space is needed; we have no idea where that space will be allocated.with current version of
__pci_bus_size_bridges, we separatepref_mem64withpref_mem32to callingpbus_size_mem.
Quoted 8 lines
so if we have pref 64bit window, we will only size pref 64 bit children under it.
and will only assign pref 64bit mem64 to them late.If the bridge does not support 64bit pref windows, and child support 64bit pref mmio, then bridge will try to use pref 32 window, in that case, .... could have
size overflow as you state follow.
This problem has nothing to do with overflow. We're concerned with IORESOURCE_MEM_64, which tells us the width of the bridge window
registers: obviously we can't put a 64-bit bus address in a 32-bit window register.
pbus_size_mem() figures out how much space we need. If we need 4GB or more and the kernel only supports 32-bit resources, we can fail immediately because we can't describe a 4GB window in a 32-bit resource.
But if we need less than 4GB, pbus_size_mem() should succeed. We might fail *later*, if we can't allocate bus addresses that fit in a 32-bit bridge window register. But in pbus_size_mem(), we have no idea where the space will come from.
So I don't think there's any point in looking at the sizes of individual BARs in pbus_size_mem(). We should only look at the total size required.
Quoted 17 lines
Quoted 6 lines
And I think there are more problems:
- I don't think we handle overflow of "size" correctly. Assume that we have BARs of 2GB, 2GB, and 8GB. If we have 32-bit resources, when we add those up, it will overflow and we'll mistakenly think we only need 8MB.
in this case we should check the overflow.
Quoted 7 lines
- We shouldn't set "
r->flags= 0". The warning says we're disabling the BAR, but this *doesn't* disable the BAR, and in fact, there is no way to disable a single BAR. What we can do is turn offPCI_COMMAND_MEMORYto disable all the memory BARs for the device. And to do that, we need to keepIORESOURCE_MEMinr->flagssopci_enable_resources()can tell that this is a memory BAR.
Quoted 3 lines
when we try to size it, means that bar is not assigned. with
r->flags= 0, means
we will ignore it all the way.
With "r->flags = 0", we will not try to assign resources to the BAR, but the hardware BAR still exists and contains some address. If the device has other memory BARs, pci_enable_resources() will turn on PCI_COMMAND_MEMORY. Now the "r->flags == 0" BAR is enabled but the address it contains might conflict with other devices. That's the problem.
To fix this, I think we need to do "r->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED".
Bjorn
Quoted 13 lines · Bjorn Helgaas
Quoted 3 lines · Yinghai Lu
when we try to size it, means that bar is not assigned. with
r->flags= 0, means
we will ignore it all the way.With "
r->flags= 0", we will not try to assign resources to the BAR, but the hardware BAR still exists and contains some address. If the device has other memory BARs,pci_enable_resources()will turn onPCI_COMMAND_MEMORY. Now the "r->flags== 0" BAR is enabled but the address it contains might conflict with other devices. That's the problem.
Quoted 2 lines
To fix this, I think we need to do "
r->flags|=IORESOURCE_UNSET|IORESOURCE_DISABLED".
No, that is not right.
Current that r->flags = 0 and reset_resource() cover the bug in pci_enable_resources() when assign and reassign resource fail.
In pci_enable_resources() if there is one resource with IO BAR or SRIOV BAR is not assigned and have UNSET, it will prevent device from being enabled.
Most drivers could work without io port.
If you change r->flags = 0 tor->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED
Those drivers would not work anymore.
I also know one driver that could work with pref mmio, but mmio is not assigned. --- silicon bug, that it exposes non-needed mmio bar.
BTW I think we may need to clear the BAR in pci_claim_resource().
Maybe Linus or Greg have more idea about this.
Do we need to get more strict? will only enable PCI_COMMAND_MEMORY unless all mmio BARs get assigned value?
Thanks
Yinghai
Bjorn Helgaas
maintainerQuoted 25 lines · Yinghai Lu
Quoted 13 lines · Bjorn Helgaas
Quoted 3 lines · Yinghai Lu
when we try to size it, means that bar is not assigned. with
r->flags= 0, means
we will ignore it all the way.With "
r->flags= 0", we will not try to assign resources to the BAR, but the hardware BAR still exists and contains some address. If the device has other memory BARs,pci_enable_resources()will turn onPCI_COMMAND_MEMORY. Now the "r->flags== 0" BAR is enabled but the address it contains might conflict with other devices. That's the problem.Quoted 2 lines
To fix this, I think we need to do "
r->flags|=IORESOURCE_UNSET|IORESOURCE_DISABLED".No, that is not right.
Current that
r->flags= 0 andreset_resource()cover the bug inpci_enable_resources()when assign and reassign resource fail.In
pci_enable_resources()if there is one resource with IO BAR or SRIOV BAR is not assigned and have UNSET, it will prevent device from being enabled.
Quoted 6 lines
Most drivers could work without io port.
If you change
r->flags= 0 tor->flags|=IORESOURCE_UNSET|IORESOURCE_DISABLEDThose drivers would not work anymore.
If a driver doesn't need IO ports, it should use pci_enable_device_mem(). Then pci_enable_device_flags() will build a BAR bitmask that does not include the IO BARs, and pci_enable_resources() will not look at the resources corresponding to those BARs at all (so it doesn't matter what r->flags is), and it will not turn on PCI_COMMAND_IO.
Quoted 2 lines
I also know one driver that could work with pref mmio, but mmio is not assigned. --- silicon bug, that it exposes non-needed mmio bar.
The driver might work, but turning on PCI_COMMAND_MEMORY when an MMIO BAR is not assigned is dangerous and could cause bus conflicts. If we can't safely turn on PCI_COMMAND_MEMORY and this driver stops working, well, I think that's tough luck.
I explained why I think it is dangerous in the paragraph "With 'r->flags = 0', ..." above. This a question of how the hardware behaves, so if you think I'm wrong, you need to point out something in the spec that says the hardware works differently than how I think it does.
Quoted 5 lines
BTW I think we may need to clear the BAR in
pci_claim_resource().Maybe Linus or Greg have more idea about this.
Do we need to get more strict? will only enablePCI_COMMAND_MEMORYunless all mmio BARs get assigned value?
I think we should turn on PCI_COMMAND_MEMORY only when the driver requests IORESOURCE_MEM and all MMIO BARs are assigned.
Bjorn
Quoted 6 lines · Bjorn Helgaas
I explained why I think it is dangerous in the paragraph "With '
r->flags= 0', ..." above. This a question of how the hardware behaves, so if you think I'm wrong, you need to point out something in the spec that says the hardware works differently than how I think it does.
If the BAR have all 0, cpu can not access it as hw will forward access to ram controller according to routing table.
Yinghai
Quoted 3 lines · Yinghai Lu
If the BAR have all 0, cpu can not access it as hw will forward access to ram controller according to routing table.
On PC's, yes (also likely with many devices - there are definitely devices out there were setting the BAR to zero disables it).
On other platforms? Not necessarily at all, and some devices have no way of disabling the BAR (even clearing the IO/MEM bits in the control/status register does *not* necessarily disable things like root bridge BARs). I'm pretty sure we had it actually happen (I'd like to say "Powerpc", but I wouldn't really bet on it)
Anyway, the rule is that "r->flags = 0" means that it doesn't exist. And that rule makes sense, and is completely unambiguous. Relying on the BAR _value_ being zero is bogus and wrong.
Probing that finds a valid BAR, but an unassigned one, will set the UNSET bit. Plus IORESOURCE_MEM or IO should also always be set for a valid bar. So a value of zero really is completely unambiguous - it does not exist. It's not unassigned, it really isn't there.
Of course, it's possible that we get this wrong, and initial probing will obviously start out with a zero value, so there is clearly one valid situation where we go from "does not exist" to something else, but in general, "r->flags == 0" really is special.
Linus
Bjorn Helgaas
maintainerQuoted 7 lines · Linus Torvalds
Quoted 3 lines · Yinghai Lu
If the BAR have all 0, cpu can not access it as hw will forward access to ram controller according to routing table.
On PC's, yes (also likely with many devices - there are definitely devices out there were setting the BAR to zero disables it).
Ooh, we might trip over this some day on a platform with host bridge address translation that maps to PCI bus address 0. I don't think we have anything in the allocator that avoids bus address 0 in that case.
Bjorn
Quoted 7 lines · Bjorn Helgaas
Quoted 5 lines · Linus Torvalds
Quoted 3 lines · Yinghai Lu
If the BAR have all 0, cpu can not access it as hw will forward access to ram controller according to routing table.
Quoted 6 lines
Quoted 2 lines
On PC's, yes (also likely with many devices - there are definitely devices out there were setting the BAR to zero disables it).
Ooh, we might trip over this some day on a platform with host bridge address translation that maps to PCI bus address 0. I don't think we have anything in the allocator that avoids bus address 0 in that case.
Anyway, I draft RFC one that replace flags = 0 with flags |= IORESOURCE_UNSET | IORESOURCE_DISABLEd.
Let's see how many drivers will fail.
Yinghai
Subject: [RFC PATCH] PCI: don't set flags to 0 when assign resource fail
make flags take IORESOURCE_UNSET | IORESOURCE_DISABLE instead.
--- drivers/pci/setup-bus.c | 49 ++++++++++++++++++++++++------------------------ drivers/pci/setup-res.c | 11 ++++++++++ 2 files changed, 36 insertions(+), 24 deletions(-)
drivers/pci/setup-bus.c +25 -24
@@ -136,7 +136,8 @@ static void pdev_sort_resources(struct pif (r->flags &IORESOURCE_PCI_FIXED )continue ; -if (!(r->flags) || r->parent) +if (!(r->flags) || r->parent || + (r->flags &IORESOURCE_DISABLED ))continue ; r_align =pci_resource_alignment (dev, r);
@@ -190,13 +191,6 @@ static void __dev_sort_resources(structpdev_sort_resources (dev, head); } -static inline void reset_resource (struct resource *res) -{ - res->start =0 ; - res->end =0 ; - res->flags =0 ; -} -/** * reassign_resources_sorted() - satisfy any additional resource requests *
@@ -242,7 +236,7 @@ static void reassign_resources_sorted(st res->start = add_res->start; res->end = res->start + add_size -1 ;if (pci_assign_resource (add_res->dev, idx)) -reset_resource (res); + res->flags |=IORESOURCE_DISABLED ; }else {resource_size_t align = add_res->min_align; res->flags |= add_res->flags &
@@ -294,7 +288,7 @@ static void assign_requested_resources_s0 /* don't care */ , 0 /* don't care */); } -reset_resource (res); + res->flags |=IORESOURCE_DISABLED ; } } }
@@ -547,7 +541,7 @@ static void pci_setup_bridge_io(struct p/* Set up the top and bottom of the PCI I/O segment for this bus. */ res = bus->resource[0 ];pcibios_resource_to_bus (bridge->bus, ®ion, res); -if (res->flags &IORESOURCE_IO ) { +if ((res->flags &IORESOURCE_IO ) && !(res->flags &IORESOURCE_UNSET )) {pci_read_config_word (bridge,PCI_IO_BASE , &l); io_base_lo = (region.start >>8 ) & io_mask; io_limit_lo = (region.end >>8 ) & io_mask;
@@ -578,7 +572,8 @@ static void pci_setup_bridge_mmio(struct/* Set up the top and bottom of the PCI Memory segment for this bus. */ res = bus->resource[1 ];pcibios_resource_to_bus (bridge->bus, ®ion, res); -if (res->flags &IORESOURCE_MEM ) { +if ((res->flags &IORESOURCE_MEM ) && + !(res->flags &IORESOURCE_UNSET )) { l = (region.start >>16 ) &0xfff0 ; l |= region.end &0xfff00000 ;dev_info (&bridge->dev," bridge window %pR\n" , res);
@@ -604,7 +599,8 @@ static void pci_setup_bridge_mmio_pref(s bu = lu =0 ; res = bus->resource[2 ];pcibios_resource_to_bus (bridge->bus, ®ion, res); -if (res->flags &IORESOURCE_PREFETCH ) { +if ((res->flags &IORESOURCE_PREFETCH ) && + !(res->flags &IORESOURCE_UNSET )) { l = (region.start >>16 ) &0xfff0 ; l |= region.end &0xfff00000 ;if (res->flags &IORESOURCE_MEM_64 ) {
@@ -661,6 +657,7 @@ static void pci_bridge_check_ranges(stru b_res = &bridge->resource[PCI_BRIDGE_RESOURCES ]; b_res[1 ].flags |=IORESOURCE_MEM ; + b_res[1 ].flags &= ~IORESOURCE_DISABLED ;pci_read_config_word (bridge,PCI_IO_BASE , &io);if (!io) {
@@ -668,8 +665,10 @@ static void pci_bridge_check_ranges(strupci_read_config_word (bridge,PCI_IO_BASE , &io);pci_write_config_word (bridge,PCI_IO_BASE ,0x0 ); } -if (io) +if (io) { b_res[0 ].flags |=IORESOURCE_IO ; + b_res[0 ].flags &= ~IORESOURCE_DISABLED ; + }/* DECchip 21050 pass 2 errata: the bridge may miss an address disconnect boundary by one PCI data phase.
@@ -686,6 +685,7 @@ static void pci_bridge_check_ranges(stru }if (pmem) { b_res[2 ].flags |=IORESOURCE_MEM |IORESOURCE_PREFETCH ; + b_res[2 ].flags &= ~IORESOURCE_DISABLED ;if ((pmem &PCI_PREF_RANGE_TYPE_MASK ) ==PCI_PREF_RANGE_TYPE_64 ) { b_res[2 ].flags |=IORESOURCE_MEM_64 ;
@@ -830,8 +830,10 @@ static void pbus_size_io(struct pci_busstruct resource *r = &dev->resource[i];unsigned long r_size; -if (r->parent || !(r->flags &IORESOURCE_IO )) +if (r->parent || !(r->flags &IORESOURCE_IO ) || + (r->flags &IORESOURCE_DISABLED ))continue ; + r_size =resource_size (r);if (r_size <0x400 )
@@ -860,7 +862,7 @@ static void pbus_size_io(struct pci_busif (b_res->start || b_res->end)dev_info (&bus->self->dev,"disabling bridge window %pR to %pR (unused)\n" , b_res, &bus->busn_res); - b_res->flags =0 ; + b_res->flags |=IORESOURCE_UNSET |IORESOURCE_DISABLED ;return ; }
@@ -947,8 +949,10 @@ static int pbus_size_mem(struct pci_busif (r->parent || ((r->flags & mask) != type && (r->flags & mask) != type2 && - (r->flags & mask) != type3)) + (r->flags & mask) != type3) || + (r->flags &IORESOURCE_DISABLED ))continue ; + r_size =resource_size (r);#ifdef CONFIG_PCI_IOV /* put SRIOV requested res to the optional list */
@@ -973,7 +977,8 @@ static int pbus_size_mem(struct pci_busif (order >=ARRAY_SIZE (aligns)) {dev_warn (&dev->dev,"disabling BAR %d: %pR (bad alignment %#llx)\n" , i, r, (unsigned long long ) align); - r->flags =0 ; + r->flags |=IORESOURCE_UNSET | +IORESOURCE_DISABLED ;continue ; } size += r_size;
@@ -1001,7 +1006,7 @@ static int pbus_size_mem(struct pci_busif (b_res->start || b_res->end)dev_info (&bus->self->dev,"disabling bridge window %pR to %pR (unused)\n" , b_res, &bus->busn_res); - b_res->flags =0 ; + b_res->flags |=IORESOURCE_UNSET |IORESOURCE_DISABLED ;return 0 ; } b_res->start = min_align;
@@ -1367,7 +1372,7 @@ static void pci_bridge_release_resources/* keep the old size */ r->end =resource_size (r) -1 ; r->start =0 ; - r->flags =0 ; + r->flags |=IORESOURCE_UNSET |IORESOURCE_DISABLED ;/* avoiding touch the one without PREF */ if (type &IORESOURCE_PREFETCH )
@@ -1622,8 +1627,6 @@ again: res->start = fail_res->start; res->end = fail_res->end; res->flags = fail_res->flags; -if (fail_res->dev->subordinate) - res->flags =0 ; }free_list (&fail_head);
@@ -1688,8 +1691,6 @@ again: res->start = fail_res->start; res->end = fail_res->end; res->flags = fail_res->flags; -if (fail_res->dev->subordinate) - res->flags =0 ; }free_list (&fail_head);
drivers/pci/setup-res.c +11 -0
@@ -362,6 +362,17 @@ int pci_enable_resources(struct pci_devcontinue ;if (r->flags &IORESOURCE_UNSET ) { +/* bridge BAR could be disabled one by one */ +if (dev->subordinate && i >=PCI_BRIDGE_RESOURCES && + i <=PCI_BRIDGE_RESOURCE_END ) +continue ; + +#ifdef CONFIG_PCI_IOV +/* SRIOV ? */ +if (i >=PCI_IOV_RESOURCES && i <=PCI_IOV_RESOURCE_END ) +continue ; +#endif +dev_err (&dev->dev,"can't enable device: BAR %d %pR not assigned\n" , i, r);return -EINVAL ;
Bjorn Helgaas
maintainerQuoted 11 lines · Yinghai Lu
Quoted 10 lines · Bjorn Helgaas
Quoted 7 lines · Linus Torvalds
Quoted 3 lines · Yinghai Lu
If the BAR have all 0, cpu can not access it as hw will forward access to ram controller according to routing table.
On PC's, yes (also likely with many devices - there are definitely devices out there were setting the BAR to zero disables it).
Quoted 6 lines
Quoted 3 lines
Ooh, we might trip over this some day on a platform with host bridge address translation that maps to PCI bus address 0. I don't think we have anything in the allocator that avoids bus address 0 in that case.
Anyway, I draft RFC one that replace flags = 0 with flags |=
IORESOURCE_UNSET|IORESOURCE_DISABLEd.
I think we should do something like this patch, but we need to clarify how we want to use the IORESOURCE flags.
Here's what I think these IORESOURCE flags mean:
IORESOURCE_MEM Hardware decodes memory accesses (when enabled)
IORESOURCE_UNSET We have not allocated space for this decoder,
although the hardware still contains some address
IORESOURCE_DISABLED This hardware decoder is disabled, e.g., a bridge
window with LIMIT < BASE, or a ROM BAR with its enable bit cleared
For PCI, this would imply:
- The resource for a standard BAR should never have IORESOURCE_DISABLED because it can't be individually disabled.
- We can turn on PCI_COMMAND_MEMORY only if every IORESOURCE_MEM resource has either (IORESOURCE_DISABLED set) or (IORESOURCE_UNSET cleared).
Quoted 219 lines
Let's see how many drivers will fail.
Yinghai
Subject: [RFC PATCH] PCI: don't set flags to 0 when assign resource fail
make flags take
IORESOURCE_UNSET|IORESOURCE_DISABLEinstead.Signed-off-by Yinghai Lu <[email protected]>--- drivers/pci/setup-bus.c | 49 ++++++++++++++++++++++++------------------------ drivers/pci/setup-res.c | 11 ++++++++++ 2 files changed, 36 insertions(+), 24 deletions(-)Index:
linux-2.6/drivers/pci/setup-bus.c
===================================================================drivers/pci/setup-bus.c +25
-24@@ -136,7 +136,8 @@ static void pdev_sort_resources(struct pif (r->flags &IORESOURCE_PCI_FIXED )continue ; -if (!(r->flags) || r->parent) +if (!(r->flags) || r->parent || + (r->flags &IORESOURCE_DISABLED ))continue ; r_align =pci_resource_alignment (dev, r);@@ -190,13 +191,6 @@ static void __dev_sort_resources(structpdev_sort_resources (dev, head); } -static inline void reset_resource (struct resource *res) -{ - res->start =0 ; - res->end =0 ; - res->flags =0 ; -} -/** * reassign_resources_sorted() - satisfy any additional resource requests * @@ -242,7 +236,7 @@ static void reassign_resources_sorted(st res->start = add_res->start; res->end = res->start + add_size -1 ;if (pci_assign_resource (add_res->dev, idx)) -reset_resource (res); + res->flags |=IORESOURCE_DISABLED ; }else {resource_size_t align = add_res->min_align; res->flags |= add_res->flags &@@ -294,7 +288,7 @@ static void assign_requested_resources_s0 /* don't care */ , 0 /* don't care */); } -reset_resource (res); + res->flags |=IORESOURCE_DISABLED ; } } }@@ -547,7 +541,7 @@ static void pci_setup_bridge_io(struct p/* Set up the top and bottom of the PCI I/O segment for this bus. */ res = bus->resource[0 ];pcibios_resource_to_bus (bridge->bus, ®ion, res); -if (res->flags &IORESOURCE_IO ) { +if ((res->flags &IORESOURCE_IO ) && !(res->flags &IORESOURCE_UNSET )) {pci_read_config_word (bridge,PCI_IO_BASE , &l); io_base_lo = (region.start >>8 ) & io_mask; io_limit_lo = (region.end >>8 ) & io_mask;@@ -578,7 +572,8 @@ static void pci_setup_bridge_mmio(struct/* Set up the top and bottom of the PCI Memory segment for this bus. */ res = bus->resource[1 ];pcibios_resource_to_bus (bridge->bus, ®ion, res); -if (res->flags &IORESOURCE_MEM ) { +if ((res->flags &IORESOURCE_MEM ) && + !(res->flags &IORESOURCE_UNSET )) { l = (region.start >>16 ) &0xfff0 ; l |= region.end &0xfff00000 ;dev_info (&bridge->dev," bridge window %pR\n" , res);@@ -604,7 +599,8 @@ static void pci_setup_bridge_mmio_pref(s bu = lu =0 ; res = bus->resource[2 ];pcibios_resource_to_bus (bridge->bus, ®ion, res); -if (res->flags &IORESOURCE_PREFETCH ) { +if ((res->flags &IORESOURCE_PREFETCH ) && + !(res->flags &IORESOURCE_UNSET )) { l = (region.start >>16 ) &0xfff0 ; l |= region.end &0xfff00000 ;if (res->flags &IORESOURCE_MEM_64 ) {@@ -661,6 +657,7 @@ static void pci_bridge_check_ranges(stru b_res = &bridge->resource[PCI_BRIDGE_RESOURCES ]; b_res[1 ].flags |=IORESOURCE_MEM ; + b_res[1 ].flags &= ~IORESOURCE_DISABLED ;pci_read_config_word (bridge,PCI_IO_BASE , &io);if (!io) {@@ -668,8 +665,10 @@ static void pci_bridge_check_ranges(strupci_read_config_word (bridge,PCI_IO_BASE , &io);pci_write_config_word (bridge,PCI_IO_BASE ,0x0 ); } -if (io) +if (io) { b_res[0 ].flags |=IORESOURCE_IO ; + b_res[0 ].flags &= ~IORESOURCE_DISABLED ; + }/* DECchip 21050 pass 2 errata: the bridge may miss an address disconnect boundary by one PCI data phase. @@ -686,6 +685,7 @@ static void pci_bridge_check_ranges(stru }if (pmem) { b_res[2 ].flags |=IORESOURCE_MEM |IORESOURCE_PREFETCH ; + b_res[2 ].flags &= ~IORESOURCE_DISABLED ;if ((pmem &PCI_PREF_RANGE_TYPE_MASK ) ==PCI_PREF_RANGE_TYPE_64 ) { b_res[2 ].flags |=IORESOURCE_MEM_64 ;@@ -830,8 +830,10 @@ static void pbus_size_io(struct pci_busstruct resource *r = &dev->resource[i];unsigned long r_size; -if (r->parent || !(r->flags &IORESOURCE_IO )) +if (r->parent || !(r->flags &IORESOURCE_IO ) || + (r->flags &IORESOURCE_DISABLED ))continue ; + r_size =resource_size (r);if (r_size <0x400 )@@ -860,7 +862,7 @@ static void pbus_size_io(struct pci_busif (b_res->start || b_res->end)dev_info (&bus->self->dev,"disabling bridge window %pR to %pR (unused)\n" , b_res, &bus->busn_res); - b_res->flags =0 ; + b_res->flags |=IORESOURCE_UNSET |IORESOURCE_DISABLED ;return ; }@@ -947,8 +949,10 @@ static int pbus_size_mem(struct pci_busif (r->parent || ((r->flags & mask) != type && (r->flags & mask) != type2 && - (r->flags & mask) != type3)) + (r->flags & mask) != type3) || + (r->flags &IORESOURCE_DISABLED ))continue ; + r_size =resource_size (r);#ifdef CONFIG_PCI_IOV /* put SRIOV requested res to the optional list */ @@ -973,7 +977,8 @@ static int pbus_size_mem(struct pci_busif (order >=ARRAY_SIZE (aligns)) {dev_warn (&dev->dev,"disabling BAR %d: %pR (bad alignment %#llx)\n" , i, r, (unsigned long long ) align); - r->flags =0 ; + r->flags |=IORESOURCE_UNSET | +IORESOURCE_DISABLED ;continue ; } size += r_size;@@ -1001,7 +1006,7 @@ static int pbus_size_mem(struct pci_busif (b_res->start || b_res->end)dev_info (&bus->self->dev,"disabling bridge window %pR to %pR (unused)\n" , b_res, &bus->busn_res); - b_res->flags =0 ; + b_res->flags |=IORESOURCE_UNSET |IORESOURCE_DISABLED ;return 0 ; } b_res->start = min_align;@@ -1367,7 +1372,7 @@ static void pci_bridge_release_resources/* keep the old size */ r->end =resource_size (r) -1 ; r->start =0 ; - r->flags =0 ; + r->flags |=IORESOURCE_UNSET |IORESOURCE_DISABLED ;/* avoiding touch the one without PREF */ if (type &IORESOURCE_PREFETCH )@@ -1622,8 +1627,6 @@ again: res->start = fail_res->start; res->end = fail_res->end; res->flags = fail_res->flags; -if (fail_res->dev->subordinate) - res->flags =0 ; }free_list (&fail_head);@@ -1688,8 +1691,6 @@ again: res->start = fail_res->start; res->end = fail_res->end; res->flags = fail_res->flags; -if (fail_res->dev->subordinate) - res->flags =0 ; }free_list (&fail_head);Index:
linux-2.6/drivers/pci/setup-res.c
===================================================================drivers/pci/setup-res.c +11
-0@@ -362,6 +362,17 @@ int pci_enable_resources(struct pci_devcontinue ;if (r->flags &IORESOURCE_UNSET ) { +/* bridge BAR could be disabled one by one */ +if (dev->subordinate && i >=PCI_BRIDGE_RESOURCES && + i <=PCI_BRIDGE_RESOURCE_END ) +continue ; + +#ifdef CONFIG_PCI_IOV +/* SRIOV ? */ +if (i >=PCI_IOV_RESOURCES && i <=PCI_IOV_RESOURCE_END ) +continue ; +#endif +dev_err (&dev->dev,"can't enable device: BAR %d %pR not assigned\n" , i, r);return -EINVAL ;