x86 PCI domain support
cover x86 PCI domain support
ACPI PCI support stopped short of supporting multiple PCI domains, which is something I need in order to support a current machine configuration, and something many will soon need, to support upcoming systems.
This is a minimal, untested implementation. But it should work, provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains (mmconfig).
Patches:
1) Fix: don't hardcode PCI domain to zero
2) Introduce struct pci_sysdata
3) Add PCI domain to struct pci_sysdata
PATCH 1/3 x86 PCI domain support: a humble fix
commit 6bc310571a421a755822f1e65815399caddcb400
Author: Jeff Garzik <[email protected]> Date: Fri Dec 2 19:20:52 2005 -0500
[x86, PCI] pass PCI domain number to PCI config read/write hooks
Don't hardcode zero, since modern x86 (with special ACPI sauce) can support multiple "PCI segments", aka PCI domains.
arch/i386/pci/common.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-)
6bc310571a421a755822f1e65815399caddcb400
arch/i386/pci/common.c +4 -2
@@ -31,12 +31,14 @@ struct pci_raw_ops *raw_pci_ops;static int pci_read (struct pci_bus *bus,unsigned int devfn,int where,int size,u32 *value) { -return raw_pci_ops->read (0 , bus->number, devfn, where, size, value); +return raw_pci_ops->read (pci_domain_nr (bus), bus->number, + devfn, where, size, value); }static int pci_write (struct pci_bus *bus,unsigned int devfn,int where,int size,u32 value) { -return raw_pci_ops->write (0 , bus->number, devfn, where, size, value); +return raw_pci_ops->write (pci_domain_nr (bus), bus->number, + devfn, where, size, value); }struct pci_ops pci_root_ops = {
PATCH 2/3 x86 PCI domain support: struct pci_sysdata
In reply to Jeff Garzikcommit a5e17f3017ae2a4bd10454057bf63bd56bdd506c
Author: Jeff Garzik <[email protected]> Date: Fri Dec 2 20:12:52 2005 -0500
[x86, PCI] Switch pci_bus::sysdata from NUMA node integer to a pointer
On x86[-64], struct pci_bus::sysdata is only used on NUMA platforms, to store the associated NUMA node.
Preparing for the future when we'll want to do other things with sysdata, struct pci_sysdata was created. An allocated structure replaces the cast-pointer-to-int NUMA usage. Updated all NUMA users.
arch/i386/pci/acpi.c | 7 ++++--- arch/i386/pci/common.c | 13 ++++++++++++- arch/x86_64/pci/k8-bus.c | 6 +++++- include/asm-i386/pci.h | 5 +++++ include/asm-i386/topology.h | 2 +- include/asm-x86_64/pci.h | 4 ++++ include/asm-x86_64/topology.h | 2 +- 7 files changed, 32 insertions(+), 7 deletions(-)
a5e17f3017ae2a4bd10454057bf63bd56bdd506c
arch/i386/pci/acpi.c +4 -3
@@ -19,9 +19,10 @@ struct pci_bus * __devinit pci_acpi_scanif (bus !=NULL ) {int pxm =acpi_get_pxm (device->handle);if (pxm >=0 ) { - bus->sysdata = (void *)(unsigned long )pxm_to_node (pxm); -printk ("bus %d -> pxm %d -> node %ld\n" , - busnum, pxm, (long )(bus->sysdata)); +struct pci_sysdata *sd = bus->sysdata; + sd->node =pxm_to_node (pxm); +printk ("bus %d -> pxm %d -> node %d\n" , + busnum, pxm, sd->node); } }#endif
arch/i386/pci/common.c +12 -1
@@ -126,6 +126,7 @@ void __devinit pcibios_fixup_bus(structstruct pci_bus *__devinit pcibios_scan_root (int busnum) {struct pci_bus *bus =NULL ; +struct pci_sysdata *sd;while ((bus =pci_find_next_bus (bus)) !=NULL ) {if (bus->number == busnum) {
@@ -134,9 +135,19 @@ struct pci_bus * __devinit pcibios_scan_ } } +/* Allocate per-root-bus (not per bus) arch-specific data. +* TODO: leak; this memory is never freed. +* It's arguable whether it's worth the trouble to care. +*/ + sd =kzalloc (sizeof (*sd),GFP_KERNEL ); +if (!sd) { +printk (KERN_ERR "PCI: OOM, not probing PCI bus %02x\n" , busnum); +return NULL ; + } +printk (KERN_DEBUG "PCI: Probing PCI hardware (bus %02x)\n" , busnum); -return pci_scan_bus_parented (NULL , busnum, &pci_root_ops,NULL ); +return pci_scan_bus_parented (NULL , busnum, &pci_root_ops, sd); }extern u8 pci_cache_line_size;
arch/x86_64/pci/k8-bus.c +5 -1
@@ -59,6 +59,8 @@ fill_mp_bus_to_cpumask(void) j <=SUBORDINATE_LDT_BUS_NUMBER (ldtbus); j++) {struct pci_bus *bus; +struct pci_sysdata *sd; +long node =NODE_ID (nid);/* Algorithm a bit dumb, but it shouldn't matter here */
@@ -67,7 +69,9 @@ fill_mp_bus_to_cpumask(void)continue ;if (!node_online (node)) node =0 ; - bus->sysdata = (void *)node; + + sd = bus->sysdata; + sd->node = node; } } }
include/asm-i386/pci.h +5 -0
@@ -4,6 +4,11 @@#include <linux/config.h> #ifdef __KERNEL__ + +struct pci_sysdata { +int node;/* NUMA node */ +}; +#include <linux/mm.h> /* for struct page */ /* Can be used to override the logic in pci_scan_bus for skipping
include/asm-i386/topology.h +1 -1
@@ -60,7 +60,7 @@ static inline int node_to_first_cpu(intreturn first_cpu (mask); } -#define pcibus_to_node (bus) ((long) (bus)->sysdata) +#define pcibus_to_node (bus) ((struct pci_sysdata *)((bus)->sysdata))->node#define pcibus_to_cpumask (bus) node_to_cpumask(pcibus_to_node(bus))/* sched_domains SD_NODE_INIT for NUMAQ machines */
include/asm-x86_64/pci.h +4 -0
@@ -6,6 +6,10 @@#ifdef __KERNEL__ +struct pci_sysdata { +int node;/* NUMA node */ +}; +#include <linux/mm.h> /* for struct page */ /* Can be used to override the logic in pci_scan_bus for skipping
include/asm-x86_64/topology.h +1 -1
@@ -25,7 +25,7 @@ extern int __node_distance(int, int);#define parent_node (node) (node)#define node_to_first_cpu (node) (__ffs(node_to_cpumask[node]))#define node_to_cpumask (node) (node_to_cpumask[node]) -#define pcibus_to_node (bus) ((long)(bus->sysdata)) +#define pcibus_to_node (bus) ((struct pci_sysdata *)((bus)->sysdata))->node#define pcibus_to_cpumask (bus) node_to_cpumask(pcibus_to_node(bus));#define numa_node_id () read_pda(nodenumber)
PATCH 3/3 x86 PCI domain support: the meat
In reply to Jeff Garzikcommit 31c93c154ca9a86a865b4ef28b7ba5d8e48f7b13
Author: Jeff Garzik <[email protected]> Date: Fri Dec 2 20:30:48 2005 -0500
[x86, PCI] add PCI domain support
* Store PCI domain in struct pci_sysdata
* Add magic inlines to pass PCI domain to read/write config hooks
* Support ACPI's notion of PCI domains (PCI segments)
arch/i386/pci/acpi.c | 20 ++++++++++++++++++-- include/asm-i386/pci.h | 14 ++++++++++++++ include/asm-x86_64/pci.h | 14 ++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-)
31c93c154ca9a86a865b4ef28b7ba5d8e48f7b13
arch/i386/pci/acpi.c +18 -2
@@ -8,18 +8,34 @@struct pci_bus *__devinit pci_acpi_scan_root (struct acpi_device *device,int domain,int busnum) {struct pci_bus *bus; +struct pci_sysdata *sd; +/* Allocate per-root-bus (not per bus) arch-specific data. +* TODO: leak; this memory is never freed. +* It's arguable whether it's worth the trouble to care. +*/ + sd =kzalloc (sizeof (*sd),GFP_KERNEL ); +if (!sd) { +printk (KERN_ERR "PCI: OOM, not probing PCI bus %02x\n" , busnum); +return NULL ; + } + +#ifdef CONFIG_PCI_DOMAINS + sd->domain = domain; +#else if (domain !=0 ) {printk (KERN_WARNING "PCI: Multiple domains not supported\n" );return NULL ; } +#endif /* CONFIG_PCI_DOMAINS */ - bus =pcibios_scan_root (busnum); + bus =pci_scan_bus_parented (NULL , busnum, &pci_root_ops, sd); +if (!bus) +kfree (sd);#ifdef CONFIG_ACPI_NUMA if (bus !=NULL ) {int pxm =acpi_get_pxm (device->handle);if (pxm >=0 ) { -struct pci_sysdata *sd = bus->sysdata; sd->node =pxm_to_node (pxm);printk ("bus %d -> pxm %d -> node %d\n" , busnum, pxm, sd->node);
include/asm-i386/pci.h +14 -0
@@ -6,9 +6,23 @@#ifdef __KERNEL__struct pci_sysdata { +int domain;/* PCI domain */ int node;/* NUMA node */ }; +#ifdef CONFIG_PCI_DOMAINS +static inline int pci_domain_nr (struct pci_bus *bus) +{ +struct pci_sysdata *sd = bus->sysdata; +return sd->domain; +} + +static inline int pci_proc_domain (struct pci_bus *bus) +{ +return pci_domain_nr (bus); +} +#endif /* CONFIG_PCI_DOMAINS */ +#include <linux/mm.h> /* for struct page */ /* Can be used to override the logic in pci_scan_bus for skipping
include/asm-x86_64/pci.h +14 -0
@@ -7,9 +7,23 @@#ifdef __KERNEL__struct pci_sysdata { +int domain;/* PCI domain */ int node;/* NUMA node */ }; +#ifdef CONFIG_PCI_DOMAINS +static inline int pci_domain_nr (struct pci_bus *bus) +{ +struct pci_sysdata *sd = bus->sysdata; +return sd->domain; +} + +static inline int pci_proc_domain (struct pci_bus *bus) +{ +return pci_domain_nr (bus); +} +#endif /* CONFIG_PCI_DOMAINS */ +#include <linux/mm.h> /* for struct page */ /* Can be used to override the logic in pci_scan_bus for skipping
Quoted 10 lines · Jeff Garzik
ACPI PCI support stopped short of supporting multiple PCI domains, which is something I need in order to support a current machine configuration, and something many will soon need, to support upcoming systems.
Patches:
1) Fix: don't hardcode PCI domain to zero
2) Introduce structpci_sysdata
3) Add PCI domain to structpci_sysdata
NOTE: This should have the DO NOT APPLY label. This is just for comment, and testing if there are brave souls.
Jeff
Quoted 9 lines · Jeff Garzik
ACPI PCI support stopped short of supporting multiple PCI domains, which is something I need in order to support a current machine configuration, and something many will soon need, to support upcoming systems.
This is a minimal, untested implementation. But it should work, provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains (mmconfig).
It looks like a good start. Thanks for doing this.
It actually needs some more fixes - e.g. falling back to type1 if the bus is not covered in MCFG (needed for the
K8 internal busses) and a workaround for buggy Asus BIOS with wrong MCFG. I have that in the works.
But your changes are needed too - or at least they are correct according to the spec. I don't know of a system that actually has different mmconfig apertures for different busses yet. The only case that's interesting right now is that some busses don't support it at all, but these don't need a seg number, just a non listing in MCFG.
Greg are you queueing this up?
-Andi
Quoted 17 lines · Andi Kleen
Quoted 8 lines · Jeff Garzik
ACPI PCI support stopped short of supporting multiple PCI domains, which is something I need in order to support a current machine configuration, and something many will soon need, to support upcoming systems.
This is a minimal, untested implementation. But it should work, provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains (mmconfig).
It looks like a good start. Thanks for doing this.
It actually needs some more fixes - e.g. falling back to type1 if the bus is not covered in MCFG (needed for the K8 internal busses) and a workaround for buggy Asus BIOS with wrong MCFG. I have that in the works.
Quoted 8 lines
But your changes are needed too - or at least they are correct according to the spec. I don't know of a system that actually has different mmconfig apertures for different busses yet. The only case that's interesting right now is that some busses don't support it at all, but these don't need a seg number, just a non listing in MCFG.
Greg are you queueing this up?
The first two patches could go in immediately, the last should probably wait a bit...
Jeff
Quoted 27 lines · Jeff Garzik
Quoted 26 lines · Andi Kleen
Quoted 8 lines · Jeff Garzik
ACPI PCI support stopped short of supporting multiple PCI domains, which is something I need in order to support a current machine configuration, and something many will soon need, to support upcoming systems.
This is a minimal, untested implementation. But it should work, provided your PCI op hooks (direct, BIOS, mmconfig) support PCI domains (mmconfig).
It looks like a good start. Thanks for doing this.
It actually needs some more fixes - e.g. falling back to type1 if the bus is not covered in MCFG (needed for the
K8 internal busses) and a workaround for buggy Asus BIOS with wrong MCFG. I have that in the works.But your changes are needed too - or at least they are correct according to the spec. I don't know of a system that actually has different mmconfig apertures for different busses yet. The only case that's interesting right now is that some busses don't support it at all, but these don't need a seg number, just a non listing in MCFG.
Quoted 4 lines
Quoted 1 line
Greg are you queueing this up?
The first two patches could go in immediately, the last should probably wait a bit...
Ok, I'll queue up all of them for testing in the next few -mm releases before sending them on.
Care to add some "Signed-off-by:" lines to the patches?
thanks,
greg k-h
Quoted 2 lines · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
What is the rush? These seem pretty late for the -rc series :)
I'll send them in after 2.6.15 is out, is that ok?
thanks,
greg k-h
Quoted 7 lines · Greg KH
Quoted 2 lines · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
What is the rush? These seem pretty late for the -rc series :)
I'll send them in after 2.6.15 is out, is that ok?
It's ok, but the two related workaround patches I posted earlier should go into .15 (because they fix broken boot on some machines)
-Andi
Quoted 11 lines · Andi Kleen
Quoted 7 lines · Greg KH
Quoted 2 lines · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
What is the rush? These seem pretty late for the -rc series :)
I'll send them in after 2.6.15 is out, is that ok?
It's ok, but the two related workaround patches I posted earlier should go into .15 (because they fix broken boot on some machines)
Ok, that's fine. Thanks for fixing those issues, I appreciate it.
greg k-h
Quoted 8 lines · Greg KH
Quoted 1 line · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
What is the rush? These seem pretty late for the -rc series :)
I'll send them in after 2.6.15 is out, is that ok?
You were supposed to read my mind :) "immediately" meant "ok for upstream when -rc cycle closes" :) The third patch I don't consider ready for upstream, -rc or no.
So post-2.6.15 is quite fine, that's what I expected.
Jeff
Quoted 7 lines · Jeff Garzik
Quoted 6 lines · Greg KH
Quoted 2 lines · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
Quoted 7 lines
Quoted 3 lines
What is the rush? These seem pretty late for the -rc series :)
I'll send them in after 2.6.15 is out, is that ok?
You were supposed to read my mind :) "immediately" meant "ok for upstream when -rc cycle closes" :) The third patch I don't consider ready for upstream, -rc or no.
Ok, thanks. But I did just include the third patch in my tree, so it will get tested in -mm. If you don't want this to happen, just let me know and I'll drop it.
thanks,
greg k-h
Quoted 12 lines · Greg KH
Quoted 10 lines · Jeff Garzik
Quoted 8 lines · Greg KH
Quoted 1 line · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
What is the rush? These seem pretty late for the -rc series :)
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
I'll send them in after 2.6.15 is out, is that ok?
You were supposed to read my mind :) "immediately" meant "ok for upstream when -rc cycle closes" :) The third patch I don't consider ready for upstream, -rc or no.
Ok, thanks. But I did just include the third patch in my tree, so it will get tested in -mm. If you don't want this to happen, just let me know and I'll drop it.
There's no ultimate harm in it, because nothing turns on CONFIG_PCI_DOMAINS in x86[-64] yet...
Jeff
Quoted 21 lines · Jeff Garzik
Quoted 20 lines · Greg KH
Quoted 16 lines · Jeff Garzik
Quoted 10 lines · Greg KH
Quoted 2 lines · Jeff Garzik
The first two patches could go in immediately, the last should probably wait a bit...
What is the rush? These seem pretty late for the -rc series :)
I'll send them in after 2.6.15 is out, is that ok?
You were supposed to read my mind :) "immediately" meant "ok for upstream when -rc cycle closes" :) The third patch I don't consider ready for upstream, -rc or no.
Quoted 6 lines
Quoted 3 lines
Ok, thanks. But I did just include the third patch in my tree, so it will get tested in -mm. If you don't want this to happen, just let me know and I'll drop it.
There's no ultimate harm in it, because nothing turns on
CONFIG_PCI_DOMAINSin x86[-64] yet...
I'm guessing that will be a follow-on patch? :)
greg k-h