x86_64: use bus conf in NB conf fun1 to get bus range on node
Subsystems (6)
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/8 x86_64: check MSR to get MMCONFIG for AMD Family 10h Opteron v3
From: Yinghai Lu <[email protected]>
so even booting kernel with acpi=off or even MCFG is not there, we still can use MMCONFIG.
---
arch/x86/pci/mmconfig-shared.c +61 -6
@@ -100,33 +100,88 @@ static const char __init *pci_mmcfg_intereturn "Intel Corporation 945G/GZ/P/PL Express Memory Controller Hub" ; } +static const char __init *pci_mmcfg_amd_fam10h (void ) +{ +u32 low, high, address; +u64 base, msr; +int i; +unsigned segnbits =0 , busnbits; + + address =MSR_FAM10H_MMIO_CONF_BASE ; +if (rdmsr_safe (address, &low, &high)) +return NULL ; + + msr = high; + msr <<=32 ; + msr |= low; + +/* mmconfig is not enable */ +if (!(msr &FAM10H_MMIO_CONF_ENABLE )) +return NULL ; + + base = msr & (FAM10H_MMIO_CONF_BASE_MASK <<FAM10H_MMIO_CONF_BASE_SHIFT ); + + busnbits = (msr >>FAM10H_MMIO_CONF_BUSRANGE_SHIFT ) & +FAM10H_MMIO_CONF_BUSRANGE_MASK ; +if (busnbits >8 ) { + segnbits = busnbits -8 ; + busnbits =8 ; + } + + pci_mmcfg_config_num = (1 << segnbits); + pci_mmcfg_config =kzalloc (sizeof (pci_mmcfg_config[0 ]) * + pci_mmcfg_config_num,GFP_KERNEL ); +if (!pci_mmcfg_config) +return NULL ; + +for (i =0 ; i < (1 << segnbits); i++) { + pci_mmcfg_config[i].address = base + (1 <<28 ) * i; + pci_mmcfg_config[i].pci_segment = i; + pci_mmcfg_config[i].start_bus_number =0 ; + pci_mmcfg_config[i].end_bus_number = (1 << busnbits) -1 ; + } + +return "AMD Family 10h NB" ; +} +struct pci_mmcfg_hostbridge_probe { +u32 bus; +u32 devfn;u32 vendor;u32 device;const char *(*probe)(void ); };static struct pci_mmcfg_hostbridge_probe pci_mmcfg_probes[]__initdata = { - {PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_E7520_MCH , pci_mmcfg_e7520 }, - {PCI_VENDOR_ID_INTEL ,PCI_DEVICE_ID_INTEL_82945G_HB , pci_mmcfg_intel_945 }, + {0 ,PCI_DEVFN (0 ,0 ),PCI_VENDOR_ID_INTEL , +PCI_DEVICE_ID_INTEL_E7520_MCH , pci_mmcfg_e7520 }, + {0 ,PCI_DEVFN (0 ,0 ),PCI_VENDOR_ID_INTEL , +PCI_DEVICE_ID_INTEL_82945G_HB , pci_mmcfg_intel_945 }, + {0 ,PCI_DEVFN (0x18 ,0 ),PCI_VENDOR_ID_AMD , +0x1200 , pci_mmcfg_amd_fam10h }, + {0xff ,PCI_DEVFN (0 ,0 ),PCI_VENDOR_ID_AMD , +0x1200 , pci_mmcfg_amd_fam10h }, };static int __init pci_mmcfg_check_hostbridge (void ) {u32 l; +u32 bus, devfn;u16 vendor, device;int i;const char *name; - pci_direct_conf1.read (0 ,0 ,PCI_DEVFN (0 ,0 ),0 ,4 , &l); - vendor = l &0xffff ; - device = (l >>16 ) &0xffff ; - pci_mmcfg_config_num =0 ; pci_mmcfg_config =NULL ; name =NULL ;for (i =0 ; !name && i <ARRAY_SIZE (pci_mmcfg_probes); i++) { + bus = pci_mmcfg_probes[i].bus; + devfn = pci_mmcfg_probes[i].devfn; + pci_direct_conf1.read (0 , bus, devfn,0 ,4 , &l); + vendor = l &0xffff ; + device = (l >>16 ) &0xffff ; +if (pci_mmcfg_probes[i].vendor == vendor && pci_mmcfg_probes[i].device == device) name = pci_mmcfg_probes[i].probe ();
PATCH 1/1 x86: skip it if Fam 10h only handle bus 0
need to apply after
x86_64: check MSR to get MMCONFIG for AMD Family 10h Opteron v3
some BIOS only let AMD fam 10h handle bus0, and nvidia mcp55/ck804 to handle other buses. at that case MCFG will cover all over them.
this patch will skip it so to use MCFG later.
arch/x86/pci/mmconfig-shared.c +8 -0
@@ -123,6 +123,14 @@ static const char __init *pci_mmcfg_amd_ busnbits = (msr >>FAM10H_MMIO_CONF_BUSRANGE_SHIFT ) &FAM10H_MMIO_CONF_BUSRANGE_MASK ; + +/* +* only handle bus 0 ? +* need to skip it +*/ +if (!busnbits) +return NULL ; +if (busnbits >8 ) { segnbits = busnbits -8 ; busnbits =8 ;
PATCH 2/8 x86_64: check and enable MMCONFIG for AMD Family 10h Opteron v3
In reply to a message not in the archiveSo we can use MMCONF when MMCONF is not set by BIOS
using TOP_MEM2 msr to get memory top, and try to scan fam10h mmio routing to make sure the range is not conflicted with some prefetch MMIO that is above 4G. (current only LinuxBIOS assign 64 bit mmio above 4G for some co-processor)
arch/x86/kernel/setup_64.c +204 -0
@@ -29,6 +29,7 @@#include <linux/crash_dump.h> #include <linux/root_dev.h> #include <linux/pci.h> +#include <asm/pci-direct.h> #include <linux/efi.h> #include <linux/acpi.h> #include <linux/kallsyms.h>
@@ -39,6 +40,7 @@#include <linux/dmi.h> #include <linux/dma-mapping.h> #include <linux/ctype.h> +#include <linux/sort.h> #include <linux/uaccess.h> #include <linux/init_ohci1394_dma.h>
@@ -577,6 +579,205 @@ static int __cpuinit nearby_node(int api }#endif +#ifdef CONFIG_PCI_MMCONFIG +struct pci_hostbridge_probe { +u32 bus; +u32 slot; +u32 vendor; +u32 device; +}; + +static u64 __cpuinitdata fam10h_pci_mmconf_base; +static int __cpuinitdata fam10h_pci_mmconf_base_status; + +static struct pci_hostbridge_probe pci_probes[]__cpuinitdata = { + {0 ,0x18 ,PCI_VENDOR_ID_AMD ,0x1200 }, + {0xff ,0 ,PCI_VENDOR_ID_AMD ,0x1200 }, +}; + +struct range { +u64 start; +u64 end; +}; + +static int __cpuinit cmp_range (const void *x1,const void *x2) +{ +const struct range *r1 = x1; +const struct range *r2 = x2; +int start1, start2; + + start1 = r1->start >>32 ; + start2 = r2->start >>32 ; + +return start1 - start2; +} + +/*[47:0] */ +/* need to avoid (0xfd<<32) and (0xfe<<32), ht used space */ +#define FAM10H_PCI_MMCONF_BASE (0xfcULL<<32) +#define BASE_VALID (b) ((b != (0xfdULL << 32)) && (b != (0xfeULL << 32))) +static void __cpuinit get_fam10h_pci_mmconf_base (void ) +{ +int i; +unsigned bus; +unsigned slot; +int found; + +u64 val; +u32 address; +u64 tom2; +u64 base =FAM10H_PCI_MMCONF_BASE ; + +int hi_mmio_num; +struct range range[8 ]; + +/* only try to get setting from BSP */ +/* -1 or 1 */ +if (fam10h_pci_mmconf_base_status) +return ; + +if (!early_pci_allowed ()) + goto fail; + + found =0 ; +for (i =0 ; i <ARRAY_SIZE (pci_probes); i++) { +u32 id; +u16 device; +u16 vendor; + + bus = pci_probes[i].bus; + slot = pci_probes[i].slot; + id =read_pci_config (bus, slot,0 ,PCI_VENDOR_ID ); + + vendor = id &0xffff ; + device = (id>>16 ) &0xffff ; +if (pci_probes[i].vendor == vendor && + pci_probes[i].device == device) { + found =1 ; +break ; + } + } + +if (!found) + goto fail; + +/* SYS_CFG */ + address =MSR_K8_SYSCFG ; +rdmsrl (address, val); + +/* TOP_MEM2 is not enabled? */ +if (!(val & (1 <<21 ))) { + tom2 =0 ; + }else { +/* TOP_MEM2 */ + address =MSR_K8_TOP_MEM2 ; +rdmsrl (address, val); + tom2 = val & (0xffffULL <<32 ); + } + +if (base <= tom2) + base = tom2 + (1ULL <<32 ); + +/* +* need to check if the range is in the high mmio range that is +* above 4G +*/ + hi_mmio_num =0 ; +for (i =0 ; i <8 ; i++) { +u32 reg; +u64 start; +u64 end; + reg =read_pci_config (bus, slot,1 ,0x80 + (i <<3 )); +if (!(reg &3 )) +continue ; + + start = (((u64 )reg) <<8 ) & (0xffULL <<32 );/* 39:16 on 31:8*/ + reg =read_pci_config (bus, slot,1 ,0x84 + (i <<3 )); + end = (((u64 )reg) <<8 ) & (0xffULL <<32 );/* 39:16 on 31:8*/ + +if (!end) +continue ; + + range[hi_mmio_num].start = start; + range[hi_mmio_num].end = end; + hi_mmio_num++; + } + +if (!hi_mmio_num) + goto out; + +/* sort the range */ +sort (range, hi_mmio_num,sizeof (struct range ), cmp_range,NULL ); + +if (range[hi_mmio_num -1 ].end < base) + goto out; +if (range[0 ].start > base) + goto out; + +/* need to find one window */ + base = range[0 ].start - (1ULL <<32 ); +if ((base > tom2) &&BASE_VALID (base)) + goto out; + base = range[hi_mmio_num -1 ].end + (1ULL <<32 ); +if ((base > tom2) &&BASE_VALID (base)) + goto out; +/* need to find window between ranges */ +if (hi_mmio_num >1 ) +for (i =0 ; i < hi_mmio_num -1 ; i++) { +if (range[i +1 ].start > (range[i].end + (1ULL <<32 ))) { + base = range[i].end + (1ULL <<32 ); +if ((base > tom2) &&BASE_VALID (base)) + goto out; + } + } + +fail: + fam10h_pci_mmconf_base_status =-1 ; +return ; +out: + fam10h_pci_mmconf_base = base; + fam10h_pci_mmconf_base_status =1 ; +} +#endif + +static void __cpuinit fam10h_check_enable_mmcfg (struct cpuinfo_x86 *c) +{ +#ifdef CONFIG_PCI_MMCONFIG +u64 val; +u32 address; + + address =MSR_FAM10H_MMIO_CONF_BASE ; +rdmsrl (address, val); + +/* try to make sure that AP's setting is identical to BSP setting */ +if (val &FAM10H_MMIO_CONF_ENABLE ) { +u64 base; + base = val & (0xffffULL <<32 ); +if (fam10h_pci_mmconf_base_status <=0 ) { + fam10h_pci_mmconf_base = base; + fam10h_pci_mmconf_base_status =1 ; +return ; + }else if (fam10h_pci_mmconf_base == base) +return ; + } + +/* +* if it is not enabled, try to enable it and assume only one segment +* with 256 buses +*/ +get_fam10h_pci_mmconf_base (); +if (fam10h_pci_mmconf_base_status <=0 ) +return ; + +printk (KERN_INFO "Enable MMCONFIG on AMD Family 10h\n" ); + val &= ~((FAM10H_MMIO_CONF_BASE_MASK <<FAM10H_MMIO_CONF_BASE_SHIFT ) | + (FAM10H_MMIO_CONF_BUSRANGE_MASK <<FAM10H_MMIO_CONF_BUSRANGE_SHIFT )); + val |= fam10h_pci_mmconf_base | (8 <<FAM10H_MMIO_CONF_BUSRANGE_SHIFT ) | +FAM10H_MMIO_CONF_ENABLE ; +wrmsrl (address, val); +#endif +} +/* * On a AMD dual core setup the lower bits of the APIC id distingush the cores. * Assumes number of cores is a power of two.
@@ -760,6 +961,9 @@ static void __cpuinit init_amd(struct cp/* MFENCE stops RDTSC speculation */ set_cpu_cap (c,X86_FEATURE_MFENCE_RDTSC ); +if (c->x86 ==0x10 ) +fam10h_check_enable_mmcfg (c); +if (amd_apic_timer_broken ()) disable_apic_timer =1 ; }
PATCH 3/8 x86_64: get mp_bus_to_node as early v4
In reply to a message not in the archivecurrent on amd k8 system with multi ht chain, the numa_node of pci devices under /sys/devices/pci0000:80/* always 0, even that chain is on node 1 or 2 or 3.
workaround: pcibus_to_node(bus) is used when we want to get node that pci_device is on.
In struct device, we already have numa_node member. and we could use dev_to_node() /set_dev_node() to get and set numa_node in the device.set_dev_node is called in pci_device_add() with pcibus_to_node(bus). and pcibus_to_node use bus->sysdata for nodeid.
the problem is when pci_add_device is called, bus->sysdata is not assigned correct nodeid yet. the result will be numa_node always is 0.
pcibios_scan_root and pci_scan_root could take sysdata. So we need to get mp_bus_to_node mapping before these two are called. and get_mp_bus_to_node could get correct node for sysdata in root bus.
in scanning of root bus, all child bus will take parent bus sysdata. So all pci_device->dev.numa_node will be assigned correctly automatically.
later we could use dev_to_node(&pci_dev->dev) to get numa_node, and we could also could make other bus specific device get the correct numa_node too.
this is one update version to pci_sysdata and jeff's pci_domain patch.
arch/i386/pci/Makefile | 1 arch/i386/pci/Makefile | 1 arch/i386/pci/acpi.c | 40 +++++-------------- arch/i386/pci/common.c | 17 ++++++-- arch/i386/pci/irq.c | 4 + arch/i386/pci/legacy.c | 6 ++ arch/i386/pci/mp_bus_to_node.c | 24 +++++++++++ arch/x86_64/pci/k8-bus.c | 83 ++++++++++++++++++++++++++++------------- include/asm-i386/pci.h | 1 include/asm-i386/topology.h | 10 ++++ include/asm-x86_64/pci.h | 1 include/asm-x86_64/topology.h | 13 ++++++ 11 files changed, 138 insertions(+), 62 deletions(-)
arch/x86/pci/Makefile_32 +1 -0
@@ -10,5 +10,6 @@ pci-y += legacy.o irq.o pci-$(CONFIG_X86_VISWS) := visws.o fixup.o pci-$(CONFIG_X86_NUMAQ) := numa.o irq.o +pci-$(CONFIG_NUMA) += mp_bus_to_node.o obj-y += $(pci-y) common.o early.o
arch/x86/pci/acpi.c +17 -10
@@ -191,7 +191,10 @@ struct pci_bus * __devinit pci_acpi_scan {struct pci_bus *bus;struct pci_sysdata *sd; +int node; +#ifdef CONFIG_ACPI_NUMA int pxm; +#endif dmi_check_system (acpi_pciprobe_dmi_table);
@@ -201,6 +204,17 @@ struct pci_bus * __devinit pci_acpi_scanreturn NULL ; } + node =-1 ; +#ifdef CONFIG_ACPI_NUMA + pxm =acpi_get_pxm (device->handle); +if (pxm >=0 ) + node =pxm_to_node (pxm); +if (node !=-1 ) +set_mp_bus_to_node (busnum, node); +else + node =get_mp_bus_to_node (busnum); +#endif +/* Allocate per-root-bus (not per bus) arch-specific data. * TODO: leak; this memory is never freed. * It's arguable whether it's worth the trouble to care.
@@ -212,22 +226,16 @@ struct pci_bus * __devinit pci_acpi_scan } sd->domain = domain; - sd->node =-1 ; - - pxm =acpi_get_pxm (device->handle); -#ifdef CONFIG_ACPI_NUMA -if (pxm >=0 ) - sd->node =pxm_to_node (pxm); -#endif + sd->node = node; bus =pci_scan_bus_parented (NULL , busnum, &pci_root_ops, sd);if (!bus)kfree (sd);#ifdef CONFIG_ACPI_NUMA -if (bus !=NULL ) { +if (bus) {if (pxm >=0 ) { -printk ("bus %d -> pxm %d -> node %d\n" , +printk (KERN_DEBUG "bus %02x -> pxm %d -> node %d\n" , busnum, pxm, sd->node); } }
@@ -235,7 +243,6 @@ struct pci_bus * __devinit pci_acpi_scanif (bus && (pci_probe &PCI_USE__CRS ))get_current_resources (device, busnum, bus); -return bus; }
arch/x86/pci/common.c +14 -4
@@ -396,9 +396,14 @@ struct pci_bus * __devinit pcibios_scan_return NULL ; } + sd->node =get_mp_bus_to_node (busnum); +printk (KERN_DEBUG "PCI: Probing PCI hardware (bus %02x)\n" , busnum); + bus =pci_scan_bus_parented (NULL , busnum, &pci_root_ops, sd); +if (!bus) +kfree (sd); -return pci_scan_bus_parented (NULL , busnum, &pci_root_ops, sd); +return bus; }extern u8 pci_cache_line_size;
@@ -541,7 +546,7 @@ void pcibios_disable_device (struct pci_pcibios_disable_irq (dev); } -struct pci_bus *__devinit pci_scan_bus_with_sysdata (int busno) +struct pci_bus *pci_scan_bus_on_node (int busno,struct pci_ops *ops,int node) {struct pci_bus *bus =NULL ;struct pci_sysdata *sd;
@@ -556,10 +561,15 @@ struct pci_bus *__devinit pci_scan_bus_wprintk (KERN_ERR "PCI: OOM, skipping PCI bus %02x\n" , busno);return NULL ; } - sd->node =-1 ; - bus =pci_scan_bus (busno, &pci_root_ops, sd); + sd->node = node; + bus =pci_scan_bus (busno, ops, sd);if (!bus)kfree (sd);return bus; } + +struct pci_bus *pci_scan_bus_with_sysdata (int busno) +{ +return pci_scan_bus_on_node (busno, &pci_root_ops,-1 ); +}
arch/x86/pci/irq.c +3 -1
@@ -136,9 +136,11 @@ static void __init pirq_peer_trick(void) busmap[e->bus] =1 ; }for (i =1 ; i <256 ; i++) { +int node;if (!busmap[i] ||pci_find_bus (0 , i))continue ; -if (pci_scan_bus_with_sysdata (i)) + node =get_mp_bus_to_node (i); +if (pci_scan_bus_on_node (i, &pci_root_ops, node))printk (KERN_INFO "PCI: Discovered primary peer " "bus %02x [IRQ]\n" , i); }
arch/x86/pci/legacy.c +3 -1
@@ -12,6 +12,7 @@static void __devinit pcibios_fixup_peer_bridges (void ) {int n, devfn; +long node;if (pcibios_last_bus <=0 || pcibios_last_bus >=0xff )return ;
@@ -21,12 +22,13 @@ static void __devinit pcibios_fixup_peeru32 l;if (pci_find_bus (0 , n))continue ; + node =get_mp_bus_to_node (n);for (devfn =0 ; devfn <256 ; devfn +=8 ) {if (!raw_pci_read (0 , n, devfn,PCI_VENDOR_ID ,2 , &l) && l !=0x0000 && l !=0xffff ) {DBG ("Found device at %02x:%02x [%04x]\n" , n, devfn, l);printk (KERN_INFO "PCI: Discovered peer bus %02x\n" , n); -pci_scan_bus_with_sysdata (n); +pci_scan_bus_on_node (n, &pci_root_ops, node);break ; } }
arch/x86/pci/mp_bus_to_node.c new file +23 -0
@@ -0,0 +1,23 @@ +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/topology.h> + +#define BUS_NR 256 + +static unsigned char mp_bus_to_node[BUS_NR ]; + +void set_mp_bus_to_node (int busnum,int node) +{ +if (busnum >=0 && busnum <BUS_NR ) + mp_bus_to_node[busnum] = (unsigned char ) node; +} + +int get_mp_bus_to_node (int busnum) +{ +int node; + +if (busnum <0 || busnum > (BUS_NR -1 )) +return 0 ; + node = mp_bus_to_node[busnum]; +return node; +}
arch/x86/pci/k8-bus_64.c +60 -26
131 lines not on this page · raw message
include/asm-x86/pci.h +2 -0
@@ -20,6 +20,8 @@ struct pci_sysdata { };/* scan a bus after allocating a pci_sysdata for it */ +extern struct pci_bus *pci_scan_bus_on_node (int busno,struct pci_ops *ops, +int node);extern struct pci_bus *pci_scan_bus_with_sysdata (int busno);static inline int pci_domain_nr (struct pci_bus *bus)
include/asm-x86/topology.h +11 -0
@@ -165,8 +165,19 @@ extern int __node_distance(int, int);#define node_distance (a, b) __node_distance(a, b)#endif +int get_mp_bus_to_node (int busnum); +void set_mp_bus_to_node (int busnum,int node); +#else /* CONFIG_NUMA */ +static inline int get_mp_bus_to_node (int busnum) +{ +return 0 ; +} +static inline void set_mp_bus_to_node (int busnum,int node) +{ +} +#include <asm-generic/topology.h> #endif
PATCH 4/8 x86_64: use bus conf in NB conf fun1 to get bus range on node
In reply to a message not in the archiveSo we use the same code with Quad core cpu as old opteron. this patch is usful when acpi=off or _PXM is not there in DSDT
arch/x86/pci/k8-bus_64.c +48 -40
@@ -12,15 +12,18 @@* RED-PEN empty cpus get reported wrong */ -#define NODE_ID_REGISTER 0x60 -#define NODE_ID (dword) (dword & 0x07) -#define LDT_BUS_NUMBER_REGISTER_0 0x94 -#define LDT_BUS_NUMBER_REGISTER_1 0xB4 -#define LDT_BUS_NUMBER_REGISTER_2 0xD4 -#define NR_LDT_BUS_NUMBER_REGISTERS 3 -#define SECONDARY_LDT_BUS_NUMBER (dword) ((dword >> 8) & 0xFF) -#define SUBORDINATE_LDT_BUS_NUMBER (dword) ((dword >> 16) & 0xFF) +#define NODE_ID (dword) ((dword>>4) & 0x07) +#define LDT_BUS_NUMBER_REGISTER_0 0xE0 +#define LDT_BUS_NUMBER_REGISTER_1 0xE4 +#define LDT_BUS_NUMBER_REGISTER_2 0xE8 +#define LDT_BUS_NUMBER_REGISTER_3 0xEC +#define NR_LDT_BUS_NUMBER_REGISTERS 4 +#define SECONDARY_LDT_BUS_NUMBER (dword) ((dword >> 16) & 0xFF) +#define SUBORDINATE_LDT_BUS_NUMBER (dword) ((dword >> 24) & 0xFF) +#define PCI_DEVICE_ID_K8HTCONFIG 0x1100 +#define PCI_DEVICE_ID_K8_10H_HTCONFIG 0x1200 +#define PCI_DEVICE_ID_K8_11H_HTCONFIG 0x1300#define BUS_NR 256
@@ -62,12 +65,19 @@ early_fill_mp_bus_to_node(void) {int i, j;unsigned slot; -u32 ldtbus, nid; +u32 ldtbus;u32 id; -static int lbnr[3 ] = { +int node; +u16 deviceid; +u16 vendorid; +int min_bus; +int max_bus; + +static int lbnr[NR_LDT_BUS_NUMBER_REGISTERS ] = {LDT_BUS_NUMBER_REGISTER_0 ,LDT_BUS_NUMBER_REGISTER_1 , -LDT_BUS_NUMBER_REGISTER_2 +LDT_BUS_NUMBER_REGISTER_2 , +LDT_BUS_NUMBER_REGISTER_3 };for (i =0 ; i <BUS_NR ; i++)
@@ -76,38 +86,36 @@ early_fill_mp_bus_to_node(void)if (!early_pci_allowed ())return -1 ; -for (slot =0x18 ; slot <0x20 ; slot++) { - id =read_pci_config (0 , slot,0 ,PCI_VENDOR_ID ); -if (id != (PCI_VENDOR_ID_AMD | (PCI_DEVICE_ID_K8HTCONFIG <<16 ))) -break ; - nid =read_pci_config (0 , slot,0 ,NODE_ID_REGISTER ); - -for (i =0 ; i <NR_LDT_BUS_NUMBER_REGISTERS ; i++) { - ldtbus =read_pci_config (0 , slot,0 , lbnr[i]); -/* -* if there are no busses hanging off of the current -* ldt link then both the secondary and subordinate -* bus number fields are set to 0. -* -* RED-PEN -* This is slightly broken because it assumes -* HT node IDs == Linux node ids, which is not always -* true. However it is probably mostly true. -*/ -if (!(SECONDARY_LDT_BUS_NUMBER (ldtbus) ==0 - &&SUBORDINATE_LDT_BUS_NUMBER (ldtbus) ==0 )) { -for (j =SECONDARY_LDT_BUS_NUMBER (ldtbus); - j <=SUBORDINATE_LDT_BUS_NUMBER (ldtbus); - j++) { -int node =NODE_ID (nid); - mp_bus_to_node[j] = (unsigned char )node; - } - } - } + slot =0x18 ; + id =read_pci_config (0 , slot,0 ,PCI_VENDOR_ID ); + + vendorid = id &0xffff ; +if (vendorid !=PCI_VENDOR_ID_AMD ) + goto out; + + deviceid = (id>>16 ) &0xffff ; +if ((deviceid !=PCI_DEVICE_ID_K8HTCONFIG ) && + (deviceid !=PCI_DEVICE_ID_K8_10H_HTCONFIG ) && + (deviceid !=PCI_DEVICE_ID_K8_11H_HTCONFIG )) + goto out; + +for (i =0 ; i <NR_LDT_BUS_NUMBER_REGISTERS ; i++) { + ldtbus =read_pci_config (0 , slot,1 , lbnr[i]); + +/* Check if that register is enabled for bus range */ +if ((ldtbus &7 ) !=3 ) +continue ; + + min_bus =SECONDARY_LDT_BUS_NUMBER (ldtbus); + max_bus =SUBORDINATE_LDT_BUS_NUMBER (ldtbus); + node =NODE_ID (ldtbus); +for (j = min_bus; j <= max_bus; j++) + mp_bus_to_node[j] = (unsigned char ) node; } +out:for (i =0 ; i <BUS_NR ; i++) { -int node = mp_bus_to_node[i]; + node = mp_bus_to_node[i];if (node >=0 )printk (KERN_DEBUG "bus: %02x to node: %02x\n" , i, node); }
PATCH 5/8 try parent numa_node at first before using default v2
In reply to a message not in the archivein the device_add, we try to use use parent numa_node. need to make sure pci root bus's bridge device numa_node is set. then we could use device->numa_node direclty for all device. and don't need to call pcibus_to_node().
drivers/pci/probe.c +3 -1
@@ -964,7 +964,6 @@ void pci_device_add(struct pci_dev *dev, dev->dev.release = pci_release_dev;pci_dev_get (dev); -set_dev_node (&dev->dev,pcibus_to_node (bus)); dev->dev.dma_mask = &dev->dma_mask; dev->dev.dma_parms = &dev->dma_parms; dev->dev.coherent_dma_mask =0xffffffffull ;
@@ -1116,6 +1115,9 @@ struct pci_bus * pci_create_bus(struct d goto dev_reg_err; b->bridge =get_device (dev); +if (!parent) +set_dev_node (b->bridge,pcibus_to_node (b)); + b->dev.class = &pcibus_class; b->dev.parent = b->bridge;sprintf (b->dev.bus_id,"%04x:%02x" ,pci_domain_nr (b), bus);
drivers/base/core.c +12 -2
@@ -788,6 +788,10 @@ int device_add(struct device *dev) parent =get_device (dev->parent);setup_parent (dev, parent); +/* use parent numa_node */ +if (parent) +set_dev_node (dev,dev_to_node (parent)); +/* first, register with generic layer. */ error =kobject_add (&dev->kobj, dev->kobj.parent,"%s" , dev->bus_id);if (error)
@@ -1339,8 +1343,11 @@ int device_move(struct device *dev, stru dev->parent = new_parent;if (old_parent)klist_remove (&dev->knode_parent); -if (new_parent) +if (new_parent) {klist_add_tail (&dev->knode_parent, &new_parent->klist_children); +set_dev_node (dev,dev_to_node (new_parent)); + } +if (!dev->class) goto out_put; error =device_move_class_links (dev, old_parent, new_parent);
@@ -1350,9 +1357,12 @@ int device_move(struct device *dev, struif (!kobject_move (&dev->kobj, &old_parent->kobj)) {if (new_parent)klist_remove (&dev->knode_parent); -if (old_parent) + dev->parent = old_parent; +if (old_parent) {klist_add_tail (&dev->knode_parent, &old_parent->klist_children); +set_dev_node (dev,dev_to_node (old_parent)); + } }cleanup_glue_dir (dev, new_parent_kobj);put_device (new_parent);
Re: [PATCH 5/8] try parent numa_node at first before using default v2
Quoted 7 lines · Yinghai Lu
in the
device_add, we try to use use parentnuma_node. need to make sure pci root bus's bridge devicenuma_nodeis set. then we could usedevice->numa_nodedireclty for all device. and don't need to callpcibus_to_node().Signed-off-by Yinghai Lu <[email protected]>
This should be split into different patches, one for the driver core, and one for the PCI stuff as they are doing totally different things here.
thanks,
greg k-h
Re: [PATCH 5/8] try parent numa_node at first before using default v2
Quoted 7 lines · Greg KH
Quoted 6 lines · Yinghai Lu
in the
device_add, we try to use use parentnuma_node. need to make sure pci root bus's bridge devicenuma_nodeis set. then we could usedevice->numa_nodedireclty for all device. and don't need to callpcibus_to_node().
Quoted 5 lines
Quoted 1 line
Signed-off-by Yinghai Lu <[email protected]>This should be split into different patches, one for the driver core, and one for the PCI stuff as they are doing totally different things here.
but one for the driver core, rely on the one for pci stuff. without the one pci stuff, the one for the driver core doesn't work.
last time you already said it's ok. I will dig the mail out if needed.
YH
PATCH 6/8 net: use numa_node in net_devcice->dev instead of parent
In reply to a message not in the archivenet/core/skbuff.c +1 -1
@@ -252,7 +252,7 @@ nodata:struct sk_buff *__netdev_alloc_skb (struct net_device *dev,unsigned int length,gfp_t gfp_mask) { -int node = dev->dev.parent ?dev_to_node (dev->dev.parent) :-1 ; +int node =dev_to_node (&dev->dev);struct sk_buff *skb; skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask | __GFP_NOTRACK,
Re: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
net/core/skbuff.c nodata: · line 255
Quoted 5 lines · Yinghai Lu
struct
sk_buff*__netdev_alloc_skb(structnet_device*dev,net/core/skbuff.c +1
-1unsigned int length, gfp_t gfp_mask) { - int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1; +int node =dev_to_node (&dev->dev);
i think this is a fix for the networking folks. (Dave Cc:-ed)
Ingo
David Miller
maintainerRe: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
From: Ingo Molnar <[email protected]>
Date: Tue, 19 Feb 2008 12:21:46 +0100
Quoted 10 lines · Yinghai Lu
net/core/skbuff.c nodata: · line 255
Quoted 6 lines
struct
sk_buff*__netdev_alloc_skb(structnet_device*dev,net/core/skbuff.c +1
-1unsigned int length, gfp_t gfp_mask) { - int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1; +int node =dev_to_node (&dev->dev);i think this is a fix for the networking folks. (Dave Cc:-ed)
It keeps getting NAK's because it's wrong.
The author of the patch hasn't convinced folks why this is really necessary, and using the net_device embedded device struct is definitely wrong here. It doesn't contain the NUMA node information, the physical device does, and that is what the parent it.
Re: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
Quoted 14 lines · David Miller
From: Ingo Molnar <[email protected]>
Date: Tue, 19 Feb 2008 12:21:46 +0100Quoted 10 lines · Yinghai Lu
net/core/skbuff.c nodata: · line 255
Quoted 6 lines
struct
sk_buff*__netdev_alloc_skb(structnet_device*dev,net/core/skbuff.c +1
-1unsigned int length, gfp_t gfp_mask) { - int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1; +int node =dev_to_node (&dev->dev);i think this is a fix for the networking folks. (Dave Cc:-ed)
Quoted 6 lines
It keeps getting NAK's because it's wrong.
The author of the patch hasn't convinced folks why this is really necessary, and using the
net_deviceembedded device struct is definitely wrong here. It doesn't contain the NUMA node information, the physical device does, and that is what the parent it.
can you check the 5/8?
that will make sure every struct device get numa_node get assigned.
YH
Re: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
Quoted 22 lines · Yinghai Lu
Quoted 12 lines · David Miller
From: Ingo Molnar <[email protected]>
Date: Tue, 19 Feb 2008 12:21:46 +0100Quoted 9 lines · Yinghai Lu
Quoted 6 lines
struct
sk_buff*__netdev_alloc_skb(structnet_device*dev,
unsigned int length,gfp_tgfp_mask)
{
- int node =dev->dev.parent?dev_to_node(dev->dev.parent): -1;
+ int node =dev_to_node(&dev->dev);Quoted 8 lines
Quoted 1 line
i think this is a fix for the networking folks. (Dave Cc:-ed)
It keeps getting NAK's because it's wrong.
The author of the patch hasn't convinced folks why this is really necessary, and using the
net_deviceembedded device struct is definitely wrong here. It doesn't contain the NUMA node information, the physical device does, and that is what the parent it.
Quoted 5 lines
can you check the 5/8?
that will make sure every struct device getnuma_nodeget assigned.YH
or need to combine 5 and 6 together?
YH
David Miller
maintainerRe: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
In reply to Yinghai LuFrom: Yinghai Lu <[email protected]>
Date: Tue, 19 Feb 2008 11:42:48 -0800
Quoted 17 lines · David Miller
Quoted 16 lines
From: Ingo Molnar <[email protected]>
Date: Tue, 19 Feb 2008 12:21:46 +0100Quoted 10 lines · Yinghai Lu
net/core/skbuff.c nodata: · line 255
Quoted 6 lines
struct
sk_buff*__netdev_alloc_skb(structnet_device*dev,net/core/skbuff.c +1
-1unsigned int length, gfp_t gfp_mask) { - int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1; +int node =dev_to_node (&dev->dev);i think this is a fix for the networking folks. (Dave Cc:-ed)
It keeps getting NAK's because it's wrong.
Quoted 7 lines
Quoted 4 lines
The author of the patch hasn't convinced folks why this is really necessary, and using the
net_deviceembedded device struct is definitely wrong here. It doesn't contain the NUMA node information, the physical device does, and that is what the parent it.can you check the 5/8?
that will make sure every struct device getnuma_nodeget assigned.
Why do we need to bother with that if the parent will have the necessary information for us here?
Re: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
Quoted 25 lines · David Miller
From: Yinghai Lu <[email protected]>
Date: Tue, 19 Feb 2008 11:42:48 -0800Quoted 22 lines · David Miller
Quoted 20 lines
From: Ingo Molnar <[email protected]>
Date: Tue, 19 Feb 2008 12:21:46 +0100Quoted 10 lines · Yinghai Lu
net/core/skbuff.c nodata: · line 255
Quoted 6 lines
struct
sk_buff*__netdev_alloc_skb(structnet_device*dev,
unsigned int length,gfp_tgfp_mask)
{net/core/skbuff.c +1
-1-int node = dev->dev.parent ?dev_to_node (dev->dev.parent) :-1 ; +int node =dev_to_node (&dev->dev);i think this is a fix for the networking folks. (Dave Cc:-ed)
It keeps getting NAK's because it's wrong.
The author of the patch hasn't convinced folks why this is really necessary, and using the
net_deviceembedded device struct is definitely wrong here. It doesn't contain the NUMA node information, the physical device does, and that is what the parent it.
Quoted 5 lines
Quoted 2 lines
can you check the 5/8?
that will make sure every struct device getnuma_nodeget assigned.Why do we need to bother with that if the parent will have the necessary information for us here?
less code?
or some kind of usb or other bus interface. may have several level...
YH
Re: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
Quoted 9 lines · Yinghai Lu
Quoted 5 lines
Quoted 2 lines
can you check the 5/8? that will make sure every struct device get
numa_nodeget assigned.Why do we need to bother with that if the parent will have the necessary information for us here?
less code?
or some kind of usb or other bus interface. may have several level...
you mean it's a small optimization: otherwise every struct device (net dev, usb...) need to go back to find another pci device (parent or host controller) to use their numa_node.
Ingo
Re: [PATCH 6/8] net: use numa_node in net_devcice->dev instead of parent
Quoted 9 lines · Ingo Molnar
Quoted 6 lines · Yinghai Lu
Quoted 5 lines
Quoted 2 lines
can you check the 5/8? that will make sure every struct device get
numa_nodeget assigned.Why do we need to bother with that if the parent will have the necessary information for us here?
Quoted 8 lines
Quoted 3 lines
less code?
or some kind of usb or other bus interface. may have several level...
you mean it's a small optimization: otherwise every struct device (net dev, usb...) need to go back to find another pci device (parent or host controller) to use their
numa_node.
Yes.
YH
PATCH 7/8 x86_64: get boot_cpu_id as early for k8_scan_nodes
In reply to a message not in the archiveWhen acpi=off or there is no SRAT defined, apicid_to_node is got from K8 Northbridge PCI configuration space in k8_scan_nodes() in arch/x86_64/mm/k8toplogy.c.
The problem is that it assumes bsp apic id is 0 at that point. For four socket system with Quad core cpus installed, all cpus apic id is offset by 4, and bsp apic id is 4.
For eight socket system with dual core cpus installed, all cpus apic id is offset by 2, and bsp apic id is 2.
We need get boot_cpu_id --- bsp apic id, before k8_scan_nodes by called.
So create early_acpi_boot_init and early_get_smp_config for get boot_cpu_id.
arch/x86_64/mm/k8topology.c | 8 +++ b/arch/i386/kernel/acpi/boot.c | 70 ++++++++++++++++++++++++++++++++ b/arch/x86_64/kernel/apic.c | 29 +++++++++++-- b/arch/x86_64/kernel/mpparse.c | 89 ++++++++++++++++++++++++++++++----------- b/arch/x86_64/kernel/setup.c | 21 +++++++++ b/include/asm-x86_64/apic.h | 2 b/include/asm-x86_64/mpspec.h | 2 b/include/linux/acpi.h | 5 ++ 8 files changed, 198 insertions(+), 28 deletions(-)
arch/x86/mm/k8topology_64.c +36 -1
@@ -13,12 +13,15 @@#include <linux/nodemask.h> #include <asm/io.h> #include <linux/pci_ids.h> +#include <linux/acpi.h> #include <asm/types.h> #include <asm/mmzone.h> #include <asm/proto.h> #include <asm/e820.h> #include <asm/pci-direct.h> #include <asm/numa.h> +#include <asm/mpspec.h> +#include <asm/apic.h> static __init int find_northbridge (void ) {
@@ -44,6 +47,30 @@ static __init int find_northbridge(void)return -1 ; } +static __init void early_get_boot_cpu_id (void ) +{ +/* +* need to get boot_cpu_id so can use that to create apicid_to_node +* in k8_scan_nodes() +*/ +/* +* Find possible boot-time SMP configuration: +*/ +early_find_smp_config (); +#ifdef CONFIG_ACPI +/* +* Read APIC information from ACPI tables. +*/ +early_acpi_boot_init (); +#endif +/* +* get boot-time SMP configuration: +*/ +if (smp_found_config) +early_get_smp_config (); +early_init_lapic_mapping (); +} +int __init k8_scan_nodes (unsigned long start,unsigned long end) {unsigned long prevbase;
@@ -56,6 +83,7 @@ int __init k8_scan_nodes(unsigned long sunsigned cores;unsigned bits;int j; +unsigned apicid_base;if (!early_pci_allowed ())return -1 ;
@@ -174,11 +202,18 @@ int __init k8_scan_nodes(unsigned long s/* use the coreid bits from early_identify_cpu */ bits = boot_cpu_data.x86_coreid_bits; cores = (1 <<bits); + apicid_base =0 ; +/* need to get boot_cpu_id early for system with apicid lifting */ +early_get_boot_cpu_id (); +if (boot_cpu_id >0 ) { +printk (KERN_INFO "BSP APIC ID: %02x\n" , boot_cpu_id); + apicid_base = boot_cpu_id; + }for (i =0 ; i <8 ; i++) {if (nodes[i].start != nodes[i].end) { nodeid = nodeids[i]; -for (j =0 ; j < cores; j++) +for (j = apicid_base; j < cores + apicid_base; j++) apicid_to_node[(nodeid << bits) + j] = i;setup_node_bootmem (i, nodes[i].start, nodes[i].end); }
arch/x86/kernel/acpi/boot.c +70 -0
88 lines not on this page · raw message
arch/x86/kernel/mpparse_64.c +66 -23
171 lines not on this page · raw message
include/linux/acpi.h +5 -0
@@ -79,6 +79,7 @@ typedef int (*acpi_table_handler) (structypedef int (*acpi_table_entry_handler ) (struct acpi_subtable_header *header,const unsigned long end);char *__acpi_map_table (unsigned long phys_addr,unsigned long size); +int early_acpi_boot_init (void );int acpi_boot_init (void );int acpi_boot_table_init (void );int acpi_numa_init (void );
@@ -235,6 +236,10 @@ int acpi_check_mem_region(resource_size_#else /* CONFIG_ACPI */ +static inline int early_acpi_boot_init (void ) +{ +return 0 ; +}static inline int acpi_boot_init (void ) {return 0 ;
include/asm-x86/mpspec.h +3 -0
@@ -26,6 +26,9 @@ extern int pic_mode; extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); +extern void early_find_smp_config (void ); +extern void early_get_smp_config (void ); +#endif extern int mp_bus_id_to_pci_bus[MAX_MP_BUSSES ];
include/asm-x86/apic.h +1 -0
@@ -124,6 +124,7 @@ extern void enable_NMI_through_LVT0(void*/ #ifdef CONFIG_X86_64 extern void setup_apic_routing (void ); +extern void early_init_lapic_mapping (void );#endif extern u8 setup_APIC_eilvt_mce (u8 vector,u8 msg_type,u8 mask);
arch/x86/kernel/apic_64.c +24 -0
@@ -861,6 +861,30 @@ static int __init detect_init_APIC(void)return 0 ; } +void __init early_init_lapic_mapping (void ) +{ +unsigned long apic_phys; + +/* +* If no local APIC can be found then go out +* : it means there is no mpatable and MADT +*/ +if (!smp_found_config) +return ; + + apic_phys = mp_lapic_addr; + +set_fixmap_nocache (FIX_APIC_BASE , apic_phys); +apic_printk (APIC_VERBOSE ,"mapped APIC to %16lx (%16lx)\n" , +APIC_BASE , apic_phys); + +/* +* Fetch the APIC ID of the BSP in case we have a +* default configuration (or the MP table is broken). +*/ + boot_cpu_id =GET_APIC_ID (apic_read (APIC_ID )); +} +/** * init_apic_mappings - initialize APIC mappings */
PATCH 8/8 x86_64: multi pci root bus with different io resource range
In reply to a message not in the archivescan AMD opteron io/mmio routing to make sure every pci root bus get correct resource range. So later pci scan could assign correct resource to device with unassigned resource.
this some kind make up for system without _CRS for multi pci root bus.
arch/x86/pci/k8-bus_64.c +368 -50
@@ -7,23 +7,31 @@/* * This discovers the pcibus <-> node mapping on AMD K8. -* -* RED-PEN need to call this again on PCI hotplug -* RED-PEN empty cpus get reported wrong +* also get peer root bus resource for io,mmio */ -#define NODE_ID (dword) ((dword>>4) & 0x07) -#define LDT_BUS_NUMBER_REGISTER_0 0xE0 -#define LDT_BUS_NUMBER_REGISTER_1 0xE4 -#define LDT_BUS_NUMBER_REGISTER_2 0xE8 -#define LDT_BUS_NUMBER_REGISTER_3 0xEC -#define NR_LDT_BUS_NUMBER_REGISTERS 4 -#define SECONDARY_LDT_BUS_NUMBER (dword) ((dword >> 16) & 0xFF) -#define SUBORDINATE_LDT_BUS_NUMBER (dword) ((dword >> 24) & 0xFF) - -#define PCI_DEVICE_ID_K8HTCONFIG 0x1100 -#define PCI_DEVICE_ID_K8_10H_HTCONFIG 0x1200 -#define PCI_DEVICE_ID_K8_11H_HTCONFIG 0x1300 + +/* +* sub bus (transparent) will use entres from 3 to store extra from root, +* so need to make sure have enought slot there, increase PCI_BUS_NUM_RESOURCES? +*/ +#define RES_NUM 16 +struct pci_root_info { +char name[12 ]; +unsigned int res_num; +struct resource res[RES_NUM ]; +int bus_min; +int bus_max; +int node; +int link; +}; + +/* 4 at this time, it may become to 32 */ +#define PCI_ROOT_NR 4 +static int pci_root_num; +static struct pci_root_info pci_root_info[PCI_ROOT_NR ]; + +#ifdef CONFIG_NUMA #define BUS_NR 256
@@ -53,6 +61,135 @@ int get_mp_bus_to_node(int busnum)return node; } +#endif + +void set_pci_bus_resources_arch_default (struct pci_bus *b) +{ +int i; +int j; +struct pci_root_info *info; + +if (!pci_root_num) +return ; + +for (i =0 ; i < pci_root_num; i++) { +if (pci_root_info[i].bus_min == b->number) +break ; + } + +if (i == pci_root_num) +return ; + + info = &pci_root_info[i]; +for (j =0 ; j < info->res_num; j++) { +struct resource *res; +struct resource *root; + + res = &info->res[j]; + b->resource[j] = res; +if (res->flags &IORESOURCE_IO ) + root = &ioport_resource; +else + root = &iomem_resource; +insert_resource (root, res); + } +} + +#define RANGE_NUM 16 + +struct res_range { +size_t start; +size_t end; +}; + +static void __init update_range (struct res_range *range,size_t start, +size_t end) +{ +int i; +int j; + +for (j =0 ; j <RANGE_NUM ; j++) { +if (!range[j].end) +continue ; +if (start == range[j].start && end < range[j].end) { + range[j].start = end +1 ; +break ; + }else if (start == range[j].start && end == range[j].end) { + range[j].start =0 ; + range[j].end =0 ; +break ; + }else if (start > range[j].start && end == range[j].end) { + range[j].end = start -1 ; +break ; + }else if (start > range[j].start && end < range[j].end) { +/* find the new spare */ +for (i =0 ; i <RANGE_NUM ; i++) { +if (range[i].end ==0 ) +break ; + } +if (i <RANGE_NUM ) { + range[i].end = range[j].end; + range[i].start = end +1 ; + }else { +printk (KERN_ERR "run of slot in ranges\n" ); + } + range[j].end = start -1 ; +break ; + } + } +} + +static void __init update_res (struct pci_root_info *info,size_t start, +size_t end,unsigned long flags,int merge) +{ +int i; +struct resource *res; + +if (!merge) + goto addit; + +/* try to merge it with old one */ +for (i =0 ; i < info->res_num; i++) { + res = &info->res[i]; +if (res->flags != flags) +continue ; +if (res->end +1 == start) { + res->end = end; +return ; + }else if (end +1 == res->start) { + res->start = start; +return ; + } + } + +addit: + +/* need to add that */ +if (info->res_num >=RES_NUM ) +return ; + + res = &info->res[info->res_num]; + res->name = info->name; + res->flags = flags; + res->start = start; + res->end = end; + res->child =NULL ; + info->res_num++; +} + +struct pci_hostbridge_probe { +u32 bus; +u32 slot; +u32 vendor; +u32 device; +}; + +static struct pci_hostbridge_probe pci_probes[]__initdata = { + {0 ,0x18 ,PCI_VENDOR_ID_AMD ,0x1100 }, + {0 ,0x18 ,PCI_VENDOR_ID_AMD ,0x1200 }, + {0xff ,0 ,PCI_VENDOR_ID_AMD ,0x1200 }, + {0 ,0x18 ,PCI_VENDOR_ID_AMD ,0x1300 }, +};/** * early_fill_mp_bus_to_node()
@@ -60,25 +197,25 @@ int get_mp_bus_to_node(int busnum)* fills the mp_bus_to_cpumask array based according to the LDT Bus Number * Registers found in the K8 northbridge */ -__init static int -early_fill_mp_bus_to_node (void ) +static int __init early_fill_mp_bus_info (void ) { -int i, j; +int i; +int j; +unsigned bus;unsigned slot; -u32 ldtbus; -u32 id; +int found;int node; -u16 deviceid; -u16 vendorid; -int min_bus; -int max_bus; - -static int lbnr[NR_LDT_BUS_NUMBER_REGISTERS ] = { -LDT_BUS_NUMBER_REGISTER_0 , -LDT_BUS_NUMBER_REGISTER_1 , -LDT_BUS_NUMBER_REGISTER_2 , -LDT_BUS_NUMBER_REGISTER_3 - }; +int link; +int def_node; +int def_link; +struct pci_root_info *info; +u32 reg; +struct resource *res; +size_t start; +size_t end; +struct res_range range[RANGE_NUM ]; +u64 val; +u32 address;for (i =0 ; i <BUS_NR ; i++) mp_bus_to_node[i] =-1 ;
@@ -86,40 +223,221 @@ early_fill_mp_bus_to_node(void)if (!early_pci_allowed ())return -1 ; - slot =0x18 ; - id =read_pci_config (0 , slot,0 ,PCI_VENDOR_ID ); + found =0 ; +for (i =0 ; i <ARRAY_SIZE (pci_probes); i++) { +u32 id; +u16 device; +u16 vendor; + + bus = pci_probes[i].bus; + slot = pci_probes[i].slot; + id =read_pci_config (bus, slot,0 ,PCI_VENDOR_ID ); + + vendor = id &0xffff ; + device = (id>>16 ) &0xffff ; +if (pci_probes[i].vendor == vendor && + pci_probes[i].device == device) { + found =1 ; +break ; + } + } - vendorid = id &0xffff ; -if (vendorid !=PCI_VENDOR_ID_AMD ) - goto out; - - deviceid = (id>>16 ) &0xffff ; -if ((deviceid !=PCI_DEVICE_ID_K8HTCONFIG ) && - (deviceid !=PCI_DEVICE_ID_K8_10H_HTCONFIG ) && - (deviceid !=PCI_DEVICE_ID_K8_11H_HTCONFIG )) - goto out; +if (!found) +return 0 ; -for (i =0 ; i <NR_LDT_BUS_NUMBER_REGISTERS ; i++) { - ldtbus =read_pci_config (0 , slot,1 , lbnr[i]); + pci_root_num =0 ; +for (i =0 ; i <4 ; i++) { +int min_bus; +int max_bus; + reg =read_pci_config (bus, slot,1 ,0xe0 + (i <<2 ));/* Check if that register is enabled for bus range */ -if ((ldtbus &7 ) !=3 ) +if ((reg &7 ) !=3 )continue ; - min_bus =SECONDARY_LDT_BUS_NUMBER (ldtbus); - max_bus =SUBORDINATE_LDT_BUS_NUMBER (ldtbus); - node =NODE_ID (ldtbus); + min_bus = (reg >>16 ) &0xff ; + max_bus = (reg >>24 ) &0xff ; + node = (reg >>4 ) &0x07 ; +#ifdef CONFIG_NUMA for (j = min_bus; j <= max_bus; j++) mp_bus_to_node[j] = (unsigned char ) node; +#endif + link = (reg >>8 ) &0x03 ; + + info = &pci_root_info[pci_root_num]; + info->bus_min = min_bus; + info->bus_max = max_bus; + info->node = node; + info->link = link; +sprintf (info->name,"PCI Bus #%02x" , min_bus); + pci_root_num++; + } + +/* get the default node and link for left over res */ + reg =read_pci_config (bus, slot,0 ,0x60 ); + def_node = (reg >>8 ) &0x07 ; + reg =read_pci_config (bus, slot,0 ,0x64 ); + def_link = (reg >>8 ) &0x03 ; + +memset (range,0 ,sizeof (range)); + range[0 ].end =0xffff ; +/* io port resource */ +for (i =0 ; i <4 ; i++) { + reg =read_pci_config (bus, slot,1 ,0xc0 + (i <<3 )); +if (!(reg &3 )) +continue ; + + start = reg &0xfff000 ; + reg =read_pci_config (bus, slot,1 ,0xc4 + (i <<3 )); + node = reg &0x07 ; + link = (reg >>4 ) &0x03 ; + end = (reg &0xfff000 ) |0xfff ; + +/* find the position */ +for (j =0 ; j < pci_root_num; j++) { + info = &pci_root_info[j]; +if (info->node == node && info->link == link) +break ; + } +if (j == pci_root_num) +continue ;/* not found */ + + info = &pci_root_info[j]; +update_res (info, start, end,IORESOURCE_IO ,0 ); +update_range (range, start, end); + } +/* add left over io port range to def node/link, [0, 0xffff] */ +/* find the position */ +for (j =0 ; j < pci_root_num; j++) { + info = &pci_root_info[j]; +if (info->node == def_node && info->link == def_link) +break ; } +if (j < pci_root_num) { + info = &pci_root_info[j]; +for (i =0 ; i <RANGE_NUM ; i++) { +if (!range[i].end) +continue ; + +update_res (info, range[i].start, range[i].end, +IORESOURCE_IO ,1 ); + } + } + +memset (range,0 ,sizeof (range)); +/* 0xfd00000000-0xffffffffff for HT */ +/* 0xfc00000000-0xfcffffffff for Family 10h mmconfig*/ + range[0 ].end =0xfbffffffffULL ; + +/* need to take out [0, TOM) for RAM*/ + address =MSR_K8_TOP_MEM1 ; +rdmsrl (address, val); + end = (val &0xffffff8000000ULL ); +printk (KERN_INFO "TOM: %016lx aka %ldM\n" , end, end>>20 ); +if (end < (1ULL <<32 )) +update_range (range,0 , end -1 ); + +/* mmio resource */ +for (i =0 ; i <8 ; i++) { + reg =read_pci_config (bus, slot,1 ,0x80 + (i <<3 )); +if (!(reg &3 )) +continue ; -out: + start = reg &0xffffff00 ;/* 39:16 on 31:8*/ + start <<=8 ; + reg =read_pci_config (bus, slot,1 ,0x84 + (i <<3 )); + node = reg &0x07 ; + link = (reg >>4 ) &0x03 ; + end = (reg &0xffffff00 ); + end <<=8 ; + end |=0xffff ; + +/* find the position */ +for (j =0 ; j < pci_root_num; j++) { + info = &pci_root_info[j]; +if (info->node == node && info->link == link) +break ; + } +if (j == pci_root_num) +continue ;/* not found */ + + info = &pci_root_info[j]; +update_res (info, start, end,IORESOURCE_MEM ,0 ); +update_range (range, start, end); + } + +/* need to take out [4G, TOM2) for RAM*/ +/* SYS_CFG */ + address =MSR_K8_SYSCFG ; +rdmsrl (address, val); +/* TOP_MEM2 is enabled? */ +if (val & (1 <<21 )) { +/* TOP_MEM2 */ + address =MSR_K8_TOP_MEM2 ; +rdmsrl (address, val); + end = (val &0xffffff8000000ULL ); +printk (KERN_INFO "TOM2: %016lx aka %ldM\n" , end, end>>20 ); +update_range (range,1ULL <<32 , end -1 ); + } + +/* +* add left over mmio range to def node/link ? +* that is tricky, just record range in from start_min to 4G +*/ +for (j =0 ; j < pci_root_num; j++) { + info = &pci_root_info[j]; +if (info->node == def_node && info->link == def_link) +break ; + } +if (j < pci_root_num) { + info = &pci_root_info[j]; + +for (i =0 ; i <RANGE_NUM ; i++) { +if (!range[i].end) +continue ; +#if 0 +/* don't use the one start less than 1M */ +if (range[i].start <0x100000 ) +continue ; +#endif +#if 0 +/* don't use last one near 4G */ +if (range[i].end ==0xffffffffULL ) +continue ; +#endif + +update_res (info, range[i].start, range[i].end, +IORESOURCE_MEM ,1 ); + } + } + +#ifdef CONFIG_NUMA for (i =0 ; i <BUS_NR ; i++) { node = mp_bus_to_node[i];if (node >=0 )printk (KERN_DEBUG "bus: %02x to node: %02x\n" , i, node); } +#endif + +for (i =0 ; i < pci_root_num; i++) { +int res_num; +int busnum; + + info = &pci_root_info[i]; + res_num = info->res_num; + busnum = info->bus_min; +printk (KERN_DEBUG "bus: [%02x,%02x] on node %x link %x\n" , + info->bus_min, info->bus_max, info->node, info->link); +for (j =0 ; j < res_num; j++) { + res = &info->res[j]; +printk (KERN_DEBUG "bus: %02x index %x %s: [%llx, %llx]\n" , + busnum, j, + (res->flags &IORESOURCE_IO )?"io port" :"mmio" , + res->start, res->end); + } + } +return 0 ; } -postcore_initcall (early_fill_mp_bus_to_node); +postcore_initcall (early_fill_mp_bus_info);
include/asm-x86/topology.h +3 -0
@@ -25,6 +25,7 @@#ifndef _ASM_X86_TOPOLOGY_H#define _ASM_X86_TOPOLOGY_H +struct pci_bus ;#ifdef CONFIG_NUMA #include <linux/cpumask.h> #include <asm/mpspec.h>
@@ -191,6 +192,8 @@ extern cpumask_t cpu_coregroup_map(int c#define topology_thread_siblings (cpu) (per_cpu(cpu_sibling_map, cpu))#endif +void set_pci_bus_resources_arch_default (struct pci_bus *b); +#ifdef CONFIG_SMP #define mc_capable () (boot_cpu_data.x86_max_cores > 1)#define smt_capable () (smp_num_siblings > 1)
drivers/pci/probe.c +6 -0
@@ -1076,6 +1076,10 @@ unsigned int pci_scan_child_bus(struct preturn max; } +void __attribute__((weak))set_pci_bus_resources_arch_default (struct pci_bus *b) +{ +} +struct pci_bus *pci_create_bus (struct device *parent,int bus,struct pci_ops *ops,void *sysdata) {
@@ -1135,6 +1139,8 @@ struct pci_bus * pci_create_bus(struct d b->resource[0 ] = &ioport_resource; b->resource[1 ] = &iomem_resource; +set_pci_bus_resources_arch_default (b); +return b; dev_create_file_err:
arch/x86/pci/Makefile_64 +1 -1
@@ -13,5 +13,5 @@ obj-y += legacy.o irq.o common.o early # mmconfig has a 64bit special obj-$(CONFIG_PCI_MMCONFIG) += mmconfig_64.o direct.o mmconfig-shared.o -obj-$(CONFIG_NUMA) += k8-bus_64.o +obj-y += k8-bus_64.o
include/linux/pci.h +1 -1
@@ -247,7 +247,7 @@ static inline void pci_add_saved_cap(str#define PCI_NUM_RESOURCES 11#ifndef PCI_BUS_NUM_RESOURCES -#define PCI_BUS_NUM_RESOURCES 8 +#define PCI_BUS_NUM_RESOURCES 16#endif #define PCI_REGION_FLAG_MASK 0x0fU/* These bits of resource flags tell us the PCI region flags */