use nr_cpus= to set nr_cpu_ids early
RFC PATCH 1/5 use nr_cpus= to set nr_cpu_ids early
on x86, before prefill_possible_map(), nr_cpu_ids will be NR_CPUS aka CONFIG_NR_CPUS
add nr_cpus= to set nr_cpu_ids. so we can simulate cpus <=8 on normal config. instead of change NR_CPUS directly.
--- arch/ia64/kernel/acpi.c | 4 ++-- arch/x86/kernel/smpboot.c | 7 ++++--- init/main.c | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-)
init/main.c +14 -0
@@ -149,6 +149,20 @@ static int __init nosmp(char *str)early_param ("nosmp" , nosmp); +/* this is hard limit */ +static int __init nrcpus (char *str) +{ +int nr_cpus; + +get_option (&str, &nr_cpus); +if (nr_cpus >0 && nr_cpus < nr_cpu_ids) + nr_cpu_ids = nr_cpus; + +return 0 ; +} + +early_param ("nr_cpus" , nrcpus); +static int __init maxcpus (char *str) {get_option (&str, &setup_max_cpus);
arch/ia64/kernel/acpi.c +2 -2
@@ -883,8 +883,8 @@ __init void prefill_possible_map(void) possible = available_cpus + additional_cpus; -if (possible >NR_CPUS ) - possible =NR_CPUS ; +if (possible > nr_cpu_ids) + possible = nr_cpu_ids;printk (KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n" , possible,max ((possible - available_cpus),0 ));
arch/x86/kernel/smpboot.c +4 -3
@@ -1213,11 +1213,12 @@ __init void prefill_possible_map(void) total_cpus =max_t (int, possible, num_processors + disabled_cpus); -if (possible >CONFIG_NR_CPUS ) { +/* nr_cpu_ids could be reduced via nr_cpus= */ +if (possible > nr_cpu_ids) {printk (KERN_WARNING "%d Processors exceeds NR_CPUS limit of %d\n" , - possible,CONFIG_NR_CPUS ); - possible =CONFIG_NR_CPUS ; + possible, nr_cpu_ids); + possible = nr_cpu_ids; } printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
RFC PATCH 2/5 x86: use dmi check to treat disabled cpus as hotplug cpus.
some systems that have disable cpus entries because same
BIOS will support 2 sockets and 4 sockets and more at same time, BIOS just leave some disable entries, but those system do not support cpu hotplug. we don't need treat disabled_cpus as hotplug cpus.
so we can make nr_cpu_ids smaller and save more space
(pcpu data allocations), and could make some systems run with logical flat instead of physical flat apic mode
-v2: change to black list instead
--- arch/x86/kernel/smpboot.c | 76 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-)
arch/x86/kernel/smpboot.c +74 -2
@@ -47,6 +47,7 @@#include <linux/bootmem.h> #include <linux/err.h> #include <linux/nmi.h> +#include <linux/dmi.h> #include <linux/tboot.h> #include <asm/acpi.h>
@@ -1180,6 +1181,59 @@ static int __init _setup_possible_cpus(c }early_param ("possible_cpus" , _setup_possible_cpus); +static __initdata int treat_disabled_cpus_as_hotplug =1 ; +static __init int hotplug_cpus_check (const struct dmi_system_id *d) +{ +printk (KERN_NOTICE "%s detected: treat disabled cpus as hotplug ones\n" , d->ident); + treat_disabled_cpus_as_hotplug =0 ; + +return 0 ; +} + +static struct dmi_system_id hotplug_cpus_dmi_table[]__initdata = { + { + .callback = hotplug_cpus_check, + .ident ="Sun Microsystems Sun Fire X4440" , + .matches = { +DMI_MATCH (DMI_SYS_VENDOR ,"Sun Microsystems" ), +DMI_MATCH (DMI_PRODUCT_NAME ,"Sun Fire X4440" ), + }, + }, + { + .callback = hotplug_cpus_check, + .ident ="Sun Microsystems Sun Fire X4240" , + .matches = { +DMI_MATCH (DMI_SYS_VENDOR ,"Sun Microsystems" ), +DMI_MATCH (DMI_PRODUCT_NAME ,"Sun Fire X4240" ), + }, + }, + { + .callback = hotplug_cpus_check, + .ident ="Sun Microsystems Sun Fire X4140" , + .matches = { +DMI_MATCH (DMI_SYS_VENDOR ,"Sun Microsystems" ), +DMI_MATCH (DMI_PRODUCT_NAME ,"Sun Fire X4140" ), + }, + }, + { + .callback = hotplug_cpus_check, + .ident ="Sun Microsystems Sun Fire X4600" , + .matches = { +DMI_MATCH (DMI_SYS_VENDOR ,"Sun Microsystems" ), +DMI_MATCH (DMI_PRODUCT_NAME ,"Sun Fire X4600" ), + }, + }, + { + .callback = hotplug_cpus_check, + .ident ="Sun Microsystems Sun Fire X4640" , + .matches = { +DMI_MATCH (DMI_SYS_VENDOR ,"Sun Microsystems" ), +DMI_MATCH (DMI_PRODUCT_NAME ,"Sun Fire X4640" ), + }, + }, + { }/* NULL entry stops DMI scanning */ +}; +/* * cpu_possible_mask should be static, it cannot change as cpu's
@@ -1206,8 +1260,26 @@ __init void prefill_possible_map(void)if (!num_processors) num_processors =1 ; -if (setup_possible_cpus ==-1 ) - possible = num_processors + disabled_cpus; +if (setup_possible_cpus ==-1 ) { + possible = num_processors; +/* +* do we have better way to detect hotplug cpus? +* +* some systems that have disable cpus entries because same +* BIOS will support 2 sockets and 4 sockets and more at +* same time, BIOS just leave some disabled entries with wild +* apicid, but those system do not support cpu hotplug. +* we don't need treat disabled_cpus as hotplug cpus. +* so we can make nr_cpu_ids smaller and save more space +* (pcpu data allocations), and could make some systems run +* with logical flat instead of physical flat apic mode +*/ +if (disabled_cpus) { +dmi_check_system (hotplug_cpus_dmi_table); +if (treat_disabled_cpus_as_hotplug) + possible += disabled_cpus; + } + }else possible = setup_possible_cpus;
RFC PATCH 3/5 x86: using logical flat for amd cpu too.
In reply to Yinghai Lunot sure it works again, could be bios fix the irq routing dest ?
tested that amd support logical flat too when cpus num <= 8 and even bsp apic id > 8
so could remove vendor check...
--- arch/x86/kernel/apic/apic.c | 11 ++--------- arch/x86/kernel/apic/probe_64.c | 13 ++----------- 2 files changed, 4 insertions(+), 20 deletions(-)
arch/x86/kernel/apic/apic.c +2 -9
@@ -1898,15 +1898,8 @@ void __cpuinit generic_processor_info(in max_physical_apicid = apicid;#ifdef CONFIG_X86_32 -switch (boot_cpu_data.x86_vendor) { -case X86_VENDOR_INTEL : -if (num_processors >8 ) - def_to_bigsmp =1 ; -break ; -case X86_VENDOR_AMD : -if (max_physical_apicid >=8 ) - def_to_bigsmp =1 ; - } +if (num_processors >8 ) + def_to_bigsmp =1 ;#endif #if defined(CONFIG_SMP ) || defined(CONFIG_X86_64 )
arch/x86/kernel/apic/probe_64.c +2 -11
@@ -67,17 +67,8 @@ void __init default_setup_apic_routing(v }#endif -if (apic == &apic_flat) { -switch (boot_cpu_data.x86_vendor) { -case X86_VENDOR_INTEL : -if (num_processors >8 ) - apic = &apic_physflat; -break ; -case X86_VENDOR_AMD : -if (max_physical_apicid >=8 ) - apic = &apic_physflat; - } - } +if (apic == &apic_flat && num_processors >8 ) + apic = &apic_physflat;printk (KERN_INFO "Setting APIC routing to %s\n" , apic->name);
RFC PATCH 4/5 x86: according to nr_cpu_ids to decide if need to leave logical flat
In reply to Yinghai Lushould use nr_cpu_ids instead of num_processor, in case we have hotplug cpus.
if current only have 8 cpus is up, but if we will have more cpus that will be hot added later, we should use physical flat at first.
nr_cpu_ids is the total cpus that could be supported.
-v2: per linus, chenage default_setup_apic_routing to _init, and call it for uni_processor
--- arch/x86/kernel/apic/apic.c | 2 -- arch/x86/kernel/apic/probe_32.c | 20 ++++++++++++++++++-- arch/x86/kernel/apic/probe_64.c | 3 ++- arch/x86/kernel/smpboot.c | 2 -- 4 files changed, 20 insertions(+), 7 deletions(-)
arch/x86/kernel/apic/probe_32.c +18 -2
@@ -52,7 +52,7 @@ static int __init print_ipi_mode(void) }late_initcall (print_ipi_mode); -void default_setup_apic_routing (void ) +static void local_default_setup_apic_routing (void ) {#ifdef CONFIG_X86_IO_APIC printk(KERN_INFO
@@ -103,7 +103,7 @@ struct apic apic_default = { .init_apic_ldr = default_init_apic_ldr, .ioapic_phys_id_map = default_ioapic_phys_id_map, - .setup_apic_routing = default_setup_apic_routing, + .setup_apic_routing = local_default_setup_apic_routing, .multi_timer_check = NULL, .apicid_to_node = default_apicid_to_node, .cpu_to_logical_apicid = default_cpu_to_logical_apicid,
@@ -207,6 +207,22 @@ void __init generic_bigsmp_probe(void) apic = &apic_bigsmp;printk (KERN_INFO "Overriding APIC driver with %s\n" , apic->name); + } + } +#endif +} + +void __init default_setup_apic_routing (void ) +{ +#ifdef CONFIG_X86_BIGSMP +/* +* make sure we go to bigsmp according to real nr_cpu_ids +*/ +if (!cmdline_apic && apic == &apic_default) { +if (nr_cpu_ids >8 ) { + apic = &apic_bigsmp; +printk (KERN_INFO "Overriding APIC driver with %s\n" , + apic->name); } }#endif
arch/x86/kernel/smpboot.c +0 -2
@@ -1084,9 +1084,7 @@ void __init native_smp_prepare_cpus(unsiset_cpu_sibling_map (0 );enable_IR_x2apic (); -#ifdef CONFIG_X86_64 default_setup_apic_routing (); -#endif if (smp_sanity_check (max_cpus) <0 ) {printk (KERN_INFO "SMP disabled\n" );
arch/x86/kernel/apic/probe_64.c +2 -1
@@ -67,7 +67,8 @@ void __init default_setup_apic_routing(v }#endif -if (apic == &apic_flat && num_processors >8 ) +/* not just num_processors, we could have hotplug cpus */ +if (apic == &apic_flat && nr_cpu_ids >8 ) apic = &apic_physflat;printk (KERN_INFO "Setting APIC routing to %s\n" , apic->name);
arch/x86/kernel/apic/apic.c +0 -2
@@ -1647,9 +1647,7 @@ int __init APIC_init_uniprocessor(void)#endif enable_IR_x2apic (); -#ifdef CONFIG_X86_64 default_setup_apic_routing (); -#endif verify_local_APIC ();connect_bsp_APIC ();
RFC PATCH 5/5 x86: make 32bit apic flat to physflat switch like 64bit
In reply to Yinghai Lukill def_to_bigsmp
and move switch from default to bigsmp at default_setup_apic_routing... so make default_setup_apic_routing more like 64 bit
also make the dmi relate code to be __init/__initdata
--- arch/x86/include/asm/apic.h | 3 - arch/x86/include/asm/mpspec.h | 1 arch/x86/kernel/acpi/boot.c | 3 - arch/x86/kernel/apic/apic.c | 5 -- arch/x86/kernel/apic/bigsmp_32.c | 48 ----------------------- arch/x86/kernel/apic/probe_32.c | 80 ++++++++++++++++++++------------------- arch/x86/kernel/mpparse.c | 4 - arch/x86/kernel/setup.c | 2 arch/x86/kernel/smpboot.c | 2 9 files changed, 46 insertions(+), 102 deletions(-)
arch/x86/include/asm/mpspec.h +0 -1
@@ -23,7 +23,6 @@ extern int pic_mode;#define MAX_IRQ_SOURCES 256 -extern unsigned int def_to_bigsmp;extern u8 apicid_2_node[];#ifdef CONFIG_X86_NUMAQ
arch/x86/kernel/apic/apic.c +0 -5
@@ -1895,11 +1895,6 @@ void __cpuinit generic_processor_info(inif (apicid > max_physical_apicid) max_physical_apicid = apicid; -#ifdef CONFIG_X86_32 -if (num_processors >8 ) - def_to_bigsmp =1 ; -#endif -#if defined(CONFIG_SMP ) || defined(CONFIG_X86_64 )early_per_cpu (x86_cpu_to_apicid, cpu) = apicid;early_per_cpu (x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/bigsmp_32.c +2 -46
81 lines not on this page · raw message
arch/x86/kernel/apic/probe_32.c +43 -37
138 lines not on this page · raw message
arch/x86/kernel/setup.c +0 -2
@@ -184,8 +184,6 @@ static void set_mca_bus(int x)#endif } -unsigned int def_to_bigsmp; -/* for MCA, but anyone else can use it if they want */ unsigned int machine_id;unsigned int machine_submodel_id;
arch/x86/kernel/smpboot.c +1 -1
@@ -947,7 +947,7 @@ static int __init smp_sanity_check(unsigpreempt_disable ();#if !defined(CONFIG_X86_BIGSMP ) && defined(CONFIG_X86_32 ) -if (def_to_bigsmp && nr_cpu_ids >8 ) { +if (apic == &apic_default && nr_cpu_ids >8 ) {unsigned int cpu;unsigned nr;
arch/x86/include/asm/apic.h +0 -3
@@ -456,9 +456,6 @@ static inline void default_wait_for_initreturn ; } -extern void generic_bigsmp_probe (void ); - -#ifdef CONFIG_X86_LOCAL_APIC #include <asm/smp.h>
arch/x86/kernel/acpi/boot.c +0 -3
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(voiif (!error) { acpi_lapic =1 ; -#ifdef CONFIG_X86_BIGSMP -generic_bigsmp_probe (); -#endif /* * Parse MADT IO-APIC entries */
arch/x86/kernel/mpparse.c +0 -4
@@ -359,10 +359,6 @@ static int __init smp_read_mpc(struct mp x86_init.mpparse.mpc_record (1 ); } -#ifdef CONFIG_X86_BIGSMP -generic_bigsmp_probe (); -#endif -if (apic->setup_apic_routing) apic->setup_apic_routing ();
Quoted 4 lines · Yinghai Lu
on x86, before
prefill_possible_map(),nr_cpu_idswill beNR_CPUSakaCONFIG_NR_CPUSadd nr_cpus= to set
nr_cpu_ids. so we can simulate cpus <=8 on normal config. instead of changeNR_CPUSdirectly.
We already have
maxcpus=x?
Why do we need this twice?
Quoted 5 lines · Christoph Lameter
Quoted 2 lines · Yinghai Lu
on x86, before
prefill_possible_map(),nr_cpu_idswill beNR_CPUSakaCONFIG_NR_CPUS
Quoted 8 lines
Quoted 2 lines
add nr_cpus= to set
nr_cpu_ids. so we can simulate cpus <=8 on normal config. instead of changeNR_CPUSdirectly.We already have
maxcpus=x?
Why do we need this twice?
maxcpus only change setup_max_cpus., and if you are using maxcpus=1, and you have 8 cpus installed, you can put other
cpus back online via /sys/interface.
nr_cpus= is hard limit nr_cpu_ids, so if you have 16 cpus installed, nr_cpus=8 will make your nr_cpu_ids=8, and you can not put back other 8 back. and apic mode could stay with logical flat. this is used to simulate some debug case. for example you have kernel support physflat, and flat, with CONFIG_NR_CPUS=255. to run that on system that only have 8 cpus, you will have apic mode in logical flat.
but if you have run the kernel on system with 32 cpus installed, it
will switch to physflat even you have maxcpus=8 appedded.
YH
Quoted 5 lines · Yinghai Lu
Quoted 1 line
Why do we need this twice?
maxcpus only change
setup_max_cpus., and if you are using maxcpus=1, and you have 8 cpus installed, you can put other
cpus back online via/sys/interface.
Hmmm.. Strange semantics since maxcpus=0 disables smp completely. No cpu can be activated later. Similar to nr_cpus ?
Quoted 8 lines
nr_cpus= is hard limit
nr_cpu_ids, so if you have 16 cpus installed, nr_cpus=8 will make your nr_cpu_ids=8, and you can not put back other 8 back. and apic mode could stay with logical flat. this is used to simulate some debug case. for example you have kernel support physflat, and flat, withCONFIG_NR_CPUS=255. to run that on system that only have 8 cpus, you will have apic mode in logical flat.
but if you have run the kernel on system with 32 cpus installed, it
will switch to physflat even you have maxcpus=8 appedded.
Ok makes sense.
Quoted 10 lines · Christoph Lameter
Quoted 5 lines · Yinghai Lu
Quoted 1 line
Why do we need this twice?
maxcpus only change
setup_max_cpus., and if you are using maxcpus=1, and you have 8 cpus installed, you can put other
cpus back online via/sys/interface.Hmmm.. Strange semantics since maxcpus=0 disables smp completely. No cpu can be activated later. Similar to
nr_cpus?
no
for SMP kernel, when nr_cpus=1 is specified, it looks like only one cpu is installed physically.
smp is still enabled.
maxcpus=0 is really caused misunderstanding.
other maxcpus=1 and other should be rename to boot_online_cpus=
Quoted 11 lines
Quoted 9 lines
nr_cpus= is hard limit
nr_cpu_ids, so if you have 16 cpus installed, nr_cpus=8 will make your nr_cpu_ids=8, and you can not put back other 8 back. and apic mode could stay with logical flat. this is used to simulate some debug case. for example you have kernel support physflat, and flat, withCONFIG_NR_CPUS=255. to run that on system that only have 8 cpus, you will have apic mode in logical flat. but if you have run the kernel on system with 32 cpus installed, it will switch to physflat even you have maxcpus=8 appedded.Ok makes sense.
good.
YH
Trouble is that NR_CPUS is used all over the place. If nr_cpu_ids < NR_CPUS at boot then there is a danger of for loops to NR_CPUS going out of
bounds.
CONFIG_NR_CPUS is used for various bitmaps so that seems to be okay.
drivers/acpi/numa.c:acpi_numa_init probably should use nr_cpu_ids instead now.
There is an octeon driver in staging that has some issues with NR_CPUS as well.
Quoted 5 lines · Christoph Lameter
Trouble is that
NR_CPUSis used all over the place. Ifnr_cpu_ids<NR_CPUSat boot then there is a danger of for loops toNR_CPUSgoing out of bounds.
Quoted 7 lines
CONFIG_NR_CPUSis used for various bitmaps so that seems to be okay.drivers/acpi/numa.c:acpi_numa_init probably should use
nr_cpu_idsinstead now.There is an octeon driver in staging that has some issues with
NR_CPUSas well.
I suspect that this is the real reason for the current behavior of 'maxcpus=', and that if all of those issues get fixed we could probably make maxcpus do what Yinghai's new 'nr_cpus=' does.
So in a perfect world, CONFIG_NR_CPUS's would never really be used, except for some fundamental static allocations/limits that are too painful to try to make dynamic.
I doubt that anybody really _cares_ about the "you can add them later" behavior of the current maxcpus thing, and I suspect that the nr_cpus semantics is what people generally would have expected.
Linus
Quoted 7 lines · Christoph Lameter
Trouble is that
NR_CPUSis used all over the place. Ifnr_cpu_ids<NR_CPUSat boot then there is a danger of for loops toNR_CPUSgoing out of
bounds.
CONFIG_NR_CPUSis used for various bitmaps so that seems to be okay.
cpumask_var?
Quoted 3 lines
drivers/acpi/numa.c:acpi_numa_init probably should use
nr_cpu_idsinstead now.
before prefill_possible_map for x86
before setup_nr_cpu_ids for other platform
nr_cpu_ids = NR_CPUS = CONFIG_NR_CPUS
Quoted 4 lines
There is an octeon driver in staging that has some issues with
NR_CPUSas well.
that should be fixed, we should use nr_cpu_ids instead for them.
YH
Quoted 3 lines · Christoph Lameter
drivers/acpi/numa.c:acpi_numa_init probably should use
nr_cpu_idsinstead now.
you are right.
YH