Skip to contentSkip to conversation

use nr_cpus= to set nr_cpu_ids early

RFC PATCH In review Yinghai Lu 3 participants 13 messages Jan 12, 2010

Subsystems (2)
lore
Share & tools
lore

Yinghai Lu

author
#0 lore raw

RFC PATCH 1/5 use nr_cpus= to set nr_cpu_ids early

on x86, before prefill_possible_map(), nr_cpu_ids will be NR_CPUS aka CONFIG_NR_CPUS

add nr_cpus= to set nr_cpu_ids. so we can simulate cpus <=8 on normal config. instead of change NR_CPUS directly.

---
 arch/ia64/kernel/acpi.c   |    4 ++--
 arch/x86/kernel/smpboot.c |    7 ++++---
 init/main.c               |   14 ++++++++++++++
 3 files changed, 20 insertions(+), 5 deletions(-)
init/main.c +14 -0
Index: linux-2.6/init/main.c
===================================================================
@@ -149,6 +149,20 @@ static int __init nosmp(char *str)
 
 early_param("nosmp", nosmp);
 
+/* this is hard limit */
+static int __init nrcpus(char *str)
+{
+	int nr_cpus;
+
+	get_option(&str, &nr_cpus);
+	if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
+		nr_cpu_ids = nr_cpus;
+
+	return 0;
+}
+
+early_param("nr_cpus", nrcpus);
+
 static int __init maxcpus(char *str)
 {
 	get_option(&str, &setup_max_cpus);
arch/ia64/kernel/acpi.c +2 -2
Index: linux-2.6/arch/ia64/kernel/acpi.c
===================================================================
@@ -883,8 +883,8 @@ __init void prefill_possible_map(void)
 
 	possible = available_cpus + additional_cpus;
 
-	if (possible > NR_CPUS)
-		possible = NR_CPUS;
+	if (possible > nr_cpu_ids)
+		possible = nr_cpu_ids;
 
 	printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
 		possible, max((possible - available_cpus), 0));
arch/x86/kernel/smpboot.c +4 -3
Index: linux-2.6/arch/x86/kernel/smpboot.c
===================================================================
@@ -1213,11 +1213,12 @@ __init void prefill_possible_map(void)
 
 	total_cpus = max_t(int, possible, num_processors + disabled_cpus);
 
-	if (possible > CONFIG_NR_CPUS) {
+	/* nr_cpu_ids could be reduced via nr_cpus= */
+	if (possible > nr_cpu_ids) {
 		printk(KERN_WARNING
 			"%d Processors exceeds NR_CPUS limit of %d\n",
-			possible, CONFIG_NR_CPUS);
-		possible = CONFIG_NR_CPUS;
+			possible, nr_cpu_ids);
+		possible = nr_cpu_ids;
 	}
 
 	printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
author
#1 lore raw

RFC PATCH 2/5 x86: use dmi check to treat disabled cpus as hotplug cpus.

some systems that have disable cpus entries because same
BIOS will support 2 sockets and 4 sockets and more at same time, BIOS just leave some disable entries, but those system do not support cpu hotplug. we don't need treat disabled_cpus as hotplug cpus.
so we can make nr_cpu_ids smaller and save more space
(pcpu data allocations), and could make some systems run with logical flat instead of physical flat apic mode

-v2: change to black list instead

---
 arch/x86/kernel/smpboot.c |   76 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 74 insertions(+), 2 deletions(-)
arch/x86/kernel/smpboot.c +74 -2
Index: linux-2.6/arch/x86/kernel/smpboot.c
===================================================================
@@ -47,6 +47,7 @@
 #include <linux/bootmem.h>
 #include <linux/err.h>
 #include <linux/nmi.h>
+#include <linux/dmi.h>
 #include <linux/tboot.h>
 
 #include <asm/acpi.h>
@@ -1180,6 +1181,59 @@ static int __init _setup_possible_cpus(c
 }
 early_param("possible_cpus", _setup_possible_cpus);
 
+static __initdata int treat_disabled_cpus_as_hotplug = 1;
+static __init int hotplug_cpus_check(const struct dmi_system_id *d)
+{
+        printk(KERN_NOTICE "%s detected: treat disabled cpus as hotplug ones\n", d->ident);
+        treat_disabled_cpus_as_hotplug = 0;
+
+        return 0;
+}
+
+static struct dmi_system_id hotplug_cpus_dmi_table[] __initdata = {
+        {
+                .callback = hotplug_cpus_check,
+                .ident = "Sun Microsystems Sun Fire X4440",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Sun Microsystems"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Sun Fire X4440"),
+                },
+        },
+        {
+                .callback = hotplug_cpus_check,
+                .ident = "Sun Microsystems Sun Fire X4240",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Sun Microsystems"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Sun Fire X4240"),
+                },
+        },
+        {
+                .callback = hotplug_cpus_check,
+                .ident = "Sun Microsystems Sun Fire X4140",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Sun Microsystems"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Sun Fire X4140"),
+                },
+        },
+        {
+                .callback = hotplug_cpus_check,
+                .ident = "Sun Microsystems Sun Fire X4600",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Sun Microsystems"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Sun Fire X4600"),
+                },
+        },
+        {
+                .callback = hotplug_cpus_check,
+                .ident = "Sun Microsystems Sun Fire X4640",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Sun Microsystems"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Sun Fire X4640"),
+                },
+        },
+        { } /* NULL entry stops DMI scanning */
+};
+
 
 /*
  * cpu_possible_mask should be static, it cannot change as cpu's
@@ -1206,8 +1260,26 @@ __init void prefill_possible_map(void)
 	if (!num_processors)
 		num_processors = 1;
 
-	if (setup_possible_cpus == -1)
-		possible = num_processors + disabled_cpus;
+	if (setup_possible_cpus == -1) {
+		possible = num_processors;
+		/*
+		 * do we have better way to detect hotplug cpus?
+		 *
+		 * some systems that have disable cpus entries because same
+		 *  BIOS will support 2 sockets and 4 sockets and more at
+		 *  same time, BIOS just leave some disabled entries with wild
+		 *  apicid, but those system do not support cpu hotplug.
+		 *  we don't need treat disabled_cpus as hotplug cpus.
+		 * so we can make nr_cpu_ids smaller and save more space
+		 *  (pcpu data allocations), and could make some systems run
+		 *  with logical flat instead of physical flat apic mode
+		 */
+		if (disabled_cpus) {
+			dmi_check_system(hotplug_cpus_dmi_table);
+			if (treat_disabled_cpus_as_hotplug)
+				possible += disabled_cpus;
+		}
+	}
 	else
 		possible = setup_possible_cpus;
 
author
#2 lore raw

RFC PATCH 3/5 x86: using logical flat for amd cpu too.

In reply to Yinghai Lu

not sure it works again, could be bios fix the irq routing dest ?

tested that amd support logical flat too when cpus num <= 8 and even bsp apic id > 8

so could remove vendor check...

---
 arch/x86/kernel/apic/apic.c     |   11 ++---------
 arch/x86/kernel/apic/probe_64.c |   13 ++-----------
 2 files changed, 4 insertions(+), 20 deletions(-)
arch/x86/kernel/apic/apic.c +2 -9
Index: linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================
@@ -1898,15 +1898,8 @@ void __cpuinit generic_processor_info(in
 		max_physical_apicid = apicid;
 
 #ifdef CONFIG_X86_32
-	switch (boot_cpu_data.x86_vendor) {
-	case X86_VENDOR_INTEL:
-		if (num_processors > 8)
-			def_to_bigsmp = 1;
-		break;
-	case X86_VENDOR_AMD:
-		if (max_physical_apicid >= 8)
-			def_to_bigsmp = 1;
-	}
+	if (num_processors > 8)
+		def_to_bigsmp = 1;
 #endif
 
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
arch/x86/kernel/apic/probe_64.c +2 -11
Index: linux-2.6/arch/x86/kernel/apic/probe_64.c
===================================================================
@@ -67,17 +67,8 @@ void __init default_setup_apic_routing(v
 	}
 #endif
 
-	if (apic == &apic_flat) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (num_processors > 8)
-				apic = &apic_physflat;
-			break;
-		case X86_VENDOR_AMD:
-			if (max_physical_apicid >= 8)
-				apic = &apic_physflat;
-		}
-	}
+	if (apic == &apic_flat && num_processors > 8)
+		apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
 
author
#3 lore raw

RFC PATCH 4/5 x86: according to nr_cpu_ids to decide if need to leave logical flat

In reply to Yinghai Lu

should use nr_cpu_ids instead of num_processor, in case we have hotplug cpus.

if current only have 8 cpus is up, but if we will have more cpus that will be hot added later, we should use physical flat at first.

nr_cpu_ids is the total cpus that could be supported.

-v2: per linus, chenage default_setup_apic_routing to _init, and call it for uni_processor

---
 arch/x86/kernel/apic/apic.c     |    2 --
 arch/x86/kernel/apic/probe_32.c |   20 ++++++++++++++++++--
 arch/x86/kernel/apic/probe_64.c |    3 ++-
 arch/x86/kernel/smpboot.c       |    2 --
 4 files changed, 20 insertions(+), 7 deletions(-)
arch/x86/kernel/apic/probe_32.c +18 -2
Index: linux-2.6/arch/x86/kernel/apic/probe_32.c
===================================================================
@@ -52,7 +52,7 @@ static int __init print_ipi_mode(void)
 }
 late_initcall(print_ipi_mode);
 
-void default_setup_apic_routing(void)
+static void local_default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_IO_APIC
 	printk(KERN_INFO
@@ -103,7 +103,7 @@ struct apic apic_default = {
 	.init_apic_ldr			= default_init_apic_ldr,
 
 	.ioapic_phys_id_map		= default_ioapic_phys_id_map,
-	.setup_apic_routing		= default_setup_apic_routing,
+	.setup_apic_routing		= local_default_setup_apic_routing,
 	.multi_timer_check		= NULL,
 	.apicid_to_node			= default_apicid_to_node,
 	.cpu_to_logical_apicid		= default_cpu_to_logical_apicid,
@@ -207,6 +207,22 @@ void __init generic_bigsmp_probe(void)
 			apic = &apic_bigsmp;
 			printk(KERN_INFO "Overriding APIC driver with %s\n",
 			       apic->name);
+		}
+	}
+#endif
+}
+
+void __init default_setup_apic_routing(void)
+{
+#ifdef CONFIG_X86_BIGSMP
+	/*
+	 * make sure we go to bigsmp according to real nr_cpu_ids
+	 */
+	if (!cmdline_apic && apic == &apic_default) {
+		if (nr_cpu_ids > 8) {
+			apic = &apic_bigsmp;
+			printk(KERN_INFO "Overriding APIC driver with %s\n",
+			       apic->name);
 		}
 	}
 #endif
arch/x86/kernel/smpboot.c +0 -2
Index: linux-2.6/arch/x86/kernel/smpboot.c
===================================================================
@@ -1084,9 +1084,7 @@ void __init native_smp_prepare_cpus(unsi
 	set_cpu_sibling_map(0);
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	if (smp_sanity_check(max_cpus) < 0) {
 		printk(KERN_INFO "SMP disabled\n");
arch/x86/kernel/apic/probe_64.c +2 -1
Index: linux-2.6/arch/x86/kernel/apic/probe_64.c
===================================================================
@@ -67,7 +67,8 @@ void __init default_setup_apic_routing(v
 	}
 #endif
 
-	if (apic == &apic_flat && num_processors > 8)
+	/* not just num_processors, we could have hotplug cpus */
+	if (apic == &apic_flat && nr_cpu_ids > 8)
 		apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
arch/x86/kernel/apic/apic.c +0 -2
Index: linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================
@@ -1647,9 +1647,7 @@ int __init APIC_init_uniprocessor(void)
 #endif
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	verify_local_APIC();
 	connect_bsp_APIC();
author
#4 lore raw

RFC PATCH 5/5 x86: make 32bit apic flat to physflat switch like 64bit

In reply to Yinghai Lu

kill def_to_bigsmp
and move switch from default to bigsmp at default_setup_apic_routing... so make default_setup_apic_routing more like 64 bit
also make the dmi relate code to be __init/__initdata

---
 arch/x86/include/asm/apic.h      |    3 -
 arch/x86/include/asm/mpspec.h    |    1 
 arch/x86/kernel/acpi/boot.c      |    3 -
 arch/x86/kernel/apic/apic.c      |    5 --
 arch/x86/kernel/apic/bigsmp_32.c |   48 -----------------------
 arch/x86/kernel/apic/probe_32.c  |   80 ++++++++++++++++++++-------------------
 arch/x86/kernel/mpparse.c        |    4 -
 arch/x86/kernel/setup.c          |    2 
 arch/x86/kernel/smpboot.c        |    2 
 9 files changed, 46 insertions(+), 102 deletions(-)
arch/x86/include/asm/mpspec.h +0 -1
Index: linux-2.6/arch/x86/include/asm/mpspec.h
===================================================================
@@ -23,7 +23,6 @@ extern int pic_mode;
 
 #define MAX_IRQ_SOURCES		256
 
-extern unsigned int def_to_bigsmp;
 extern u8 apicid_2_node[];
 
 #ifdef CONFIG_X86_NUMAQ
arch/x86/kernel/apic/apic.c +0 -5
Index: linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================
@@ -1895,11 +1895,6 @@ void __cpuinit generic_processor_info(in
 	if (apicid > max_physical_apicid)
 		max_physical_apicid = apicid;
 
-#ifdef CONFIG_X86_32
-	if (num_processors > 8)
-		def_to_bigsmp = 1;
-#endif
-
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
 	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/bigsmp_32.c +2 -46

81 lines not on this page · raw message

arch/x86/kernel/apic/probe_32.c +43 -37

138 lines not on this page · raw message

arch/x86/kernel/setup.c +0 -2
Index: linux-2.6/arch/x86/kernel/setup.c
===================================================================
@@ -184,8 +184,6 @@ static void set_mca_bus(int x)
 #endif
 }
 
-unsigned int def_to_bigsmp;
-
 /* for MCA, but anyone else can use it if they want */
 unsigned int machine_id;
 unsigned int machine_submodel_id;
arch/x86/kernel/smpboot.c +1 -1
Index: linux-2.6/arch/x86/kernel/smpboot.c
===================================================================
@@ -947,7 +947,7 @@ static int __init smp_sanity_check(unsig
 	preempt_disable();
 
 #if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
-	if (def_to_bigsmp && nr_cpu_ids > 8) {
+	if (apic == &apic_default && nr_cpu_ids > 8) {
 		unsigned int cpu;
 		unsigned nr;
 
arch/x86/include/asm/apic.h +0 -3
Index: linux-2.6/arch/x86/include/asm/apic.h
===================================================================
@@ -456,9 +456,6 @@ static inline void default_wait_for_init
 	return;
 }
 
-extern void generic_bigsmp_probe(void);
-
-
 #ifdef CONFIG_X86_LOCAL_APIC
 
 #include <asm/smp.h>
arch/x86/kernel/acpi/boot.c +0 -3
Index: linux-2.6/arch/x86/kernel/acpi/boot.c
===================================================================
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(voi
 		if (!error) {
 			acpi_lapic = 1;
 
-#ifdef CONFIG_X86_BIGSMP
-			generic_bigsmp_probe();
-#endif
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
arch/x86/kernel/mpparse.c +0 -4
Index: linux-2.6/arch/x86/kernel/mpparse.c
===================================================================
@@ -359,10 +359,6 @@ static int __init smp_read_mpc(struct mp
 		x86_init.mpparse.mpc_record(1);
 	}
 
-#ifdef CONFIG_X86_BIGSMP
-	generic_bigsmp_probe();
-#endif
-
 	if (apic->setup_apic_routing)
 		apic->setup_apic_routing();
 
#5 lore raw
In reply to Yinghai Lu
Quoted 4 lines · Yinghai Lu

on x86, before prefill_possible_map(), nr_cpu_ids will be NR_CPUS aka CONFIG_NR_CPUS

add nr_cpus= to set nr_cpu_ids. so we can simulate cpus <=8 on normal config. instead of change NR_CPUS directly.

We already have

maxcpus=x?

Why do we need this twice?

author
#6 lore raw
Quoted 5 lines · Christoph Lameter
Quoted 2 lines · Yinghai Lu

on x86, before prefill_possible_map(), nr_cpu_ids will be NR_CPUS aka CONFIG_NR_CPUS

Quoted 8 lines
Quoted 2 lines

add nr_cpus= to set nr_cpu_ids. so we can simulate cpus <=8 on normal config. instead of change NR_CPUS directly.

We already have

maxcpus=x?

Why do we need this twice?

maxcpus only change setup_max_cpus., and if you are using maxcpus=1, and you have 8 cpus installed, you can put other
cpus back online via /sys/interface.

nr_cpus= is hard limit nr_cpu_ids, so if you have 16 cpus installed, nr_cpus=8 will make your nr_cpu_ids=8, and you can not put back other 8 back. and apic mode could stay with logical flat. this is used to simulate some debug case. for example you have kernel support physflat, and flat, with CONFIG_NR_CPUS=255. to run that on system that only have 8 cpus, you will have apic mode in logical flat.
but if you have run the kernel on system with 32 cpus installed, it
will switch to physflat even you have maxcpus=8 appedded.

YH

#7 lore raw
Quoted 5 lines · Yinghai Lu
Quoted 1 line

Why do we need this twice?

maxcpus only change setup_max_cpus., and if you are using maxcpus=1, and you have 8 cpus installed, you can put other
cpus back online via /sys/interface.

Hmmm.. Strange semantics since maxcpus=0 disables smp completely. No cpu can be activated later. Similar to nr_cpus ?

Quoted 8 lines

nr_cpus= is hard limit nr_cpu_ids, so if you have 16 cpus installed, nr_cpus=8 will make your nr_cpu_ids=8, and you can not put back other 8 back. and apic mode could stay with logical flat. this is used to simulate some debug case. for example you have kernel support physflat, and flat, with CONFIG_NR_CPUS=255. to run that on system that only have 8 cpus, you will have apic mode in logical flat.
but if you have run the kernel on system with 32 cpus installed, it
will switch to physflat even you have maxcpus=8 appedded.

Ok makes sense.

author
#8 lore raw
Quoted 10 lines · Christoph Lameter
Quoted 5 lines · Yinghai Lu
Quoted 1 line

Why do we need this twice?

maxcpus only change setup_max_cpus., and if you are using maxcpus=1, and you have 8 cpus installed, you can put other
cpus back online via /sys/interface.

Hmmm.. Strange semantics since maxcpus=0 disables smp completely. No cpu can be activated later. Similar to nr_cpus ?

no
for SMP kernel, when nr_cpus=1 is specified, it looks like only one cpu is installed physically.
smp is still enabled.

maxcpus=0 is really caused misunderstanding.

other maxcpus=1 and other should be rename to boot_online_cpus=

Quoted 11 lines
Quoted 9 lines

nr_cpus= is hard limit nr_cpu_ids, so if you have 16 cpus installed, nr_cpus=8 will make your nr_cpu_ids=8, and you can not put back other 8 back. and apic mode could stay with logical flat. this is used to simulate some debug case. for example you have kernel support physflat, and flat, with CONFIG_NR_CPUS=255. to run that on system that only have 8 cpus, you will have apic mode in logical flat. but if you have run the kernel on system with 32 cpus installed, it will switch to physflat even you have maxcpus=8 appedded.

Ok makes sense.

good.

YH

#9 lore raw
In reply to Yinghai Lu

Trouble is that NR_CPUS is used all over the place. If nr_cpu_ids < NR_CPUS at boot then there is a danger of for loops to NR_CPUS going out of
bounds.

CONFIG_NR_CPUS is used for various bitmaps so that seems to be okay.

drivers/acpi/numa.c:acpi_numa_init probably should use nr_cpu_ids instead now.

There is an octeon driver in staging that has some issues with NR_CPUS as well.

#10 lore raw
Quoted 5 lines · Christoph Lameter

Trouble is that NR_CPUS is used all over the place. If nr_cpu_ids < NR_CPUS at boot then there is a danger of for loops to NR_CPUS going out of bounds.

Quoted 7 lines

CONFIG_NR_CPUS is used for various bitmaps so that seems to be okay.

drivers/acpi/numa.c:acpi_numa_init probably should use nr_cpu_ids instead now.

There is an octeon driver in staging that has some issues with NR_CPUS as well.

I suspect that this is the real reason for the current behavior of 'maxcpus=', and that if all of those issues get fixed we could probably make maxcpus do what Yinghai's new 'nr_cpus=' does.

So in a perfect world, CONFIG_NR_CPUS's would never really be used, except for some fundamental static allocations/limits that are too painful to try to make dynamic.

I doubt that anybody really _cares_ about the "you can add them later" behavior of the current maxcpus thing, and I suspect that the nr_cpus semantics is what people generally would have expected.

Linus

author
#11 lore raw
In reply to Christoph Lameter
Quoted 7 lines · Christoph Lameter

Trouble is that NR_CPUS is used all over the place. If nr_cpu_ids < NR_CPUS at boot then there is a danger of for loops to NR_CPUS going out of
bounds.

CONFIG_NR_CPUS is used for various bitmaps so that seems to be okay.

cpumask_var?

Quoted 3 lines

drivers/acpi/numa.c:acpi_numa_init probably should use nr_cpu_ids instead now.

before prefill_possible_map for x86
before setup_nr_cpu_ids for other platform

nr_cpu_ids = NR_CPUS = CONFIG_NR_CPUS

Quoted 4 lines

There is an octeon driver in staging that has some issues with NR_CPUS as well.

that should be fixed, we should use nr_cpu_ids instead for them.

YH