Skip to contentSkip to conversation

x86, apic: use physical mode for IBM summit platforms

PATCH In review Suresh Siddha 3 participants 11 messages Jan 18, 2010 – Feb 10, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 x86, apic: use physical mode for IBM summit platforms

Chris McDermott from IBM confirmed that hurricane chipset in IBM summit platforms doesn't support logical flat mode. Irrespective of the other things like apic_id's, total number of logical cpu's, Linux kernel should default to physical mode for this system. 32bit kernel does so using the OEM checks for the IBM summit platform. Add a similar OEM platform check for the 64bit kernel too. Otherwise the linux kernel boot can hang on this platform under certain bios/platform settings.

---
 arch/x86/kernel/apic/apic_flat_64.c |    5 +++++
 1 file changed, 5 insertions(+)
arch/x86/kernel/apic/apic_flat_64.c +5 -0
Index: tip/arch/x86/kernel/apic/apic_flat_64.c
===================================================================
@@ -240,6 +240,11 @@ static int physflat_acpi_madt_oem_check(
 		printk(KERN_DEBUG "system APIC only can use physical flat");
 		return 1;
 	}
+
+	if (!strncmp(oem_id, "IBM", 3) && !strncmp(oem_table_id, "EXA", 3)) {
+		printk(KERN_DEBUG "IBM Summit detected, will use apic physical");
+		return 1;
+	}
 #endif
 
 	return 0;
#1 lore raw

PATCH 2/2 x86, apic: use logical flat for systems with <= 8 logical cpus

We can use logical flat mode if there are <= 8 logical cpu's (irrespective of physical apic id values). This will enable simplified and efficient IPI and device interrupt routing on such platforms.

This has been tested to work on both Intel and AMD platforms. Exceptions like IBM summit platform which can't use logical flat mode are addressed by using OEM platform checks.

Cc Ananth N Mavinakayanahalli <[email protected]>
Cc Chris McDermott <[email protected]>
---
 arch/x86/kernel/apic/apic.c     |   15 +--------------
 arch/x86/kernel/apic/probe_64.c |    8 +++-----
 2 files changed, 4 insertions(+), 19 deletions(-)
arch/x86/kernel/apic/apic.c +1 -14
Index: tip/arch/x86/kernel/apic/apic.c
===================================================================
@@ -61,12 +61,6 @@ unsigned int boot_cpu_physical_apicid = 
 
 /*
  * The highest APIC ID seen during enumeration.
- *
- * This determines the messaging protocol we can use: if all APIC IDs
- * are in the 0 ... 7 range, then we can use logical addressing which
- * has some performance advantages (better broadcasting).
- *
- * If there's an APIC ID above 8, we use physical addressing.
  */
 unsigned int max_physical_apicid;
 
@@ -1898,14 +1892,7 @@ void __cpuinit generic_processor_info(in
 		max_physical_apicid = apicid;
 
 #ifdef CONFIG_X86_32
-	/*
-	 * Would be preferable to switch to bigsmp when CONFIG_HOTPLUG_CPU=y
-	 * but we need to work other dependencies like SMP_SUSPEND etc
-	 * before this can be done without some confusion.
-	 * if (CPU_HOTPLUG_ENABLED || num_processors > 8)
-	 *       - Ashok Raj <[email protected]>
-	 */
-	if (max_physical_apicid >= 8) {
+	if (num_processors > 8) {
 		switch (boot_cpu_data.x86_vendor) {
 		case X86_VENDOR_INTEL:
 			if (!APIC_XAPIC(version)) {
arch/x86/kernel/apic/probe_64.c +3 -5
Index: tip/arch/x86/kernel/apic/probe_64.c
===================================================================
@@ -64,15 +64,13 @@ void __init default_setup_apic_routing(v
 			apic = &apic_x2apic_phys;
 		else
 			apic = &apic_x2apic_cluster;
-		printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
 	}
 #endif
 
-	if (apic == &apic_flat) {
-		if (max_physical_apicid >= 8)
+	if (apic == &apic_flat && num_processors > 8)
 			apic = &apic_physflat;
-		printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
-	}
+
+	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
 
 	if (is_vsmp_box()) {
 		/* need to update phys_pkg_id */
#2 lore raw

[tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Commit-ID:  418f727639cfc8f03068aae09f473c493c759b9b
Gitweb:     http://git.kernel.org/tip/418f727639cfc8f03068aae09f473c493c759b9b
Author:     Suresh Siddha <[email protected]>
AuthorDate: Tue, 9 Feb 2010 09:18:35 -0800
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Tue, 9 Feb 2010 09:29:17 -0800

x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug, we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

LKML-Reference: <[email protected]>

---
 arch/x86/kernel/apic/apic.c     |    2 +-
 arch/x86/kernel/apic/probe_64.c |    2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/apic/apic.c +1 -1
index 3987e44..fc1d579 100644
@@ -1892,7 +1892,7 @@ void __cpuinit generic_processor_info(int apicid, int version)
 		max_physical_apicid = apicid;
 
 #ifdef CONFIG_X86_32
-	if (num_processors > 8) {
+	if (num_possible_cpus() > 8) {
 		switch (boot_cpu_data.x86_vendor) {
 		case X86_VENDOR_INTEL:
 			if (!APIC_XAPIC(version)) {
arch/x86/kernel/apic/probe_64.c +1 -1
index 450fe20..83e9be4 100644
@@ -67,7 +67,7 @@ void __init default_setup_apic_routing(void)
 	}
 #endif
 
-	if (apic == &apic_flat && num_processors > 8)
+	if (apic == &apic_flat && num_possible_cpus() > 8)
 			apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
#3 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Quoted 31 lines · tip-bot for Suresh Siddha
Commit-ID:  418f727639cfc8f03068aae09f473c493c759b9b
Gitweb:     http://git.kernel.org/tip/418f727639cfc8f03068aae09f473c493c759b9b
Author:     Suresh Siddha <[email protected]>
AuthorDate: Tue, 9 Feb 2010 09:18:35 -0800
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Tue, 9 Feb 2010 09:29:17 -0800

x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug, we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

LKML-Reference: <[email protected]>

---
 arch/x86/kernel/apic/apic.c     |    2 +-
 arch/x86/kernel/apic/probe_64.c |    2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/apic/apic.c +0 -0
index 3987e44..fc1d579 100644

arch/x86/kernel/apic/apic.c void __cpuinit generic_processor_info(int apicid, int version) · line 1895

Quoted 5 lines
arch/x86/kernel/apic/apic.c +1 -1
@@ -1892,7 +1892,7 @@ void __cpuinit generic_processor_info(int apicid, int version)
 		max_physical_apicid = apicid;
 
 #ifdef CONFIG_X86_32
-	if (num_processors > 8) {
+	if (num_possible_cpus() > 8) {

for 32bit you can not use this function yet. that only can be used after prefill_possible_map()

Quoted 8 lines
arch/x86/kernel/apic/apic.c +0 -0
 		switch (boot_cpu_data.x86_vendor) {
 		case X86_VENDOR_INTEL:
 			if (!APIC_XAPIC(version)) {
arch/x86/kernel/apic/probe_64.c +0 -0
index 450fe20..83e9be4 100644

arch/x86/kernel/apic/probe_64.c void __init default_setup_apic_routing(void) · line 73

Quoted 8 lines
arch/x86/kernel/apic/probe_64.c +1 -1
@@ -67,7 +67,7 @@ void __init default_setup_apic_routing(void)
 	}
 #endif
 
-	if (apic == &apic_flat && num_processors > 8)
+	if (apic == &apic_flat && num_possible_cpus() > 8)
 			apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);

please check
http://patchwork.kernel.org/patch/74536/
http://patchwork.kernel.org/patch/74533/
http://patchwork.kernel.org/patch/74525/
http://patchwork.kernel.org/patch/74524/

YH

author Acked-by Shaohui Zheng
#4 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Quoted 6 lines · Yinghai Lu

Review

Quoted 3 lines

#ifdef CONFIG_X86_32

(quoted diff) +1 -1
-	if (num_processors > 8) {
+	if (num_possible_cpus() > 8) {

for 32bit you can not use this function yet. that only can be used after prefill_possible_map()

Yinghai, Agreed. How about the appended patch? I tested and it works. Peter wants a small quick fix for this issue (to resolve the boot issue reported in the virtualization guest context) so that we can queue it for .33 and .32 kernels (as some distributions will be based on these kernels).

Can you please Ack if it is ok?

thanks,
suresh
---

From: Suresh Siddha <[email protected]>
Subject: x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug(with bios listing not enabled but possible cpus as disabled cpus in MADT), we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

32bit apic code can use more cleanups (like the removal of vendor checks in 32bit default_setup_apic_routing()) and more unifications with 64bit code. Yinghai has some patches in works already. This patch addresses the boot issue that is reported in the virtualization guest context.

---

arch/x86/kernel/acpi/boot.c +0 -5
index 036d28a..0acbcdf 100644
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(void)
 		if (!error) {
 			acpi_lapic = 1;
 
-#ifdef CONFIG_X86_BIGSMP
-			generic_bigsmp_probe();
-#endif
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
@@ -1197,8 +1194,6 @@ static void __init acpi_process_madt(void)
 				acpi_ioapic = 1;
 
 				smp_found_config = 1;
-				if (apic->setup_apic_routing)
-					apic->setup_apic_routing();
 			}
 		}
 		if (error == -EINVAL) {
arch/x86/kernel/apic/apic.c +0 -17
index a85f216..6e29b2a 100644
@@ -1641,9 +1641,7 @@ int __init APIC_init_uniprocessor(void)
 #endif
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	verify_local_APIC();
 	connect_bsp_APIC();
@@ -1891,21 +1889,6 @@ void __cpuinit generic_processor_info(int apicid, int version)
 	if (apicid > max_physical_apicid)
 		max_physical_apicid = apicid;
 
-#ifdef CONFIG_X86_32
-	if (num_processors > 8) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (!APIC_XAPIC(version)) {
-				def_to_bigsmp = 0;
-				break;
-			}
-			/* If P4 and above fall through */
-		case X86_VENDOR_AMD:
-			def_to_bigsmp = 1;
-		}
-	}
-#endif
-
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
 	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/probe_32.c +26 -1
index 1a6559f..88e78ae 100644
@@ -54,6 +54,31 @@ late_initcall(print_ipi_mode);
 
 void default_setup_apic_routing(void)
 {
+	int version = apic_version[boot_cpu_physical_apicid];
+
+	if (num_possible_cpus() > 8) {
+		switch (boot_cpu_data.x86_vendor) {
+		case X86_VENDOR_INTEL:
+			if (!APIC_XAPIC(version)) {
+				def_to_bigsmp = 0;
+				break;
+			}
+			/* If P4 and above fall through */
+		case X86_VENDOR_AMD:
+			def_to_bigsmp = 1;
+		}
+	}
+
+#ifdef CONFIG_X86_BIGSMP
+	generic_bigsmp_probe();
+#endif
+
+	if (apic->setup_apic_routing)
+		apic->setup_apic_routing();
+}
+
+void setup_apic_flat_routing(void)
+{
 #ifdef CONFIG_X86_IO_APIC
 	printk(KERN_INFO
 		"Enabling APIC mode:  Flat.  Using %d I/O APICs\n",
@@ -103,7 +128,7 @@ struct apic apic_default = {
 	.init_apic_ldr			= default_init_apic_ldr,
 
 	.ioapic_phys_id_map		= default_ioapic_phys_id_map,
-	.setup_apic_routing		= default_setup_apic_routing,
+	.setup_apic_routing		= setup_apic_flat_routing,
 	.multi_timer_check		= NULL,
 	.apicid_to_node			= default_apicid_to_node,
 	.cpu_to_logical_apicid		= default_cpu_to_logical_apicid,
arch/x86/kernel/apic/probe_64.c +1 -1
index 450fe20..83e9be4 100644
@@ -67,7 +67,7 @@ void __init default_setup_apic_routing(void)
 	}
 #endif
 
-	if (apic == &apic_flat && num_processors > 8)
+	if (apic == &apic_flat && num_possible_cpus() > 8)
 			apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
arch/x86/kernel/mpparse.c +0 -7
index 40b54ce..a2c1edd 100644
@@ -359,13 +359,6 @@ static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
 		x86_init.mpparse.mpc_record(1);
 	}
 
-#ifdef CONFIG_X86_BIGSMP
-	generic_bigsmp_probe();
-#endif
-
-	if (apic->setup_apic_routing)
-		apic->setup_apic_routing();
-
 	if (!num_processors)
 		printk(KERN_ERR "MPTABLE: no processors registered!\n");
 	return num_processors;
arch/x86/kernel/smpboot.c +0 -2
index b2ebcba..c08829a 100644
@@ -1087,9 +1087,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	set_cpu_sibling_map(0);
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	if (smp_sanity_check(max_cpus) < 0) {
 		printk(KERN_INFO "SMP disabled\n");
#5 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Quoted 118 lines · Suresh Siddha
Quoted 6 lines · Yinghai Lu

Review

Quoted 3 lines

#ifdef CONFIG_X86_32

(quoted diff) +1 -1
-	if (num_processors > 8) {
+	if (num_possible_cpus() > 8) {

for 32bit you can not use this function yet. that only can be used after prefill_possible_map()

Yinghai, Agreed. How about the appended patch? I tested and it works. Peter wants a small quick fix for this issue (to resolve the boot issue reported in the virtualization guest context) so that we can queue it for .33 and .32 kernels (as some distributions will be based on these kernels).

Can you please Ack if it is ok?

thanks,
suresh
---

From: Suresh Siddha <[email protected]>
Subject: x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug(with bios listing not enabled but possible cpus as disabled cpus in MADT), we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

32bit apic code can use more cleanups (like the removal of vendor checks in 32bit default_setup_apic_routing()) and more unifications with 64bit code. Yinghai has some patches in works already. This patch addresses the boot issue that is reported in the virtualization guest context.

---

arch/x86/kernel/acpi/boot.c +0 -5
index 036d28a..0acbcdf 100644
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(void)
 		if (!error) {
 			acpi_lapic = 1;
 
-#ifdef CONFIG_X86_BIGSMP
-			generic_bigsmp_probe();
-#endif
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
@@ -1197,8 +1194,6 @@ static void __init acpi_process_madt(void)
 				acpi_ioapic = 1;
 
 				smp_found_config = 1;
-				if (apic->setup_apic_routing)
-					apic->setup_apic_routing();
 			}
 		}
 		if (error == -EINVAL) {
arch/x86/kernel/apic/apic.c +0 -17
index a85f216..6e29b2a 100644
@@ -1641,9 +1641,7 @@ int __init APIC_init_uniprocessor(void)
 #endif
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	verify_local_APIC();
 	connect_bsp_APIC();
@@ -1891,21 +1889,6 @@ void __cpuinit generic_processor_info(int apicid, int version)
 	if (apicid > max_physical_apicid)
 		max_physical_apicid = apicid;
 
-#ifdef CONFIG_X86_32
-	if (num_processors > 8) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (!APIC_XAPIC(version)) {
-				def_to_bigsmp = 0;
-				break;
-			}
-			/* If P4 and above fall through */
-		case X86_VENDOR_AMD:
-			def_to_bigsmp = 1;
-		}
-	}
-#endif
-
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
 	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/probe_32.c +11 -0
index 1a6559f..88e78ae 100644
@@ -54,6 +54,31 @@ late_initcall(print_ipi_mode);
 
 void default_setup_apic_routing(void)
 {
+	int version = apic_version[boot_cpu_physical_apicid];
+
+	if (num_possible_cpus() > 8) {
+		switch (boot_cpu_data.x86_vendor) {
+		case X86_VENDOR_INTEL:
+			if (!APIC_XAPIC(version)) {
+				def_to_bigsmp = 0;
+				break;
+			}
+			/* If P4 and above fall through */
+		case X86_VENDOR_AMD:

arch/x86/kernel/apic/probe_32.c late_initcall(print_ipi_mode); · line 77

Quoted 10 lines
arch/x86/kernel/apic/probe_32.c +10 -0
+			def_to_bigsmp = 1;
+		}
+	}
+
+#ifdef CONFIG_X86_BIGSMP
+	generic_bigsmp_probe();
+#endif
+
+	if (apic->setup_apic_routing)
+		apic->setup_apic_routing();

the main difference between this patch and
http://patchwork.kernel.org/patch/74525/

is moving apic->setup_apic_routing calling.

wonder if it will affect subarch other than logical flat and physical flat.

YH

Quoted 67 lines
arch/x86/kernel/apic/probe_32.c +5 -1
+}
+
+void setup_apic_flat_routing(void)
+{
 #ifdef CONFIG_X86_IO_APIC
 	printk(KERN_INFO
 		"Enabling APIC mode:  Flat.  Using %d I/O APICs\n",
@@ -103,7 +128,7 @@ struct apic apic_default = {
 	.init_apic_ldr			= default_init_apic_ldr,
 
 	.ioapic_phys_id_map		= default_ioapic_phys_id_map,
-	.setup_apic_routing		= default_setup_apic_routing,
+	.setup_apic_routing		= setup_apic_flat_routing,
 	.multi_timer_check		= NULL,
 	.apicid_to_node			= default_apicid_to_node,
 	.cpu_to_logical_apicid		= default_cpu_to_logical_apicid,
arch/x86/kernel/apic/probe_64.c +1 -1
index 450fe20..83e9be4 100644
@@ -67,7 +67,7 @@ void __init default_setup_apic_routing(void)
 	}
 #endif
 
-	if (apic == &apic_flat && num_processors > 8)
+	if (apic == &apic_flat && num_possible_cpus() > 8)
 			apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
arch/x86/kernel/mpparse.c +0 -7
index 40b54ce..a2c1edd 100644
@@ -359,13 +359,6 @@ static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
 		x86_init.mpparse.mpc_record(1);
 	}
 
-#ifdef CONFIG_X86_BIGSMP
-	generic_bigsmp_probe();
-#endif
-
-	if (apic->setup_apic_routing)
-		apic->setup_apic_routing();
-
 	if (!num_processors)
 		printk(KERN_ERR "MPTABLE: no processors registered!\n");
 	return num_processors;
arch/x86/kernel/smpboot.c +0 -3
index b2ebcba..c08829a 100644
@@ -1087,9 +1087,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	set_cpu_sibling_map(0);
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	if (smp_sanity_check(max_cpus) < 0) {
 		printk(KERN_INFO "SMP disabled\n");
 
 
--

To unsubscribe from this list: send the line "unsubscribe linux-tip-commits" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#6 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Quoted 133 lines · Yinghai Lu
Quoted 118 lines · Suresh Siddha
Quoted 6 lines · Yinghai Lu

Review

Quoted 3 lines

#ifdef CONFIG_X86_32

(quoted diff) +1 -1
-	if (num_processors > 8) {
+	if (num_possible_cpus() > 8) {

for 32bit you can not use this function yet. that only can be used after prefill_possible_map()

Yinghai, Agreed. How about the appended patch? I tested and it works. Peter wants a small quick fix for this issue (to resolve the boot issue reported in the virtualization guest context) so that we can queue it for .33 and .32 kernels (as some distributions will be based on these kernels).

Can you please Ack if it is ok?

thanks,
suresh
---

From: Suresh Siddha <[email protected]>
Subject: x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug(with bios listing not enabled but possible cpus as disabled cpus in MADT), we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

32bit apic code can use more cleanups (like the removal of vendor checks in 32bit default_setup_apic_routing()) and more unifications with 64bit code. Yinghai has some patches in works already. This patch addresses the boot issue that is reported in the virtualization guest context.

---

arch/x86/kernel/acpi/boot.c +0 -5
index 036d28a..0acbcdf 100644
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(void)
 		if (!error) {
 			acpi_lapic = 1;
 
-#ifdef CONFIG_X86_BIGSMP
-			generic_bigsmp_probe();
-#endif
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
@@ -1197,8 +1194,6 @@ static void __init acpi_process_madt(void)
 				acpi_ioapic = 1;
 
 				smp_found_config = 1;
-				if (apic->setup_apic_routing)
-					apic->setup_apic_routing();
 			}
 		}
 		if (error == -EINVAL) {
arch/x86/kernel/apic/apic.c +0 -17
index a85f216..6e29b2a 100644
@@ -1641,9 +1641,7 @@ int __init APIC_init_uniprocessor(void)
 #endif
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	verify_local_APIC();
 	connect_bsp_APIC();
@@ -1891,21 +1889,6 @@ void __cpuinit generic_processor_info(int apicid, int version)
 	if (apicid > max_physical_apicid)
 		max_physical_apicid = apicid;
 
-#ifdef CONFIG_X86_32
-	if (num_processors > 8) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (!APIC_XAPIC(version)) {
-				def_to_bigsmp = 0;
-				break;
-			}
-			/* If P4 and above fall through */
-		case X86_VENDOR_AMD:
-			def_to_bigsmp = 1;
-		}
-	}
-#endif
-
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
 	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/probe_32.c +11 -0
index 1a6559f..88e78ae 100644
@@ -54,6 +54,31 @@ late_initcall(print_ipi_mode);
 
 void default_setup_apic_routing(void)
 {
+	int version = apic_version[boot_cpu_physical_apicid];
+
+	if (num_possible_cpus() > 8) {
+		switch (boot_cpu_data.x86_vendor) {
+		case X86_VENDOR_INTEL:
+			if (!APIC_XAPIC(version)) {
+				def_to_bigsmp = 0;
+				break;
+			}
+			/* If P4 and above fall through */
+		case X86_VENDOR_AMD:

arch/x86/kernel/apic/probe_32.c late_initcall(print_ipi_mode); · line 77

Quoted 10 lines
arch/x86/kernel/apic/probe_32.c +10 -0
+			def_to_bigsmp = 1;
+		}
+	}
+
+#ifdef CONFIG_X86_BIGSMP
+	generic_bigsmp_probe();
+#endif
+
+	if (apic->setup_apic_routing)
+		apic->setup_apic_routing();

the main difference between this patch and
http://patchwork.kernel.org/patch/74525/

Quoted 3 lines

is moving apic->setup_apic_routing calling.

wonder if it will affect subarch other than logical flat and physical flat.

other subarch setup_apic_routing is just pr_info...

so your patch should be ok.

YH

#7 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Quoted 5 lines · Yinghai Lu

other subarch setup_apic_routing is just pr_info...

so your patch should be ok.

Great, can I take that as an Acked-by: or Reviewed-by: from you?

I really appreciate the review.

-hpa

#8 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

Quoted 9 lines · H. Peter Anvin
Quoted 5 lines · Yinghai Lu

other subarch setup_apic_routing is just pr_info...

so your patch should be ok.

Great, can I take that as an Acked-by: or Reviewed-by: from you?

sure.

#9 lore raw

Re: [tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

In reply to Suresh Siddha
Quoted 104 lines · Suresh Siddha
Quoted 6 lines · Yinghai Lu

Review

Quoted 3 lines

#ifdef CONFIG_X86_32

(quoted diff) +1 -1
-	if (num_processors > 8) {
+	if (num_possible_cpus() > 8) {

for 32bit you can not use this function yet. that only can be used after prefill_possible_map()

Yinghai, Agreed. How about the appended patch? I tested and it works. Peter wants a small quick fix for this issue (to resolve the boot issue reported in the virtualization guest context) so that we can queue it for .33 and .32 kernels (as some distributions will be based on these kernels).

Can you please Ack if it is ok?

thanks,
suresh
---

From: Suresh Siddha <[email protected]>
Subject: x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug(with bios listing not enabled but possible cpus as disabled cpus in MADT), we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

32bit apic code can use more cleanups (like the removal of vendor checks in 32bit default_setup_apic_routing()) and more unifications with 64bit code. Yinghai has some patches in works already. This patch addresses the boot issue that is reported in the virtualization guest context.

---

arch/x86/kernel/acpi/boot.c +0 -5
index 036d28a..0acbcdf 100644
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(void)
 		if (!error) {
 			acpi_lapic = 1;
 
-#ifdef CONFIG_X86_BIGSMP
-			generic_bigsmp_probe();
-#endif
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
@@ -1197,8 +1194,6 @@ static void __init acpi_process_madt(void)
 				acpi_ioapic = 1;
 
 				smp_found_config = 1;
-				if (apic->setup_apic_routing)
-					apic->setup_apic_routing();
 			}
 		}
 		if (error == -EINVAL) {
arch/x86/kernel/apic/apic.c +0 -17
index a85f216..6e29b2a 100644
@@ -1641,9 +1641,7 @@ int __init APIC_init_uniprocessor(void)
 #endif
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	verify_local_APIC();
 	connect_bsp_APIC();
@@ -1891,21 +1889,6 @@ void __cpuinit generic_processor_info(int apicid, int version)
 	if (apicid > max_physical_apicid)
 		max_physical_apicid = apicid;
 
-#ifdef CONFIG_X86_32
-	if (num_processors > 8) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (!APIC_XAPIC(version)) {
-				def_to_bigsmp = 0;
-				break;
-			}
-			/* If P4 and above fall through */
-		case X86_VENDOR_AMD:
-			def_to_bigsmp = 1;
-		}
-	}
-#endif
-
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
 	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/probe_32.c +0 -0
index 1a6559f..88e78ae 100644

arch/x86/kernel/apic/probe_32.c late_initcall(print_ipi_mode); · line 55

Quoted 2 lines
arch/x86/kernel/apic/probe_32.c +0 -0
@@ -54,6 +54,31 @@ late_initcall(print_ipi_mode);
 
 void default_setup_apic_routing(void)

__init

Quoted 15 lines
arch/x86/kernel/apic/probe_32.c +14 -0
 {
+	int version = apic_version[boot_cpu_physical_apicid];
+
+	if (num_possible_cpus() > 8) {
+		switch (boot_cpu_data.x86_vendor) {
+		case X86_VENDOR_INTEL:
+			if (!APIC_XAPIC(version)) {
+				def_to_bigsmp = 0;
+				break;
+			}
+			/* If P4 and above fall through */
+		case X86_VENDOR_AMD:
+			def_to_bigsmp = 1;
+		}
+	}

arch/x86/kernel/apic/probe_32.c late_initcall(print_ipi_mode); · line 80

Quoted 10 lines
arch/x86/kernel/apic/probe_32.c +10 -0
+
+#ifdef CONFIG_X86_BIGSMP
+	generic_bigsmp_probe();
+#endif
+
+	if (apic->setup_apic_routing)
+		apic->setup_apic_routing();
+}
+
+void setup_apic_flat_routing(void)

static?

YH

#10 lore raw

[tip:x86/urgent] x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

In reply to Suresh Siddha
Commit-ID:  681ee44d40d7c93b42118320e4620d07d8704fd6
Gitweb:     http://git.kernel.org/tip/681ee44d40d7c93b42118320e4620d07d8704fd6
Author:     Suresh Siddha <[email protected]>
AuthorDate: Tue, 9 Feb 2010 18:01:44 -0800
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Tue, 9 Feb 2010 20:51:11 -0800

x86, apic: Don't use logical-flat mode when CPU hotplug may exceed 8 CPUs

We need to fall back from logical-flat APIC mode to physical-flat mode when we have more than 8 CPUs. However, in the presence of CPU hotplug(with bios listing not enabled but possible cpus as disabled cpus in MADT), we have to consider the number of possible CPUs rather than the number of current CPUs; otherwise we may cross the 8-CPU boundary when CPUs are added later.

32bit apic code can use more cleanups (like the removal of vendor checks in 32bit default_setup_apic_routing()) and more unifications with 64bit code. Yinghai has some patches in works already. This patch addresses the boot issue that is reported in the virtualization guest context.

[ hpa: incorporated function annotation feedback from Yinghai Lu ]

LKML-Reference: <[email protected]>

---
 arch/x86/kernel/acpi/boot.c     |    5 -----
 arch/x86/kernel/apic/apic.c     |   17 -----------------
 arch/x86/kernel/apic/probe_32.c |   29 +++++++++++++++++++++++++++--
 arch/x86/kernel/apic/probe_64.c |    2 +-
 arch/x86/kernel/mpparse.c       |    7 -------
 arch/x86/kernel/smpboot.c       |    2 --
 6 files changed, 28 insertions(+), 34 deletions(-)
arch/x86/kernel/acpi/boot.c +0 -5
index 036d28a..0acbcdf 100644
@@ -1185,9 +1185,6 @@ static void __init acpi_process_madt(void)
 		if (!error) {
 			acpi_lapic = 1;
 
-#ifdef CONFIG_X86_BIGSMP
-			generic_bigsmp_probe();
-#endif
 			/*
 			 * Parse MADT IO-APIC entries
 			 */
@@ -1197,8 +1194,6 @@ static void __init acpi_process_madt(void)
 				acpi_ioapic = 1;
 
 				smp_found_config = 1;
-				if (apic->setup_apic_routing)
-					apic->setup_apic_routing();
 			}
 		}
 		if (error == -EINVAL) {
arch/x86/kernel/apic/apic.c +0 -17
index 3987e44..dfca210 100644
@@ -1641,9 +1641,7 @@ int __init APIC_init_uniprocessor(void)
 #endif
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	verify_local_APIC();
 	connect_bsp_APIC();
@@ -1891,21 +1889,6 @@ void __cpuinit generic_processor_info(int apicid, int version)
 	if (apicid > max_physical_apicid)
 		max_physical_apicid = apicid;
 
-#ifdef CONFIG_X86_32
-	if (num_processors > 8) {
-		switch (boot_cpu_data.x86_vendor) {
-		case X86_VENDOR_INTEL:
-			if (!APIC_XAPIC(version)) {
-				def_to_bigsmp = 0;
-				break;
-			}
-			/* If P4 and above fall through */
-		case X86_VENDOR_AMD:
-			def_to_bigsmp = 1;
-		}
-	}
-#endif
-
 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
 	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
arch/x86/kernel/apic/probe_32.c +27 -2
index 1a6559f..99d2fe0 100644
@@ -52,7 +52,32 @@ static int __init print_ipi_mode(void)
 }
 late_initcall(print_ipi_mode);
 
-void default_setup_apic_routing(void)
+void __init default_setup_apic_routing(void)
+{
+	int version = apic_version[boot_cpu_physical_apicid];
+
+	if (num_possible_cpus() > 8) {
+		switch (boot_cpu_data.x86_vendor) {
+		case X86_VENDOR_INTEL:
+			if (!APIC_XAPIC(version)) {
+				def_to_bigsmp = 0;
+				break;
+			}
+			/* If P4 and above fall through */
+		case X86_VENDOR_AMD:
+			def_to_bigsmp = 1;
+		}
+	}
+
+#ifdef CONFIG_X86_BIGSMP
+	generic_bigsmp_probe();
+#endif
+
+	if (apic->setup_apic_routing)
+		apic->setup_apic_routing();
+}
+
+static void setup_apic_flat_routing(void)
 {
 #ifdef CONFIG_X86_IO_APIC
 	printk(KERN_INFO
@@ -103,7 +128,7 @@ struct apic apic_default = {
 	.init_apic_ldr			= default_init_apic_ldr,
 
 	.ioapic_phys_id_map		= default_ioapic_phys_id_map,
-	.setup_apic_routing		= default_setup_apic_routing,
+	.setup_apic_routing		= setup_apic_flat_routing,
 	.multi_timer_check		= NULL,
 	.apicid_to_node			= default_apicid_to_node,
 	.cpu_to_logical_apicid		= default_cpu_to_logical_apicid,
arch/x86/kernel/apic/probe_64.c +1 -1
index 450fe20..83e9be4 100644
@@ -67,7 +67,7 @@ void __init default_setup_apic_routing(void)
 	}
 #endif
 
-	if (apic == &apic_flat && num_processors > 8)
+	if (apic == &apic_flat && num_possible_cpus() > 8)
 			apic = &apic_physflat;
 
 	printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
arch/x86/kernel/mpparse.c +0 -7
index 40b54ce..a2c1edd 100644
@@ -359,13 +359,6 @@ static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
 		x86_init.mpparse.mpc_record(1);
 	}
 
-#ifdef CONFIG_X86_BIGSMP
-	generic_bigsmp_probe();
-#endif
-
-	if (apic->setup_apic_routing)
-		apic->setup_apic_routing();
-
 	if (!num_processors)
 		printk(KERN_ERR "MPTABLE: no processors registered!\n");
 	return num_processors;
arch/x86/kernel/smpboot.c +0 -2
index 678d0b8..b4e870c 100644
@@ -1083,9 +1083,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	set_cpu_sibling_map(0);
 
 	enable_IR_x2apic();
-#ifdef CONFIG_X86_64
 	default_setup_apic_routing();
-#endif
 
 	if (smp_sanity_check(max_cpus) < 0) {
 		printk(KERN_INFO "SMP disabled\n");