Skip to contentSkip to conversation

ACPI based physical CPU hotplug support for x86_64

PATCH In review Ashok Raj 2 participants 7 messages Feb 28, 2006 – Mar 1, 2006 ACPI

lore
Share & tools
lore

Ashok Raj

author
#0 lore raw

cover ACPI based physical CPU hotplug support for x86_64

Hi Andi/Andrew

Here are a set of patches to implement physical cpu hotplug for x86_64 arch.

Physical cpu hotplug support is already available for ia64, these set of patches extend support for x86_64 as well.

Patches are split as follows. Please help the first 3 patches stage in -mm before considering for mainline inclusion. The last 2 patches are purely for testing.

1. remove-processor-entries-on-eject - Minor bugfix to current base code.
2. remove-unused-lapic-entry - Remove unnecessary lapic definition
3. x86_64-physical-cpu-hotplug.patch - Physical cpu hotplug support for x86_64

The following two patches are not for mainline inclusion. They exist so we could test the code paths to test physical hotplug without need for a platform that is capable of hotplug.

Procedure to do this is included in sci emulate patches.
4. x86_64-limit-cpus - Limit cpus present to emulate Physical
hotplug.
5. sci_emu.patch - Code to emulate SCI notify on a object.

Cheers,
ashok

author
#1 lore raw

PATCH 1/5 Remove entries in /sys/firmware/acpi for processor also.

Processor entries under /sys/firmware/acpi/namespace/ACPI/_SB/CPU* were not being removed due to acpi_bus_trim not asking for it just for processors. Without which a new hot-add after a remove doesnt call appropriate init functions resulting in subsequent hot-add for same cpu failing.

Not clear why we restricted that to non-processor devices.

-----------------------------------------------------

 drivers/acpi/scan.c |    5 +----
 1 files changed, 1 insertion(+), 4 deletions(-)
drivers/acpi/scan.c +1 -4
Index: linux-2.6.16-rc4-mm1/drivers/acpi/scan.c
===================================================================
@@ -434,10 +434,7 @@ acpi_eject_store(struct acpi_device *dev
 	islockable = device->flags.lockable;
 	handle = device->handle;
 
-	if (type == ACPI_TYPE_PROCESSOR)
-		result = acpi_bus_trim(device, 0);
-	else
-		result = acpi_bus_trim(device, 1);
+	result = acpi_bus_trim(device, 1);
 
 	if (!result)
 		result = acpi_eject_operation(handle, islockable);

--

author
#2 lore raw

PATCH 2/5 Remove unnecessary lapic definition from acpidef.h

In reply to Ashok Raj

Dont know why this exists... just happened to trip me when i used a variable name with lapic, and until i looked at the pre-processed output couldnt figure out we had a lame definition like this.

Hope iam not breaking anything here..

------------------------------------------------

 include/asm-i386/apicdef.h   |    1 -
 include/asm-x86_64/apicdef.h |    2 --
 2 files changed, 3 deletions(-)
include/asm-i386/apicdef.h +0 -1
Index: linux-2.6.16-rc1-mm4/include/asm-i386/apicdef.h
===================================================================
@@ -120,7 +120,6 @@
  */
 #define u32 unsigned int
 
-#define lapic ((volatile struct local_apic *)APIC_BASE)
 
 struct local_apic {
 
include/asm-x86_64/apicdef.h +0 -2
Index: linux-2.6.16-rc1-mm4/include/asm-x86_64/apicdef.h
===================================================================
@@ -136,8 +136,6 @@
  */
 #define u32 unsigned int
 
-#define lapic ((volatile struct local_apic *)APIC_BASE)
-
 struct local_apic {
 
 /*000*/	struct { u32 __reserved[4]; } __reserved_01;

--

#3 lore raw

Re: [patch 2/5] Remove unnecessary lapic definition from acpidef.h

Quoted 5 lines · Ashok Raj

Dont know why this exists... just happened to trip me when i used a variable name with lapic, and until i looked at the pre-processed output couldnt figure out we had a lame definition like this.

Hope iam not breaking anything here..

No, the file is a mess anyways.

Quoted 12 lines

------------------------------------------------

 include/asm-i386/apicdef.h   |    1 -
 include/asm-x86_64/apicdef.h |    2 --
 2 files changed, 3 deletions(-)

Index: linux-2.6.16-rc1-mm4/include/asm-i386/apicdef.h
===================================================================

include/asm-i386/apicdef.h line 121

Quoted 2 lines
include/asm-i386/apicdef.h +0 -0
@@ -120,7 +120,6 @@
  */
 #define u32 unsigned int

Like this should go too.

include/asm-i386/apicdef.h line 122

Quoted 2 lines
include/asm-i386/apicdef.h +0 -1
-#define lapic ((volatile struct local_apic *)APIC_BASE)

I'll take a stab at cleaning it up.

Thanks,
-Andi

author
#4 lore raw

PATCH 3/5 Support physical cpu hotplug for x86_64.

In reply to Ashok Raj

This patch enables ACPI based physical CPU hotplug support for x86_64. Implements acpi_map_lsapic() and acpi_unmap_lsapic() to support physical cpu hotplug.

------------------------------------------------------------------

 arch/i386/kernel/acpi/boot.c |   69 ++++++++++++++++++++++++++++++++++++++++---
 1 files changed, 65 insertions(+), 4 deletions(-)
arch/i386/kernel/acpi/boot.c +65 -4
Index: linux-2.6.16-rc4-mm1/arch/i386/kernel/acpi/boot.c
===================================================================
@@ -25,6 +25,7 @@
 
 #include <linux/init.h>
 #include <linux/config.h>
+#include <linux/cpumask.h>
 #include <linux/acpi.h>
 #include <linux/efi.h>
 #include <linux/module.h>
@@ -505,16 +506,76 @@ EXPORT_SYMBOL(acpi_register_gsi);
 #ifdef CONFIG_ACPI_HOTPLUG_CPU
 int acpi_map_lsapic(acpi_handle handle, int *pcpu)
 {
-	/* TBD */
-	return -EINVAL;
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	struct acpi_table_lapic *lapic;
+	cpumask_t tmp_map, new_map;
+	u8 physid;
+	int cpu;
+
+	if (ACPI_FAILURE(acpi_evaluate_object(handle, "_MAT", NULL, &buffer)))
+		return -EINVAL;
+
+	if (!buffer.length || !buffer.pointer)
+		return -EINVAL;
+
+	obj = buffer.pointer;
+	if (obj->type != ACPI_TYPE_BUFFER ||
+	    obj->buffer.length < sizeof(*lapic)) {
+		acpi_os_free(buffer.pointer);
+		return -EINVAL;
+	}
+
+	lapic = (struct acpi_table_lapic *)obj->buffer.pointer;
+
+	if ((lapic->header.type != ACPI_MADT_LAPIC) ||
+	    (!lapic->flags.enabled)) {
+		acpi_os_free(buffer.pointer);
+		return -EINVAL;
+	}
+
+	physid = lapic->id;
+
+	acpi_os_free(buffer.pointer);
+	buffer.length = ACPI_ALLOCATE_BUFFER;
+	buffer.pointer = NULL;
+
+	tmp_map = cpu_present_map;
+	mp_register_lapic(physid, lapic->flags.enabled);
+
+	/*
+	 * If mp_register_lapic successfully generates a new logical cpu
+	 * number, then the following will get us exactly what was mapped
+	 */
+	cpus_andnot(new_map, cpu_present_map, tmp_map);
+	if (cpus_empty(new_map)) {
+		printk ("Unable to map lapic to logical cpu number\n");
+		return -EINVAL;
+	}
+
+	cpu = first_cpu(new_map);
+
+	*pcpu = cpu;
+	return 0;
 }
 
 EXPORT_SYMBOL(acpi_map_lsapic);
 
 int acpi_unmap_lsapic(int cpu)
 {
-	/* TBD */
-	return -EINVAL;
+	int i;
+
+	for_each_cpu(i) {
+		if (x86_acpiid_to_apicid[i] == x86_cpu_to_apicid[cpu]) {
+			x86_acpiid_to_apicid[i] = -1;
+			break;
+		}
+	}
+	x86_cpu_to_apicid[cpu] = -1;
+	cpu_clear(cpu, cpu_present_map);
+	num_processors--;
+
+	return (0);
 }
 
 EXPORT_SYMBOL(acpi_unmap_lsapic);

--

author
#5 lore raw

PATCH 4/5 Patch to limit present cpus to fake cpu hot-add testing.

In reply to Ashok Raj

[NOTE: Not for mainline inclusion.. only necessary for testing purposes]

Patch to limit cpu_present_map to less that what is physically present. This is used to test physical CPU hotplug by hiding entries being created and emulate a hot-add of the CPU.

enable CONFIG_LIMIT_CPUS, and use cmdline option limit_cpus=X, where X indicates number of CPUs to mark present when booted.

for e.g with limit_cpus=2 on a 4 way system, you will only notice 2 cpus in /sys/devices/system/cpu/.

-------------------------------------------------------------

 arch/i386/kernel/acpi/boot.c |    9 ++++++++-
 arch/x86_64/Kconfig          |    9 +++++++++
 arch/x86_64/kernel/mpparse.c |    2 +-
 arch/x86_64/kernel/smpboot.c |   26 ++++++++++++++++++++++++++
 init/main.c                  |   22 ++++++++++++++++++++++
 5 files changed, 66 insertions(+), 2 deletions(-)
arch/i386/kernel/acpi/boot.c +8 -1
Index: linux-2.6.16-rc4-mm1/arch/i386/kernel/acpi/boot.c
===================================================================
@@ -540,6 +540,12 @@ int acpi_map_lsapic(acpi_handle handle, 
 	buffer.length = ACPI_ALLOCATE_BUFFER;
 	buffer.pointer = NULL;
 
+#ifdef CONFIG_LIMIT_CPUS
+	cpus_complement(tmp_map, cpu_present_map);
+	cpu = first_cpu(tmp_map);
+	physid = bios_cpu_apicid[cpu];
+	printk ("Limit-cpus... activating CPU %d APIC_ID = %d\n", cpu, physid);
+#endif
 	tmp_map = cpu_present_map;
 	mp_register_lapic(physid, lapic->flags.enabled);
 
@@ -563,8 +569,8 @@ EXPORT_SYMBOL(acpi_map_lsapic);
 
 int acpi_unmap_lsapic(int cpu)
 {
+#ifndef CONFIG_LIMIT_CPUS
 	int i;
-
 	for_each_cpu(i) {
 		if (x86_acpiid_to_apicid[i] == x86_cpu_to_apicid[cpu]) {
 			x86_acpiid_to_apicid[i] = -1;
@@ -572,6 +578,7 @@ int acpi_unmap_lsapic(int cpu)
 		}
 	}
 	x86_cpu_to_apicid[cpu] = -1;
+#endif
 	cpu_clear(cpu, cpu_present_map);
 	num_processors--;
 
arch/x86_64/Kconfig +9 -0
Index: linux-2.6.16-rc4-mm1/arch/x86_64/Kconfig
===================================================================
@@ -360,6 +360,15 @@ config HOTPLUG_CPU
 		Say N if you want to disable CPU hotplug.
 
 
+config LIMIT_CPUS
+   bool "Limits the present CPUs  at boot time"
+   depends on HOTPLUG_CPU
+   default n
+   ---help---
+   At boot time pass command line option "limit_cpus=xx" to start with
+   xx number of cpus. The cpu_present_map is trimmed to hide these CPUs
+   so they can be made to appear hotplugged for testing purposes.
+
 config HPET_TIMER
 	bool
 	default y
arch/x86_64/kernel/mpparse.c +1 -1
Index: linux-2.6.16-rc4-mm1/arch/x86_64/kernel/mpparse.c
===================================================================
@@ -65,7 +65,7 @@ unsigned long mp_lapic_addr = 0;
 /* Processor that is doing the boot up */
 unsigned int boot_cpu_id = -1U;
 /* Internal processor count */
-unsigned int num_processors __initdata = 0;
+unsigned int num_processors __cpuinitdata = 0;
 
 unsigned disabled_cpus __initdata;
 
arch/x86_64/kernel/smpboot.c +26 -0
Index: linux-2.6.16-rc4-mm1/arch/x86_64/kernel/smpboot.c
===================================================================
@@ -1006,6 +1006,32 @@ __init void prefill_possible_map(void)
 	for (i = 0; i < possible; i++)
 		cpu_set(i, cpu_possible_map);
 }
+
+#ifdef CONFIG_LIMIT_CPUS
+void __init limit_cpu_present_map(void)
+{
+   extern unsigned int limit_cpus;
+   char str[NR_CPUS];
+   unsigned int cnt;
+
+   if (num_processors <= limit_cpus)
+       return;
+
+   printk (KERN_INFO "cpu_present_map = %s\n",\
+       cpumask_scnprintf(str, NR_CPUS, cpu_present_map)? str : NULL);
+
+   printk (KERN_INFO "Limiting to %d CPUs\n", limit_cpus);
+
+   for (cnt=limit_cpus; (cnt < num_processors); cnt++)
+   {
+       cpu_clear(cnt, cpu_present_map);
+   }
+   printk(KERN_INFO "cpu_present_map = %s\n",
+       cpumask_scnprintf(str, NR_CPUS, cpu_present_map)? str : NULL);
+
+}
+#endif
+
 #endif
 
 /*
init/main.c +22 -0
Index: linux-2.6.16-rc4-mm1/init/main.c
===================================================================
@@ -151,6 +151,23 @@ static int __init maxcpus(char *str)
 
 __setup("maxcpus=", maxcpus);
 
+#ifdef CONFIG_LIMIT_CPUS
+unsigned int limit_cpus=NR_CPUS;
+
+static int __init
+set_limit_cpus(char *str)
+{
+   int ncpus;
+   get_option (&str, &ncpus);
+   limit_cpus = ncpus;
+   printk (KERN_INFO "Limiting cpus present count to %d\n", ncpus);
+   return 1;
+}
+
+__setup("limit_cpus=", set_limit_cpus);
+#endif
+
+
 static char * argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
 char * envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
 static const char *panic_later, *panic_param;
@@ -357,6 +374,11 @@ static void __init smp_init(void)
 {
 	unsigned int i;
 
+#ifdef CONFIG_LIMIT_CPUS
+   extern __init void limit_cpu_present_map(void);
+   limit_cpu_present_map();
+#endif
+
 	/* FIXME: This should be done in userspace --RR */
 	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)

--

author
#6 lore raw

PATCH 5/5 Enable SCI_EMULATE to manually simulate physical hotplug testing.

In reply to Ashok Raj

[Note: Not for mainline inclusion, required only for testing purposes]

Emulate an ACPI SCI interrupt to emulate a hot-plug event. Useful for testing ACPI based hot-plug on systems that don't have the necessary firmware support.

Enable CONFIG_ACPI_SCI_EMULATE on kernel compile.

Now you will notice /proc/acpi/sci/notify when new kernel is booted.

echo "\_SB.CPU4 1" > /proc/acpi/sci/notify to trigger a hot-add of CPU4.

You will now notice an entry /sys/firmware/acpi/namespace/ACPI/_SB/CPU4 if the namespace had an entry CPU4 under _SB scope. If the entry had a _EJ0 method, you will also notice a file "eject" under the CPU4 directory.

How to test physical cpu hotplug?

0. You need a ACPI based MP platform to perform tne test.

1. Get a dump of DSDT that you will modify to emulate CPU hotplug. #acpidump -t DSDT -b -o DSDT

2. Disassemble it, so you can modify the output.

#iasl -d DSDT

This will create DSDT.dsl

3. Locate Scope _SB to add your new processor entries as below.

        Processor (CPU4, 0x05, 0x00000, 0x00) {
                Name(_HID,EISAID("INT0000"))
                Name(_UID,5)
                Name (L0st,0)
                Method(_STA,0,NotSerialized) {
                        If ( LEqual(L0st,1) ) {
                                Return (0x0f)
                        } else {
                                Return (0x00)
                        }
                }
                Method(_PXM) {
                        Return(0)
                }
                Method(_PS0,0,NotSerialized) {
                        Store(1,L0st)
                }
                Method(_PS3,0,NotSerialized) {
                        Store(0,L0st)
                }
                Method(_EJ0,1,NotSerialized) {
                        Store(0,L0st)
                }
                Method (_MAT) {
                        Name(MAT,Buffer() {
                        0x00,           //Local APIC
                        0x08,           //Length
                        0x05,           //ACPI Processor ID
                        0xC2,           //Local APIC ID
                        0x01,           //Enabled
                        0x00,
                        0x00,
                        0x00
                        })
                        Return (MAT)
                }

4. Now compile and generate a hex file for inclusion #iasl -tc DSDT.dsl

This will produce a file DSDT.hex which you will include in your kernel kernel.

5. Rename it to some .h file for convenience.

You need to set CONFIG_STANDALONE=y

it will ask you custom dsdt file name, put that file in drivers/acpi and provide that name when you do a
#make oldconfig.

6. Compile and boot your new kernel with limit_cpus=xxx

follow steps above to fake the notify for add, and remove cpu.

-----------------------------------------------------------------

 drivers/acpi/Kconfig          |   10 +++
 drivers/acpi/bus.c            |  138 ++++++++++++++++++++++++++++++++++++++++++
 drivers/acpi/processor_core.c |   21 +++++-
 drivers/acpi/scan.c           |   35 ++++++++++
 4 files changed, 199 insertions(+), 5 deletions(-)
drivers/acpi/Kconfig +10 -0
Index: linux-2.6.16-rc4-mm1/drivers/acpi/Kconfig
===================================================================
@@ -333,4 +333,14 @@ config ACPI_HOTPLUG_MEMORY
 		$>modprobe acpi_memhotplug 
 endif	# ACPI
 
+config ACPI_SCI_EMULATE
+	bool "ACPI SCI Event Emulation Support"
+	depends on ACPI
+	default n
+	help
+	  This will enable your system to emulate sci hotplug event
+	  notification through proc file system. For example user needs to
+	  echo "XXX 0" > /proc/acpi/sci/notify (where, XXX is a target ACPI
+	  device object name present under \_SB scope).
+
 endmenu
drivers/acpi/bus.c +138 -0

153 lines not on this page · raw message

drivers/acpi/processor_core.c +17 -4
Index: linux-2.6.16-rc4-mm1/drivers/acpi/processor_core.c
===================================================================
@@ -549,12 +549,14 @@ static int acpi_processor_start(struct a
 	 * ACPI id of processors can be reported wrongly by the BIOS.
 	 * Don't trust it blindly
 	 */
+#ifndef CONFIG_ACPI_SCI_EMULATE
 	if (processor_device_array[pr->id] != NULL &&
 	    processor_device_array[pr->id] != (void *)device) {
 		ACPI_WARNING((AE_INFO, "BIOS reporting wrong ACPI id"
 			    "for the processor"));
 		return_VALUE(-ENODEV);
 	}
+#endif
 	processor_device_array[pr->id] = (void *)device;
 
 	processors[pr->id] = pr;
@@ -678,9 +680,6 @@ static int acpi_processor_remove(struct 
 /****************************************************************************
  * 	Acpi processor hotplug support 				       	    *
  ****************************************************************************/
-
-static int is_processor_present(acpi_handle handle);
-
 static int is_processor_present(acpi_handle handle)
 {
 	acpi_status status;
@@ -734,7 +733,10 @@ acpi_processor_hotplug_notify(acpi_handl
 {
 	struct acpi_processor *pr;
 	struct acpi_device *device = NULL;
-	int result;
+	int result = 0;
+#ifdef CONFIG_ACPI_SCI_EMULATE
+	extern int is_PSX_present(acpi_handle handle, int number);
+#endif
 
 	ACPI_FUNCTION_TRACE("acpi_processor_hotplug_notify");
 
@@ -745,6 +747,16 @@ acpi_processor_hotplug_notify(acpi_handl
 		       (event == ACPI_NOTIFY_BUS_CHECK) ?
 		       "ACPI_NOTIFY_BUS_CHECK" : "ACPI_NOTIFY_DEVICE_CHECK");
 
+#ifdef CONFIG_ACPI_SCI_EMULATE
+		/*
+		 * Evaluate _PS0 to make _STA return Present Status
+		 */
+		if (is_PSX_present(handle, 0))
+			result = acpi_evaluate_object(handle, "_PS0", NULL, NULL);
+		if (ACPI_SUCCESS(result)) {
+			printk ("Called method _PS0, _STA should return Present\n");
+		}
+#endif
 		if (!is_processor_present(handle))
 			break;
 
@@ -865,6 +877,7 @@ static int acpi_processor_handle_eject(s
 	acpi_unmap_lsapic(pr->id);
 	return (0);
 }
+
 #else
 static acpi_status acpi_processor_hotadd_init(acpi_handle handle, int *p_cpu)
 {
drivers/acpi/scan.c +34 -1
Index: linux-2.6.16-rc4-mm1/drivers/acpi/scan.c
===================================================================
@@ -371,15 +371,48 @@ setup_sys_fs_device_files(struct acpi_de
 		(*(func)) (&dev->kobj, &acpi_device_attr_eject.attr);
 }
 
+#ifdef CONFIG_ACPI_SCI_EMULATE
+int is_PSX_present(acpi_handle handle, int number)
+{
+	acpi_status status = AE_OK;
+	acpi_handle temp = NULL;
+
+	ACPI_FUNCTION_TRACE("is_PSX_present");
+
+	switch (number) {
+	case 0:
+		status = acpi_get_handle(handle, "_PS0", &temp);
+		break;
+	case 3:
+		status = acpi_get_handle(handle, "_PS3", &temp);
+		break;
+	}
+
+	if (ACPI_SUCCESS(status)) {
+		printk ("Found _PS%d in Processor Scope\n", number);
+		return_VALUE(1);
+	}
+	else
+		return_VALUE(0);
+}
+#endif
+
 static int acpi_eject_operation(acpi_handle handle, int lockable)
 {
 	struct acpi_object_list arg_list;
 	union acpi_object arg;
 	acpi_status status = AE_OK;
 
+#ifdef CONFIG_ACPI_SCI_EMULATE
 	/*
-	 * TBD: evaluate _PS3?
+	 * evaluate _PS3 to make _STA return Not-Present.
 	 */
+	if (is_PSX_present(handle, 3))
+		status = acpi_evaluate_object(handle, "_PS3", NULL, NULL);
+	if (ACPI_SUCCESS(status)) {
+		printk ("Called method _PS3, _STA should return Not-Present\n");
+	}
+#endif
 
 	if (lockable) {
 		arg_list.count = 1;

--