arm64: initial CPU hotplug support

PATCH Superseded Mark Rutland 2 participants 8 messages Jul 22, 2013 – Aug 2, 2013 arm64

Share & tools

Mark Rutland

author reviewer

2013-07-22 15:37 UTC #0 lore raw

v2 cover arm64: initial CPU hotplug support

The following patches add basic HOTPLUG_CPU support to arm64, which combined with appropriate firmware (e.g. [1]) can be used to power CPUs up and down dynamically.

Currently this clashes with Xen support on arm64, but there's already a patch to fix that up [2], so hopefully that will get solved during the merge process.

I've tested this series with the bootwrapper PSCI implementation I've placed on linux-arm.org [1] and a modified foundation model dts with a psci node and each CPU's enable-method set to "psci", using a shell while repeatedly cycling all cpus off and on:

for C in $(seq 0 3); do

	./cyclichotplug.sh $C >/dev/null 2>&1 &

done

---->8----
#!/bin/sh

# cyclichotplug.sh

CPU=$1;

if [ -z "$CPU" ]; then

	printf "Usage: $0 <cpu id>\n";
	exit 1;

ONLINEFILE=/sys/devices/system/cpu/cpu$CPU/online;

while true; do

	echo 0 > $ONLINEFILE;
	echo 1 > $ONLINEFILE;

done
---->8----

Since v1 [3]:
* Rebased to v3.11-rc2 to solve cpuinit removal conflicts.
* Removed failure path for cpu_die, it causes more problems than it solves.
* Removed cpu_kill, we don't currently need it.
* Test for cpu_die in op_cpu_disable to fail early and survive when there's no mechanism for hot unplug.
* Change pr_err on failed cpu_die to a pr_crit.
* Removed dependency on HOTPLUG, which has been obliterated.

Thanks,
Mark.

[1] http://linux-arm.org/git?p=boot-wrapper-aarch64.git;a=shortlog;h=refs/tags/simple-psci
[2] http://lists.infradead.org/pipermail/linux-arm-kernel/2013-July/185237.html
[3] http://lists.infradead.org/pipermail/linux-arm-kernel/2013-July/182880.html

Mark Rutland (5):
arm64: reorganise smp_enable_ops
arm64: factor out spin-table boot method
arm64: read enable-method for CPU0
arm64: add CPU_HOTPLUG infrastructure
arm64: add PSCI CPU_OFF-based hotplug support

 arch/arm64/Kconfig                 |   7 ++
 arch/arm64/include/asm/irq.h       |   1 +
 arch/arm64/include/asm/smp.h       |  36 +++++--
 arch/arm64/kernel/cputable.c       |   2 +-
 arch/arm64/kernel/head.S           |  12 ++-
 arch/arm64/kernel/irq.c            |  61 +++++++++++
 arch/arm64/kernel/process.c        |   7 ++
 arch/arm64/kernel/smp.c            | 201 ++++++++++++++++++++++---------------
 arch/arm64/kernel/smp_psci.c       |  54 ++++++++--
 arch/arm64/kernel/smp_spin_table.c |  85 +++++++++++++++-
 arch/arm64/kernel/vmlinux.lds.S    |   1 -
 11 files changed, 360 insertions(+), 107 deletions(-)

Signature · 1 line

1.8.1.1

Mark Rutland

author reviewer

2013-07-22 15:37 UTC #1 lore raw

PATCH v2 1/5 arm64: reorganise smp_enable_ops

For hotplug support, we're going to want a place to store operations that do more than bring CPUs online, and it makes sense to group these with our current smp_enable_ops.

This patch renames smp_enable_ops to smp_ops to make the intended use of the structure clearer. While we're at it, fix up instances of the cpu parameter to be an unsigned int, drop the init markings and rename the *_cpu functions to cpu_* to reduce future churn when smp_operations is extended.

Signed-off-by Mark Rutland <[email protected]>

---
 arch/arm64/include/asm/smp.h       | 10 +++++-----
 arch/arm64/kernel/smp.c            | 24 ++++++++++++------------
 arch/arm64/kernel/smp_psci.c       | 10 +++++-----
 arch/arm64/kernel/smp_spin_table.c | 10 +++++-----
 4 files changed, 27 insertions(+), 27 deletions(-)

arch/arm64/include/asm/smp.h +5 -5

index 4b8023c..90626b6 100644
@@ -68,13 +68,13 @@ extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
 
 struct device_node;
 
-struct smp_enable_ops {
+struct smp_operations {
 	const char	*name;
-	int		(*init_cpu)(struct device_node *, int);
-	int		(*prepare_cpu)(int);
+	int		(*cpu_init)(struct device_node *, unsigned int);
+	int		(*cpu_prepare)(unsigned int);
 };
 
-extern const struct smp_enable_ops smp_spin_table_ops;
-extern const struct smp_enable_ops smp_psci_ops;
+extern const struct smp_operations smp_spin_table_ops;
+extern const struct smp_operations smp_psci_ops;
 
 #endif /* ifndef __ASM_SMP_H */

arch/arm64/kernel/smp.c +12 ~~-12~~

index fee5cce..533f405 100644
@@ -236,17 +236,17 @@ void __init smp_prepare_boot_cpu(void)
 
 static void (*smp_cross_call)(const struct cpumask *, unsigned int);
 
-static const struct smp_enable_ops *enable_ops[] __initconst = {
+static const struct smp_operations *supported_smp_ops[] __initconst = {
 	&smp_spin_table_ops,
 	&smp_psci_ops,
 	NULL,
 };
 
-static const struct smp_enable_ops *smp_enable_ops[NR_CPUS];
+static const struct smp_operations *smp_ops[NR_CPUS];
 
-static const struct smp_enable_ops * __init smp_get_enable_ops(const char *name)
+static const struct smp_operations * __init smp_get_ops(const char *name)
 {
-	const struct smp_enable_ops **ops = enable_ops;
+	const struct smp_operations **ops = supported_smp_ops;
 
 	while (*ops) {
 		if (!strcmp(name, (*ops)->name))

@@ -267,7 +267,7 @@ void __init smp_init_cpus(void)
 {
 	const char *enable_method;
 	struct device_node *dn = NULL;
-	int i, cpu = 1;
+	unsigned int i, cpu = 1;
 	bool bootcpu_valid = false;
 
 	while ((dn = of_find_node_by_type(dn, "cpu"))) {

@@ -346,15 +346,15 @@ void __init smp_init_cpus(void)
 			goto next;
 		}
 
-		smp_enable_ops[cpu] = smp_get_enable_ops(enable_method);
+		smp_ops[cpu] = smp_get_ops(enable_method);
 
-		if (!smp_enable_ops[cpu]) {
+		if (!smp_ops[cpu]) {
 			pr_err("%s: invalid enable-method property: %s\n",
 			       dn->full_name, enable_method);
 			goto next;
 		}
 
-		if (smp_enable_ops[cpu]->init_cpu(dn, cpu))
+		if (smp_ops[cpu]->cpu_init(dn, cpu))
 			goto next;
 
 		pr_debug("cpu logical map 0x%llx\n", hwid);

@@ -384,8 +384,8 @@ next:
 
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
-	int cpu, err;
-	unsigned int ncores = num_possible_cpus();
+	int err;
+	unsigned int cpu, ncores = num_possible_cpus();
 
 	/*
 	 * are we trying to boot more cores than exist?

@@ -412,10 +412,10 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 		if (cpu == smp_processor_id())
 			continue;
 
-		if (!smp_enable_ops[cpu])
+		if (!smp_ops[cpu])
 			continue;
 
-		err = smp_enable_ops[cpu]->prepare_cpu(cpu);
+		err = smp_ops[cpu]->cpu_prepare(cpu);
 		if (err)
 			continue;

arch/arm64/kernel/smp_psci.c +5 -5

index 0c53330..2f0d3dd 100644
@@ -23,12 +23,12 @@
 #include <asm/psci.h>
 #include <asm/smp_plat.h>
 
-static int __init smp_psci_init_cpu(struct device_node *dn, int cpu)
+static int smp_psci_cpu_init(struct device_node *dn, unsigned int cpu)
 {
 	return 0;
 }
 
-static int __init smp_psci_prepare_cpu(int cpu)
+static int smp_psci_cpu_prepare(unsigned int cpu)
 {
 	int err;

@@ -46,8 +46,8 @@ static int __init smp_psci_prepare_cpu(int cpu)
 	return 0;
 }
 
-const struct smp_enable_ops smp_psci_ops __initconst = {
+const struct smp_operations smp_psci_ops = {
 	.name		= "psci",
-	.init_cpu	= smp_psci_init_cpu,
-	.prepare_cpu	= smp_psci_prepare_cpu,
+	.cpu_init	= smp_psci_cpu_init,
+	.cpu_prepare	= smp_psci_cpu_prepare,
 };

arch/arm64/kernel/smp_spin_table.c +5 -5

index 7c35fa6..5fecffc 100644
@@ -24,7 +24,7 @@
 
 static phys_addr_t cpu_release_addr[NR_CPUS];
 
-static int __init smp_spin_table_init_cpu(struct device_node *dn, int cpu)
+static int smp_spin_table_cpu_init(struct device_node *dn, unsigned int cpu)
 {
 	/*
 	 * Determine the address from which the CPU is polling.

@@ -40,7 +40,7 @@ static int __init smp_spin_table_init_cpu(struct device_node *dn, int cpu)
 	return 0;
 }
 
-static int __init smp_spin_table_prepare_cpu(int cpu)
+static int smp_spin_table_cpu_prepare(unsigned int cpu)
 {
 	void **release_addr;

@@ -59,8 +59,8 @@ static int __init smp_spin_table_prepare_cpu(int cpu)
 	return 0;
 }
 
-const struct smp_enable_ops smp_spin_table_ops __initconst = {
+const struct smp_operations smp_spin_table_ops = {
 	.name		= "spin-table",
-	.init_cpu 	= smp_spin_table_init_cpu,
-	.prepare_cpu	= smp_spin_table_prepare_cpu,
+	.cpu_init	= smp_spin_table_cpu_init,
+	.cpu_prepare	= smp_spin_table_cpu_prepare,
 };

Signature · 1 line

1.8.1.1

Mark Rutland

author reviewer

2013-07-22 15:37 UTC #2 lore raw

PATCH v2 2/5 arm64: factor out spin-table boot method

In reply to Mark Rutland

The arm64 kernel has an internal holding pen, which is necessary for some systems where we can't bring CPUs online individually and must hold multiple CPUs in a safe area until the kernel is able to handle them. The current SMP infrastructure for arm64 is closely coupled to this holding pen, and alternative boot methods must launch CPUs into the pen, from whence they are launched into the kernel proper.

With PSCI (and possibly other future boot methods), we can bring CPUs online individually, and need not perform the secondary_holding_pen dance. Instead, this patch factors the holding pen management code out to the spin-table boot method code, as it is the only boot method requiring the pen.

A new entry point for secondaries, secondary_entry is added for other boot methods to use, which bypasses the holding pen and its associated overhead when bringing CPUs online. The smp.pen.text section is also removed, as the pen can live in head.text without problem.

The smp_operations structure is extended with two new functions, cpu_boot and cpu_postboot, for bringing a cpu into the kernel and performing any post-boot cleanup required by a bootmethod (e.g. resetting the secondary_holding_pen_release to INVALID_HWID).

Signed-off-by Mark Rutland <[email protected]>

---
 arch/arm64/include/asm/smp.h       | 17 ++++++++-
 arch/arm64/kernel/head.S           | 12 +++++-
 arch/arm64/kernel/smp.c            | 67 +++-------------------------------
 arch/arm64/kernel/smp_psci.c       | 16 ++++----
 arch/arm64/kernel/smp_spin_table.c | 75 ++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/vmlinux.lds.S    |  1 -
 6 files changed, 115 insertions(+), 73 deletions(-)

arch/arm64/include/asm/smp.h +15 -2

index 90626b6..af39644 100644
@@ -60,8 +60,7 @@ struct secondary_data {
 	void *stack;
 };
 extern struct secondary_data secondary_data;
-extern void secondary_holding_pen(void);
-extern volatile unsigned long secondary_holding_pen_release;
+extern void secondary_entry(void);
 
 extern void arch_send_call_function_single_ipi(int cpu);
 extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);

@@ -70,8 +69,22 @@ struct device_node;
 
 struct smp_operations {
 	const char	*name;
+	/*
+	 * Check devicetree data for cpu
+	 */
 	int		(*cpu_init)(struct device_node *, unsigned int);
+	/*
+	 * Test if cpu is present and bootable
+	 */
 	int		(*cpu_prepare)(unsigned int);
+	/*
+	 * Boot cpu into the kernel
+	 */
+	int		(*cpu_boot)(unsigned int);
+	/*
+	 * Performs post-boot cleanup
+	 */
+	void		(*cpu_postboot)(void);
 };
 
 extern const struct smp_operations smp_spin_table_ops;

arch/arm64/kernel/head.S +10 -2

index 53dcae4..3532ca6 100644
@@ -217,7 +217,6 @@ ENTRY(__boot_cpu_mode)
 	.quad	PAGE_OFFSET
 
 #ifdef CONFIG_SMP
-	.pushsection    .smp.pen.text, "ax"
 	.align	3
 1:	.quad	.
 	.quad	secondary_holding_pen_release

@@ -242,7 +241,16 @@ pen:	ldr	x4, [x3]
 	wfe
 	b	pen
 ENDPROC(secondary_holding_pen)
-	.popsection
+
+	/*
+	 * Secondary entry point that jumps straight into the kernel. Only to
+	 * be used where CPUs are brought online dynamically by the kernel.
+	 */
+ENTRY(secondary_entry)
+	bl	__calc_phys_offset		// x2=phys offset
+	bl	el2_setup			// Drop to EL1
+	b	secondary_startup
+ENDPROC(secondary_entry)
 
 ENTRY(secondary_startup)
 	/*

arch/arm64/kernel/smp.c +6 ~~-61~~

98 lines not on this page · raw message

arch/arm64/kernel/smp_psci.c +9 -7

index 2f0d3dd..20499bc 100644
@@ -30,24 +30,26 @@ static int smp_psci_cpu_init(struct device_node *dn, unsigned int cpu)
 
 static int smp_psci_cpu_prepare(unsigned int cpu)
 {
-	int err;
-
 	if (!psci_ops.cpu_on) {
 		pr_err("psci: no cpu_on method, not booting CPU%d\n", cpu);
 		return -ENODEV;
 	}
 
-	err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_holding_pen));
-	if (err) {
+	return 0;
+}
+
+static int smp_psci_cpu_boot(unsigned int cpu)
+{
+	int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_entry));
+	if (err)
 		pr_err("psci: failed to boot CPU%d (%d)\n", cpu, err);
-		return err;
-	}
 
-	return 0;
+	return err;
 }
 
 const struct smp_operations smp_psci_ops = {
 	.name		= "psci",
 	.cpu_init	= smp_psci_cpu_init,
 	.cpu_prepare	= smp_psci_cpu_prepare,
+	.cpu_boot	= smp_psci_cpu_boot,
 };

arch/arm64/kernel/smp_spin_table.c +75 -0

96 lines not on this page · raw message

arch/arm64/kernel/vmlinux.lds.S +0 -1

index f5e5574..d8ca8d9 100644
@@ -54,7 +54,6 @@ SECTIONS
 	}
 	.text : {			/* Real text segment		*/
 		_stext = .;		/* Text and read-only data	*/
-			*(.smp.pen.text)
 			__exception_text_start = .;
 			*(.exception.text)
 			__exception_text_end = .;

Signature · 1 line

1.8.1.1

Santosh Shilimkar

2013-07-23 15:17 UTC #3 lore raw

PATCH v2 2/5 arm64: factor out spin-table boot method

Quoted 7 lines · Mark Rutland

The arm64 kernel has an internal holding pen, which is necessary for some systems where we can't bring CPUs online individually and must hold multiple CPUs in a safe area until the kernel is able to handle them. The current SMP infrastructure for arm64 is closely coupled to this holding pen, and alternative boot methods must launch CPUs into the pen, from whence they are launched into the kernel proper.

s/whence/where

Quoted 41 lines

With PSCI (and possibly other future boot methods), we can bring CPUs online individually, and need not perform the secondary_holding_pen dance. Instead, this patch factors the holding pen management code out to the spin-table boot method code, as it is the only boot method requiring the pen.
A new entry point for secondaries, secondary_entry is added for other boot methods to use, which bypasses the holding pen and its associated overhead when bringing CPUs online. The smp.pen.text section is also removed, as the pen can live in head.text without problem.
The smp_operations structure is extended with two new functions, cpu_boot and cpu_postboot, for bringing a cpu into the kernel and performing any post-boot cleanup required by a bootmethod (e.g. resetting the secondary_holding_pen_release to INVALID_HWID).
Signed-off-by Mark Rutland <[email protected]>
---
 arch/arm64/include/asm/smp.h       | 17 ++++++++-
 arch/arm64/kernel/head.S           | 12 +++++-
 arch/arm64/kernel/smp.c            | 67 +++-------------------------------
 arch/arm64/kernel/smp_psci.c       | 16 ++++----
 arch/arm64/kernel/smp_spin_table.c | 75 ++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/vmlinux.lds.S    |  1 -
 6 files changed, 115 insertions(+), 73 deletions(-)
arch/arm64/include/asm/smp.h +1 -2
index 90626b6..af39644 100644
@@ -60,8 +60,7 @@ struct secondary_data {
 	void *stack;
 };
 extern struct secondary_data secondary_data;
-extern void secondary_holding_pen(void);
-extern volatile unsigned long secondary_holding_pen_release;
+extern void secondary_entry(void);
 
 extern void arch_send_call_function_single_ipi(int cpu);
 extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);

arch/arm64/include/asm/smp.h struct device_node; · line 74

Quoted 6 lines

arch/arm64/include/asm/smp.h +3 -0

@@ -70,8 +69,22 @@ struct device_node;
 
 struct smp_operations {
 	const char	*name;
+	/*
+	 * Check devicetree data for cpu
+	 */

All these comments can go above the struct as mentioned in kerneldoc.

Quoted 36 lines

arch/arm64/include/asm/smp.h +11 -0

 	int		(*cpu_init)(struct device_node *, unsigned int);
+	/*
+	 * Test if cpu is present and bootable
+	 */
 	int		(*cpu_prepare)(unsigned int);
+	/*
+	 * Boot cpu into the kernel
+	 */
+	int		(*cpu_boot)(unsigned int);
+	/*
+	 * Performs post-boot cleanup
+	 */
+	void		(*cpu_postboot)(void);
 };
 
 extern const struct smp_operations smp_spin_table_ops;

arch/arm64/kernel/head.S +3 -2

index 53dcae4..3532ca6 100644
@@ -217,7 +217,6 @@ ENTRY(__boot_cpu_mode)
 	.quad	PAGE_OFFSET
 
 #ifdef CONFIG_SMP
-	.pushsection    .smp.pen.text, "ax"
 	.align	3
 1:	.quad	.
 	.quad	secondary_holding_pen_release

@@ -242,7 +241,16 @@ pen:	ldr	x4, [x3]
 	wfe
 	b	pen
 ENDPROC(secondary_holding_pen)
-	.popsection
+
+	/*
+	 * Secondary entry point that jumps straight into the kernel. Only to

arch/arm64/kernel/head.S pen: ldr x4, [x3] · line 256

Quoted 10 lines

arch/arm64/kernel/head.S +7 -0

+	 * be used where CPUs are brought online dynamically by the kernel.
+	 */
+ENTRY(secondary_entry)
+	bl	__calc_phys_offset		// x2=phys offset
+	bl	el2_setup			// Drop to EL1
+	b	secondary_startup
+ENDPROC(secondary_entry)
 
 ENTRY(secondary_startup)
 	/*

Other than that, rest of the patch looks fine too me.

regards,
Santosh

Mark Rutland

author reviewer

2013-08-02 14:14 UTC #4 lore raw

PATCH v2 2/5 arm64: factor out spin-table boot method

Quoted 9 lines · Santosh Shilimkar

Quoted 7 lines · Mark Rutland
The arm64 kernel has an internal holding pen, which is necessary for some systems where we can't bring CPUs online individually and must hold multiple CPUs in a safe area until the kernel is able to handle them. The current SMP infrastructure for arm64 is closely coupled to this holding pen, and alternative boot methods must launch CPUs into the pen, from whence they are launched into the kernel proper.
s/whence/where

Will fix up.

Quoted 43 lines

Quoted 43 lines
With PSCI (and possibly other future boot methods), we can bring CPUs online individually, and need not perform the secondary_holding_pen dance. Instead, this patch factors the holding pen management code out to the spin-table boot method code, as it is the only boot method requiring the pen.
A new entry point for secondaries, secondary_entry is added for other boot methods to use, which bypasses the holding pen and its associated overhead when bringing CPUs online. The smp.pen.text section is also removed, as the pen can live in head.text without problem.
The smp_operations structure is extended with two new functions, cpu_boot and cpu_postboot, for bringing a cpu into the kernel and performing any post-boot cleanup required by a bootmethod (e.g. resetting the secondary_holding_pen_release to INVALID_HWID).
Signed-off-by Mark Rutland <[email protected]>
---
 arch/arm64/include/asm/smp.h       | 17 ++++++++-
 arch/arm64/kernel/head.S           | 12 +++++-
 arch/arm64/kernel/smp.c            | 67 +++-------------------------------
 arch/arm64/kernel/smp_psci.c       | 16 ++++----
 arch/arm64/kernel/smp_spin_table.c | 75 ++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/vmlinux.lds.S    |  1 -
 6 files changed, 115 insertions(+), 73 deletions(-)
arch/arm64/include/asm/smp.h +1 -2
index 90626b6..af39644 100644
@@ -60,8 +60,7 @@ struct secondary_data {
 	void *stack;
 };
 extern struct secondary_data secondary_data;
-extern void secondary_holding_pen(void);
-extern volatile unsigned long secondary_holding_pen_release;
+extern void secondary_entry(void);
 
 extern void arch_send_call_function_single_ipi(int cpu);
 extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
@@ -70,8 +69,22 @@ struct device_node;
 

Quoted 7 lines

arch/arm64/include/asm/smp.h struct device_node; · line 74
Quoted 5 lines
arch/arm64/include/asm/smp.h +3 -0
 struct smp_operations {
 	const char	*name;
+	/*
+	 * Check devicetree data for cpu
+	 */
All these comments can go above the struct as mentioned in kerneldoc.

Ok, I'll take a look at the kerneldoc documentation.

Quoted 45 lines

arch/arm64/include/asm/smp.h +11 -0

 	int		(*cpu_init)(struct device_node *, unsigned int);
+	/*
+	 * Test if cpu is present and bootable
+	 */
 	int		(*cpu_prepare)(unsigned int);
+	/*
+	 * Boot cpu into the kernel
+	 */
+	int		(*cpu_boot)(unsigned int);
+	/*
+	 * Performs post-boot cleanup
+	 */
+	void		(*cpu_postboot)(void);
 };
 
 extern const struct smp_operations smp_spin_table_ops;

arch/arm64/kernel/head.S +10 -2

index 53dcae4..3532ca6 100644
@@ -217,7 +217,6 @@ ENTRY(__boot_cpu_mode)
 	.quad	PAGE_OFFSET
 
 #ifdef CONFIG_SMP
-	.pushsection    .smp.pen.text, "ax"
 	.align	3
 1:	.quad	.
 	.quad	secondary_holding_pen_release

@@ -242,7 +241,16 @@ pen:	ldr	x4, [x3]
 	wfe
 	b	pen
 ENDPROC(secondary_holding_pen)
-	.popsection
+
+	/*
+	 * Secondary entry point that jumps straight into the kernel. Only to
+	 * be used where CPUs are brought online dynamically by the kernel.
+	 */
+ENTRY(secondary_entry)
+	bl	__calc_phys_offset		// x2=phys offset
+	bl	el2_setup			// Drop to EL1
+	b	secondary_startup
+ENDPROC(secondary_entry)

Quoted 3 lines

arch/arm64/kernel/head.S pen: ldr x4, [x3] · line 256
Quoted 2 lines
arch/arm64/kernel/head.S +0 -0
 ENTRY(secondary_startup)
 	/*
Other than that, rest of the patch looks fine too me.

Great!

Thanks,
Mark.

Mark Rutland

author reviewer

2013-07-22 15:37 UTC #5 lore raw

PATCH v2 3/5 arm64: read enable-method for CPU0

In reply to Mark Rutland

With the advent of CPU_HOTPLUG, the enable-method property for CPU0 may tells us something useful (i.e. how to hotplug it back on), so we must read it along with all the enable-method for all the other CPUs.

This patch ensures that CPU0's enable-method property is read.

Signed-off-by Mark Rutland <[email protected]>

---
 arch/arm64/kernel/smp.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

arch/arm64/kernel/smp.c +7 -8

index 72c2823..f6ce2ae 100644
@@ -254,6 +254,8 @@ void __init smp_init_cpus(void)
 			}
 		}
 
+		enable_method = of_get_property(dn, "enable-method", NULL);
+
 		/*
 		 * The numbering scheme requires that the boot CPU
 		 * must be assigned logical id 0. Record it so that

@@ -269,11 +271,12 @@ void __init smp_init_cpus(void)
 
 			bootcpu_valid = true;
 
+			if (enable_method)
+				smp_ops[0] = smp_get_ops(enable_method);
+
 			/*
-			 * cpu_logical_map has already been
-			 * initialized and the boot cpu doesn't need
-			 * the enable-method so continue without
-			 * incrementing cpu.
+			 * cpu_logical_map has already been initialized so
+			 * continue without incrementing cpu.
 			 */
 			continue;
 		}

@@ -281,10 +284,6 @@ void __init smp_init_cpus(void)
 		if (cpu >= NR_CPUS)
 			goto next;
 
-		/*
-		 * We currently support only the "spin-table" enable-method.
-		 */
-		enable_method = of_get_property(dn, "enable-method", NULL);
 		if (!enable_method) {
 			pr_err("%s: missing enable-method property\n",
 				dn->full_name);

Signature · 1 line

1.8.1.1

Mark Rutland

author reviewer

2013-07-22 15:37 UTC #6 lore raw

PATCH v2 4/5 arm64: add CPU_HOTPLUG infrastructure

In reply to Mark Rutland

This patch adds the basic infrastructure necessary to support CPU_HOTPLUG on arm64, based on the arm implementation. Actual hotplug support will depend on an implementation's smp_operations (e.g. PSCI).

Signed-off-by Mark Rutland <[email protected]>

---
 arch/arm64/Kconfig           |  7 ++++
 arch/arm64/include/asm/irq.h |  1 +
 arch/arm64/include/asm/smp.h |  9 ++++
 arch/arm64/kernel/cputable.c |  2 +-
 arch/arm64/kernel/irq.c      | 61 ++++++++++++++++++++++++++++
 arch/arm64/kernel/process.c  |  7 ++++
 arch/arm64/kernel/smp.c      | 97 ++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 183 insertions(+), 1 deletion(-)

arch/arm64/Kconfig +7 -0

index 9737e97..5ce4ccb 100644
@@ -158,6 +158,13 @@ config NR_CPUS
 	default "8" if ARCH_XGENE
 	default "4"
 
+config HOTPLUG_CPU
+	bool "Support for hot-pluggable CPUs"
+	depends on SMP
+	help
+	  Say Y here to experiment with turning CPUs off and on.  CPUs
+	  can be controlled through /sys/devices/system/cpu.
+
 source kernel/Kconfig.preempt
 
 config HZ

arch/arm64/include/asm/irq.h +1 -0

index 0332fc0..e1f7ecd 100644
@@ -4,6 +4,7 @@
 #include <asm-generic/irq.h>
 
 extern void (*handle_arch_irq)(struct pt_regs *);
+extern void migrate_irqs(void);
 extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
 
 #endif

arch/arm64/include/asm/smp.h +9 -0

index af39644..844925d 100644
@@ -67,6 +67,11 @@ extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
 
 struct device_node;
 
+extern int __cpu_disable(void);
+
+extern void __cpu_die(unsigned int cpu);
+extern void cpu_die(void);
+
 struct smp_operations {
 	const char	*name;
 	/*

@@ -85,6 +90,10 @@ struct smp_operations {
 	 * Performs post-boot cleanup
 	 */
 	void		(*cpu_postboot)(void);
+#ifdef CONFIG_HOTPLUG_CPU
+	int  (*cpu_disable)(unsigned int cpu);
+	void (*cpu_die)(unsigned int cpu);
+#endif
 };
 
 extern const struct smp_operations smp_spin_table_ops;

arch/arm64/kernel/cputable.c +1 -1

index 63cfc4a..fd3993c 100644
@@ -22,7 +22,7 @@
 
 extern unsigned long __cpu_setup(void);
 
-struct cpu_info __initdata cpu_table[] = {
+struct cpu_info cpu_table[] = {
 	{
 		.cpu_id_val	= 0x000f0000,
 		.cpu_id_mask	= 0x000f0000,

arch/arm64/kernel/irq.c +61 -0

64 lines not on this page · raw message

arch/arm64/kernel/process.c +7 -0

index 1788bf6..4835a33 100644
@@ -102,6 +102,13 @@ void arch_cpu_idle(void)
 	local_irq_enable();
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+void arch_cpu_idle_dead(void)
+{
+       cpu_die();
+}
+#endif
+
 void machine_shutdown(void)
 {
 #ifdef CONFIG_SMP

arch/arm64/kernel/smp.c +97 -0

103 lines not on this page · raw message

Signature · 1 line

1.8.1.1

Mark Rutland

author reviewer

2013-07-22 15:37 UTC #7 lore raw

PATCH v2 5/5 arm64: add PSCI CPU_OFF-based hotplug support

In reply to Mark Rutland

This patch adds support for using PSCI CPU_OFF calls for CPU hotplug. With this code it is possible to hot unplug CPUs with "psci" as their boot-method, as long as there's an appropriate cpu_off function id specified in the psci node.

Signed-off-by Mark Rutland <[email protected]>

---
 arch/arm64/kernel/smp_psci.c | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

arch/arm64/kernel/smp_psci.c +30 -0

index 20499bc..06123de 100644
@@ -47,9 +47,39 @@ static int smp_psci_cpu_boot(unsigned int cpu)
 	return err;
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+static int smp_psci_cpu_disable(unsigned int cpu)
+{
+	/* Fail early if we don't have CPU_OFF support */
+	if (!psci_ops.cpu_off)
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+static void smp_psci_cpu_die(unsigned int cpu)
+{
+	int ret;
+	/*
+	 * There are no known implementations of PSCI actually using the
+	 * power state field, pass a sensible default for now.
+	 */
+	struct psci_power_state state = {
+		.type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
+	};
+
+	ret = psci_ops.cpu_off(state);
+
+	pr_crit("psci: unable to power off CPU%u (%d)", cpu, ret);
+}
+#endif
+
 const struct smp_operations smp_psci_ops = {
 	.name		= "psci",
 	.cpu_init	= smp_psci_cpu_init,
 	.cpu_prepare	= smp_psci_cpu_prepare,
 	.cpu_boot	= smp_psci_cpu_boot,
+#ifdef CONFIG_HOTPLUG_CPU
+	.cpu_disable	= smp_psci_cpu_disable,
+	.cpu_die	= smp_psci_cpu_die,
+#endif
 };

Signature · 1 line

1.8.1.1