enable nr_cpus for powerpc
Subsystems (3)
v5 cover enable nr_cpus for powerpc
This topic has a very long history. It comes from Mahesh Salgaonkar <[email protected]> For v3: https://patchwork.ozlabs.org/patch/834860/
I hope we can acquire it for "kexec -p" soon.
V4->V5:
improve the [1/3] implementation based on Benjamin's suggestion.
Mahesh Salgaonkar (1):
ppc64 boot: Wait for boot cpu to show up if nr_cpus limit is about to
hit.
Pingfan Liu (2):
powerpc, cpu: partially unbind the mapping between cpu logical id and
its seq in dt
powerpc, cpu: handling the special case when boot_cpuid greater than
nr_cpus
arch/powerpc/include/asm/paca.h | 3 +++ arch/powerpc/include/asm/smp.h | 2 ++ arch/powerpc/kernel/paca.c | 19 ++++++++++++++----- arch/powerpc/kernel/prom.c | 27 ++++++++++++++++----------- arch/powerpc/kernel/setup-common.c | 35 ++++++++++++++++++++++++++++++++--- 5 files changed, 67 insertions(+), 19 deletions(-)
Signature · 1 line
2.7.4
PATCH v5 1/3 powerpc, cpu: partially unbind the mapping between cpu logical id and its seq in dt
For kexec -p, the boot cpu can be not the cpu0, this causes the problem to alloc paca[]. In theory, there is no requirement to assign cpu's logical id as its present seq by device tree. But we have something like cpu_first_thread_sibling(), which makes assumption on the mapping inside a core. Hence partially changing the mapping, i.e. unbind the mapping of core while keep the mapping inside a core. After this patch, boot-cpu will always be mapped into the range [0,threads_per_core).
--- arch/powerpc/include/asm/smp.h | 2 ++ arch/powerpc/kernel/prom.c | 27 ++++++++++++++++----------- arch/powerpc/kernel/setup-common.c | 28 +++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 14 deletions(-)
arch/powerpc/include/asm/smp.h +2 -0
@@ -30,6 +30,8 @@#include <asm/percpu.h> extern int boot_cpuid; +extern int boot_cpuhwid; +extern int boot_dn_nthreads;extern int spinning_secondaries;extern void cpu_die (void );
arch/powerpc/kernel/prom.c +16 -11
@@ -315,8 +315,7 @@ static int __init early_init_dt_scan_cpus(unsigned long node,const __be32 *intserv;int i, nthreads;int len; -int found =-1 ; -int found_thread =0 ; +bool found = false;/* We are scanning "cpu" nodes only */ if (type ==NULL ||strcmp (type,"cpu" ) !=0 )
@@ -341,8 +340,12 @@ static int __init early_init_dt_scan_cpus(unsigned long node,if (fdt_version (initial_boot_params) >=2 ) {if (be32_to_cpu (intserv[i]) ==fdt_boot_cpuid_phys (initial_boot_params)) { - found = boot_cpu_count; - found_thread = i; +/* always map the boot-cpu logical id into the +* the range of [0, thread_per_core) +*/ + boot_cpuid = i; + boot_dn_nthreads = nthreads; + found = true; } }else {/*
@@ -351,8 +354,11 @@ static int __init early_init_dt_scan_cpus(unsigned long node,* off secondary threads. */ if (of_get_flat_dt_prop (node, -"linux,boot-cpu" ,NULL ) !=NULL ) - found = boot_cpu_count; +"linux,boot-cpu" ,NULL ) !=NULL ) { + boot_cpuid = i; + boot_dn_nthreads = nthreads; + found = true; + } }#ifdef CONFIG_SMP /* logical cpu id is always 0 on UP kernels */
@@ -361,13 +367,12 @@ static int __init early_init_dt_scan_cpus(unsigned long node, }/* Not the boot CPU */ -if (found <0 ) +if (!found)return 0 ; -DBG ("boot cpu: logical %d physical %d\n" , found, -be32_to_cpu (intserv[found_thread])); - boot_cpuid = found; -set_hard_smp_processor_id (found,be32_to_cpu (intserv[found_thread])); + boot_cpuhwid =be32_to_cpu (intserv[boot_cpuid]); +DBG ("boot cpu: logical %d physical %d\n" , boot_cpuid, boot_cpuhwid); +set_hard_smp_processor_id (boot_cpuid, boot_cpuhwid);/* * PAPR defines "logical" PVR values for cpus that
arch/powerpc/kernel/setup-common.c +25 -3
@@ -86,7 +86,11 @@ struct machdep_calls *machine_id;EXPORT_SYMBOL (machine_id);int boot_cpuid =-1 ; +int boot_cpuhwid =-1 ; +int boot_dn_nthreads =-1 ;EXPORT_SYMBOL_GPL (boot_cpuid); +EXPORT_SYMBOL_GPL (boot_cpuhwid); +EXPORT_SYMBOL_GPL (boot_dn_nthreads);/* * These are used in binfmt_elf.c to put aux entries on the stack
@@ -461,6 +465,9 @@ void __init smp_setup_cpu_maps(void)struct device_node *dn;int cpu =0 ;int nthreads =1 ; +int online_num =0 ; +int next_cpu = boot_dn_nthreads; +bool btdn_handled = false;DBG ("smp_setup_cpu_maps()\n" );
@@ -488,7 +495,21 @@ void __init smp_setup_cpu_maps(void) nthreads = len /sizeof (int ); -for (j =0 ; j < nthreads && cpu < nr_cpu_ids; j++) { +if (boot_cpuid < nthreads && +be32_to_cpu (intserv[boot_cpuid]) == boot_cpuhwid) { + cpu =0 ; + btdn_handled = true; + }else if ( !btdn_handled && + nr_cpu_ids - online_num <= boot_dn_nthreads) +/* online threads in boot dn before using up quota */ +continue ; +else { + cpu = next_cpu; + next_cpu += nthreads; + } + + +for (j =0 ; j < nthreads && online_num < nr_cpu_ids; j++) {bool avail; DBG(" thread %d -> cpu %d (hard id %d)\n",
@@ -503,12 +524,13 @@ void __init smp_setup_cpu_maps(void)set_hard_smp_processor_id (cpu,be32_to_cpu (intserv[j]));set_cpu_possible (cpu, true); cpu++; + online_num++; } - -if (cpu >= nr_cpu_ids) { +if (online_num >= nr_cpu_ids) {of_node_put (dn);break ; } + }/* If no SMT supported, nthreads is forced to 1 */
Signature · 1 line
2.7.4
kbuild test robot
build botRe: [PATCHv5 1/3] powerpc, cpu: partially unbind the mapping between cpu logical id and its seq in dt
Thank you for the patch! Yet something to improve:
Hi Pingfan,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on v4.16-rc4]
[also build test ERROR on next-20180319]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Pingfan-Liu/enable-nr_cpus-for-powerpc/20180318-041117
config: powerpc-tqm8548_defconfig (attached as .config)
compiler: powerpc-linux-gnu-gcc (Debian 7.2.0-11) 7.2.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=powerpc
All errors (new ones prefixed by >>):
>> arch/powerpc/kernel/prom.c:79:23: error: 'boot_cpu_count' defined but not used [-Werror=unused-variable]
static int __initdata boot_cpu_count;
^~~~~~~~~~~~~~cc1: all warnings being treated as errors
vim +/boot_cpu_count +79 arch/powerpc/kernel/prom.c
9b6b563c Paul Mackerras 2005-10-06 71 9b6b563c Paul Mackerras 2005-10-06 72 #ifdef CONFIG_PPC64 28897731 Olof Johansson 2006-04-12 73 int __initdata iommu_is_off; 9b6b563c Paul Mackerras 2005-10-06 74 int __initdata iommu_force_on; cf00a8d1 Paul Mackerras 2005-10-31 75 unsigned long tce_alloc_start, tce_alloc_end; cd3db0c4 Benjamin Herrenschmidt 2010-07-06 76 u64 ppc64_rma_size; 9b6b563c Paul Mackerras 2005-10-06 77 #endif 03bf469a Benjamin Herrenschmidt 2011-05-11 78 static phys_addr_t first_memblock_size;
7ac87abb Matt Evans 2011-05-25 @79 static int __initdata boot_cpu_count;9b6b563c Paul Mackerras 2005-10-06 80
:::::: The code at line 79 was first introduced by commit
:::::: 7ac87abb8166b99584149fcfb2efef5773a078e9 powerpc: Fix early boot accounting of CPUs
:::::: TO: Matt Evans <[email protected]>
:::::: CC: Benjamin Herrenschmidt <[email protected]>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
PATCH v5 2/3 powerpc, cpu: handling the special case when boot_cpuid greater than nr_cpus
In reply to Pingfan LiuFor kexec -p, after boot_cpuid is mapping into the range of [0, threads_per_core), then if nr_cpus is small, we will have the bitmap [0,..., nr_cpus, ..., boot_cpuid, ...). This patch chooses cpus inside the range of [boot_cpuid - nr_cpus +1, ..., boot_cpuid] to be online.
With this patch and the next, on a P9 machine with thread_per_core=4, and set nr_cpus=2 for the crash kernel.
After
taskset -c 11 sh -c "echo c > /proc/sysrq-trigger"
Then
kdump:/sys/devices/system/cpu# cat possible
2-3
kdump:/sys/devices/system/cpu# cat present
2-3
kdump:/sys/devices/system/cpu# cat online
2-3--- arch/powerpc/kernel/setup-common.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
arch/powerpc/kernel/setup-common.c +9 -2
@@ -468,6 +468,7 @@ void __init smp_setup_cpu_maps(void)int online_num =0 ;int next_cpu = boot_dn_nthreads;bool btdn_handled = false; +int first_threadid =0 ;DBG ("smp_setup_cpu_maps()\n" );
@@ -497,7 +498,12 @@ void __init smp_setup_cpu_maps(void)if (boot_cpuid < nthreads &&be32_to_cpu (intserv[boot_cpuid]) == boot_cpuhwid) { - cpu =0 ; +/* choose a bunch of continous threads */ +if (boot_cpuid > nr_cpu_ids -1 ) { + first_threadid = boot_cpuid - nr_cpu_ids +1 ; + } +/* keep the mapping of logical and thread */ + cpu = first_threadid; btdn_handled = true; }else if ( !btdn_handled && nr_cpu_ids - online_num <= boot_dn_nthreads)
@@ -509,7 +515,8 @@ void __init smp_setup_cpu_maps(void) } -for (j =0 ; j < nthreads && online_num < nr_cpu_ids; j++) { +for (j = first_threadid; + j < nthreads && online_num < nr_cpu_ids; j++) {bool avail; DBG(" thread %d -> cpu %d (hard id %d)\n",
Signature · 1 line
2.7.4
PATCH v5 3/3 ppc64 boot: Wait for boot cpu to show up if nr_cpus limit is about to hit.
In reply to Pingfan LiuFrom: Mahesh Salgaonkar <[email protected]>
The kernel boot parameter 'nr_cpus=' allows one to specify number of possible cpus in the system. In the normal scenario the first cpu (cpu0) that shows up is the boot cpu and hence it gets covered under nr_cpus limit.
But this assumption will be broken in kdump scenario where kdump kenrel after a crash can boot up on an non-zero boot cpu. The paca structure allocation depends on value of nr_cpus and is indexed using logical cpu ids. This definetly will be an issue if boot cpu id > nr_cpus
This patch modifies allocate_pacas() and smp_setup_cpu_maps() to accommodate boot cpu for the case where boot_cpuid > nr_cpu_ids.
This change would help to reduce the memory reservation requirement for kdump on ppc64.
cpu id mapping and stick to the semantics of nr_cpus)
--- arch/powerpc/include/asm/paca.h | 3 +++ arch/powerpc/kernel/paca.c | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-)
arch/powerpc/include/asm/paca.h +3 -0
@@ -49,6 +49,9 @@ extern unsigned int debug_smp_processor_id(void); /* from linux/smp.h */#define get_lppaca () (get_paca()->lppaca_ptr)#define get_slb_shadow () (get_paca()->slb_shadow_ptr) +/* Maximum number of threads per core. */ +#define MAX_SMT 8 +struct task_struct ;/*
arch/powerpc/kernel/paca.c +14 -5
@@ -209,6 +209,7 @@ void __init allocate_pacas(void) {u64 limit;int cpu; +unsigned int nr_cpus_aligned;#ifdef CONFIG_PPC_BOOK3S_64 /*
@@ -220,20 +221,28 @@ void __init allocate_pacas(void) limit = ppc64_rma_size;#endif - paca_size =PAGE_ALIGN (sizeof (struct paca_struct ) * nr_cpu_ids); +/* +* Alloc the paca[] align up to SMT threads. +* This will help us to prepare for a situation where +* boot cpu id > nr_cpus. +* We keep the semantics of nr_cpus in kernel cmdline, but +* waste a bit memory +*/ + nr_cpus_aligned =_ALIGN_UP (nr_cpu_ids,MAX_SMT ); + paca_size =PAGE_ALIGN (sizeof (struct paca_struct ) * nr_cpus_aligned); paca =__va (memblock_alloc_base (paca_size,PAGE_SIZE , limit));memset (paca,0 , paca_size);printk (KERN_DEBUG "Allocated %u bytes for %u pacas at %p\n" , - paca_size, nr_cpu_ids, paca); + paca_size, nr_cpus_aligned, paca); -allocate_lppacas (nr_cpu_ids, limit); +allocate_lppacas (nr_cpus_aligned, limit); -allocate_slb_shadows (nr_cpu_ids, limit); +allocate_slb_shadows (nr_cpus_aligned, limit);/* Can't use for_each_*_cpu, as they aren't functional yet */ -for (cpu =0 ; cpu < nr_cpu_ids; cpu++) +for (cpu =0 ; cpu < nr_cpus_aligned; cpu++)initialise_paca (&paca[cpu], cpu); }
Signature · 1 line
2.7.4
Maintainers, ping? Any comment?
Thanks
Quoted 27 lines · Pingfan Liu
This topic has a very long history. It comes from Mahesh Salgaonkar <[email protected]> For v3: https://patchwork.ozlabs.org/patch/834860/
I hope we can acquire it for "kexec -p" soon.
V4->V5:
improve the [1/3] implementation based on Benjamin's suggestion.Mahesh Salgaonkar (1):
ppc64 boot: Wait for boot cpu to show up ifnr_cpuslimit is about to
hit.Pingfan Liu (2):
powerpc, cpu: partially unbind the mapping between cpu logical id and
its seq in dt
powerpc, cpu: handling the special case whenboot_cpuidgreater than
nr_cpusarch/powerpc/include/asm/paca.h | 3 +++ arch/powerpc/include/asm/smp.h | 2 ++ arch/powerpc/kernel/paca.c | 19 ++++++++++++++----- arch/powerpc/kernel/prom.c | 27 ++++++++++++++++----------- arch/powerpc/kernel/setup-common.c | 35 ++++++++++++++++++++++++++++++++--- 5 files changed, 67 insertions(+), 19 deletions(-)--
2.7.4
v5 cover enable nr_cpus for powerpc
Le 23/03/2018 ? 06:28, Pingfan Liu a ?crit?:
Quoted 1 line
Maintainers, ping? Any comment?
This series doesn't apply anymore, and it has build failure report from the robot.
If it's still relevant, please rebase and fix.
Christophe
Quoted 31 lines
Thanks
Quoted 27 lines · Pingfan Liu
This topic has a very long history. It comes from Mahesh Salgaonkar <[email protected]> For v3: https://patchwork.ozlabs.org/patch/834860/
I hope we can acquire it for "kexec -p" soon.
V4->V5:
improve the [1/3] implementation based on Benjamin's suggestion.Mahesh Salgaonkar (1):
ppc64 boot: Wait for boot cpu to show up ifnr_cpuslimit is about to
hit.Pingfan Liu (2):
powerpc, cpu: partially unbind the mapping between cpu logical id and
its seq in dt
powerpc, cpu: handling the special case whenboot_cpuidgreater than
nr_cpusarch/powerpc/include/asm/paca.h | 3 +++ arch/powerpc/include/asm/smp.h | 2 ++ arch/powerpc/kernel/paca.c | 19 ++++++++++++++----- arch/powerpc/kernel/prom.c | 27 ++++++++++++++++----------- arch/powerpc/kernel/setup-common.c | 35 ++++++++++++++++++++++++++++++++--- 5 files changed, 67 insertions(+), 19 deletions(-)--
2.7.4