reboot: hotplug cpus in migrate_to_reboot_cpu()
RFC PATCH 1/1 reboot: hotplug cpus in migrate_to_reboot_cpu()
Currently system reboots use arch specific codes (eg. smp_send_stop) to offline non reboot cpus. Some arch like arm64, arm, and x86... set offline masks to cpu without really offlining them. Thus it causes some race condition and kernel warning comes out sometimes when system reboots. We can do cpu hotplug in migrate_to_reboot_cpu() to avoid this issue.
---
kernel warnings at reboot:
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/
--- kernel/cpu.c | 35 +++++++++++++++++++++++++++++++++++ kernel/reboot.c | 18 ------------------ 2 files changed, 35 insertions(+), 18 deletions(-)
kernel/cpu.c +35 -0
@@ -31,6 +31,7 @@#include <linux/relay.h> #include <linux/slab.h> #include <linux/percpu-rwsem.h> +#include <linux/reboot.h> #include <trace/events/power.h> #define CREATE_TRACE_POINTS
@@ -1366,6 +1367,40 @@ int __boot_cpu_id;#endif /* CONFIG_SMP */ +void migrate_to_reboot_cpu (void ) +{ +/* The boot cpu is always logical cpu 0 */ +int cpu = reboot_cpu; + +/* Make certain the cpu I'm about to reboot on is online */ +if (!cpu_online (cpu)) + cpu =cpumask_first (cpu_online_mask); + +/* Prevent races with other tasks migrating this task */ + current->flags |=PF_NO_SETAFFINITY ; + +/* Make certain I only run on the appropriate processor */ +set_cpus_allowed_ptr (current,cpumask_of (cpu)); + +/* Hotplug other cpus if possible */ +if (IS_ENABLED (CONFIG_HOTPLUG_CPU )) { +int i, err; + +cpu_maps_update_begin (); + +for_each_online_cpu (i) { +if (i == cpu) +continue ; + err =_cpu_down (i,0 ,CPUHP_OFFLINE ); +if (err) +pr_info ("Failed to offline cpu %d\n" , i); + } + cpu_hotplug_disabled++; + +cpu_maps_update_done (); + } +} +/* Boot processor state steps */ static struct cpuhp_step cpuhp_hp_states[] = { [CPUHP_OFFLINE ] = {
kernel/reboot.c +0 -18
@@ -215,24 +215,6 @@ void do_kernel_restart(char *cmd)atomic_notifier_call_chain (&restart_handler_list, reboot_mode, cmd); } -void migrate_to_reboot_cpu (void ) -{ -/* The boot cpu is always logical cpu 0 */ -int cpu = reboot_cpu; - -cpu_hotplug_disable (); - -/* Make certain the cpu I'm about to reboot on is online */ -if (!cpu_online (cpu)) - cpu =cpumask_first (cpu_online_mask); - -/* Prevent races with other tasks migrating this task */ - current->flags |=PF_NO_SETAFFINITY ; - -/* Make certain I only run on the appropriate processor */ -set_cpus_allowed_ptr (current,cpumask_of (cpu)); -} -/** * kernel_restart - reboot the system * @cmd: pointer to buffer containing command to execute for restart
Signature · 1 line
2.23.0.444.g18eeb5a265-goog
kbuild test robot
build bot[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please…
Hi Hsin-Yi,
[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on linus/master]
[cannot apply to v5.4-rc1 next-20191002]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Hsin-Yi-Wang/reboot-hotplug-cpus-in-migrate_to_reboot_cpu/20191003-104721
config: i386-tinyconfig (attached as .config)
compiler: gcc-7 (Debian 7.4.0-13) 7.4.0
reproduce:
# save the attached .config to linux build tree
make ARCH=i386 If you fix the issue, kindly add following tag
All errors (new ones prefixed by >>):
kernel/cpu.c: In function 'migrate_to_reboot_cpu':
>> kernel/cpu.c:1394:10: error: implicit declaration of function '_cpu_down'; did you mean 'cpu_data'? [-Werror=implicit-function-declaration]
err = _cpu_down(i, 0, CPUHP_OFFLINE);
^~~~~~~~~
cpu_data
>> kernel/cpu.c:1398:3: error: 'cpu_hotplug_disabled' undeclared (first use in this function); did you mean 'cpu_hotplug_disable'?
cpu_hotplug_disabled++;
^~~~~~~~~~~~~~~~~~~~
cpu_hotplug_disable kernel/cpu.c:1398:3: note: each undeclared identifier is reported only once for each function it appears in
cc1: some warnings being treated as errors
vim +1394 kernel/cpu.c
1369
1370 void migrate_to_reboot_cpu(void)
1371 {
1372 /* The boot cpu is always logical cpu 0 */
1373 int cpu = reboot_cpu;
1374
1375 /* Make certain the cpu I'm about to reboot on is online */
1376 if (!cpu_online(cpu))
1377 cpu = cpumask_first(cpu_online_mask);
1378
1379 /* Prevent races with other tasks migrating this task */
1380 current->flags |= PF_NO_SETAFFINITY;
1381
1382 /* Make certain I only run on the appropriate processor */
1383 set_cpus_allowed_ptr(current, cpumask_of(cpu));
1384
1385 /* Hotplug other cpus if possible */
1386 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
1387 int i, err;
1388
1389 cpu_maps_update_begin();
1390
1391 for_each_online_cpu(i) {
1392 if (i == cpu)
1393 continue;
> 1394 err = _cpu_down(i, 0, CPUHP_OFFLINE);
1395 if (err)
1396 pr_info("Failed to offline cpu %d\n", i);
1397 }
> 1398 cpu_hotplug_disabled++; 1399
1400 cpu_maps_update_done();
1401 }
1402 }
1403
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
kbuild test robot
build bot[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please…
Hi Hsin-Yi,
[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on linus/master]
[cannot apply to v5.4-rc1 next-20191002]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Hsin-Yi-Wang/reboot-hotplug-cpus-in-migrate_to_reboot_cpu/20191003-104721
config: riscv-allmodconfig (attached as .config)
compiler: riscv64-linux-gcc (GCC) 7.4.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.4.0 make.cross ARCH=riscv
If you fix the issue, kindly add following tag
All errors (new ones prefixed by >>):
kernel/cpu.c: In function 'migrate_to_reboot_cpu':
>> kernel/cpu.c:1394:10: error: implicit declaration of function '_cpu_down'; did you mean '_cpu_up'? [-Werror=implicit-function-declaration]
err = _cpu_down(i, 0, CPUHP_OFFLINE);
^~~~~~~~~
_cpu_upcc1: some warnings being treated as errors
vim +1394 kernel/cpu.c
1369
1370 void migrate_to_reboot_cpu(void)
1371 {
1372 /* The boot cpu is always logical cpu 0 */
1373 int cpu = reboot_cpu;
1374
1375 /* Make certain the cpu I'm about to reboot on is online */
1376 if (!cpu_online(cpu))
1377 cpu = cpumask_first(cpu_online_mask);
1378
1379 /* Prevent races with other tasks migrating this task */
1380 current->flags |= PF_NO_SETAFFINITY;
1381
1382 /* Make certain I only run on the appropriate processor */
1383 set_cpus_allowed_ptr(current, cpumask_of(cpu));
1384
1385 /* Hotplug other cpus if possible */
1386 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
1387 int i, err;
1388
1389 cpu_maps_update_begin();
1390
1391 for_each_online_cpu(i) {
1392 if (i == cpu)
1393 continue;
> 1394 err = _cpu_down(i, 0, CPUHP_OFFLINE);
1395 if (err)
1396 pr_info("Failed to offline cpu %d\n", i);
1397 }
1398 cpu_hotplug_disabled++;
1399
1400 cpu_maps_update_done();
1401 }
1402 }
1403
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
Quoted 40 lines · Hsin-Yi Wang
Currently system reboots use arch specific codes (eg.
smp_send_stop) to offline non reboot cpus. Some arch like arm64, arm, and x86... set offline masks to cpu without really offlining them. Thus it causes some race condition and kernel warning comes out sometimes when system reboots. We can do cpu hotplug inmigrate_to_reboot_cpu()to avoid this issue.Signed-off-by Hsin-Yi Wang <[email protected]>---
kernel warnings at reboot:
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/--- kernel/cpu.c | 35 +++++++++++++++++++++++++++++++++++ kernel/reboot.c | 18 ------------------ 2 files changed, 35 insertions(+), 18 deletions(-)kernel/cpu.c +8
-0@@ -31,6 +31,7 @@#include <linux/relay.h> #include <linux/slab.h> #include <linux/percpu-rwsem.h> +#include <linux/reboot.h> #include <trace/events/power.h> #define CREATE_TRACE_POINTS @@ -1366,6 +1367,40 @@ int __boot_cpu_id;#endif /* CONFIG_SMP */ +void migrate_to_reboot_cpu (void ) +{ +/* The boot cpu is always logical cpu 0 */ +int cpu = reboot_cpu; + +/* Make certain the cpu I'm about to reboot on is online */ +if (!cpu_online (cpu))
kernel/cpu.c int __boot_cpu_id; · line 1386
Quoted 10 lines
kernel/cpu.c +10
-0+ cpu =cpumask_first (cpu_online_mask); + +/* Prevent races with other tasks migrating this task */ + current->flags |=PF_NO_SETAFFINITY ; + +/* Make certain I only run on the appropriate processor */ +set_cpus_allowed_ptr (current,cpumask_of (cpu)); + +/* Hotplug other cpus if possible */ +if (IS_ENABLED (CONFIG_HOTPLUG_CPU )) {
Should use #ifdef CONFIG_HOTPLUG_CPU here. Will fix in the next version if this patch is reasonable.
(Reported-by: kbuild test robot <[email protected]>)
Quoted 52 lines
kernel/cpu.c +17
-0+int i, err; + +cpu_maps_update_begin (); + +for_each_online_cpu (i) { +if (i == cpu) +continue ; + err =_cpu_down (i,0 ,CPUHP_OFFLINE ); +if (err) +pr_info ("Failed to offline cpu %d\n" , i); + } + cpu_hotplug_disabled++; + +cpu_maps_update_done (); + } +} +/* Boot processor state steps */ static struct cpuhp_step cpuhp_hp_states[] = { [CPUHP_OFFLINE ] = {kernel/reboot.c +0
-19@@ -215,24 +215,6 @@ void do_kernel_restart(char *cmd)atomic_notifier_call_chain (&restart_handler_list, reboot_mode, cmd); } -void migrate_to_reboot_cpu (void ) -{ -/* The boot cpu is always logical cpu 0 */ -int cpu = reboot_cpu; - -cpu_hotplug_disable (); - -/* Make certain the cpu I'm about to reboot on is online */ -if (!cpu_online (cpu)) - cpu =cpumask_first (cpu_online_mask); - -/* Prevent races with other tasks migrating this task */ - current->flags |=PF_NO_SETAFFINITY ; - -/* Make certain I only run on the appropriate processor */ -set_cpus_allowed_ptr (current,cpumask_of (cpu)); -} -/** * kernel_restart - reboot the system * @cmd: pointer to buffer containing command to execute for restart -- 2.23.0.444.g18eeb5a265-goog