Skip to contentSkip to conversation

reboot: hotplug cpus in migrate_to_reboot_cpu()

RFC PATCH Superseded Hsin-Yi Wang 2 participants 4 messages Oct 3, 2019 CPU hotplug

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 reboot: hotplug cpus in migrate_to_reboot_cpu()

Currently system reboots use arch specific codes (eg. smp_send_stop) to offline non reboot cpus. Some arch like arm64, arm, and x86... set offline masks to cpu without really offlining them. Thus it causes some race condition and kernel warning comes out sometimes when system reboots. We can do cpu hotplug in migrate_to_reboot_cpu() to avoid this issue.

---
kernel warnings at reboot:
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/

---
 kernel/cpu.c    | 35 +++++++++++++++++++++++++++++++++++
 kernel/reboot.c | 18 ------------------
 2 files changed, 35 insertions(+), 18 deletions(-)
kernel/cpu.c +35 -0
index fc28e17940e0..2f4d51fe91e3 100644
@@ -31,6 +31,7 @@
 #include <linux/relay.h>
 #include <linux/slab.h>
 #include <linux/percpu-rwsem.h>
+#include <linux/reboot.h>
 
 #include <trace/events/power.h>
 #define CREATE_TRACE_POINTS
@@ -1366,6 +1367,40 @@ int __boot_cpu_id;
 
 #endif /* CONFIG_SMP */
 
+void migrate_to_reboot_cpu(void)
+{
+	/* The boot cpu is always logical cpu 0 */
+	int cpu = reboot_cpu;
+
+	/* Make certain the cpu I'm about to reboot on is online */
+	if (!cpu_online(cpu))
+		cpu = cpumask_first(cpu_online_mask);
+
+	/* Prevent races with other tasks migrating this task */
+	current->flags |= PF_NO_SETAFFINITY;
+
+	/* Make certain I only run on the appropriate processor */
+	set_cpus_allowed_ptr(current, cpumask_of(cpu));
+
+	/* Hotplug other cpus if possible */
+	if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
+		int i, err;
+
+		cpu_maps_update_begin();
+
+		for_each_online_cpu(i) {
+			if (i == cpu)
+				continue;
+			err = _cpu_down(i, 0, CPUHP_OFFLINE);
+			if (err)
+				pr_info("Failed to offline cpu %d\n", i);
+		}
+		cpu_hotplug_disabled++;
+
+		cpu_maps_update_done();
+	}
+}
+
 /* Boot processor state steps */
 static struct cpuhp_step cpuhp_hp_states[] = {
 	[CPUHP_OFFLINE] = {
kernel/reboot.c +0 -18
index c4d472b7f1b4..f0046be34a60 100644
@@ -215,24 +215,6 @@ void do_kernel_restart(char *cmd)
 	atomic_notifier_call_chain(&restart_handler_list, reboot_mode, cmd);
 }
 
-void migrate_to_reboot_cpu(void)
-{
-	/* The boot cpu is always logical cpu 0 */
-	int cpu = reboot_cpu;
-
-	cpu_hotplug_disable();
-
-	/* Make certain the cpu I'm about to reboot on is online */
-	if (!cpu_online(cpu))
-		cpu = cpumask_first(cpu_online_mask);
-
-	/* Prevent races with other tasks migrating this task */
-	current->flags |= PF_NO_SETAFFINITY;
-
-	/* Make certain I only run on the appropriate processor */
-	set_cpus_allowed_ptr(current, cpumask_of(cpu));
-}
-
 /**
  *	kernel_restart - reboot the system
  *	@cmd: pointer to buffer containing command to execute for restart
Signature · 1 line
2.23.0.444.g18eeb5a265-goog
#1 lore raw
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please…

Hi Hsin-Yi,

[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on linus/master]
[cannot apply to v5.4-rc1 next-20191002]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url: https://github.com/0day-ci/linux/commits/Hsin-Yi-Wang/reboot-hotplug-cpus-in-migrate_to_reboot_cpu/20191003-104721
config: i386-tinyconfig (attached as .config)
compiler: gcc-7 (Debian 7.4.0-13) 7.4.0
reproduce:

        # save the attached .config to linux build tree
        make ARCH=i386 

If you fix the issue, kindly add following tag

All errors (new ones prefixed by >>):

kernel/cpu.c: In function 'migrate_to_reboot_cpu':

>> kernel/cpu.c:1394:10: error: implicit declaration of function '_cpu_down'; did you mean 'cpu_data'? [-Werror=implicit-function-declaration]
       err = _cpu_down(i, 0, CPUHP_OFFLINE);
             ^~~~~~~~~
             cpu_data
>> kernel/cpu.c:1398:3: error: 'cpu_hotplug_disabled' undeclared (first use in this function); did you mean 'cpu_hotplug_disable'?
      cpu_hotplug_disabled++;
      ^~~~~~~~~~~~~~~~~~~~
      cpu_hotplug_disable

kernel/cpu.c:1398:3: note: each undeclared identifier is reported only once for each function it appears in
cc1: some warnings being treated as errors

vim +1394 kernel/cpu.c

1369
1370 void migrate_to_reboot_cpu(void)
1371 {
1372 /* The boot cpu is always logical cpu 0 */
1373 int cpu = reboot_cpu;
1374
1375 /* Make certain the cpu I'm about to reboot on is online */
1376 if (!cpu_online(cpu))
1377 cpu = cpumask_first(cpu_online_mask);
1378
1379 /* Prevent races with other tasks migrating this task */
1380 current->flags |= PF_NO_SETAFFINITY;
1381
1382 /* Make certain I only run on the appropriate processor */
1383 set_cpus_allowed_ptr(current, cpumask_of(cpu));
1384
1385 /* Hotplug other cpus if possible */
1386 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
1387 int i, err;
1388
1389 cpu_maps_update_begin();
1390
1391 for_each_online_cpu(i) {
1392 if (i == cpu)
1393 continue;

> 1394				err = _cpu_down(i, 0, CPUHP_OFFLINE);
  1395				if (err)
  1396					pr_info("Failed to offline cpu %d\n", i);
  1397			}
> 1398			cpu_hotplug_disabled++;

1399
1400 cpu_maps_update_done();
1401 }
1402 }
1403

---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation

#2 lore raw
In reply to Hsin-Yi Wang
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please…

Hi Hsin-Yi,

[FYI, it's a private test report for your RFC patch.]
[auto build test ERROR on linus/master]
[cannot apply to v5.4-rc1 next-20191002]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url: https://github.com/0day-ci/linux/commits/Hsin-Yi-Wang/reboot-hotplug-cpus-in-migrate_to_reboot_cpu/20191003-104721
config: riscv-allmodconfig (attached as .config)
compiler: riscv64-linux-gcc (GCC) 7.4.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.4.0 make.cross ARCH=riscv

If you fix the issue, kindly add following tag

All errors (new ones prefixed by >>):

kernel/cpu.c: In function 'migrate_to_reboot_cpu':

>> kernel/cpu.c:1394:10: error: implicit declaration of function '_cpu_down'; did you mean '_cpu_up'? [-Werror=implicit-function-declaration]
       err = _cpu_down(i, 0, CPUHP_OFFLINE);
             ^~~~~~~~~
             _cpu_up

cc1: some warnings being treated as errors

vim +1394 kernel/cpu.c

1369
1370 void migrate_to_reboot_cpu(void)
1371 {
1372 /* The boot cpu is always logical cpu 0 */
1373 int cpu = reboot_cpu;
1374
1375 /* Make certain the cpu I'm about to reboot on is online */
1376 if (!cpu_online(cpu))
1377 cpu = cpumask_first(cpu_online_mask);
1378
1379 /* Prevent races with other tasks migrating this task */
1380 current->flags |= PF_NO_SETAFFINITY;
1381
1382 /* Make certain I only run on the appropriate processor */
1383 set_cpus_allowed_ptr(current, cpumask_of(cpu));
1384
1385 /* Hotplug other cpus if possible */
1386 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
1387 int i, err;
1388
1389 cpu_maps_update_begin();
1390
1391 for_each_online_cpu(i) {
1392 if (i == cpu)
1393 continue;

> 1394				err = _cpu_down(i, 0, CPUHP_OFFLINE);

1395 if (err)
1396 pr_info("Failed to offline cpu %d\n", i);
1397 }
1398 cpu_hotplug_disabled++;
1399
1400 cpu_maps_update_done();
1401 }
1402 }
1403

---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation

#3 lore raw
In reply to Hsin-Yi Wang
Quoted 40 lines · Hsin-Yi Wang

Currently system reboots use arch specific codes (eg. smp_send_stop) to offline non reboot cpus. Some arch like arm64, arm, and x86... set offline masks to cpu without really offlining them. Thus it causes some race condition and kernel warning comes out sometimes when system reboots. We can do cpu hotplug in migrate_to_reboot_cpu() to avoid this issue.

---
kernel warnings at reboot:
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/

---
 kernel/cpu.c    | 35 +++++++++++++++++++++++++++++++++++
 kernel/reboot.c | 18 ------------------
 2 files changed, 35 insertions(+), 18 deletions(-)
kernel/cpu.c +8 -0
index fc28e17940e0..2f4d51fe91e3 100644
@@ -31,6 +31,7 @@
 #include <linux/relay.h>
 #include <linux/slab.h>
 #include <linux/percpu-rwsem.h>
+#include <linux/reboot.h>
 
 #include <trace/events/power.h>
 #define CREATE_TRACE_POINTS
@@ -1366,6 +1367,40 @@ int __boot_cpu_id;
 
 #endif /* CONFIG_SMP */
 
+void migrate_to_reboot_cpu(void)
+{
+       /* The boot cpu is always logical cpu 0 */
+       int cpu = reboot_cpu;
+
+       /* Make certain the cpu I'm about to reboot on is online */
+       if (!cpu_online(cpu))

kernel/cpu.c int __boot_cpu_id; · line 1386

Quoted 10 lines
kernel/cpu.c +10 -0
+               cpu = cpumask_first(cpu_online_mask);
+
+       /* Prevent races with other tasks migrating this task */
+       current->flags |= PF_NO_SETAFFINITY;
+
+       /* Make certain I only run on the appropriate processor */
+       set_cpus_allowed_ptr(current, cpumask_of(cpu));
+
+       /* Hotplug other cpus if possible */
+       if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {

Should use #ifdef CONFIG_HOTPLUG_CPU here. Will fix in the next version if this patch is reasonable.
(Reported-by: kbuild test robot <[email protected]>)

Quoted 52 lines
kernel/cpu.c +17 -0
+               int i, err;
+
+               cpu_maps_update_begin();
+
+               for_each_online_cpu(i) {
+                       if (i == cpu)
+                               continue;
+                       err = _cpu_down(i, 0, CPUHP_OFFLINE);
+                       if (err)
+                               pr_info("Failed to offline cpu %d\n", i);
+               }
+               cpu_hotplug_disabled++;
+
+               cpu_maps_update_done();
+       }
+}
+
 /* Boot processor state steps */
 static struct cpuhp_step cpuhp_hp_states[] = {
        [CPUHP_OFFLINE] = {
kernel/reboot.c +0 -19
index c4d472b7f1b4..f0046be34a60 100644
@@ -215,24 +215,6 @@ void do_kernel_restart(char *cmd)
        atomic_notifier_call_chain(&restart_handler_list, reboot_mode, cmd);
 }
 
-void migrate_to_reboot_cpu(void)
-{
-       /* The boot cpu is always logical cpu 0 */
-       int cpu = reboot_cpu;
-
-       cpu_hotplug_disable();
-
-       /* Make certain the cpu I'm about to reboot on is online */
-       if (!cpu_online(cpu))
-               cpu = cpumask_first(cpu_online_mask);
-
-       /* Prevent races with other tasks migrating this task */
-       current->flags |= PF_NO_SETAFFINITY;
-
-       /* Make certain I only run on the appropriate processor */
-       set_cpus_allowed_ptr(current, cpumask_of(cpu));
-}
-
 /**
  *     kernel_restart - reboot the system
  *     @cmd: pointer to buffer containing command to execute for restart
--

2.23.0.444.g18eeb5a265-goog