reboot: support hotplug CPUs before reboot
Subsystems (14)
PATCH v4 1/1 reboot: support hotplug CPUs before reboot
Currently system reboots uses architecture specific codes (smp_send_stop) to offline non reboot CPUs. Most architecture's implementation is looping through all non reboot online CPUs and call ipi function to each of them. Some architecture like arm64, arm, and x86... would set offline masks to cpu without really offline them. This causes some race condition and kernel warning comes out sometimes when system reboots.
This patch adds a config ARCH_OFFLINE_CPUS_ON_REBOOT, which would hotplug cpus in migrate_to_reboot_cpu(). If non reboot cpus are all offlined here, the loop for checking online cpus would be an empty loop. If architecture don't enable this config, or some cpus somehow fails to offline, it would fallback to ipi function.
Opt in this config for architectures that support CONFIG_HOTPLUG_CPU.
---
Change from v3:
* Opt in config for architectures that support CONFIG_HOTPLUG_CPU
* Merge function offline_secondary_cpus() and freeze_secondary_cpus() with an additional flag.
Change from v2:
* Add another config instead of configed by CONFIG_HOTPLUG_CPU
--- arch/Kconfig | 5 +++++ arch/arm/Kconfig | 1 + arch/arm64/Kconfig | 1 + arch/arm64/kernel/hibernate.c | 2 +- arch/csky/Kconfig | 1 + arch/ia64/Kconfig | 1 + arch/mips/Kconfig | 1 + arch/parisc/Kconfig | 1 + arch/powerpc/Kconfig | 1 + arch/s390/Kconfig | 1 + arch/sh/Kconfig | 1 + arch/sparc/Kconfig | 1 + arch/x86/Kconfig | 1 + arch/xtensa/Kconfig | 1 + drivers/power/reset/sc27xx-poweroff.c | 2 +- include/linux/cpu.h | 9 ++++++--- kernel/cpu.c | 17 +++++++++++------ kernel/reboot.c | 8 ++++++++ 18 files changed, 44 insertions(+), 11 deletions(-)
arch/Kconfig +5 -0
@@ -255,6 +255,11 @@ config ARCH_HAS_UNCACHED_SEGMENT select ARCH_HAS_DMA_PREP_COHERENT bool +# Select to do a full hotplug on secondary CPUs before reboot. +config ARCH_OFFLINE_CPUS_ON_REBOOT + bool "Support for hotplug CPUs before reboot" + depends on HOTPLUG_CPU + # Select if arch init_task must go in the __init_task_data section config ARCH_TASK_STRUCT_ON_STACK bool
arch/arm/Kconfig +1 -0
@@ -28,6 +28,7 @@ config ARM select ARCH_KEEP_MEMBLOCK if HAVE_ARCH_PFN_VALID || KEXEC select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMW
arch/arm64/Kconfig +1 -0
@@ -61,6 +61,7 @@ config ARM64 select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION select ARCH_KEEP_MEMBLOCK + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS
arch/arm64/kernel/hibernate.c +1 -1
@@ -581,5 +581,5 @@ int hibernate_resume_nonboot_cpu_disable(void)return -ENODEV ; } -return freeze_secondary_cpus (sleep_cpu); +return freeze_secondary_cpus (sleep_cpu, false); }
arch/csky/Kconfig +1 -0
@@ -5,6 +5,7 @@ config CSKY select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2 select COMMON_CLK
arch/ia64/Kconfig +1 -0
@@ -10,6 +10,7 @@ config IA64 bool select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ACPI select ACPI_NUMA if NUMA select ARCH_SUPPORTS_ACPI
arch/mips/Kconfig +1 -0
@@ -8,6 +8,7 @@ config MIPS select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAS_FORTIFY_SOURCE + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SUPPORTS_UPROBES select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if 64BIT
arch/parisc/Kconfig +1 -0
@@ -13,6 +13,7 @@ config PARISC select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_NO_SG_CHAIN + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SUPPORTS_MEMORY_FAILURE select RTC_CLASS select RTC_DRV_GENERIC
arch/powerpc/Kconfig +1 -0
@@ -142,6 +142,7 @@ config PPC select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_USE_BUILTIN_BSWAP
arch/s390/Kconfig +1 -0
@@ -102,6 +102,7 @@ config S390 select ARCH_INLINE_WRITE_UNLOCK_IRQ select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE select ARCH_KEEP_MEMBLOCK + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SAVE_PAGE_KEYS if HIBERNATION select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW
arch/sh/Kconfig +1 -0
@@ -18,6 +18,7 @@ config SUPERH select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A) select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select PERF_USE_VMALLOC select HAVE_DEBUG_KMEMLEAK select HAVE_KERNEL_GZIP
arch/sparc/Kconfig +1 -0
@@ -30,6 +30,7 @@ config SPARC select RTC_SYSTOHC select HAVE_ARCH_JUMP_LABEL if SPARC64 select GENERIC_IRQ_SHOW + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_WANT_IPC_PARSE_VERSION select GENERIC_PCI_IOMAP select HAVE_NMI_WATCHDOG if SPARC64
arch/x86/Kconfig +1 -0
@@ -85,6 +85,7 @@ config X86 select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_STACKWALK select ARCH_SUPPORTS_ACPI select ARCH_SUPPORTS_ATOMIC_RMW
arch/xtensa/Kconfig +1 -0
@@ -7,6 +7,7 @@ config XTENSA select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU select ARCH_HAS_UNCACHED_SEGMENT if MMU + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_FRAME_POINTERS
drivers/power/reset/sc27xx-poweroff.c +1 -1
@@ -30,7 +30,7 @@ static void sc27xx_poweroff_shutdown(void)#ifdef CONFIG_PM_SLEEP_SMP int cpu =smp_processor_id (); -freeze_secondary_cpus (cpu); +freeze_secondary_cpus (cpu, false);#endif }
include/linux/cpu.h +6 -3
@@ -137,11 +137,14 @@ static inline void cpu_hotplug_done(void) { cpus_write_unlock(); }static inline void get_online_cpus (void ) {cpus_read_lock (); }static inline void put_online_cpus (void ) {cpus_read_unlock (); } +#if defined(CONFIG_PM_SLEEP_SMP ) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT ) +extern int freeze_secondary_cpus (int primary,bool reboot); +#endif +#ifdef CONFIG_PM_SLEEP_SMP -extern int freeze_secondary_cpus (int primary);static inline int disable_nonboot_cpus (void ) { -return freeze_secondary_cpus (0 ); +return freeze_secondary_cpus (0 , false); }extern void enable_nonboot_cpus (void );
@@ -152,7 +155,7 @@ static inline int suspend_disable_secondary_cpus(void)if (IS_ENABLED (CONFIG_PM_SLEEP_SMP_NONZERO_CPU )) cpu =-1 ; -return freeze_secondary_cpus (cpu); +return freeze_secondary_cpus (cpu, false); }static inline void suspend_enable_secondary_cpus (void ) {
kernel/cpu.c +11 -6
@@ -1209,10 +1209,10 @@ int cpu_up(unsigned int cpu) }EXPORT_SYMBOL_GPL (cpu_up); -#ifdef CONFIG_PM_SLEEP_SMP +#if defined(CONFIG_PM_SLEEP_SMP ) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT )static cpumask_var_t frozen_cpus; -int freeze_secondary_cpus (int primary) +int freeze_secondary_cpus (int primary,bool reboot) {int cpu, error =0 ;
@@ -1237,20 +1237,25 @@ int freeze_secondary_cpus(int primary)if (cpu == primary)continue ; -if (pm_wakeup_pending ()) { +#ifdef CONFIG_PM_SLEEP +if (!reboot &&pm_wakeup_pending ()) {pr_info ("Wakeup pending. Abort CPU freeze\n" ); error = -EBUSY ;break ; } +#endif -trace_suspend_resume (TPS ("CPU_OFF" ), cpu, true); +if (!reboot) +trace_suspend_resume (TPS ("CPU_OFF" ), cpu, true); error =_cpu_down (cpu,1 ,CPUHP_OFFLINE ); -trace_suspend_resume (TPS ("CPU_OFF" ), cpu, false); +if (!reboot) +trace_suspend_resume (TPS ("CPU_OFF" ), cpu, false);if (!error)cpumask_set_cpu (cpu, frozen_cpus);else {pr_err ("Error taking CPU%d down: %d\n" , cpu, error); -break ; +if (!reboot) +break ; } }
kernel/reboot.c +8 -0
@@ -7,6 +7,7 @@#define pr_fmt (fmt) "reboot: " fmt +#include <linux/cpu.h> #include <linux/ctype.h> #include <linux/export.h> #include <linux/kexec.h>
@@ -220,7 +221,9 @@ void migrate_to_reboot_cpu(void)/* The boot cpu is always logical cpu 0 */ int cpu = reboot_cpu; +#if !IS_ENABLED (CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT )cpu_hotplug_disable (); +#endif /* Make certain the cpu I'm about to reboot on is online */ if (!cpu_online (cpu))
@@ -231,6 +234,11 @@ void migrate_to_reboot_cpu(void)/* Make certain I only run on the appropriate processor */ set_cpus_allowed_ptr (current,cpumask_of (cpu)); + +#if IS_ENABLED (CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT ) +/* Hotplug other cpus if possible */ +freeze_secondary_cpus (cpu, true); +#endif }/**
Signature · 1 line
2.25.0.rc1.283.g88dfdc4193-goog
Quoted 13 lines · Hsin-Yi Wang
Currently system reboots uses architecture specific codes (
smp_send_stop) to offline non reboot CPUs. Most architecture's implementation is looping through all non reboot online CPUs and call ipi function to each of them. Some architecture like arm64, arm, and x86... would set offline masks to cpu without really offline them. This causes some race condition and kernel warning comes out sometimes when system reboots.This patch adds a config
ARCH_OFFLINE_CPUS_ON_REBOOT, which would hotplug cpus inmigrate_to_reboot_cpu(). If non reboot cpus are all offlined here, the loop for checking online cpus would be an empty loop. If architecture don't enable this config, or some cpus somehow fails to offline, it would fallback to ipi function.
Quoted 8 lines
Opt in this config for architectures that support
CONFIG_HOTPLUG_CPU.Signed-off-by Hsin-Yi Wang <[email protected]>---
Change from v3:
* Opt in config for architectures that supportCONFIG_HOTPLUG_CPU
* Merge functionoffline_secondary_cpus()andfreeze_secondary_cpus()with an additional flag.
I haven't commented on the idea itself, actually (I'm not sure I'm the right person though but as I'm already looking at the code...)
On x86, the custom IPI/NMI solution we have now is more 'lightweight' and here you suggest we switch to the full-blown CPUHP machinery instead. We seem to be already using it for suspend/hibernation, however, I would expect it to be way less tested: on servers, for example, less people do suspend/resume than reboot :-)
The existing IPI/NMI code stays in place so nothing (in theory) should break -- unles some cpuhp_ hook does something really weird. But your change will definitely contribute to improving these hooks' quality in the long run :-)
I have nothing to say on other arches and I don't see anyone from e.g. ARM on the CC: list, not good.
Quoted 29 lines
Change from v2:
* Add another config instead of configed byCONFIG_HOTPLUG_CPU--- arch/Kconfig | 5 +++++ arch/arm/Kconfig | 1 + arch/arm64/Kconfig | 1 + arch/arm64/kernel/hibernate.c | 2 +- arch/csky/Kconfig | 1 + arch/ia64/Kconfig | 1 + arch/mips/Kconfig | 1 + arch/parisc/Kconfig | 1 + arch/powerpc/Kconfig | 1 + arch/s390/Kconfig | 1 + arch/sh/Kconfig | 1 + arch/sparc/Kconfig | 1 + arch/x86/Kconfig | 1 + arch/xtensa/Kconfig | 1 + drivers/power/reset/sc27xx-poweroff.c | 2 +- include/linux/cpu.h | 9 ++++++--- kernel/cpu.c | 17 +++++++++++------ kernel/reboot.c | 8 ++++++++ 18 files changed, 44 insertions(+), 11 deletions(-)arch/Kconfig +0
-0
arch/Kconfig config ARCH_HAS_UNCACHED_SEGMENT · line 262
Quoted 8 lines
arch/Kconfig +5
-0@@ -255,6 +255,11 @@ config ARCH_HAS_UNCACHED_SEGMENT select ARCH_HAS_DMA_PREP_COHERENT bool +# Select to do a full hotplug on secondary CPUs before reboot. +config ARCH_OFFLINE_CPUS_ON_REBOOT + bool "Support for hotplug CPUs before reboot" + depends on HOTPLUG_CPU +
Nit: what we're actually doing is not 'hotplug', it's either 'hotunplug' or 'offlining'.
Quoted 219 lines
arch/Kconfig +0
-0# Select if arch init_task must go in the __init_task_data section config ARCH_TASK_STRUCT_ON_STACK boolarch/arm/Kconfig +1
-0@@ -28,6 +28,7 @@ config ARM select ARCH_KEEP_MEMBLOCK if HAVE_ARCH_PFN_VALID || KEXEC select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMWarch/arm64/Kconfig +1
-0@@ -61,6 +61,7 @@ config ARM64 select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION select ARCH_KEEP_MEMBLOCK + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKSarch/arm64/kernel/hibernate.c +1
-1@@ -581,5 +581,5 @@ int hibernate_resume_nonboot_cpu_disable(void)return -ENODEV ; } -return freeze_secondary_cpus (sleep_cpu); +return freeze_secondary_cpus (sleep_cpu, false); }arch/csky/Kconfig +1
-0@@ -5,6 +5,7 @@ config CSKY select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2 select COMMON_CLKarch/ia64/Kconfig +1
-0@@ -10,6 +10,7 @@ config IA64 bool select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ACPI select ACPI_NUMA if NUMA select ARCH_SUPPORTS_ACPIarch/mips/Kconfig +1
-0@@ -8,6 +8,7 @@ config MIPS select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAS_FORTIFY_SOURCE + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SUPPORTS_UPROBES select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if 64BITarch/parisc/Kconfig +1
-0@@ -13,6 +13,7 @@ config PARISC select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_NO_SG_CHAIN + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SUPPORTS_MEMORY_FAILURE select RTC_CLASS select RTC_DRV_GENERICarch/powerpc/Kconfig +1
-0@@ -142,6 +142,7 @@ config PPC select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_USE_BUILTIN_BSWAParch/s390/Kconfig +1
-0@@ -102,6 +102,7 @@ config S390 select ARCH_INLINE_WRITE_UNLOCK_IRQ select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE select ARCH_KEEP_MEMBLOCK + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SAVE_PAGE_KEYS if HIBERNATION select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMWarch/sh/Kconfig +1
-0@@ -18,6 +18,7 @@ config SUPERH select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A) select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select PERF_USE_VMALLOC select HAVE_DEBUG_KMEMLEAK select HAVE_KERNEL_GZIParch/sparc/Kconfig +1
-0@@ -30,6 +30,7 @@ config SPARC select RTC_SYSTOHC select HAVE_ARCH_JUMP_LABEL if SPARC64 select GENERIC_IRQ_SHOW + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_WANT_IPC_PARSE_VERSION select GENERIC_PCI_IOMAP select HAVE_NMI_WATCHDOG if SPARC64arch/x86/Kconfig +1
-0@@ -85,6 +85,7 @@ config X86 select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_STACKWALK select ARCH_SUPPORTS_ACPI select ARCH_SUPPORTS_ATOMIC_RMWarch/xtensa/Kconfig +1
-0@@ -7,6 +7,7 @@ config XTENSA select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU select ARCH_HAS_UNCACHED_SEGMENT if MMU + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_FRAME_POINTERSdrivers/power/reset/sc27xx-poweroff.c +1
-1@@ -30,7 +30,7 @@ static void sc27xx_poweroff_shutdown(void)#ifdef CONFIG_PM_SLEEP_SMP int cpu =smp_processor_id (); -freeze_secondary_cpus (cpu); +freeze_secondary_cpus (cpu, false);#endif }include/linux/cpu.h +6
-3@@ -137,11 +137,14 @@ static inline void cpu_hotplug_done(void) { cpus_write_unlock(); }static inline void get_online_cpus (void ) {cpus_read_lock (); }static inline void put_online_cpus (void ) {cpus_read_unlock (); } +#if defined(CONFIG_PM_SLEEP_SMP ) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT ) +extern int freeze_secondary_cpus (int primary,bool reboot); +#endif +#ifdef CONFIG_PM_SLEEP_SMP -extern int freeze_secondary_cpus (int primary);static inline int disable_nonboot_cpus (void ) { -return freeze_secondary_cpus (0 ); +return freeze_secondary_cpus (0 , false); }extern void enable_nonboot_cpus (void );@@ -152,7 +155,7 @@ static inline int suspend_disable_secondary_cpus(void)if (IS_ENABLED (CONFIG_PM_SLEEP_SMP_NONZERO_CPU )) cpu =-1 ; -return freeze_secondary_cpus (cpu); +return freeze_secondary_cpus (cpu, false); }static inline void suspend_enable_secondary_cpus (void ) {kernel/cpu.c +2
-2@@ -1209,10 +1209,10 @@ int cpu_up(unsigned int cpu) }EXPORT_SYMBOL_GPL (cpu_up); -#ifdef CONFIG_PM_SLEEP_SMP +#if defined(CONFIG_PM_SLEEP_SMP ) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT )static cpumask_var_t frozen_cpus; -int freeze_secondary_cpus (int primary) +int freeze_secondary_cpus (int primary,bool reboot) {int cpu, error =0 ;
kernel/cpu.c int freeze_secondary_cpus(int primary) · line 1241
Quoted 6 lines
kernel/cpu.c +2
-1@@ -1237,20 +1237,25 @@ int freeze_secondary_cpus(int primary)if (cpu == primary)continue ; -if (pm_wakeup_pending ()) { +#ifdef CONFIG_PM_SLEEP +if (!reboot &&pm_wakeup_pending ()) {
I would've moveed pm_wakeup_pending() check to callers (disable_nonboot_cpus()/suspend_disable_secondary_cpus()) now.
kernel/cpu.c int freeze_secondary_cpus(int primary) · line 1252
Quoted 13 lines
kernel/cpu.c +5
-2pr_info ("Wakeup pending. Abort CPU freeze\n" ); error = -EBUSY ; break; } +#endif -trace_suspend_resume (TPS ("CPU_OFF" ), cpu, true); +if (!reboot) +trace_suspend_resume (TPS ("CPU_OFF" ), cpu, true); error =_cpu_down (cpu,1 ,CPUHP_OFFLINE ); -trace_suspend_resume (TPS ("CPU_OFF" ), cpu, false); +if (!reboot) +trace_suspend_resume (TPS ("CPU_OFF" ), cpu, false);
Does it actually hurt if we keep this for reboot case?
kernel/cpu.c int freeze_secondary_cpus(int primary) · line 1258
Quoted 7 lines
kernel/cpu.c +2
-1if (!error)cpumask_set_cpu (cpu, frozen_cpus);else {pr_err ("Error taking CPU%d down: %d\n" , cpu, error); -break ; +if (!reboot) +break ;
I would've added a comment like "When rebooting, try to offline as many CPUs as possible".
Quoted 37 lines
kernel/cpu.c +0
-0} }kernel/reboot.c +8
-0@@ -7,6 +7,7 @@#define pr_fmt (fmt) "reboot: " fmt +#include <linux/cpu.h> #include <linux/ctype.h> #include <linux/export.h> #include <linux/kexec.h> @@ -220,7 +221,9 @@ void migrate_to_reboot_cpu(void)/* The boot cpu is always logical cpu 0 */ int cpu = reboot_cpu; +#if !IS_ENABLED (CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT )cpu_hotplug_disable (); +#endif /* Make certain the cpu I'm about to reboot on is online */ if (!cpu_online (cpu))@@ -231,6 +234,11 @@ void migrate_to_reboot_cpu(void)/* Make certain I only run on the appropriate processor */ set_cpus_allowed_ptr (current,cpumask_of (cpu)); + +#if IS_ENABLED (CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT ) +/* Hotplug other cpus if possible */ +freeze_secondary_cpus (cpu, true); +#endif }/**
Signature · 1 line
Vitaly
Quoted 34 lines · Vitaly Kuznetsov
Quoted 13 lines · Hsin-Yi Wang
Currently system reboots uses architecture specific codes (
smp_send_stop) to offline non reboot CPUs. Most architecture's implementation is looping through all non reboot online CPUs and call ipi function to each of them. Some architecture like arm64, arm, and x86... would set offline masks to cpu without really offline them. This causes some race condition and kernel warning comes out sometimes when system reboots.This patch adds a config
ARCH_OFFLINE_CPUS_ON_REBOOT, which would hotplug cpus inmigrate_to_reboot_cpu(). If non reboot cpus are all offlined here, the loop for checking online cpus would be an empty loop. If architecture don't enable this config, or some cpus somehow fails to offline, it would fallback to ipi function.Quoted 8 lines
Opt in this config for architectures that support
CONFIG_HOTPLUG_CPU.Signed-off-by Hsin-Yi Wang <[email protected]>---
Change from v3:
* Opt in config for architectures that supportCONFIG_HOTPLUG_CPU
* Merge functionoffline_secondary_cpus()andfreeze_secondary_cpus()with an additional flag.I haven't commented on the idea itself, actually (I'm not sure I'm the right person though but as I'm already looking at the code...)
On x86, the custom IPI/NMI solution we have now is more 'lightweight' and here you suggest we switch to the full-blown CPUHP machinery instead. We seem to be already using it for suspend/hibernation, however, I would expect it to be way less tested: on servers, for example, less people do suspend/resume than reboot :-)
Quoted 8 lines
The existing IPI/NMI code stays in place so nothing (in theory) should break -- unles some
cpuhp_hook does something really weird. But your change will definitely contribute to improving these hooks' quality in the long run :-)I have nothing to say on other arches and I don't see anyone from e.g. ARM on the CC: list, not good.
I'll resend to CC ARM as well, thanks.
The patch idea came from there are some warnings during reboot seen on x86 and arm, arm64.
(x86) https://lore.kernel.org/lkml/[email protected]/
(arm64) https://lkml.org/lkml/2012/8/22/3, resend version
https://lore.kernel.org/patchwork/patch/1117201/
Quoted 263 lines
Quoted 29 lines
Change from v2:
* Add another config instead of configed byCONFIG_HOTPLUG_CPU--- arch/Kconfig | 5 +++++ arch/arm/Kconfig | 1 + arch/arm64/Kconfig | 1 + arch/arm64/kernel/hibernate.c | 2 +- arch/csky/Kconfig | 1 + arch/ia64/Kconfig | 1 + arch/mips/Kconfig | 1 + arch/parisc/Kconfig | 1 + arch/powerpc/Kconfig | 1 + arch/s390/Kconfig | 1 + arch/sh/Kconfig | 1 + arch/sparc/Kconfig | 1 + arch/x86/Kconfig | 1 + arch/xtensa/Kconfig | 1 + drivers/power/reset/sc27xx-poweroff.c | 2 +- include/linux/cpu.h | 9 ++++++--- kernel/cpu.c | 17 +++++++++++------ kernel/reboot.c | 8 ++++++++ 18 files changed, 44 insertions(+), 11 deletions(-)arch/Kconfig +0
-0arch/Kconfig config ARCH_HAS_UNCACHED_SEGMENT · line 262
Quoted 8 lines
arch/Kconfig +5
-0@@ -255,6 +255,11 @@ config ARCH_HAS_UNCACHED_SEGMENT select ARCH_HAS_DMA_PREP_COHERENT bool +# Select to do a full hotplug on secondary CPUs before reboot. +config ARCH_OFFLINE_CPUS_ON_REBOOT + bool "Support for hotplug CPUs before reboot" + depends on HOTPLUG_CPU +Nit: what we're actually doing is not 'hotplug', it's either 'hotunplug' or 'offlining'.
Quoted 222 lines
arch/Kconfig +0
-0# Select if arch init_task must go in the __init_task_data section config ARCH_TASK_STRUCT_ON_STACK boolarch/arm/Kconfig +1
-0@@ -28,6 +28,7 @@ config ARM select ARCH_KEEP_MEMBLOCK if HAVE_ARCH_PFN_VALID || KEXEC select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMWarch/arm64/Kconfig +1
-0@@ -61,6 +61,7 @@ config ARM64 select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION select ARCH_KEEP_MEMBLOCK + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKSarch/arm64/kernel/hibernate.c +1
-1@@ -581,5 +581,5 @@ int hibernate_resume_nonboot_cpu_disable(void)return -ENODEV ; } -return freeze_secondary_cpus (sleep_cpu); +return freeze_secondary_cpus (sleep_cpu, false); }arch/csky/Kconfig +1
-0@@ -5,6 +5,7 @@ config CSKY select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2 select COMMON_CLKarch/ia64/Kconfig +1
-0@@ -10,6 +10,7 @@ config IA64 bool select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ACPI select ACPI_NUMA if NUMA select ARCH_SUPPORTS_ACPIarch/mips/Kconfig +1
-0@@ -8,6 +8,7 @@ config MIPS select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAS_FORTIFY_SOURCE + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SUPPORTS_UPROBES select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if 64BITarch/parisc/Kconfig +1
-0@@ -13,6 +13,7 @@ config PARISC select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_NO_SG_CHAIN + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SUPPORTS_MEMORY_FAILURE select RTC_CLASS select RTC_DRV_GENERICarch/powerpc/Kconfig +1
-0@@ -142,6 +142,7 @@ config PPC select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_USE_BUILTIN_BSWAParch/s390/Kconfig +1
-0@@ -102,6 +102,7 @@ config S390 select ARCH_INLINE_WRITE_UNLOCK_IRQ select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE select ARCH_KEEP_MEMBLOCK + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_SAVE_PAGE_KEYS if HIBERNATION select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMWarch/sh/Kconfig +1
-0@@ -18,6 +18,7 @@ config SUPERH select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A) select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select PERF_USE_VMALLOC select HAVE_DEBUG_KMEMLEAK select HAVE_KERNEL_GZIParch/sparc/Kconfig +1
-0@@ -30,6 +30,7 @@ config SPARC select RTC_SYSTOHC select HAVE_ARCH_JUMP_LABEL if SPARC64 select GENERIC_IRQ_SHOW + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_WANT_IPC_PARSE_VERSION select GENERIC_PCI_IOMAP select HAVE_NMI_WATCHDOG if SPARC64arch/x86/Kconfig +1
-0@@ -85,6 +85,7 @@ config X86 select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_STACKWALK select ARCH_SUPPORTS_ACPI select ARCH_SUPPORTS_ATOMIC_RMWarch/xtensa/Kconfig +1
-0@@ -7,6 +7,7 @@ config XTENSA select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU select ARCH_HAS_UNCACHED_SEGMENT if MMU + select ARCH_OFFLINE_CPUS_ON_REBOOT if HOTPLUG_CPU select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_FRAME_POINTERSdrivers/power/reset/sc27xx-poweroff.c +1
-1@@ -30,7 +30,7 @@ static void sc27xx_poweroff_shutdown(void)#ifdef CONFIG_PM_SLEEP_SMP int cpu =smp_processor_id (); -freeze_secondary_cpus (cpu); +freeze_secondary_cpus (cpu, false);#endif }include/linux/cpu.h +6
-3@@ -137,11 +137,14 @@ static inline void cpu_hotplug_done(void) { cpus_write_unlock(); }static inline void get_online_cpus (void ) {cpus_read_lock (); }static inline void put_online_cpus (void ) {cpus_read_unlock (); } +#if defined(CONFIG_PM_SLEEP_SMP ) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT ) +extern int freeze_secondary_cpus (int primary,bool reboot); +#endif +#ifdef CONFIG_PM_SLEEP_SMP -extern int freeze_secondary_cpus (int primary);static inline int disable_nonboot_cpus (void ) { -return freeze_secondary_cpus (0 ); +return freeze_secondary_cpus (0 , false); }extern void enable_nonboot_cpus (void );@@ -152,7 +155,7 @@ static inline int suspend_disable_secondary_cpus(void)if (IS_ENABLED (CONFIG_PM_SLEEP_SMP_NONZERO_CPU )) cpu =-1 ; -return freeze_secondary_cpus (cpu); +return freeze_secondary_cpus (cpu, false); }static inline void suspend_enable_secondary_cpus (void ) {kernel/cpu.c +2
-2@@ -1209,10 +1209,10 @@ int cpu_up(unsigned int cpu) }EXPORT_SYMBOL_GPL (cpu_up); -#ifdef CONFIG_PM_SLEEP_SMP +#if defined(CONFIG_PM_SLEEP_SMP ) || defined(CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT )static cpumask_var_t frozen_cpus; -int freeze_secondary_cpus (int primary) +int freeze_secondary_cpus (int primary,bool reboot) {int cpu, error =0 ;@@ -1237,20 +1237,25 @@ int freeze_secondary_cpus(int primary)if (cpu == primary)continue ;
Quoted 7 lines
kernel/cpu.c int freeze_secondary_cpus(int primary) · line 1241
Quoted 3 lines
kernel/cpu.c +2
-1-if (pm_wakeup_pending ()) { +#ifdef CONFIG_PM_SLEEP +if (!reboot &&pm_wakeup_pending ()) {I would've moveed
pm_wakeup_pending()check to callers (disable_nonboot_cpus()/suspend_disable_secondary_cpus()) now.
pm_wakeup_pending() is in for_each_online_cpu() loop. Quoting from commit message of a66d955e910:
"CPU hotplug is a slow operation, so it makes sense to check for wakeup pending in the freezer loop before bringing down the next CPU. This improves the system suspend abort latency significantly."
In reboot case don't have to check for wakeup pending condition. But for suspend it should check between bringing down CPUs.
Quoted 14 lines
kernel/cpu.c int freeze_secondary_cpus(int primary) · line 1252
Quoted 13 lines
kernel/cpu.c +5
-2pr_info ("Wakeup pending. Abort CPU freeze\n" ); error = -EBUSY ; break; } +#endif -trace_suspend_resume (TPS ("CPU_OFF" ), cpu, true); +if (!reboot) +trace_suspend_resume (TPS ("CPU_OFF" ), cpu, true); error =_cpu_down (cpu,1 ,CPUHP_OFFLINE ); -trace_suspend_resume (TPS ("CPU_OFF" ), cpu, false); +if (!reboot) +trace_suspend_resume (TPS ("CPU_OFF" ), cpu, false);
Quoted 2 lines
Does it actually hurt if we keep this for reboot case?
Okay I'll keep them with reboot case.
Quoted 53 lines
kernel/cpu.c int freeze_secondary_cpus(int primary) · line 1258
Quoted 7 lines
kernel/cpu.c +2
-1if (!error)cpumask_set_cpu (cpu, frozen_cpus);else {pr_err ("Error taking CPU%d down: %d\n" , cpu, error); -break ; +if (!reboot) +break ;I would've added a comment like "When rebooting, try to offline as many CPUs as possible".
Quoted 27 lines
kernel/cpu.c +0
-0} }kernel/reboot.c +3
-0@@ -7,6 +7,7 @@#define pr_fmt (fmt) "reboot: " fmt +#include <linux/cpu.h> #include <linux/ctype.h> #include <linux/export.h> #include <linux/kexec.h> @@ -220,7 +221,9 @@ void migrate_to_reboot_cpu(void)/* The boot cpu is always logical cpu 0 */ int cpu = reboot_cpu; +#if !IS_ENABLED (CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT )cpu_hotplug_disable (); +#endif /* Make certain the cpu I'm about to reboot on is online */ if (!cpu_online (cpu))@@ -231,6 +234,11 @@ void migrate_to_reboot_cpu(void)kernel/reboot.c void migrate_to_reboot_cpu(void) · line 244
Quoted 10 lines
kernel/reboot.c +5
-0/* Make certain I only run on the appropriate processor */ set_cpus_allowed_ptr (current,cpumask_of (cpu)); + +#if IS_ENABLED (CONFIG_ARCH_OFFLINE_CPUS_ON_REBOOT ) +/* Hotplug other cpus if possible */ +freeze_secondary_cpus (cpu, true); +#endif }/** --
Vitaly
kbuild test robot
build botThank you for the patch! Yet something to improve:
Hi Hsin-Yi,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on next-20200110]
[also build test ERROR on v5.5-rc6]
[cannot apply to arm64/for-next/core arm/for-next ia64/next hp-parisc/for-next powerpc/next s390/features sparc/master tip/x86/core linus/master sparc-next/master v5.5-rc6 v5.5-rc5 v5.5-rc4] [if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Hsin-Yi-Wang/reboot-support-hotplug-CPUs-before-reboot/20200114-143340 base: 6c09d7dbb7d366122d0218bc7487e0a1e6cca6ed config: powerpc-rhel-kconfig (attached as .config) compiler: powerpc64le-linux-gcc (GCC) 7.5.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.5.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag
All errors (new ones prefixed by >>):
>> kernel/cpu.c:1286:6: error: redefinition of 'enable_nonboot_cpus'
void enable_nonboot_cpus(void)
^~~~~~~~~~~~~~~~~~~
In file included from kernel/cpu.c:16:0:
include/linux/cpu.h:167:20: note: previous definition of 'enable_nonboot_cpus' was here
static inline void enable_nonboot_cpus(void) {}
^~~~~~~~~~~~~~~~~~~vim +/enable_nonboot_cpus +1286 kernel/cpu.c
d0af9eed5aa91b Suresh Siddha 2009-08-19 1285
71cf5aeeb8e215 Mathias Krause 2015-07-19 @1286 void enable_nonboot_cpus(void)
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1287 {
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1288 int cpu, error;
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1289
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1290 /* Allow everyone to use the CPU hotplug again */
d221938c049f48 Gautham R Shenoy 2008-01-25 1291 cpu_maps_update_begin();
01b41159066531 Lianwei Wang 2016-06-09 1292 __cpu_hotplug_enable();
e0b582ec56f1a1 Rusty Russell 2009-01-01 1293 if (cpumask_empty(frozen_cpus))
1d64b9cb1dc2a7 Rafael J. Wysocki 2007-04-01 1294 goto out;
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1295
84117da5b79ffb Fabian Frederick 2014-06-04 1296 pr_info("Enabling non-boot CPUs ...\n");
d0af9eed5aa91b Suresh Siddha 2009-08-19 1297
d0af9eed5aa91b Suresh Siddha 2009-08-19 1298 arch_enable_nonboot_cpus_begin();
d0af9eed5aa91b Suresh Siddha 2009-08-19 1299
e0b582ec56f1a1 Rusty Russell 2009-01-01 1300 for_each_cpu(cpu, frozen_cpus) {
bb3632c6101b2f Todd E Brandt 2014-06-06 1301 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
af1f40457da6f7 Thomas Gleixner 2016-02-26 1302 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
bb3632c6101b2f Todd E Brandt 2014-06-06 1303 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1304 if (!error) {
84117da5b79ffb Fabian Frederick 2014-06-04 1305 pr_info("CPU%d is up\n", cpu);
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1306 continue;
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1307 }
84117da5b79ffb Fabian Frederick 2014-06-04 1308 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
e3920fb42c8ddf Rafael J. Wysocki 2006-09-25 1309 }
d0af9eed5aa91b Suresh Siddha 2009-08-19 1310
d0af9eed5aa91b Suresh Siddha 2009-08-19 1311 arch_enable_nonboot_cpus_end();
d0af9eed5aa91b Suresh Siddha 2009-08-19 1312
e0b582ec56f1a1 Rusty Russell 2009-01-01 1313 cpumask_clear(frozen_cpus);
1d64b9cb1dc2a7 Rafael J. Wysocki 2007-04-01 1314 out:
d221938c049f48 Gautham R Shenoy 2008-01-25 1315 cpu_maps_update_done();
^1da177e4c3f41 Linus Torvalds 2005-04-16 1316 }
e0b582ec56f1a1 Rusty Russell 2009-01-01 1317 :::::: The code at line 1286 was first introduced by commit
:::::: 71cf5aeeb8e2154efda5f40be50c925f15057755 kernel, cpu: Remove bogus __ref annotations
:::::: TO: Mathias Krause <[email protected]> :::::: CC: Greg Kroah-Hartman <[email protected]>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org Intel Corporation