Skip to contentSkip to conversation

Bug during kexec...not all cpus are stopped

RFC PATCH In review Alok Kataria 4 participants 16 messages Oct 8, 2010 – Oct 21, 2010

Subsystems (7)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 Bug during kexec...not all cpus are stopped

Before starting the new kernel kexec calls machine_shutdown to stop all the cpus, which internally calls native_smp_send_stop. AFAIU, kexec expects that all the cpus are now halted after that call returns. Now, looking at the code for native_smp_send_stop, it assumes that all the processors have processed the REBOOT ipi in 1 second after the IPI was sent.
native_smp_send_stop()
---------------------------------------------------------

	apic->send_IPI_allbutself(REBOOT_VECTOR);

        /* Don't wait longer than a second */
        wait = USEC_PER_SEC;
        while (num_online_cpus() > 1 && wait--)
        	udelay(1);

---------------------------------------------------------

It just returns after that 1 second irrespective of whether all cpus were halted or not. This brings up a issue in the kexec case, since we can have the BSP starting the new kernel and AP's still processing the REBOOT IPI simultaneously.

Many distribution kernels use kexec to load the newly installed kernel during the installation phase, in virtualized environment with the host heavily overcommitted, we have seen some instances when vcpu fails to process the IPI in the allotted 1 sec and as a result the AP's end up accessing uninitialized state (the BSP has already gone ahead with setting up the new state) and causing GPF's.

IMO, kexec expects machine_shutdown to return only after all cpus are stopped.

The patch below should fix the issue, comments ??

--
machine_shutdown now takes a parameter "wait", if it is true, it waits until all the cpus are halted. All the callers except kexec still fallback to the earlier version of the shutdown call, where it just waited for max 1 sec before returning.

arch/x86/include/asm/reboot.h +2 -2
Index: linux-2.6/arch/x86/include/asm/reboot.h
===================================================================
@@ -9,7 +9,7 @@ struct machine_ops {
 	void (*restart)(char *cmd);
 	void (*halt)(void);
 	void (*power_off)(void);
-	void (*shutdown)(void);
+	void (*shutdown)(int wait);
 	void (*crash_shutdown)(struct pt_regs *);
 	void (*emergency_restart)(void);
 };
@@ -17,7 +17,7 @@ struct machine_ops {
 extern struct machine_ops machine_ops;
 
 void native_machine_crash_shutdown(struct pt_regs *regs);
-void native_machine_shutdown(void);
+void native_machine_shutdown(int wait);
 void machine_real_restart(const unsigned char *code, int length);
 
 typedef void (*nmi_shootdown_cb)(int, struct die_args*);
arch/x86/include/asm/smp.h +3 -3
Index: linux-2.6/arch/x86/include/asm/smp.h
===================================================================
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*smp_send_stop)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -71,9 +71,9 @@ extern void set_cpu_sibling_map(int cpu)
 #endif
 extern struct smp_ops smp_ops;
 
-static inline void smp_send_stop(void)
+static inline void smp_send_stop(int wait)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.smp_send_stop(wait);
 }
 
 static inline void smp_prepare_boot_cpu(void)
arch/x86/kernel/kvmclock.c +2 -2
Index: linux-2.6/arch/x86/kernel/kvmclock.c
===================================================================
@@ -174,10 +174,10 @@ static void kvm_crash_shutdown(struct pt
 }
 #endif
 
-static void kvm_shutdown(void)
+static void kvm_shutdown(int wait)
 {
 	native_write_msr(msr_kvm_system_time, 0, 0);
-	native_machine_shutdown();
+	native_machine_shutdown(wait);
 }
 
 void __init kvmclock_init(void)
arch/x86/kernel/reboot.c +7 -7
Index: linux-2.6/arch/x86/kernel/reboot.c
===================================================================
@@ -616,7 +616,7 @@ static void native_machine_emergency_res
 	}
 }
 
-void native_machine_shutdown(void)
+void native_machine_shutdown(int wait)
 {
 	/* Stop the cpus and apics */
 #ifdef CONFIG_SMP
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	smp_send_stop(wait);
 #endif
 
 	lapic_shutdown();
@@ -670,14 +670,14 @@ static void native_machine_restart(char 
 	printk("machine restart\n");
 
 	if (!reboot_force)
-		machine_shutdown();
+		machine_shutdown(0);
 	__machine_emergency_restart(0);
 }
 
 static void native_machine_halt(void)
 {
 	/* stop other cpus and apics */
-	machine_shutdown();
+	machine_shutdown(0);
 
 	tboot_shutdown(TB_SHUTDOWN_HALT);
 
@@ -689,7 +689,7 @@ static void native_machine_power_off(voi
 {
 	if (pm_power_off) {
 		if (!reboot_force)
-			machine_shutdown();
+			machine_shutdown(0);
 		pm_power_off();
 	}
 	/* a fallback in case there is no PM info available */
@@ -712,9 +712,9 @@ void machine_power_off(void)
 	machine_ops.power_off();
 }
 
-void machine_shutdown(void)
+void machine_shutdown(int wait)
 {
-	machine_ops.shutdown();
+	machine_ops.shutdown(wait);
 }
 
 void machine_emergency_restart(void)
arch/x86/kernel/smp.c +5 -5
Index: linux-2.6/arch/x86/kernel/smp.c
===================================================================
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_smp_send_stop(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,9 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/* Don't wait longer than a second if this not a synchronous call */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
arch/x86/xen/enlighten.c +1 -1
Index: linux-2.6/arch/x86/xen/enlighten.c
===================================================================
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	smp_send_stop(0);
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
arch/x86/xen/smp.c +1 -1
Index: linux-2.6/arch/x86/xen/smp.c
===================================================================
@@ -400,7 +400,7 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_smp_send_stop(int wait)
 {
 	smp_call_function(stop_self, NULL, 0);
 }
drivers/s390/char/sclp_quiesce.c +1 -1
Index: linux-2.6/drivers/s390/char/sclp_quiesce.c
===================================================================
@@ -29,7 +29,7 @@ static void do_machine_quiesce(void)
 {
 	psw_t quiesce_psw;
 
-	smp_send_stop();
+	smp_send_stop(0);
 	quiesce_psw.mask = PSW_BASE_BITS | PSW_MASK_WAIT;
 	quiesce_psw.addr = 0xfff;
 	__load_psw(quiesce_psw);
include/linux/reboot.h +1 -1
Index: linux-2.6/include/linux/reboot.h
===================================================================
@@ -51,7 +51,7 @@ extern void machine_restart(char *cmd);
 extern void machine_halt(void);
 extern void machine_power_off(void);
 
-extern void machine_shutdown(void);
+extern void machine_shutdown(int wait);
 struct pt_regs;
 extern void machine_crash_shutdown(struct pt_regs *);
 
include/linux/smp.h +2 -2
Index: linux-2.6/include/linux/smp.h
===================================================================
@@ -43,7 +43,7 @@ int smp_call_function_single(int cpuid, 
 /*
  * stops all CPUs but the current one:
  */
-extern void smp_send_stop(void);
+extern void smp_send_stop(int wait);
 
 /*
  * sends a 'reschedule' event to another CPU:
@@ -116,7 +116,7 @@ extern unsigned int setup_max_cpus;
 
 #else /* !SMP */
 
-static inline void smp_send_stop(void) { }
+static inline void smp_send_stop(int wait) { }
 
 /*
  *	These macros fold the SMP functionality into a single CPU system
kernel/kexec.c +1 -1
Index: linux-2.6/kernel/kexec.c
===================================================================
@@ -1538,7 +1538,7 @@ int kernel_kexec(void)
 	{
 		kernel_restart_prepare(NULL);
 		printk(KERN_EMERG "Starting new kernel\n");
-		machine_shutdown();
+		machine_shutdown(1);
 	}
 
 	machine_kexec(kexec_image);
kernel/panic.c +1 -1
Index: linux-2.6/kernel/panic.c
===================================================================
@@ -94,7 +94,7 @@ NORET_TYPE void panic(const char * fmt, 
 	 * unfortunately means it may not be hardened to work in a panic
 	 * situation.
 	 */
-	smp_send_stop();
+	smp_send_stop(0);
 
 	atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
 
#1 lore raw

Copying a few more maintainers on the thread...

Quoted 296 lines · Alok Kataria

Before starting the new kernel kexec calls machine_shutdown to stop all the cpus, which internally calls native_smp_send_stop. AFAIU, kexec expects that all the cpus are now halted after that call returns. Now, looking at the code for native_smp_send_stop, it assumes that all the processors have processed the REBOOT ipi in 1 second after the IPI was sent.
native_smp_send_stop()
---------------------------------------------------------

	apic->send_IPI_allbutself(REBOOT_VECTOR);

        /* Don't wait longer than a second */
        wait = USEC_PER_SEC;
        while (num_online_cpus() > 1 && wait--)
        	udelay(1);

---------------------------------------------------------

It just returns after that 1 second irrespective of whether all cpus were halted or not. This brings up a issue in the kexec case, since we can have the BSP starting the new kernel and AP's still processing the REBOOT IPI simultaneously.

Many distribution kernels use kexec to load the newly installed kernel during the installation phase, in virtualized environment with the host heavily overcommitted, we have seen some instances when vcpu fails to process the IPI in the allotted 1 sec and as a result the AP's end up accessing uninitialized state (the BSP has already gone ahead with setting up the new state) and causing GPF's.

IMO, kexec expects machine_shutdown to return only after all cpus are stopped.

The patch below should fix the issue, comments ??

--
machine_shutdown now takes a parameter "wait", if it is true, it waits until all the cpus are halted. All the callers except kexec still fallback to the earlier version of the shutdown call, where it just waited for max 1 sec before returning.

Index: linux-2.6/arch/x86/include/asm/reboot.h
===================================================================

arch/x86/include/asm/reboot.h +2 -2
@@ -9,7 +9,7 @@ struct machine_ops {
 	void (*restart)(char *cmd);
 	void (*halt)(void);
 	void (*power_off)(void);
-	void (*shutdown)(void);
+	void (*shutdown)(int wait);
 	void (*crash_shutdown)(struct pt_regs *);
 	void (*emergency_restart)(void);
 };
@@ -17,7 +17,7 @@ struct machine_ops {
 extern struct machine_ops machine_ops;
 
 void native_machine_crash_shutdown(struct pt_regs *regs);
-void native_machine_shutdown(void);
+void native_machine_shutdown(int wait);
 void machine_real_restart(const unsigned char *code, int length);
 
 typedef void (*nmi_shootdown_cb)(int, struct die_args*);

Index: linux-2.6/arch/x86/include/asm/smp.h
===================================================================

arch/x86/include/asm/smp.h +3 -3
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*smp_send_stop)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -71,9 +71,9 @@ extern void set_cpu_sibling_map(int cpu)
 #endif
 extern struct smp_ops smp_ops;
 
-static inline void smp_send_stop(void)
+static inline void smp_send_stop(int wait)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.smp_send_stop(wait);
 }
 
 static inline void smp_prepare_boot_cpu(void)

Index: linux-2.6/arch/x86/kernel/kvmclock.c
===================================================================

arch/x86/kernel/kvmclock.c +2 -2
@@ -174,10 +174,10 @@ static void kvm_crash_shutdown(struct pt
 }
 #endif
 
-static void kvm_shutdown(void)
+static void kvm_shutdown(int wait)
 {
 	native_write_msr(msr_kvm_system_time, 0, 0);
-	native_machine_shutdown();
+	native_machine_shutdown(wait);
 }
 
 void __init kvmclock_init(void)

Index: linux-2.6/arch/x86/kernel/reboot.c
===================================================================

arch/x86/kernel/reboot.c +7 -7
@@ -616,7 +616,7 @@ static void native_machine_emergency_res
 	}
 }
 
-void native_machine_shutdown(void)
+void native_machine_shutdown(int wait)
 {
 	/* Stop the cpus and apics */
 #ifdef CONFIG_SMP
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	smp_send_stop(wait);
 #endif
 
 	lapic_shutdown();
@@ -670,14 +670,14 @@ static void native_machine_restart(char 
 	printk("machine restart\n");
 
 	if (!reboot_force)
-		machine_shutdown();
+		machine_shutdown(0);
 	__machine_emergency_restart(0);
 }
 
 static void native_machine_halt(void)
 {
 	/* stop other cpus and apics */
-	machine_shutdown();
+	machine_shutdown(0);
 
 	tboot_shutdown(TB_SHUTDOWN_HALT);
 
@@ -689,7 +689,7 @@ static void native_machine_power_off(voi
 {
 	if (pm_power_off) {
 		if (!reboot_force)
-			machine_shutdown();
+			machine_shutdown(0);
 		pm_power_off();
 	}
 	/* a fallback in case there is no PM info available */
@@ -712,9 +712,9 @@ void machine_power_off(void)
 	machine_ops.power_off();
 }
 
-void machine_shutdown(void)
+void machine_shutdown(int wait)
 {
-	machine_ops.shutdown();
+	machine_ops.shutdown(wait);
 }
 
 void machine_emergency_restart(void)

Index: linux-2.6/arch/x86/kernel/smp.c
===================================================================

arch/x86/kernel/smp.c +5 -5
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_smp_send_stop(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,9 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/* Don't wait longer than a second if this not a synchronous call */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 

Index: linux-2.6/arch/x86/xen/enlighten.c
===================================================================

arch/x86/xen/enlighten.c +1 -1
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	smp_send_stop(0);
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))

Index: linux-2.6/arch/x86/xen/smp.c
===================================================================

arch/x86/xen/smp.c +1 -1
@@ -400,7 +400,7 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_smp_send_stop(int wait)
 {
 	smp_call_function(stop_self, NULL, 0);
 }

Index: linux-2.6/drivers/s390/char/sclp_quiesce.c
===================================================================

drivers/s390/char/sclp_quiesce.c +1 -1
@@ -29,7 +29,7 @@ static void do_machine_quiesce(void)
 {
 	psw_t quiesce_psw;
 
-	smp_send_stop();
+	smp_send_stop(0);
 	quiesce_psw.mask = PSW_BASE_BITS | PSW_MASK_WAIT;
 	quiesce_psw.addr = 0xfff;
 	__load_psw(quiesce_psw);

Index: linux-2.6/include/linux/reboot.h
===================================================================

include/linux/reboot.h +1 -1
@@ -51,7 +51,7 @@ extern void machine_restart(char *cmd);
 extern void machine_halt(void);
 extern void machine_power_off(void);
 
-extern void machine_shutdown(void);
+extern void machine_shutdown(int wait);
 struct pt_regs;
 extern void machine_crash_shutdown(struct pt_regs *);
 

Index: linux-2.6/include/linux/smp.h
===================================================================

include/linux/smp.h +2 -2
@@ -43,7 +43,7 @@ int smp_call_function_single(int cpuid, 
 /*
  * stops all CPUs but the current one:
  */
-extern void smp_send_stop(void);
+extern void smp_send_stop(int wait);
 
 /*
  * sends a 'reschedule' event to another CPU:
@@ -116,7 +116,7 @@ extern unsigned int setup_max_cpus;
 
 #else /* !SMP */
 
-static inline void smp_send_stop(void) { }
+static inline void smp_send_stop(int wait) { }
 
 /*
  *	These macros fold the SMP functionality into a single CPU system

Index: linux-2.6/kernel/kexec.c
===================================================================

kernel/kexec.c +1 -1
@@ -1538,7 +1538,7 @@ int kernel_kexec(void)
 	{
 		kernel_restart_prepare(NULL);
 		printk(KERN_EMERG "Starting new kernel\n");
-		machine_shutdown();
+		machine_shutdown(1);
 	}
 
 	machine_kexec(kexec_image);

Index: linux-2.6/kernel/panic.c
===================================================================

kernel/panic.c +1 -1
@@ -94,7 +94,7 @@ NORET_TYPE void panic(const char * fmt, 
 	 * unfortunately means it may not be hardened to work in a panic
 	 * situation.
 	 */
-	smp_send_stop();
+	smp_send_stop(0);
 
 	atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
 
#2 lore raw
Quoted 40 lines · Alok Kataria

Copying a few more maintainers on the thread...

Quoted 38 lines · Alok Kataria

Before starting the new kernel kexec calls machine_shutdown to stop all the cpus, which internally calls native_smp_send_stop. AFAIU, kexec expects that all the cpus are now halted after that call returns. Now, looking at the code for native_smp_send_stop, it assumes that all the processors have processed the REBOOT ipi in 1 second after the IPI was sent.
native_smp_send_stop()
---------------------------------------------------------

	apic->send_IPI_allbutself(REBOOT_VECTOR);

        /* Don't wait longer than a second */
        wait = USEC_PER_SEC;
        while (num_online_cpus() > 1 && wait--)
        	udelay(1);

---------------------------------------------------------

It just returns after that 1 second irrespective of whether all cpus were halted or not. This brings up a issue in the kexec case, since we can have the BSP starting the new kernel and AP's still processing the REBOOT IPI simultaneously.

Many distribution kernels use kexec to load the newly installed kernel during the installation phase, in virtualized environment with the host heavily overcommitted, we have seen some instances when vcpu fails to process the IPI in the allotted 1 sec and as a result the AP's end up accessing uninitialized state (the BSP has already gone ahead with setting up the new state) and causing GPF's.

IMO, kexec expects machine_shutdown to return only after all cpus are stopped.

The patch below should fix the issue, comments ??

--
machine_shutdown now takes a parameter "wait", if it is true, it waits until all the cpus are halted. All the callers except kexec still fallback to the earlier version of the shutdown call, where it just waited for max 1 sec before returning.

The approach seems reasonable. However on every path except for the panic path I would wait for all of the cpus to stop, as that is what those code paths are expecting. Until last year smp_send_stop always waited until all of the cpus stopped. So I think changing machine_shutdown should not need to happen.

This patch cannot be used as is because it changes the parameters to smp_send_stop() and there are more than just x86 implementations out there.

Let me propose a slightly different tactic. We need to separate the panic path from the normal path to avoid confusion. At the generic level smp_send_stop is exclusively used for the panic path. So we should keep that, and rename the x86 non-panic path function something else.

Can you rename the x86 smp_send_stop function say stop_all_other_cpus().

At which point we could implement smp_send_stop as:
stop_all_other_cpus(0);

And everywhere else would call stop_all_other_cpus(1) waiting for the cpus to actually stop.

I really think we want to wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

Eric

Quoted 257 lines
Quoted 257 lines

Index: linux-2.6/arch/x86/include/asm/reboot.h
===================================================================

arch/x86/include/asm/reboot.h +2 -2
@@ -9,7 +9,7 @@ struct machine_ops {
 	void (*restart)(char *cmd);
 	void (*halt)(void);
 	void (*power_off)(void);
-	void (*shutdown)(void);
+	void (*shutdown)(int wait);
 	void (*crash_shutdown)(struct pt_regs *);
 	void (*emergency_restart)(void);
 };
@@ -17,7 +17,7 @@ struct machine_ops {
 extern struct machine_ops machine_ops;
 
 void native_machine_crash_shutdown(struct pt_regs *regs);
-void native_machine_shutdown(void);
+void native_machine_shutdown(int wait);
 void machine_real_restart(const unsigned char *code, int length);
 
 typedef void (*nmi_shootdown_cb)(int, struct die_args*);

Index: linux-2.6/arch/x86/include/asm/smp.h
===================================================================

arch/x86/include/asm/smp.h +3 -3
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*smp_send_stop)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -71,9 +71,9 @@ extern void set_cpu_sibling_map(int cpu)
 #endif
 extern struct smp_ops smp_ops;
 
-static inline void smp_send_stop(void)
+static inline void smp_send_stop(int wait)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.smp_send_stop(wait);
 }
 
 static inline void smp_prepare_boot_cpu(void)

Index: linux-2.6/arch/x86/kernel/kvmclock.c
===================================================================

arch/x86/kernel/kvmclock.c +2 -2
@@ -174,10 +174,10 @@ static void kvm_crash_shutdown(struct pt
 }
 #endif
 
-static void kvm_shutdown(void)
+static void kvm_shutdown(int wait)
 {
 	native_write_msr(msr_kvm_system_time, 0, 0);
-	native_machine_shutdown();
+	native_machine_shutdown(wait);
 }
 
 void __init kvmclock_init(void)

Index: linux-2.6/arch/x86/kernel/reboot.c
===================================================================

arch/x86/kernel/reboot.c +7 -7
@@ -616,7 +616,7 @@ static void native_machine_emergency_res
 	}
 }
 
-void native_machine_shutdown(void)
+void native_machine_shutdown(int wait)
 {
 	/* Stop the cpus and apics */
 #ifdef CONFIG_SMP
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	smp_send_stop(wait);
 #endif
 
 	lapic_shutdown();
@@ -670,14 +670,14 @@ static void native_machine_restart(char 
 	printk("machine restart\n");
 
 	if (!reboot_force)
-		machine_shutdown();
+		machine_shutdown(0);
 	__machine_emergency_restart(0);
 }
 
 static void native_machine_halt(void)
 {
 	/* stop other cpus and apics */
-	machine_shutdown();
+	machine_shutdown(0);
 
 	tboot_shutdown(TB_SHUTDOWN_HALT);
 
@@ -689,7 +689,7 @@ static void native_machine_power_off(voi
 {
 	if (pm_power_off) {
 		if (!reboot_force)
-			machine_shutdown();
+			machine_shutdown(0);
 		pm_power_off();
 	}
 	/* a fallback in case there is no PM info available */
@@ -712,9 +712,9 @@ void machine_power_off(void)
 	machine_ops.power_off();
 }
 
-void machine_shutdown(void)
+void machine_shutdown(int wait)
 {
-	machine_ops.shutdown();
+	machine_ops.shutdown(wait);
 }
 
 void machine_emergency_restart(void)

Index: linux-2.6/arch/x86/kernel/smp.c
===================================================================

arch/x86/kernel/smp.c +5 -5
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_smp_send_stop(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,9 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/* Don't wait longer than a second if this not a synchronous call */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 

Index: linux-2.6/arch/x86/xen/enlighten.c
===================================================================

arch/x86/xen/enlighten.c +1 -1
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	smp_send_stop(0);
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))

Index: linux-2.6/arch/x86/xen/smp.c
===================================================================

arch/x86/xen/smp.c +1 -1
@@ -400,7 +400,7 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_smp_send_stop(int wait)
 {
 	smp_call_function(stop_self, NULL, 0);
 }

Index: linux-2.6/drivers/s390/char/sclp_quiesce.c
===================================================================

drivers/s390/char/sclp_quiesce.c +1 -1
@@ -29,7 +29,7 @@ static void do_machine_quiesce(void)
 {
 	psw_t quiesce_psw;
 
-	smp_send_stop();
+	smp_send_stop(0);
 	quiesce_psw.mask = PSW_BASE_BITS | PSW_MASK_WAIT;
 	quiesce_psw.addr = 0xfff;
 	__load_psw(quiesce_psw);

Index: linux-2.6/include/linux/reboot.h
===================================================================

include/linux/reboot.h +1 -1
@@ -51,7 +51,7 @@ extern void machine_restart(char *cmd);
 extern void machine_halt(void);
 extern void machine_power_off(void);
 
-extern void machine_shutdown(void);
+extern void machine_shutdown(int wait);
 struct pt_regs;
 extern void machine_crash_shutdown(struct pt_regs *);
 

Index: linux-2.6/include/linux/smp.h
===================================================================

include/linux/smp.h +2 -2
@@ -43,7 +43,7 @@ int smp_call_function_single(int cpuid, 
 /*
  * stops all CPUs but the current one:
  */
-extern void smp_send_stop(void);
+extern void smp_send_stop(int wait);
 
 /*
  * sends a 'reschedule' event to another CPU:
@@ -116,7 +116,7 @@ extern unsigned int setup_max_cpus;
 
 #else /* !SMP */
 
-static inline void smp_send_stop(void) { }
+static inline void smp_send_stop(int wait) { }
 
 /*
  *	These macros fold the SMP functionality into a single CPU system

Index: linux-2.6/kernel/kexec.c
===================================================================

kernel/kexec.c +1 -1
@@ -1538,7 +1538,7 @@ int kernel_kexec(void)
 	{
 		kernel_restart_prepare(NULL);
 		printk(KERN_EMERG "Starting new kernel\n");
-		machine_shutdown();
+		machine_shutdown(1);
 	}
 
 	machine_kexec(kexec_image);

Index: linux-2.6/kernel/panic.c
===================================================================

kernel/panic.c +1 -1
@@ -94,7 +94,7 @@ NORET_TYPE void panic(const char * fmt, 
 	 * unfortunately means it may not be hardened to work in a panic
 	 * situation.
 	 */
-	smp_send_stop();
+	smp_send_stop(0);
 
 	atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
 
#3 lore raw

Hi Eric,

Thanks for taking a look.

Quoted 59 lines · Eric W. Biederman
Quoted 40 lines · Alok Kataria

Copying a few more maintainers on the thread...

Quoted 38 lines · Alok Kataria

Before starting the new kernel kexec calls machine_shutdown to stop all the cpus, which internally calls native_smp_send_stop. AFAIU, kexec expects that all the cpus are now halted after that call returns. Now, looking at the code for native_smp_send_stop, it assumes that all the processors have processed the REBOOT ipi in 1 second after the IPI was sent.
native_smp_send_stop()
---------------------------------------------------------

     apic->send_IPI_allbutself(REBOOT_VECTOR);

        /* Don't wait longer than a second */
        wait = USEC_PER_SEC;
        while (num_online_cpus() > 1 && wait--)
             udelay(1);

---------------------------------------------------------

It just returns after that 1 second irrespective of whether all cpus were halted or not. This brings up a issue in the kexec case, since we can have the BSP starting the new kernel and AP's still processing the REBOOT IPI simultaneously.

Many distribution kernels use kexec to load the newly installed kernel during the installation phase, in virtualized environment with the host heavily overcommitted, we have seen some instances when vcpu fails to process the IPI in the allotted 1 sec and as a result the AP's end up accessing uninitialized state (the BSP has already gone ahead with setting up the new state) and causing GPF's.

IMO, kexec expects machine_shutdown to return only after all cpus are stopped.

The patch below should fix the issue, comments ??

--
machine_shutdown now takes a parameter "wait", if it is true, it waits until all the cpus are halted. All the callers except kexec still fallback to the earlier version of the shutdown call, where it just waited for max 1 sec before returning.

The approach seems reasonable. However on every path except for the panic path I would wait for all of the cpus to stop, as that is what those code paths are expecting. Until last year smp_send_stop always waited until all of the cpus stopped. So I think changing machine_shutdown should not need to happen.

This patch cannot be used as is because it changes the parameters to smp_send_stop() and there are more than just x86 implementations out there.

Let me propose a slightly different tactic. We need to separate the panic path from the normal path to avoid confusion. At the generic level smp_send_stop is exclusively used for the panic path. So we should keep that, and rename the x86 non-panic path function something else.

Quoted 7 lines

Can you rename the x86 smp_send_stop function say stop_all_other_cpus().

At which point we could implement smp_send_stop as:
stop_all_other_cpus(0);

And everywhere else would call stop_all_other_cpus(1) waiting for the cpus to actually stop.

Done, I have added a stop_other_cpus function which calls smp_ops.stop_other_cpus(1)

Quoted 4 lines

I really think we want to wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

Now it does, except in the panic case.

Jeremy, I have modified the xen_reboot bits too so that it now waits until all cpus are actually stopped, please take a look.

--

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop untill all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.
We now wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

Cc Eric W. Biederman <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>
arch/x86/include/asm/smp.h +7 -2
Index: linux-2.6/arch/x86/include/asm/smp.h
===================================================================
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*stop_other_cpus)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -73,7 +73,12 @@ extern struct smp_ops smp_ops;
 
 static inline void smp_send_stop(void)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.stop_other_cpus(0);
+}
+
+static inline void stop_other_cpus(void)
+{
+	smp_ops.stop_other_cpus(1);
 }
 
 static inline void smp_prepare_boot_cpu(void)
arch/x86/kernel/reboot.c +1 -1
Index: linux-2.6/arch/x86/kernel/reboot.c
===================================================================
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	lapic_shutdown();
arch/x86/kernel/smp.c +9 -6
Index: linux-2.6/arch/x86/kernel/smp.c
===================================================================
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_stop_other_cpus(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,12 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/*
+		 * Don't wait longer than a second if the caller
+		 * didn't ask us to wait.
+		 */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
@@ -227,7 +230,7 @@ struct smp_ops smp_ops = {
 	.smp_prepare_cpus	= native_smp_prepare_cpus,
 	.smp_cpus_done		= native_smp_cpus_done,
 
-	.smp_send_stop		= native_smp_send_stop,
+	.stop_other_cpus	= native_stop_other_cpus,
 	.smp_send_reschedule	= native_smp_send_reschedule,
 
 	.cpu_up			= native_cpu_up,
arch/x86/xen/enlighten.c +1 -1
Index: linux-2.6/arch/x86/xen/enlighten.c
===================================================================
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
arch/x86/xen/smp.c +3 -3
Index: linux-2.6/arch/x86/xen/smp.c
===================================================================
@@ -400,9 +400,9 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_stop_other_cpus(int wait)
 {
-	smp_call_function(stop_self, NULL, 0);
+	smp_call_function(stop_self, NULL, wait);
 }
 
 static void xen_smp_send_reschedule(int cpu)
@@ -470,7 +470,7 @@ static const struct smp_ops xen_smp_ops 
 	.cpu_disable = xen_cpu_disable,
 	.play_dead = xen_play_dead,
 
-	.smp_send_stop = xen_smp_send_stop,
+	.stop_other_cpus = xen_stop_other_cpus,
 	.smp_send_reschedule = xen_smp_send_reschedule,
 
 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
include/linux/smp.h +2 -1
Index: linux-2.6/include/linux/smp.h
===================================================================
@@ -116,7 +116,8 @@ extern unsigned int setup_max_cpus;
 
 #else /* !SMP */
 
-static inline void smp_send_stop(void) { }
+static inline void smp_send_stop() { }
+static inline void stop_other_cpus() { }
 
 /*
  *	These macros fold the SMP functionality into a single CPU system
#4 lore raw
Quoted 86 lines · Alok Kataria

Hi Eric,

Thanks for taking a look.

Quoted 59 lines · Eric W. Biederman
Quoted 40 lines · Alok Kataria

Copying a few more maintainers on the thread...

Quoted 38 lines · Alok Kataria

Before starting the new kernel kexec calls machine_shutdown to stop all the cpus, which internally calls native_smp_send_stop. AFAIU, kexec expects that all the cpus are now halted after that call returns. Now, looking at the code for native_smp_send_stop, it assumes that all the processors have processed the REBOOT ipi in 1 second after the IPI was sent.
native_smp_send_stop()
---------------------------------------------------------

     apic->send_IPI_allbutself(REBOOT_VECTOR);

        /* Don't wait longer than a second */
        wait = USEC_PER_SEC;
        while (num_online_cpus() > 1 && wait--)
             udelay(1);

---------------------------------------------------------

It just returns after that 1 second irrespective of whether all cpus were halted or not. This brings up a issue in the kexec case, since we can have the BSP starting the new kernel and AP's still processing the REBOOT IPI simultaneously.

Many distribution kernels use kexec to load the newly installed kernel during the installation phase, in virtualized environment with the host heavily overcommitted, we have seen some instances when vcpu fails to process the IPI in the allotted 1 sec and as a result the AP's end up accessing uninitialized state (the BSP has already gone ahead with setting up the new state) and causing GPF's.

IMO, kexec expects machine_shutdown to return only after all cpus are stopped.

The patch below should fix the issue, comments ??

--
machine_shutdown now takes a parameter "wait", if it is true, it waits until all the cpus are halted. All the callers except kexec still fallback to the earlier version of the shutdown call, where it just waited for max 1 sec before returning.

The approach seems reasonable. However on every path except for the panic path I would wait for all of the cpus to stop, as that is what those code paths are expecting. Until last year smp_send_stop always waited until all of the cpus stopped. So I think changing machine_shutdown should not need to happen.

This patch cannot be used as is because it changes the parameters to smp_send_stop() and there are more than just x86 implementations out there.

Let me propose a slightly different tactic. We need to separate the panic path from the normal path to avoid confusion. At the generic level smp_send_stop is exclusively used for the panic path. So we should keep that, and rename the x86 non-panic path function something else.

Quoted 7 lines

Can you rename the x86 smp_send_stop function say stop_all_other_cpus().

At which point we could implement smp_send_stop as:
stop_all_other_cpus(0);

And everywhere else would call stop_all_other_cpus(1) waiting for the cpus to actually stop.

Done, I have added a stop_other_cpus function which calls smp_ops.stop_other_cpus(1)

Quoted 4 lines

I really think we want to wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

Now it does, except in the panic case.

Jeremy, I have modified the xen_reboot bits too so that it now waits until all cpus are actually stopped, please take a look.

--

Quoted 8 lines

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop untill all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.
We now wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

One little nit.

Quoted 129 lines
Cc Eric W. Biederman <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>

Index: linux-2.6/arch/x86/include/asm/smp.h
===================================================================

arch/x86/include/asm/smp.h +7 -2
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*stop_other_cpus)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -73,7 +73,12 @@ extern struct smp_ops smp_ops;
 
 static inline void smp_send_stop(void)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.stop_other_cpus(0);
+}
+
+static inline void stop_other_cpus(void)
+{
+	smp_ops.stop_other_cpus(1);
 }
 
 static inline void smp_prepare_boot_cpu(void)

Index: linux-2.6/arch/x86/kernel/reboot.c
===================================================================

arch/x86/kernel/reboot.c +1 -1
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	lapic_shutdown();

Index: linux-2.6/arch/x86/kernel/smp.c
===================================================================

arch/x86/kernel/smp.c +9 -6
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_stop_other_cpus(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,12 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/*
+		 * Don't wait longer than a second if the caller
+		 * didn't ask us to wait.
+		 */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
@@ -227,7 +230,7 @@ struct smp_ops smp_ops = {
 	.smp_prepare_cpus	= native_smp_prepare_cpus,
 	.smp_cpus_done		= native_smp_cpus_done,
 
-	.smp_send_stop		= native_smp_send_stop,
+	.stop_other_cpus	= native_stop_other_cpus,
 	.smp_send_reschedule	= native_smp_send_reschedule,
 
 	.cpu_up			= native_cpu_up,

Index: linux-2.6/arch/x86/xen/enlighten.c
===================================================================

arch/x86/xen/enlighten.c +1 -1
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))

Index: linux-2.6/arch/x86/xen/smp.c
===================================================================

arch/x86/xen/smp.c +3 -3
@@ -400,9 +400,9 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_stop_other_cpus(int wait)
 {
-	smp_call_function(stop_self, NULL, 0);
+	smp_call_function(stop_self, NULL, wait);
 }
 
 static void xen_smp_send_reschedule(int cpu)
@@ -470,7 +470,7 @@ static const struct smp_ops xen_smp_ops 
 	.cpu_disable = xen_cpu_disable,
 	.play_dead = xen_play_dead,
 
-	.smp_send_stop = xen_smp_send_stop,
+	.stop_other_cpus = xen_stop_other_cpus,
 	.smp_send_reschedule = xen_smp_send_reschedule,
 
 	.send_call_func_ipi = xen_smp_send_call_function_ipi,

Index: linux-2.6/include/linux/smp.h
===================================================================

include/linux/smp.h extern unsigned int setup_max_cpus; · line 120

Quoted 6 lines
include/linux/smp.h +2 -1
@@ -116,7 +116,8 @@ extern unsigned int setup_max_cpus;
 
 #else /* !SMP */
 
-static inline void smp_send_stop(void) { }
+static inline void smp_send_stop() { }
+static inline void stop_other_cpus() { }

You shouldn't change smp.h.

For the moment stop_other_cpus() is x86 specific so we shouldn't impose it in other architectures.

Also note removing the explicit void is a bad idea in C because that is equivalent to saying: smp_send_stop(...) { } Which allows any set of parameters you can think of.

Eric

#5 lore raw
Quoted 1 line · Eric W. Biederman

Alok Kataria <[email protected]> writes:

<snip>
...

Quoted 14 lines

include/linux/smp.h 2010-10-11 12:07:19.000000000 -0700 extern unsigned int setup_max_cpus; · line 120

Quoted 11 lines

Index: linux-2.6/include/linux/smp.h
===================================================================

include/linux/smp.h 2010-10-11 12:07:19.000000000 -0700 renamed from include/linux/smp.h 2010-10-11 11:56:40.000000000 -0700 +2 -1
@@ -116,7 +116,8 @@ extern unsigned int setup_max_cpus;
 
 #else /* !SMP */
 
-static inline void smp_send_stop(void) { }
+static inline void smp_send_stop() { }
+static inline void stop_other_cpus() { }

You shouldn't change smp.h.

Quoted 7 lines

For the moment stop_other_cpus() is x86 specific so we shouldn't impose it in other architectures.

Also note removing the explicit void is a bad idea in C because that is equivalent to saying: smp_send_stop(...) { } Which allows any set of parameters you can think of.

You are right, that hunk was leftover from an earlier change. Thanks for the eagle eyes. Below is the refreshed patch.

Alok

--
x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop untill all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.
We now wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

Cc Eric W. Biederman <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>
arch/x86/include/asm/smp.h +7 -2
Index: linux-2.6/arch/x86/include/asm/smp.h
===================================================================
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*stop_other_cpus)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -73,7 +73,12 @@ extern struct smp_ops smp_ops;
 
 static inline void smp_send_stop(void)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.stop_other_cpus(0);
+}
+
+static inline void stop_other_cpus(void)
+{
+	smp_ops.stop_other_cpus(1);
 }
 
 static inline void smp_prepare_boot_cpu(void)
arch/x86/kernel/reboot.c +1 -1
Index: linux-2.6/arch/x86/kernel/reboot.c
===================================================================
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	lapic_shutdown();
arch/x86/kernel/smp.c +9 -6
Index: linux-2.6/arch/x86/kernel/smp.c
===================================================================
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_stop_other_cpus(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,12 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/*
+		 * Don't wait longer than a second if the caller
+		 * didn't ask us to wait.
+		 */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
@@ -227,7 +230,7 @@ struct smp_ops smp_ops = {
 	.smp_prepare_cpus	= native_smp_prepare_cpus,
 	.smp_cpus_done		= native_smp_cpus_done,
 
-	.smp_send_stop		= native_smp_send_stop,
+	.stop_other_cpus	= native_stop_other_cpus,
 	.smp_send_reschedule	= native_smp_send_reschedule,
 
 	.cpu_up			= native_cpu_up,
arch/x86/xen/enlighten.c +1 -1
Index: linux-2.6/arch/x86/xen/enlighten.c
===================================================================
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
arch/x86/xen/smp.c +3 -3
Index: linux-2.6/arch/x86/xen/smp.c
===================================================================
@@ -400,9 +400,9 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_stop_other_cpus(int wait)
 {
-	smp_call_function(stop_self, NULL, 0);
+	smp_call_function(stop_self, NULL, wait);
 }
 
 static void xen_smp_send_reschedule(int cpu)
@@ -470,7 +470,7 @@ static const struct smp_ops xen_smp_ops 
 	.cpu_disable = xen_cpu_disable,
 	.play_dead = xen_play_dead,
 
-	.smp_send_stop = xen_smp_send_stop,
+	.stop_other_cpus = xen_stop_other_cpus,
 	.smp_send_reschedule = xen_smp_send_reschedule,
 
 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
#7 lore raw
In reply to Alok Kataria
Quoted 92 lines · Alok Kataria

Hi Eric,

Thanks for taking a look.

Quoted 59 lines · Eric W. Biederman
Quoted 40 lines · Alok Kataria

Copying a few more maintainers on the thread...

Quoted 38 lines · Alok Kataria

Before starting the new kernel kexec calls machine_shutdown to stop all the cpus, which internally calls native_smp_send_stop. AFAIU, kexec expects that all the cpus are now halted after that call returns. Now, looking at the code for native_smp_send_stop, it assumes that all the processors have processed the REBOOT ipi in 1 second after the IPI was sent.
native_smp_send_stop()
---------------------------------------------------------

     apic->send_IPI_allbutself(REBOOT_VECTOR);

        /* Don't wait longer than a second */
        wait = USEC_PER_SEC;
        while (num_online_cpus() > 1 && wait--)
             udelay(1);

---------------------------------------------------------

It just returns after that 1 second irrespective of whether all cpus were halted or not. This brings up a issue in the kexec case, since we can have the BSP starting the new kernel and AP's still processing the REBOOT IPI simultaneously.

Many distribution kernels use kexec to load the newly installed kernel during the installation phase, in virtualized environment with the host heavily overcommitted, we have seen some instances when vcpu fails to process the IPI in the allotted 1 sec and as a result the AP's end up accessing uninitialized state (the BSP has already gone ahead with setting up the new state) and causing GPF's.

IMO, kexec expects machine_shutdown to return only after all cpus are stopped.

The patch below should fix the issue, comments ??

--
machine_shutdown now takes a parameter "wait", if it is true, it waits until all the cpus are halted. All the callers except kexec still fallback to the earlier version of the shutdown call, where it just waited for max 1 sec before returning.

The approach seems reasonable. However on every path except for the panic path I would wait for all of the cpus to stop, as that is what those code paths are expecting. Until last year smp_send_stop always waited until all of the cpus stopped. So I think changing machine_shutdown should not need to happen.

This patch cannot be used as is because it changes the parameters to smp_send_stop() and there are more than just x86 implementations out there.

Let me propose a slightly different tactic. We need to separate the panic path from the normal path to avoid confusion. At the generic level smp_send_stop is exclusively used for the panic path. So we should keep that, and rename the x86 non-panic path function something else.

Quoted 7 lines

Can you rename the x86 smp_send_stop function say stop_all_other_cpus().

At which point we could implement smp_send_stop as:
stop_all_other_cpus(0);

And everywhere else would call stop_all_other_cpus(1) waiting for the cpus to actually stop.

Done, I have added a stop_other_cpus function which calls smp_ops.stop_other_cpus(1)

Quoted 4 lines

I really think we want to wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

Now it does, except in the panic case.

Jeremy, I have modified the xen_reboot bits too so that it now waits until all cpus are actually stopped, please take a look.

--

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop untill all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.

Quoted 3 lines

We now wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

I don't think that kdump path uses smp_send_stop().

IIUC, on x86, we directly send NMI to other cpus.

native_machine_crash_shutdown()
kdump_nmi_shootdown_cpus()

        nmi_shootdown_cpus()
           smp_send_nmi_allbutself
                apic->send_IPI_allbutself(NMI_VECTOR);

So above description should be limited to only panic() path.

On a side note, I am wondering why panic() and kdump path can't share the shutdown routine.

Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#8 lore raw
Quoted 10 lines · Vivek Goyal
Quoted 7 lines · Alok Kataria

--

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop untill all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.

Quoted 5 lines
Quoted 3 lines

We now wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

I don't think that kdump path uses smp_send_stop().

Right, this should be removed from the patch description.

Belows is the new patch description
----------------------------------------------
x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop untill all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.
We now wait for the cpus to stop in all cases except for panic where we expect things to be broken and we are doing our best to make things work anyway.
----------------------------------------------

Quoted 5 lines

IIUC, on x86, we directly send NMI to other cpus.

native_machine_crash_shutdown()
kdump_nmi_shootdown_cpus()

Quoted 8 lines
        nmi_shootdown_cpus()
           smp_send_nmi_allbutself
                apic->send_IPI_allbutself(NMI_VECTOR);

So above description should be limited to only panic() path.

On a side note, I am wondering why panic() and kdump path can't share the shutdown routine.

The comment in native_stop_other_cpus (native_smp_send_stop earlier) explains why

/*
* Use an own vector here because smp_call_function
* does lots of things not suitable in a panic situation.
* On most systems we could also use an NMI here,
* but there are a few systems around where NMI
* is problematic so stay with an non NMI for now
* (this implies we cannot stop CPUs spinning with irq off
* currently)
*/

Thanks,
Alok

Quoted 2 lines

Vivek

#9 lore raw
In reply to Vivek Goyal
Quoted 3 lines · Vivek Goyal

On Mon, Oct 11, 2010 at 12:41:23PM -0700, Alok Kataria wrote:

I don't think that kdump path uses smp_send_stop().

It doesn't.

Quoted 9 lines

IIUC, on x86, we directly send NMI to other cpus.

native_machine_crash_shutdown()
kdump_nmi_shootdown_cpus()

        nmi_shootdown_cpus()
           smp_send_nmi_allbutself
                apic->send_IPI_allbutself(NMI_VECTOR);

So above description should be limited to only panic() path.

Is it actually confusing? With respect to documenting the line of thinking it seems reasonable.

Quoted 2 lines

On a side note, I am wondering why panic() and kdump path can't share the shutdown routine.

Hysterical raisins. Andi's change to smp_send_stop says that NMIs not working on some boxes. When someone wants to weed through all of the insanity it would probably be good to get the panic and the kdump paths sharing code. For now simply separating panic and reboot should be enough, and it lets the code evolve where it needs to.

Eric

#10 lore raw
Quoted 16 lines · Eric W. Biederman
Quoted 3 lines · Vivek Goyal

On Mon, Oct 11, 2010 at 12:41:23PM -0700, Alok Kataria wrote:

I don't think that kdump path uses smp_send_stop().

It doesn't.

Quoted 8 lines

IIUC, on x86, we directly send NMI to other cpus.

native_machine_crash_shutdown()
kdump_nmi_shootdown_cpus()

        nmi_shootdown_cpus()
           smp_send_nmi_allbutself
                apic->send_IPI_allbutself(NMI_VECTOR);
Quoted 5 lines
Quoted 1 line

So above description should be limited to only panic() path.

Is it actually confusing? With respect to documenting the line of thinking it seems reasonable.

No, just wanted to point out that let us modify the changelog to remove keyword "kdump" from it.

Quoted 9 lines
Quoted 2 lines

On a side note, I am wondering why panic() and kdump path can't share the shutdown routine.

Hysterical raisins. Andi's change to smp_send_stop says that NMIs not working on some boxes. When someone wants to weed through all of the insanity it would probably be good to get the panic and the kdump paths sharing code. For now simply separating panic and reboot should be enough, and it lets the code evolve where it needs to.

Ok. Agreed that atleast conceptually kdump and panic() path should share the code. But that's a different problem altogether and this patch can go in.

Thanks
Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#11 lore raw
Quoted 36 lines · Vivek Goyal
Quoted 16 lines · Eric W. Biederman
Quoted 3 lines · Vivek Goyal

On Mon, Oct 11, 2010 at 12:41:23PM -0700, Alok Kataria wrote:

I don't think that kdump path uses smp_send_stop().

It doesn't.

Quoted 8 lines

IIUC, on x86, we directly send NMI to other cpus.

native_machine_crash_shutdown()
kdump_nmi_shootdown_cpus()

        nmi_shootdown_cpus()
           smp_send_nmi_allbutself
                apic->send_IPI_allbutself(NMI_VECTOR);
Quoted 5 lines
Quoted 1 line

So above description should be limited to only panic() path.

Is it actually confusing? With respect to documenting the line of thinking it seems reasonable.

No, just wanted to point out that let us modify the changelog to remove keyword "kdump" from it.

Quoted 9 lines
Quoted 2 lines

On a side note, I am wondering why panic() and kdump path can't share the shutdown routine.

Hysterical raisins. Andi's change to smp_send_stop says that NMIs not working on some boxes. When someone wants to weed through all of the insanity it would probably be good to get the panic and the kdump paths sharing code. For now simply separating panic and reboot should be enough, and it lets the code evolve where it needs to.

Quoted 2 lines

Ok. Agreed that atleast conceptually kdump and panic() path should share the code. But that's a different problem altogether and this patch can go in.

Okay now that we all agree, let me repost a patch with the updated changelog, this fits on top of tip/master.

--

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop until all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.
We now wait for the cpus to stop in all cases except for panic where we expect things to be broken and we are doing our best to make things work anyway.

Cc Eric W. Biederman <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>
arch/x86/include/asm/smp.h +7 -2
Index: linux-x86-tree.git/arch/x86/include/asm/smp.h
===================================================================
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*stop_other_cpus)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -73,7 +73,12 @@ extern struct smp_ops smp_ops;
 
 static inline void smp_send_stop(void)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.stop_other_cpus(0);
+}
+
+static inline void stop_other_cpus(void)
+{
+	smp_ops.stop_other_cpus(1);
 }
 
 static inline void smp_prepare_boot_cpu(void)
arch/x86/kernel/reboot.c +1 -1
Index: linux-x86-tree.git/arch/x86/kernel/reboot.c
===================================================================
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	lapic_shutdown();
arch/x86/kernel/smp.c +9 -6
Index: linux-x86-tree.git/arch/x86/kernel/smp.c
===================================================================
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_stop_other_cpus(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,12 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/*
+		 * Don't wait longer than a second if the caller
+		 * didn't ask us to wait.
+		 */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
@@ -227,7 +230,7 @@ struct smp_ops smp_ops = {
 	.smp_prepare_cpus	= native_smp_prepare_cpus,
 	.smp_cpus_done		= native_smp_cpus_done,
 
-	.smp_send_stop		= native_smp_send_stop,
+	.stop_other_cpus	= native_stop_other_cpus,
 	.smp_send_reschedule	= native_smp_send_reschedule,
 
 	.cpu_up			= native_cpu_up,
arch/x86/xen/enlighten.c +1 -1
Index: linux-x86-tree.git/arch/x86/xen/enlighten.c
===================================================================
@@ -1019,7 +1019,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
arch/x86/xen/smp.c +3 -3
Index: linux-x86-tree.git/arch/x86/xen/smp.c
===================================================================
@@ -400,9 +400,9 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_stop_other_cpus(int wait)
 {
-	smp_call_function(stop_self, NULL, 0);
+	smp_call_function(stop_self, NULL, wait);
 }
 
 static void xen_smp_send_reschedule(int cpu)
@@ -470,7 +470,7 @@ static const struct smp_ops xen_smp_ops 
 	.cpu_disable = xen_cpu_disable,
 	.play_dead = xen_play_dead,
 
-	.smp_send_stop = xen_smp_send_stop,
+	.stop_other_cpus = xen_stop_other_cpus,
 	.smp_send_reschedule = xen_smp_send_reschedule,
 
 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
#12 lore raw
Quoted 37 lines · Alok Kataria
Quoted 36 lines · Vivek Goyal
Quoted 16 lines · Eric W. Biederman
Quoted 3 lines · Vivek Goyal

On Mon, Oct 11, 2010 at 12:41:23PM -0700, Alok Kataria wrote:

I don't think that kdump path uses smp_send_stop().

It doesn't.

Quoted 8 lines

IIUC, on x86, we directly send NMI to other cpus.

native_machine_crash_shutdown()
kdump_nmi_shootdown_cpus()

        nmi_shootdown_cpus()
           smp_send_nmi_allbutself
                apic->send_IPI_allbutself(NMI_VECTOR);
Quoted 5 lines
Quoted 1 line

So above description should be limited to only panic() path.

Is it actually confusing? With respect to documenting the line of thinking it seems reasonable.

No, just wanted to point out that let us modify the changelog to remove keyword "kdump" from it.

Quoted 9 lines
Quoted 2 lines

On a side note, I am wondering why panic() and kdump path can't share the shutdown routine.

Hysterical raisins. Andi's change to smp_send_stop says that NMIs not working on some boxes. When someone wants to weed through all of the insanity it would probably be good to get the panic and the kdump paths sharing code. For now simply separating panic and reboot should be enough, and it lets the code evolve where it needs to.

Quoted 5 lines
Quoted 2 lines

Ok. Agreed that atleast conceptually kdump and panic() path should share the code. But that's a different problem altogether and this patch can go in.

Okay now that we all agree, let me repost a patch with the updated changelog, this fits on top of tip/master.

Hi Ingo, HPA

I don't think this patch was picked up for tip, now that the 2.6.37 merge window is open can you please pick this up push it upstream. This patch fixes a legitimate regression, which was introduced during 2.6.30, by commit id 4ef702c10b5df18ab04921fc252c26421d4d6c75.

Thanks,
Alok

Quoted 137 lines

--

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop until all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel.
We now wait for the cpus to stop in all cases except for panic where we expect things to be broken and we are doing our best to make things work anyway.

Cc Eric W. Biederman <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>

Index: linux-x86-tree.git/arch/x86/include/asm/smp.h
===================================================================

arch/x86/include/asm/smp.h +7 -2
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*stop_other_cpus)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -73,7 +73,12 @@ extern struct smp_ops smp_ops;
 
 static inline void smp_send_stop(void)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.stop_other_cpus(0);
+}
+
+static inline void stop_other_cpus(void)
+{
+	smp_ops.stop_other_cpus(1);
 }
 
 static inline void smp_prepare_boot_cpu(void)

Index: linux-x86-tree.git/arch/x86/kernel/reboot.c
===================================================================

arch/x86/kernel/reboot.c +1 -1
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	lapic_shutdown();

Index: linux-x86-tree.git/arch/x86/kernel/smp.c
===================================================================

arch/x86/kernel/smp.c +9 -6
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(voi
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_stop_other_cpus(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,12 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/*
+		 * Don't wait longer than a second if the caller
+		 * didn't ask us to wait.
+		 */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
@@ -227,7 +230,7 @@ struct smp_ops smp_ops = {
 	.smp_prepare_cpus	= native_smp_prepare_cpus,
 	.smp_cpus_done		= native_smp_cpus_done,
 
-	.smp_send_stop		= native_smp_send_stop,
+	.stop_other_cpus	= native_stop_other_cpus,
 	.smp_send_reschedule	= native_smp_send_reschedule,
 
 	.cpu_up			= native_cpu_up,

Index: linux-x86-tree.git/arch/x86/xen/enlighten.c
===================================================================

arch/x86/xen/enlighten.c +1 -1
@@ -1019,7 +1019,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))

Index: linux-x86-tree.git/arch/x86/xen/smp.c
===================================================================

arch/x86/xen/smp.c +3 -3
@@ -400,9 +400,9 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_stop_other_cpus(int wait)
 {
-	smp_call_function(stop_self, NULL, 0);
+	smp_call_function(stop_self, NULL, wait);
 }
 
 static void xen_smp_send_reschedule(int cpu)
@@ -470,7 +470,7 @@ static const struct smp_ops xen_smp_ops 
 	.cpu_disable = xen_cpu_disable,
 	.play_dead = xen_play_dead,
 
-	.smp_send_stop = xen_smp_send_stop,
+	.stop_other_cpus = xen_stop_other_cpus,
 	.smp_send_reschedule = xen_smp_send_reschedule,
 
 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
#13 lore raw
Quoted 6 lines · Alok Kataria

I don't think this patch was picked up for tip, now that the 2.6.37 merge window is open can you please pick this up push it upstream. This patch fixes a legitimate regression, which was introduced during 2.6.30, by commit id 4ef702c10b5df18ab04921fc252c26421d4d6c75.

It probably would have helped if the patch had had a proper patch header and so on, and *in particular* not buried in a tree with [RFC PATCH]. RFC strongly implies that the patch is intended as a base for discussion, and is explicitly not intended to be committed.

-hpa

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#14 lore raw
Quoted 8 lines · H. Peter Anvin
Quoted 6 lines · Alok Kataria

I don't think this patch was picked up for tip, now that the 2.6.37 merge window is open can you please pick this up push it upstream. This patch fixes a legitimate regression, which was introduced during 2.6.30, by commit id 4ef702c10b5df18ab04921fc252c26421d4d6c75.

Quoted 4 lines

It probably would have helped if the patch had had a proper patch header and so on, and *in particular* not buried in a tree with [RFC PATCH]. RFC strongly implies that the patch is intended as a base for discussion, and is explicitly not intended to be committed.

I see, I have sent another mail with the patch.

Thanks,
Alok

Quoted 2 lines

-hpa

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#15 lore raw
Quoted 10 lines · Alok Kataria
Quoted 8 lines · H. Peter Anvin
Quoted 6 lines · Alok Kataria

I don't think this patch was picked up for tip, now that the 2.6.37 merge window is open can you please pick this up push it upstream. This patch fixes a legitimate regression, which was introduced during 2.6.30, by commit id 4ef702c10b5df18ab04921fc252c26421d4d6c75.

Quoted 7 lines
Quoted 4 lines

It probably would have helped if the patch had had a proper patch header and so on, and *in particular* not buried in a tree with [RFC PATCH]. RFC strongly implies that the patch is intended as a base for discussion, and is explicitly not intended to be committed.

I see, I have sent another mail with the patch.

I already took the patch and am running it through compile tests, but that was mostly for future reference.

-hpa

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#6 lore raw

[tip:x86/urgent] x86, kexec: Make sure to stop all CPUs before exiting the kernel

In reply to Alok Kataria
Commit-ID:  76fac077db6b34e2c6383a7b4f3f4f7b7d06d8ce
Gitweb:     http://git.kernel.org/tip/76fac077db6b34e2c6383a7b4f3f4f7b7d06d8ce
Author:     Alok Kataria <[email protected]>
AuthorDate: Mon, 11 Oct 2010 14:37:08 -0700
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 21 Oct 2010 13:30:44 -0700

x86, kexec: Make sure to stop all CPUs before exiting the kernel

x86 smp_ops now has a new op, stop_other_cpus which takes a parameter "wait" this allows the caller to specify if it wants to stop until all the cpus have processed the stop IPI. This is required specifically for the kexec case where we should wait for all the cpus to be stopped before starting the new kernel. We now wait for the cpus to stop in all cases except for panic/kdump where we expect things to be broken and we are doing our best to make things work anyway.

This patch fixes a legitimate regression, which was introduced during 2.6.30, by commit id 4ef702c10b5df18ab04921fc252c26421d4d6c75.

LKML-Reference: <[email protected]>

Cc Eric W. Biederman <[email protected]>
Cc Jeremy Fitzhardinge <[email protected]>
Cc <[email protected]> v2.6.30-36
---
 arch/x86/include/asm/smp.h |    9 +++++++--
 arch/x86/kernel/reboot.c   |    2 +-
 arch/x86/kernel/smp.c      |   15 +++++++++------
 arch/x86/xen/enlighten.c   |    2 +-
 arch/x86/xen/smp.c         |    6 +++---
 5 files changed, 21 insertions(+), 13 deletions(-)
arch/x86/include/asm/smp.h +7 -2
index 4cfc908..4c2f63c 100644
@@ -50,7 +50,7 @@ struct smp_ops {
 	void (*smp_prepare_cpus)(unsigned max_cpus);
 	void (*smp_cpus_done)(unsigned max_cpus);
 
-	void (*smp_send_stop)(void);
+	void (*stop_other_cpus)(int wait);
 	void (*smp_send_reschedule)(int cpu);
 
 	int (*cpu_up)(unsigned cpu);
@@ -73,7 +73,12 @@ extern struct smp_ops smp_ops;
 
 static inline void smp_send_stop(void)
 {
-	smp_ops.smp_send_stop();
+	smp_ops.stop_other_cpus(0);
+}
+
+static inline void stop_other_cpus(void)
+{
+	smp_ops.stop_other_cpus(1);
 }
 
 static inline void smp_prepare_boot_cpu(void)
arch/x86/kernel/reboot.c +1 -1
index e3af342..76a0d71 100644
@@ -641,7 +641,7 @@ void native_machine_shutdown(void)
 	/* O.K Now that I'm on the appropriate processor,
 	 * stop all of the others.
 	 */
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	lapic_shutdown();
arch/x86/kernel/smp.c +9 -6
index d801210..513deac 100644
@@ -159,10 +159,10 @@ asmlinkage void smp_reboot_interrupt(void)
 	irq_exit();
 }
 
-static void native_smp_send_stop(void)
+static void native_stop_other_cpus(int wait)
 {
 	unsigned long flags;
-	unsigned long wait;
+	unsigned long timeout;
 
 	if (reboot_force)
 		return;
@@ -179,9 +179,12 @@ static void native_smp_send_stop(void)
 	if (num_online_cpus() > 1) {
 		apic->send_IPI_allbutself(REBOOT_VECTOR);
 
-		/* Don't wait longer than a second */
-		wait = USEC_PER_SEC;
-		while (num_online_cpus() > 1 && wait--)
+		/*
+		 * Don't wait longer than a second if the caller
+		 * didn't ask us to wait.
+		 */
+		timeout = USEC_PER_SEC;
+		while (num_online_cpus() > 1 && (wait || timeout--))
 			udelay(1);
 	}
 
@@ -227,7 +230,7 @@ struct smp_ops smp_ops = {
 	.smp_prepare_cpus	= native_smp_prepare_cpus,
 	.smp_cpus_done		= native_smp_cpus_done,
 
-	.smp_send_stop		= native_smp_send_stop,
+	.stop_other_cpus	= native_stop_other_cpus,
 	.smp_send_reschedule	= native_smp_send_reschedule,
 
 	.cpu_up			= native_cpu_up,
arch/x86/xen/enlighten.c +1 -1
index 7d46c84..44f8086 100644
@@ -1018,7 +1018,7 @@ static void xen_reboot(int reason)
 	struct sched_shutdown r = { .reason = reason };
 
 #ifdef CONFIG_SMP
-	smp_send_stop();
+	stop_other_cpus();
 #endif
 
 	if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
arch/x86/xen/smp.c +3 -3
index 25f232b..f4d0100 100644
@@ -400,9 +400,9 @@ static void stop_self(void *v)
 	BUG();
 }
 
-static void xen_smp_send_stop(void)
+static void xen_stop_other_cpus(int wait)
 {
-	smp_call_function(stop_self, NULL, 0);
+	smp_call_function(stop_self, NULL, wait);
 }
 
 static void xen_smp_send_reschedule(int cpu)
@@ -470,7 +470,7 @@ static const struct smp_ops xen_smp_ops __initdata = {
 	.cpu_disable = xen_cpu_disable,
 	.play_dead = xen_play_dead,
 
-	.smp_send_stop = xen_smp_send_stop,
+	.stop_other_cpus = xen_stop_other_cpus,
 	.smp_send_reschedule = xen_smp_send_reschedule,
 
 	.send_call_func_ipi = xen_smp_send_call_function_ipi,