Skip to contentSkip to conversation

i386/x86-64: Fix timer SMP bootup race

PATCH In review Andi Kleen 3 participants 15 messages Jan 15, 2005 – Jan 17, 2005 CPU hotplug

lore
Share & tools
lore

Andi Kleen

author
#0 lore raw

PATCH 1/1 i386/x86-64: Fix timer SMP bootup race

Fix boot up SMP race in timer setup on i386/x86-64.

This fixes a long standing race in 2.6 i386/x86-64 SMP boot. The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

The race only happens when a CPU takes a long time to boot (e.g. very slow console output with debugging enabled).

To fix I added a new cpu notifier notifier command CPU_UP_PREPARE_EARLY that is called before the secondary CPU is started. timer.c uses that now to initialize the per CPU timers early before the other CPU runs any Linux code.

i386 and x86-64 have been fixed to use this.

For compatibility with other architectures I made timer.c handle both (initialization both with CPU_UP_PREPARE and CPU_UP_PREPARE_EARLY) with a flag. This could be cleaned up later.

arch/i386/kernel/smpboot.c +4 -0
Index: linux/arch/i386/kernel/smpboot.c
===================================================================
@@ -44,6 +44,7 @@
 #include <linux/smp_lock.h>
 #include <linux/irq.h>
 #include <linux/bootmem.h>
+#include <linux/cpu.h>
 
 #include <linux/delay.h>
 #include <linux/mc146818rtc.h>
@@ -766,6 +767,9 @@
 	idle = fork_idle(cpu);
 	if (IS_ERR(idle))
 		panic("failed fork for CPU %d", cpu);
+
+	cpu_up_early(cpu);
+
 	idle->thread.eip = (unsigned long) start_secondary;
 	/* start_eip had better be page-aligned! */
 	start_eip = setup_trampoline();
kernel/timer.c +6 -0
Index: linux/kernel/timer.c
===================================================================
@@ -86,6 +86,7 @@
 
 /* Fake initialization */
 static DEFINE_PER_CPU(tvec_base_t, tvec_bases) = { SPIN_LOCK_UNLOCKED };
+static DEFINE_PER_CPU(int, timer_inited) = 0; 
 
 static void check_timer_failed(struct timer_list *timer)
 {
@@ -1281,6 +1282,9 @@
 {
 	int j;
 	tvec_base_t *base;
+	
+	if (per_cpu(timer_inited, cpu))
+		return;
        
 	base = &per_cpu(tvec_bases, cpu);
 	spin_lock_init(&base->lock);
@@ -1294,6 +1298,7 @@
 		INIT_LIST_HEAD(base->tv1.vec + j);
 
 	base->timer_jiffies = jiffies;
+	per_cpu(timer_inited, cpu) = 1;
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
@@ -1367,6 +1372,7 @@
 {
 	long cpu = (long)hcpu;
 	switch(action) {
+	case CPU_UP_PREPARE_EARLY:
 	case CPU_UP_PREPARE:
 		init_timers_cpu(cpu);
 		break;
include/linux/notifier.h +1 -0
Index: linux/include/linux/notifier.h
===================================================================
@@ -70,6 +70,7 @@
 #define CPU_DOWN_PREPARE	0x0005 /* CPU (unsigned)v going down */
 #define CPU_DOWN_FAILED		0x0006 /* CPU (unsigned)v NOT going down */
 #define CPU_DEAD		0x0007 /* CPU (unsigned)v dead */
+#define CPU_UP_PREPARE_EARLY    0x0008 /* CPU (unsigned)v will be going up soon */
 
 #endif /* __KERNEL__ */
 #endif /* _LINUX_NOTIFIER_H */
arch/x86_64/kernel/smpboot.c +5 -0
Index: linux/arch/x86_64/kernel/smpboot.c
===================================================================
@@ -44,6 +44,7 @@
 #include <linux/bootmem.h>
 #include <linux/thread_info.h>
 #include <linux/module.h>
+#include <linux/cpu.h>
 
 #include <linux/delay.h>
 #include <linux/mc146818rtc.h>
@@ -566,6 +567,10 @@
 	idle = fork_idle(cpu);
 	if (IS_ERR(idle))
 		panic("failed fork for CPU %d", cpu);
+	
+	/* Initialize timer early */
+	cpu_up_early(cpu);
+
 	x86_cpu_to_apicid[cpu] = apicid;
 
 	cpu_pda[cpu].pcurrent = idle;
kernel/cpu.c +22 -0
Index: linux/kernel/cpu.c
===================================================================
@@ -191,3 +191,25 @@
 	up(&cpucontrol);
 	return ret;
 }
+
+/* CPU will be going up soon */
+int __devinit cpu_up_early(unsigned int cpu)
+{
+	void *hcpu = (void *)(long)cpu;
+	int ret;
+
+	if ((ret = down_interruptible(&cpucontrol)) != 0)
+		return ret;
+
+	ret = notifier_call_chain(&cpu_chain, CPU_UP_PREPARE_EARLY, hcpu);
+	if (ret == NOTIFY_BAD) {
+		printk("%s: attempt to prepare CPU %u failed\n",
+				__FUNCTION__, cpu);
+		ret = -EINVAL;
+		
+		/* no cancel done here for now */
+	}
+	
+	up(&cpucontrol);
+	return ret;
+}
#1 lore raw
Quoted 9 lines · Andi Kleen

Fix boot up SMP race in timer setup on i386/x86-64.

This fixes a long standing race in 2.6 i386/x86-64 SMP boot. The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

Quoted 7 lines

The race only happens when a CPU takes a long time to boot (e.g. very slow console output with debugging enabled).

To fix I added a new cpu notifier notifier command CPU_UP_PREPARE_EARLY that is called before the secondary CPU is started. timer.c uses that now to initialize the per CPU timers early before the other CPU runs any Linux code.

Andi, that's horrible. I suspect you know it's horrible and were hoping someone would fix it properly. The semantics of CPU_UP_PREPARE are supposed to do this already.

The cause of this bug is that (1) i386 and x86_64 actually bring the secondary CPUs up at boot before the core code officially brings them up using cpu_up(), after the appropriate callbacks, and (2) they call into core code tp process timer interrupts before they've been officially brought up.

The former is because I just added a shim rather than rewriting the x86 boot process, because it would have broken too much. The fix is do the boot process properly, or to suppress the call to do_timer before the CPU is actually "up".

Sorry,
Rusty.

Signature · 1 line
A bad analogy is like a leaky screwdriver -- Richard Braakman
author
#2 lore raw
Quoted 13 lines · Rusty Russell
Quoted 12 lines · Andi Kleen

Fix boot up SMP race in timer setup on i386/x86-64.

This fixes a long standing race in 2.6 i386/x86-64 SMP boot. The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

The race only happens when a CPU takes a long time to boot (e.g. very slow console output with debugging enabled).

Quoted 8 lines
Quoted 4 lines

To fix I added a new cpu notifier notifier command CPU_UP_PREPARE_EARLY that is called before the secondary CPU is started. timer.c uses that now to initialize the per CPU timers early before the other CPU runs any Linux code.

Andi, that's horrible. I suspect you know it's horrible and were hoping someone would fix it properly. The semantics of CPU_UP_PREPARE are supposed to do this already.

I shortly considered redoing the boot process, but then it looked too risky to me.

e.g. I guess on x86-64 it wouldn't be that difficult, just a bit of work, but on i386 with all the weird hardware it could be quite destabilizing. But doing it on x86-64 only is not a good solution.

Quoted 11 lines

The cause of this bug is that (1) i386 and x86_64 actually bring the secondary CPUs up at boot before the core code officially brings them up using cpu_up(), after the appropriate callbacks, and (2) they call into core code tp process timer interrupts before they've been officially brought up.

The former is because I just added a shim rather than rewriting the x86 boot process, because it would have broken too much. The fix is do the boot process properly, or to suppress the call to do_timer before the CPU is actually "up".

I don't see a better short term solution.

If you had done it properly in 2.5 it would be working and tested by now ;-) , but doing it in the middle of 2.6 would seem a bit misplaced to me.

-Andi

#3 lore raw
Quoted 6 lines · Andi Kleen

I shortly considered redoing the boot process, but then it looked too risky to me.

e.g. I guess on x86-64 it wouldn't be that difficult, just a bit of work, but on i386 with all the weird hardware it could be quite destabilizing. But doing it on x86-64 only is not a good solution.

Well, architectures which support CPU hotplug have had to fix their boot process anyway, and most are fairly trivial.

Quoted 6 lines
Quoted 6 lines

The cause of this bug is that (1) i386 and x86_64 actually bring the secondary CPUs up at boot before the core code officially brings them up using cpu_up(), after the appropriate callbacks, and (2) they call into core code tp process timer interrupts before they've been officially brought up.

Quoted 6 lines
Quoted 4 lines

The former is because I just added a shim rather than rewriting the x86 boot process, because it would have broken too much. The fix is do the boot process properly, or to suppress the call to do_timer before the CPU is actually "up".

I don't see a better short term solution.

Ick, see patch.

Quoted 3 lines

If you had done it properly in 2.5 it would be working and tested by now ;-) , but doing it in the middle of 2.6 would seem a bit misplaced to me.

Linus would not have taken the patch, because it would have broken too much. Cleaning up the x86 boot sequence is a project in itself, which needs to be done, but not by me 8)

Rusty.

Name: Fix x86 calling timers before CPU is supposed to be up
Status: Untested

Quoted 1 line

From Andi Kleen:

The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

The race only happens when a CPU takes a long time to boot (e.g. very slow console output with debugging enabled).

The reason is that the x86 boot code actually bring up all CPUs immediately, then simply unleashes them when the core code calls __cpu_up(). Unfortunately, they enable interrupts to call calibrate_delay(): if they get a timer interrupt during that, they call into the core timer code which isn't set up for it yet.

include/asm-i386/mach-voyager/do_timer.h +5 -1
Index: linux-2.6.11-rc1-bk2-Misc/include/asm-i386/mach-voyager/do_timer.h
===================================================================
@@ -3,7 +3,11 @@
 
 static inline void do_timer_interrupt_hook(struct pt_regs *regs)
 {
-	do_timer(regs);
+	/* i386 brings up CPU before core is setup. */
+	if (unlikely(!cpu_online(smp_processor_id())))
+		jiffies64++;
+	else
+		do_timer(regs);
 #ifndef CONFIG_SMP
 	update_process_times(user_mode(regs));
 #endif
include/asm-i386/mach-default/do_timer.h +5 -1
Index: linux-2.6.11-rc1-bk2-Misc/include/asm-i386/mach-default/do_timer.h
===================================================================
@@ -15,7 +15,11 @@
 
 static inline void do_timer_interrupt_hook(struct pt_regs *regs)
 {
-	do_timer(regs);
+	/* i386 brings up CPU before core is setup. */
+	if (unlikely(!cpu_online(smp_processor_id())))
+		jiffies64++;
+	else
+		do_timer(regs);
 #ifndef CONFIG_SMP
 	update_process_times(user_mode(regs));
 #endif
include/asm-i386/mach-visws/do_timer.h +5 -1
Index: linux-2.6.11-rc1-bk2-Misc/include/asm-i386/mach-visws/do_timer.h
===================================================================
@@ -8,7 +8,11 @@
 	/* Clear the interrupt */
 	co_cpu_write(CO_CPU_STAT,co_cpu_read(CO_CPU_STAT) & ~CO_STAT_TIMEINTR);
 
-	do_timer(regs);
+	/* i386 brings up CPU before core is setup. */
+	if (unlikely(!cpu_online(smp_processor_id())))
+		jiffies64++;
+	else
+		do_timer(regs);
 #ifndef CONFIG_SMP
 	update_process_times(user_mode(regs));
 #endif
Signature · 1 line
A bad analogy is like a leaky screwdriver -- Richard Braakman
#4 lore raw
Quoted 9 lines · Rusty Russell

static inline void do_timer_interrupt_hook(struct pt_regs *regs) {
- do_timer(regs);
+ /* i386 brings up CPU before core is setup. */
+ if (unlikely(!cpu_online(smp_processor_id())))
+ jiffies64++;
+ else
+ do_timer(regs);

I thik I preferred Andi's approach - it adds code which is largely dropped from the memory. This patch adds a test-n-branch to every timer interrupt..

author
#5 lore raw
In reply to Rusty Russell
Quoted 9 lines · Rusty Russell
Quoted 6 lines

I shortly considered redoing the boot process, but then it looked too risky to me.

e.g. I guess on x86-64 it wouldn't be that difficult, just a bit of work, but on i386 with all the weird hardware it could be quite destabilizing. But doing it on x86-64 only is not a good solution.

Well, architectures which support CPU hotplug have had to fix their boot process anyway, and most are fairly trivial.

The problem is not doing the work, but testing it.

Quoted 7 lines
Quoted 3 lines

If you had done it properly in 2.5 it would be working and tested by now ;-) , but doing it in the middle of 2.6 would seem a bit misplaced to me.

Linus would not have taken the patch, because it would have broken too much. Cleaning up the x86 boot sequence is a project in itself, which needs to be done, but not by me 8)

I think my patch is better. It at least keeps all the
baggage out of the normal run paths. Doing this check at each timer interrupt doesn't make much sense.

-Andi

#6 lore raw
Quoted 3 lines · Andi Kleen

I think my patch is better. It at least keeps all the
baggage out of the normal run paths. Doing this check at each timer interrupt doesn't make much sense.

It doesn't penalize the architectures which do the right thing already. If this weren't i386 we were talking about...

But adding a bizarro "pre-prepare" notifier verged on nonsensical 8(. I prefer an explicit "init_timers_early()" call as a workaround; I'll code that up and test tomorrow, when I'm back in the office with an SMP box to test.

I'm also not clear on why we need to enable interrupts around calibrate_delay() on secondary processors, but I'll try that too and find out 8)

Thanks,
Rusty.

Signature · 1 line
A bad analogy is like a leaky screwdriver -- Richard Braakman
author
#7 lore raw
Quoted 9 lines · Rusty Russell
Quoted 3 lines · Andi Kleen

I think my patch is better. It at least keeps all the
baggage out of the normal run paths. Doing this check at each timer interrupt doesn't make much sense.

It doesn't penalize the architectures which do the right thing already. If this weren't i386 we were talking about...

But adding a bizarro "pre-prepare" notifier verged on nonsensical 8(. I

I don't see the big issue. Preparse is just not as early as you thought.

Quoted 7 lines

prefer an explicit "init_timers_early()" call as a workaround; I'll code that up and test tomorrow, when I'm back in the office with an SMP box to test.

I'm also not clear on why we need to enable interrupts around calibrate_delay() on secondary processors, but I'll try that too and find out 8)

Because you cannot calibrate the timer without a timer tick.

Even if you changed that it wouldn't help because the race can as well happen in the idle loop on the secondaries.

-Andi

#8 lore raw
Quoted 4 lines · Andi Kleen
Quoted 1 line · Rusty Russell

But adding a bizarro "pre-prepare" notifier verged on nonsensical 8(. I

I don't see the big issue. Preparse is just not as early as you thought.

State machine for non-boot CPUs currently goes:

------------

|            v
|	CPU_UP_PREPARE <-----
|	     |		     |
|	     v		     |
|	__cpu_up() ---> CPU_UP_CANCELED
|	     |
|	     v
|	CPU_ONLINE
|	     |
|	     v
|	CPU_DOWN_PREPARE <----
|	     |		      |
|	     v	              |
|	__cpu_down()---> CPU_DOWN_FAILED
|	     |
|	     v
|	CPU_DEAD
|	     |

------------

You've tacked a CPU_UP_PREPARE_EARLY above this state for two architectures, which only gets called at boot, and only is used by timers, to fix the fact that those archs don't actually follow this state machine yet.

And sure enough, you've added a potential bug, since timer_jiffies doesn't get reset to jiffies when a CPU comes back up. Really, a specific hack as below is required.

Quoted 5 lines
Quoted 3 lines

I'm also not clear on why we need to enable interrupts around calibrate_delay() on secondary processors, but I'll try that too and find out 8)

Because you cannot calibrate the timer without a timer tick.

AFAICT from a quick reading, that's not true. CPU0 will bump the jiffies, which is all you need: local interrupts not required.

Quoted 2 lines

Even if you changed that it wouldn't help because the race can as well happen in the idle loop on the secondaries.

Again, I don't think so: the other CPUs are sitting in start_secondary() waiting for __cpu_up().

As I said, I will try it and see. It would certainly be the nicest solution. If not, I'll code a "timer_smp_boot_init()" function for x86 myself.

Cheers,
Rusty.

Signature · 1 line
A bad analogy is like a leaky screwdriver -- Richard Braakman
#9 lore raw
In reply to Andi Kleen
Quoted 7 lines · Andi Kleen

This fixes a long standing race in 2.6 i386/x86-64 SMP boot.
The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

Why don't we just not call calibrate_delay() on the secondaries? It doesn't seem to do anything. That way we can leave local interrupts disabled.

If for some reason we still want the bogomips printk, call calibrate_delay() from the CPU_UP_PREPARE handler?

author
#10 lore raw
Quoted 9 lines · Andrew Morton
Quoted 7 lines · Andi Kleen

This fixes a long standing race in 2.6 i386/x86-64 SMP boot.
The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

Quoted 3 lines

Why don't we just not call calibrate_delay() on the secondaries? It doesn't seem to do anything. That way we can leave local interrupts disabled.

It's used for the "accumulative bogomips". To quote Alan:

    /*
     * Allow the user to impress friends.
     */

But taking it away doesn't help because the timer startup on the BP and the secondaries going into the idle loop isn't synchronized. You could add a synchronization step, but it would be far more complicated than fixing the ordering like I did.

-Andi

#11 lore raw
Quoted 17 lines · Andi Kleen
Quoted 9 lines · Andrew Morton
Quoted 7 lines · Andi Kleen

This fixes a long standing race in 2.6 i386/x86-64 SMP boot.
The per CPU timers would only get initialized after an secondary CPU was running. But during initialization the secondary CPU would already enable interrupts to compute the jiffies. When a per CPU timer fired in this window it would run into a BUG in timer.c because the timer heap for that CPU wasn't fully initialized.

Quoted 3 lines

Why don't we just not call calibrate_delay() on the secondaries? It doesn't seem to do anything. That way we can leave local interrupts disabled.

It's used for the "accumulative bogomips". To quote Alan:

Quoted 8 lines
    /*
     * Allow the user to impress friends.
     */

But taking it away doesn't help because the timer startup on the BP and the secondaries going into the idle loop isn't synchronized. You could add a synchronization step, but it would be far more complicated than fixing the ordering like I did.

I don't get it. By the time the secondaries enter the idle loop, they've already run init_timers_cpu() anyway. You patch doesn't address a secondary taking a timer interrupt prior to the BP having run init_timers(), does it?

author
#12 lore raw
Quoted 2 lines

I don't get it. By the time the secondaries enter the idle loop, they've already run init_timers_cpu() anyway. You patch doesn't address a

The notifier uns only after smp_prepare_cpus and then all the synchronization is long done.

Quoted 2 lines

secondary taking a timer interrupt prior to the BP having run init_timers(), does it?

It initializes the timer of a CPU before it is even touched.

-Andi

#13 lore raw
In reply to Andi Kleen
Quoted 1 line · Andi Kleen

Fix boot up SMP race in timer setup on i386/x86-64.

How's this? Didn't do x86-64, but tested on i386.

Rusty.

Name: x86: no interrupts from secondary CPUs until officially online
Status: Tested on dual Pentium II, 2.6.11-rc1-bk2

Andi Kleen reported a problem where a very slow boot caused the timer interrupt on a secondary CPU to go off before the CPU was actually brought up by the core code, so the CPU_PREPARE notifier hadn't been called, so the per-cpu timer code wasn't set up.

This was caused by enabling interrupts around calibrate_delay() on secondary CPUs, which is not actually neccessary (interrupts on CPU 0 increments jiffies, which is all that is required). So delay enabling interrupts until the actual __cpu_up() call for that CPU.

arch/i386/kernel/smpboot.c +5 -3
Index: linux-2.6.11-rc1-bk2-Misc/arch/i386/kernel/smpboot.c
===================================================================
@@ -383,8 +383,6 @@
 	setup_local_APIC();
 	map_cpu_to_logical_apicid();
 
-	local_irq_enable();
-
 	/*
 	 * Get our bogomips.
 	 */
@@ -397,7 +395,7 @@
  	smp_store_cpu_info(cpuid);
 
 	disable_APIC_timer();
-	local_irq_disable();
+
 	/*
 	 * Allow the master to continue.
 	 */
@@ -439,6 +438,10 @@
 	 */
 	local_flush_tlb();
 	cpu_set(smp_processor_id(), cpu_online_map);
+
+	/* We can take interrupts now: we're officially "up". */
+	local_irq_enable();
+
 	wmb();
 	cpu_idle();
 }
arch/i386/kernel/apic.c +0 -2
Index: linux-2.6.11-rc1-bk2-Misc/arch/i386/kernel/apic.c
===================================================================
@@ -1046,9 +1046,7 @@
 
 void __init setup_secondary_APIC_clock(void)
 {
-	local_irq_disable(); /* FIXME: Do we need this? --RR */
 	setup_APIC_timer(calibration_result);
-	local_irq_enable();
 }
 
 void __init disable_APIC_timer(void)
Signature · 1 line
A bad analogy is like a leaky screwdriver -- Richard Braakman
author
#14 lore raw
Quoted 4 lines · Rusty Russell
Quoted 1 line · Andi Kleen

Fix boot up SMP race in timer setup on i386/x86-64.

How's this? Didn't do x86-64, but tested on i386.

Looks ok to me. You're right that the commenced check should do the necessary synchronization before the idle loop is entered.

-Andi