RCU fix
Linus,
Please pull the latest core-fixes-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git core-fixes-for-linus
This fixes a long-existent RCU / scheduler interaction bug resulting in long bootup delays (sometimes hangs) that has become more prominent in .29 and which has been tracked down recently.
I didnt like the fix of adding yet another boot state flag but neither of the alternatives looked too appealing either:
http://lkml.org/lkml/2009/2/25/321
So i went with this one.
But we can clean up this code if anyone has a good idea how to do it better. The main source of the fragility is the fact that we run the 'idle' thread as the init task for quite some time. But we cannot really create a real init task until we have a number of core kernel facilities up and running.
Thanks,
Ingo
------------------>
Paul E. McKenney (1):
rcu: Teach RCU that idle task is not quiscent state at boot
include/linux/rcuclassic.h | 6 ++++++ include/linux/rcupdate.h | 4 ++++ include/linux/rcupreempt.h | 15 +++++++++++++++ include/linux/rcutree.h | 6 ++++++ init/main.c | 3 ++- kernel/rcuclassic.c | 4 ++-- kernel/rcupdate.c | 12 ++++++++++++ kernel/rcupreempt.c | 3 +++ kernel/rcutree.c | 4 ++-- 9 files changed, 52 insertions(+), 5 deletions(-)
include/linux/rcuclassic.h +6 -0
@@ -181,4 +181,10 @@ extern long rcu_batches_completed_bh(void);#define rcu_enter_nohz () do { } while (0)#define rcu_exit_nohz () do { } while (0) +/* A context switch is a grace period for rcuclassic. */ +static inline int rcu_blocking_is_gp (void ) +{ +return num_online_cpus () ==1 ; +} +#endif /* __LINUX_RCUCLASSIC_H */
include/linux/rcupdate.h +4 -0
@@ -52,6 +52,9 @@ struct rcu_head {void (*func)(struct rcu_head *head); }; +/* Internal to kernel, but needed by rcupreempt.h. */ +extern int rcu_scheduler_active; +#if defined(CONFIG_CLASSIC_RCU )#include <linux/rcuclassic.h> #elif defined(CONFIG_TREE_RCU )
@@ -265,6 +268,7 @@ extern void rcu_barrier_sched(void);/* Internal to kernel */ extern void rcu_init (void ); +extern void rcu_scheduler_starting (void );extern int rcu_needs_cpu (int cpu);#endif /* __LINUX_RCUPDATE_H */
include/linux/rcupreempt.h +15 -0
@@ -142,4 +142,19 @@ static inline void rcu_exit_nohz(void)#define rcu_exit_nohz () do { } while (0)#endif /* CONFIG_NO_HZ */ +/* +* A context switch is a grace period for rcupreempt synchronize_rcu() +* only during early boot, before the scheduler has been initialized. +* So, how the heck do we get a context switch? Well, if the caller +* invokes synchronize_rcu(), they are willing to accept a context +* switch, so we simply pretend that one happened. +* +* After boot, there might be a blocked or preempted task in an RCU +* read-side critical section, so we cannot then take the fastpath. +*/ +static inline int rcu_blocking_is_gp (void ) +{ +return num_online_cpus () ==1 && !rcu_scheduler_active; +} +#endif /* __LINUX_RCUPREEMPT_H */
include/linux/rcutree.h +6 -0
@@ -326,4 +326,10 @@ static inline void rcu_exit_nohz(void) }#endif /* CONFIG_NO_HZ */ +/* A context switch is a grace period for rcutree. */ +static inline int rcu_blocking_is_gp (void ) +{ +return num_online_cpus () ==1 ; +} +#endif /* __LINUX_RCUTREE_H */
init/main.c +2 -1
@@ -97,7 +97,7 @@ static inline void mark_rodata_ro(void) { }extern void tc_init (void );#endif -enum system_states system_state; +enum system_states system_state__read_mostly ;EXPORT_SYMBOL (system_state);/*
@@ -463,6 +463,7 @@ static noinline void __init_refok rest_init(void)* at least once to get things moving: */ init_idle_bootup_task (current); +rcu_scheduler_starting ();preempt_enable_no_resched ();schedule ();preempt_disable ();
kernel/rcuclassic.c +2 -2
@@ -679,8 +679,8 @@ int rcu_needs_cpu(int cpu)void rcu_check_callbacks (int cpu,int user) {if (user || - (idle_cpu (cpu) && !in_softirq () && -hardirq_count () <= (1 <<HARDIRQ_SHIFT ))) { + (idle_cpu (cpu) && rcu_scheduler_active && + !in_softirq () &&hardirq_count () <= (1 <<HARDIRQ_SHIFT ))) {/* * Get here if this CPU took its interrupt from user
kernel/rcupdate.c +12 -0
@@ -44,6 +44,7 @@#include <linux/cpu.h> #include <linux/mutex.h> #include <linux/module.h> +#include <linux/kernel_stat.h> enum rcu_barrier {RCU_BARRIER_STD ,
@@ -55,6 +56,7 @@ static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};static atomic_t rcu_barrier_cpu_count;static DEFINE_MUTEX (rcu_barrier_mutex);static struct completion rcu_barrier_completion; +int rcu_scheduler_active__read_mostly ;/* * Awaken the corresponding synchronize_rcu() instance now that a
@@ -80,6 +82,10 @@ void wakeme_after_rcu(struct rcu_head *head)void synchronize_rcu (void ) {struct rcu_synchronize rcu; + +if (rcu_blocking_is_gp ()) +return ; +init_completion (&rcu.completion);/* Will wake me after RCU finished. */ call_rcu (&rcu.head, wakeme_after_rcu);
@@ -175,3 +181,9 @@ void __init rcu_init(void)__rcu_init (); } +void rcu_scheduler_starting (void ) +{ +WARN_ON (num_online_cpus () !=1 ); +WARN_ON (nr_context_switches () >0 ); + rcu_scheduler_active =1 ; +}
kernel/rcupreempt.c +3 -0
@@ -1181,6 +1181,9 @@ void __synchronize_sched(void) {struct rcu_synchronize rcu; +if (num_online_cpus () ==1 ) +return ;/* blocking is gp if only one CPU! */ +init_completion (&rcu.completion);/* Will wake me after RCU finished. */ call_rcu_sched (&rcu.head, wakeme_after_rcu);
kernel/rcutree.c +2 -2
@@ -948,8 +948,8 @@ static void rcu_do_batch(struct rcu_data *rdp)void rcu_check_callbacks (int cpu,int user) {if (user || - (idle_cpu (cpu) && !in_softirq () && -hardirq_count () <= (1 <<HARDIRQ_SHIFT ))) { + (idle_cpu (cpu) && rcu_scheduler_active && + !in_softirq () &&hardirq_count () <= (1 <<HARDIRQ_SHIFT ))) {/* * Get here if this CPU took its interrupt from user