rcu: fix grace-period-stall bug on large systems with CPU hotplug
PATCH 1/2 rcu: fix grace-period-stall bug on large systems with CPU hotplug
When the last CPU of a given leaf rcu_node structure goes offline, all of the tasks queued on that leaf rcu_node structure (due to having blocked in their current RCU read-side critical sections) are requeued onto the root rcu_node structure. This requeuing is carried out by rcu_preempt_offline_tasks(). However, it is possible that these queued tasks are the only thing preventing the leaf rcu_node structure from reporting a quiescent state up the rcu_node hierarchy. Unfortunately, the old code would fail to do this reporting, resulting in a grace-period stall given the following sequence of events:
1. Kernel built for more than 32 CPUs on 32-bit systems or for more
than 64 CPUs on 64-bit systems, so that there is more than one rcu_node structure. (Or CONFIG_RCU_FANOUT is artificially set to a number smaller than CONFIG_NR_CPUS.)
2. The kernel is built with CONFIG_TREE_PREEMPT_RCU.
3. A task running on a CPU associated with a given leaf rcu_node
structure blocks while in an RCU read-side critical section -and- that CPU has not yet passed through a quiescent state for the current RCU grace period. This will cause the task to be queued on the leaf rcu_node's blocked_tasks[] array, in particular, on the element of this array corresponding to the current grace period.
4. Each of the remaining CPUs corresponding to this same leaf rcu_node
structure pass through a quiescent state. However, the task is still in its RCU read-side critical section, so these quiescent states cannot be reported further up the rcu_node hierarchy. Nevertheless, all bits in the leaf rcu_node structure's ->qsmask field are now zero.
5. Each of the remaining CPUs go offline. (The events in step
#4 and #5 can happen in any order as long as each CPU passes through a quiescent state before going offline.)
6. When the last CPU goes offline, __rcu_offline_cpu() will invoke
rcu_preempt_offline_tasks(), which will move the task to the root rcu_node structure, but without reporting a quiescent state up the rcu_node hierarchy (and this failure to report a quiescent state is the bug).
But because this leaf rcu_node structure's ->qsmask field is already zero and its ->block_tasks[] entries are all empty, force_quiescent_state() will skip this rcu_node structure.
Therefore, grace periods are now hung.
This patch abstracts some code out of rcu_read_unlock_special(), calling the result task_quiet() by analogy with cpu_quiet(), and invokes task_quiet() from both rcu_read_lock_special() and __rcu_offline_cpu(). Invoking task_quiet() from __rcu_offline_cpu() reports the quiescent state up the rcu_node hierarchy, fixing the bug. This ends up requiring a separate lock_class_key per level of the rcu_node hierarchy, which this patch also provides.
--- kernel/rcutree.c | 40 +++++++++++---------- kernel/rcutree.h | 3 ++ kernel/rcutree_plugin.h | 85 +++++++++++++++++++++++++++++++++------------- 3 files changed, 85 insertions(+), 43 deletions(-)
kernel/rcutree.c +21 -19
@@ -51,7 +51,7 @@/* Data structures. */ -static struct lock_class_key rcu_root_class; +static struct lock_class_key rcu_node_class[NUM_RCU_LVLS ];#define RCU_STATE_INITIALIZER (name) { \ .level = { &name.node[0] }, \
@@ -936,6 +936,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) {unsigned long flags;unsigned long mask; +int need_quiet =0 ;struct rcu_data *rdp = rsp->rda[cpu];struct rcu_node *rnp;
@@ -949,29 +950,30 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)spin_lock (&rnp->lock);/* irqs already disabled. */ rnp->qsmaskinit &= ~mask;if (rnp->qsmaskinit !=0 ) { -spin_unlock (&rnp->lock);/* irqs remain disabled. */ +if (rnp != rdp->mynode) +spin_unlock (&rnp->lock);/* irqs remain disabled. */ break ; } - -/* -* If there was a task blocking the current grace period, -* and if all CPUs have checked in, we need to propagate -* the quiescent state up the rcu_node hierarchy. But that -* is inconvenient at the moment due to deadlock issues if -* this should end the current grace period. So set the -* offlined CPU's bit in ->qsmask in order to force the -* next force_quiescent_state() invocation to clean up this -* mess in a deadlock-free manner. -*/ -if (rcu_preempt_offline_tasks (rsp, rnp, rdp) && !rnp->qsmask) - rnp->qsmask |= mask; - +if (rnp == rdp->mynode) + need_quiet =rcu_preempt_offline_tasks (rsp, rnp, rdp); +else +spin_unlock (&rnp->lock);/* irqs remain disabled. */ mask = rnp->grpmask; -spin_unlock (&rnp->lock);/* irqs remain disabled. */ rnp = rnp->parent; }while (rnp !=NULL ); -spin_unlock_irqrestore (&rsp->onofflock, flags); +/* +* We still hold the leaf rcu_node structure lock here, and +* irqs are still disabled. The reason for this subterfuge is +* because invoking task_quiet() with ->onofflock held leads +* to deadlock. +*/ +spin_unlock (&rsp->onofflock);/* irqs remain disabled. */ + rnp = rdp->mynode; +if (need_quiet) +task_quiet (rnp, flags); +else +spin_unlock_irqrestore (&rnp->lock, flags);rcu_adopt_orphan_cbs (rsp); }
@@ -1731,6 +1733,7 @@ static void __init rcu_init_one(struct rcu_state *rsp) rnp = rsp->level[i];for (j =0 ; j < rsp->levelcnt[i]; j++, rnp++) {spin_lock_init (&rnp->lock); +lockdep_set_class (&rnp->lock, &rcu_node_class[i]); rnp->gpnum =0 ; rnp->qsmask =0 ; rnp->qsmaskinit =0 ;
@@ -1753,7 +1756,6 @@ static void __init rcu_init_one(struct rcu_state *rsp)INIT_LIST_HEAD (&rnp->blocked_tasks[1 ]); } } -lockdep_set_class (&rcu_get_root (rsp)->lock, &rcu_root_class); }/*
kernel/rcutree.h +3 -0
@@ -305,6 +305,9 @@ static void rcu_bootup_announce(void);long rcu_batches_completed (void );static void rcu_preempt_note_context_switch (int cpu);static int rcu_preempted_readers (struct rcu_node *rnp); +#ifdef CONFIG_HOTPLUG_CPU +static void task_quiet (struct rcu_node *rnp,unsigned long flags); +#endif /* #ifdef CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_RCU_CPU_STALL_DETECTOR static void rcu_print_task_stall (struct rcu_node *rnp);#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
kernel/rcutree_plugin.h +61 -24
134 lines not on this page · raw message
Signature · 1 line
1.5.2.5
PATCH 2/2 rcu: eliminate unneeded function wrapping
From: Paul E. McKenney <[email protected]>
The functions rcu_init() is a wrapper for __rcu_init(), and also sets up the CPU-hotplug notifier for rcu_barrier_cpu_hotplug(). But TINY_RCU doesn't need CPU-hotplug notification, and the rcu_barrier_cpu_hotplug() is a simple wrapper for rcu_cpu_notify(). So push rcu_init() out to kernel/rcutree.c and kernel/rcutiny.c and get rid of the wrapper function rcu_barrier_cpu_hotplug().
--- include/linux/rcutiny.h | 2 -- include/linux/rcutree.h | 3 --- kernel/rcupdate.c | 22 ---------------------- kernel/rcutiny.c | 11 +---------- kernel/rcutree.c | 17 ++++++++++++++--- 5 files changed, 15 insertions(+), 40 deletions(-)
include/linux/rcutiny.h +0 -2
@@ -38,7 +38,6 @@ void rcu_bh_qs(int cpu);#define rcu_init_sched () do { } while (0)extern void rcu_check_callbacks (int cpu,int user); -extern void __rcu_init (void );/* * Return the number of grace periods.
@@ -69,7 +68,6 @@ static inline void synchronize_rcu_bh_expedited(void) }struct notifier_block ; -extern int rcu_cpu_notify (struct notifier_block *self,unsigned long action,void *hcpu);#ifdef CONFIG_NO_HZ
include/linux/rcutree.h +0 -3
@@ -34,8 +34,6 @@ struct notifier_block;extern void rcu_sched_qs (int cpu);extern void rcu_bh_qs (int cpu); -extern int rcu_cpu_notify (struct notifier_block *self, -unsigned long action,void *hcpu);extern int rcu_needs_cpu (int cpu);extern int rcu_expedited_torture_stats (char *page);
@@ -83,7 +81,6 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched_expedited (); } -extern void __rcu_init (void );extern void rcu_check_callbacks (int cpu,int user);extern long rcu_batches_completed (void );
kernel/rcupdate.c +0 -22
@@ -161,28 +161,6 @@ EXPORT_SYMBOL_GPL(synchronize_rcu_bh);#endif /* #ifndef CONFIG_TINY_RCU */ -static int __cpuinit rcu_barrier_cpu_hotplug (struct notifier_block *self, -unsigned long action,void *hcpu) -{ -return rcu_cpu_notify (self, action, hcpu); -} - -void __init rcu_init (void ) -{ -int i; - -__rcu_init (); -cpu_notifier (rcu_barrier_cpu_hotplug,0 ); - -/* -* We don't need protection against CPU-hotplug here because -* this is called early in boot, before either interrupts -* or the scheduler are operational. -*/ -for_each_online_cpu (i) -rcu_barrier_cpu_hotplug (NULL ,CPU_UP_PREPARE , (void *)(long )i); -} -void rcu_scheduler_starting (void ) {WARN_ON (num_online_cpus () !=1 );
kernel/rcutiny.c +1 -10
@@ -178,15 +178,6 @@ static void rcu_process_callbacks(struct softirq_action *unused) }/* -* Null function to handle CPU being onlined. Longer term, we want to -* make TINY_RCU avoid using rcupdate.c, but later... -*/ -int rcu_cpu_notify (struct notifier_block *self,unsigned long action,void *hcpu) -{ -return NOTIFY_OK ; -} - -/* * Wait for a grace period to elapse. But it is illegal to invoke * synchronize_sched() from within an RCU read-side critical section. * Therefore, any legal call to synchronize_sched() is a quiescent
@@ -285,7 +276,7 @@ void rcu_barrier_sched(void) }EXPORT_SYMBOL_GPL (rcu_barrier_sched); -void __rcu_init (void ) +void __init rcu_init (void ) {open_softirq (RCU_SOFTIRQ , rcu_process_callbacks); }
kernel/rcutree.c +14 -3
@@ -1644,8 +1644,8 @@ static void __cpuinit rcu_online_cpu(int cpu)/* * Handle CPU online/offline notification events. */ -int __cpuinit rcu_cpu_notify (struct notifier_block *self, -unsigned long action,void *hcpu) +static int __cpuinit rcu_cpu_notify (struct notifier_block *self, +unsigned long action,void *hcpu) {long cpu = (long )hcpu;
@@ -1781,8 +1781,10 @@ do { \ } \ }while (0 ) -void __init __rcu_init(void) +void __init rcu_init(void) { +int i; +rcu_bootup_announce ();#ifdef CONFIG_RCU_CPU_STALL_DETECTOR printk (KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n" );
@@ -1791,6 +1793,15 @@ void __init __rcu_init(void)RCU_INIT_FLAVOR (&rcu_bh_state, rcu_bh_data);__rcu_init_preempt ();open_softirq (RCU_SOFTIRQ , rcu_process_callbacks); + +/* +* We don't need protection against CPU-hotplug here because +* this is called early in boot, before either interrupts +* or the scheduler are operational. +*/ +cpu_notifier (rcu_cpu_notify,0 ); +for_each_online_cpu (i) +rcu_cpu_notify (NULL ,CPU_UP_PREPARE , (void *)(long )i); }#include "rcutree_plugin.h"
Signature · 1 line
1.5.2.5