Expedited grace-period updates
cover Expedited grace-period updates
Hello!
This series contains updates and fixes to expedited grace-period handling:
1. Consolidate nearly identical code from synchronize_rcu_expedited()
and synchronize_sched_expedited().
2. Drive expedited grace periods from workqueue rather than user task.
3. Stop disabling expedited RCU CPU stall warnings, enabled by
#2 above.
4. Make expedited RCU CPU stall warnings respond
to the panic_on_rcu_stall sysctl setting and the rcupdate.rcu_cpu_stall_suppress kernel boot parameter.
5. Exclude RCU-offline CPUs from expedited grace periods based on
RCU's internal state.
6. Use RCU's online-CPU state for expedited-grace-period IPI retry.
Thanx, Paul
------------------------------------------------------------------------
tree.h | 1 tree_exp.h | 134 ++++++++++++++++++++++++++++++++++++----------------------- tree_trace.c | 7 +-- 3 files changed, 87 insertions(+), 55 deletions(-)
PATCH 1/6 rcu: Consolidate expedited grace period machinery
The functions synchronize_rcu_expedited() and synchronize_sched_expedite() have nearly identical code. This commit therefore consolidates this code into a new _synchronize_rcu_expedited() function.
--- kernel/rcu/tree_exp.h | 62 ++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 33 deletions(-)
kernel/rcu/tree_exp.h +29 -33
@@ -516,6 +516,33 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)mutex_unlock (&rsp->exp_wake_mutex); } +/* +* Given an rcu_state pointer and a smp_call_function() handler, kick +* off the specified flavor of expedited grace period. +*/ +static void _synchronize_rcu_expedited (struct rcu_state *rsp, +smp_call_func_t func) +{ +unsigned long s; + +/* If expedited grace periods are prohibited, fall back to normal. */ +if (rcu_gp_is_normal ()) { +wait_rcu_gp (rsp->call); +return ; + } + +/* Take a snapshot of the sequence number. */ + s =rcu_exp_gp_seq_snap (rsp); +if (exp_funnel_lock (rsp, s)) +return ;/* Someone else did our work for us. */ + +/* Initialize the rcu_node tree in preparation for the wait. */ +sync_rcu_exp_select_cpus (rsp, func); + +/* Wait and clean up, including waking everyone. */ +rcu_exp_wait_wake (rsp, s); +} +/** * synchronize_sched_expedited - Brute-force RCU-sched grace period *
@@ -534,29 +561,13 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)*/ void synchronize_sched_expedited (void ) { -unsigned long s;struct rcu_state *rsp = &rcu_sched_state;/* If only one CPU, this is automatically a grace period. */ if (rcu_blocking_is_gp ())return ; -/* If expedited grace periods are prohibited, fall back to normal. */ -if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu_sched); -return ; - } - -/* Take a snapshot of the sequence number. */ - s =rcu_exp_gp_seq_snap (rsp); -if (exp_funnel_lock (rsp, s)) -return ;/* Someone else did our work for us. */ - -/* Initialize the rcu_node tree in preparation for the wait. */ -sync_rcu_exp_select_cpus (rsp, sync_sched_exp_handler); - -/* Wait and clean up, including waking everyone. */ -rcu_exp_wait_wake (rsp, s); +_synchronize_rcu_expedited (rsp, sync_sched_exp_handler); }EXPORT_SYMBOL_GPL (synchronize_sched_expedited);
@@ -620,23 +631,8 @@ static void sync_rcu_exp_handler(void *info)void synchronize_rcu_expedited (void ) {struct rcu_state *rsp = rcu_state_p; -unsigned long s; -/* If expedited grace periods are prohibited, fall back to normal. */ -if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); -return ; - } - - s =rcu_exp_gp_seq_snap (rsp); -if (exp_funnel_lock (rsp, s)) -return ;/* Someone else did our work for us. */ - -/* Initialize the rcu_node tree in preparation for the wait. */ -sync_rcu_exp_select_cpus (rsp, sync_rcu_exp_handler); - -/* Wait for ->blkd_tasks lists to drain, then wake everyone up. */ -rcu_exp_wait_wake (rsp, s); +_synchronize_rcu_expedited (rsp, sync_rcu_exp_handler); }EXPORT_SYMBOL_GPL (synchronize_rcu_expedited);
Signature · 1 line
2.5.2
PATCH 2/6 rcu: Drive expedited grace periods from workqueue
In reply to Paul E. McKenneyThe current implementation of expedited grace periods has the user task drive the grace period. This works, but has downsides: (1) The user task must awaken tasks piggybacking on this grace period, which can result in latencies rivaling that of the grace period itself, and
(2) User tasks can receive signals, which interfere with RCU CPU stall warnings.
This commit therefore uses workqueues to drive the grace periods, so that the user task need not do the awakening. A subsequent commit will remove the now-unnecessary code allowing for signals.
--- kernel/rcu/tree.h | 1 + kernel/rcu/tree_exp.h | 48 ++++++++++++++++++++++++++++++++++++++++++------ kernel/rcu/tree_trace.c | 7 ++++--- 3 files changed, 47 insertions(+), 9 deletions(-)
kernel/rcu/tree.h +1 -0
@@ -400,6 +400,7 @@ struct rcu_data {#ifdef CONFIG_RCU_FAST_NO_HZ struct rcu_head oom_head;#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ +atomic_long_t exp_workdone0;/* # done by workqueue. */ atomic_long_t exp_workdone1;/* # done by others #1. */ atomic_long_t exp_workdone2;/* # done by others #2. */ atomic_long_t exp_workdone3;/* # done by others #3. */
kernel/rcu/tree_exp.h +42 -6
72 lines not on this page · raw message
kernel/rcu/tree_trace.c +4 -3
@@ -185,16 +185,17 @@ static int show_rcuexp(struct seq_file *m, void *v)int cpu;struct rcu_state *rsp = (struct rcu_state *)m->private;struct rcu_data *rdp; -unsigned long s1 =0 , s2 =0 , s3 =0 ; +unsigned long s0 =0 , s1 =0 , s2 =0 , s3 =0 ;for_each_possible_cpu (cpu) { rdp =per_cpu_ptr (rsp->rda, cpu); + s0 +=atomic_long_read (&rdp->exp_workdone0); s1 +=atomic_long_read (&rdp->exp_workdone1); s2 +=atomic_long_read (&rdp->exp_workdone2); s3 +=atomic_long_read (&rdp->exp_workdone3); } -seq_printf (m,"s=%lu wd1=%lu wd2=%lu wd3=%lu n=%lu enq=%d sc=%lu\n" , - rsp->expedited_sequence, s1, s2, s3, +seq_printf (m,"s=%lu wd0=%lu wd1=%lu wd2=%lu wd3=%lu n=%lu enq=%d sc=%lu\n" , + rsp->expedited_sequence, s0, s1, s2, s3,atomic_long_read (&rsp->expedited_normal),atomic_read (&rsp->expedited_need_qs), rsp->expedited_sequence /2 );
Signature · 1 line
2.5.2
PATCH 3/6 rcu: Stop disabling expedited RCU CPU stall warnings
In reply to Paul E. McKenneyNow that RCU expedited grace periods are always driven by a workqueue, there is no need to account for signal reception, and thus no need to disable expedited RCU CPU stall warnings due to signal reception. This commit therefore removes the signal-reception checks, leaving a WARN_ON() to catch possible future bugs.
--- kernel/rcu/tree_exp.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-)
kernel/rcu/tree_exp.h +1 -6
@@ -427,12 +427,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) jiffies_stall);if (ret >0 ||sync_rcu_preempt_exp_done (rnp_root))return ; -if (ret <0 ) { -/* Hit a signal, disable CPU stall warnings. */ -swait_event (rsp->expedited_wq, -sync_rcu_preempt_exp_done (rnp_root)); -return ; - } +WARN_ON (ret <0 );/* workqueues should not be signaled. */ pr_err ("INFO: %s detected expedited stalls on CPUs/tasks: {" , rsp->name); ndetected =0 ;
Signature · 1 line
2.5.2
PATCH 4/6 rcu: Make expedited RCU CPU stall warnings respond to controls
In reply to Paul E. McKenneyThe expedited RCU CPU stall warnings currently responds to neither the panic_on_rcu_stall sysctl setting nor the rcupdate.rcu_cpu_stall_suppress kernel boot parameter. This commit therefore updates teh expedited code to respond to these controls.
--- kernel/rcu/tree_exp.h | 3 +++ 1 file changed, 3 insertions(+)
kernel/rcu/tree_exp.h +3 -0
@@ -428,6 +428,9 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp)if (ret >0 ||sync_rcu_preempt_exp_done (rnp_root))return ;WARN_ON (ret <0 );/* workqueues should not be signaled. */ +if (rcu_cpu_stall_suppress) +continue ; +panic_on_rcu_stall ();pr_err ("INFO: %s detected expedited stalls on CPUs/tasks: {" , rsp->name); ndetected =0 ;
Signature · 1 line
2.5.2
PATCH 5/6 rcu: Exclude RCU-offline CPUs from expedited grace periods
In reply to Paul E. McKenneyThe expedited RCU grace periods currently rely on a failure indication from smp_call_function_single() to determine that a given CPU is offline. This works after a fashion, but is more contorted and less precise than relying on RCU's internal state. This commit therefore takes a first step towards relying on internal state.
--- kernel/rcu/tree_exp.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/rcu/tree_exp.h +2 -1
@@ -359,7 +359,8 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);if (raw_smp_processor_id () == cpu || - !(atomic_add_return (0 , &rdtp->dynticks) &0x1 )) + !(atomic_add_return (0 , &rdtp->dynticks) &0x1 ) || + !(rnp->qsmaskinitnext & rdp->grpmask)) mask_ofl_test |= rdp->grpmask; } mask_ofl_ipi = rnp->expmask & ~mask_ofl_test;
Signature · 1 line
2.5.2
PATCH 6/6 rcu: Use RCU's online-CPU state for expedited IPI retry
In reply to Paul E. McKenneyThis commit improves the accuracy of the interaction between CPU hotplug operations and RCU's expedited grace periods by using RCU's online-CPU state to determine when failed IPIs should be retried.
--- kernel/rcu/tree_exp.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)
kernel/rcu/tree_exp.h +5 -6
@@ -385,17 +385,16 @@ retry_ipi: mask_ofl_ipi &= ~mask;continue ; } -/* Failed, raced with offline. */ +/* Failed, raced with CPU hotplug operation. */ raw_spin_lock_irqsave_rcu_node (rnp, flags); -if (cpu_online (cpu) && +if ((rnp->qsmaskinitnext & mask) && (rnp->expmask & mask)) { +/* Online, so delay for a bit and try again. */ raw_spin_unlock_irqrestore_rcu_node (rnp, flags);schedule_timeout_uninterruptible (1 ); -if (cpu_online (cpu) && - (rnp->expmask & mask)) - goto retry_ipi; -raw_spin_lock_irqsave_rcu_node (rnp, flags); + goto retry_ipi; } +/* CPU really is offline, so we can ignore it. */ if (!(rnp->expmask & mask)) mask_ofl_ipi &= ~mask;raw_spin_unlock_irqrestore_rcu_node (rnp, flags);
Signature · 1 line
2.5.2