Enable tick for nohz_full CPUs not responding to expedited GP
Subsystems (2)
Paul E. McKenney
maintainerPATCH 1/1 Enable tick for nohz_full CPUs not responding to expedited GP
rcu: Enable tick for nohz_full CPUs slow to provide expedited QS
An expedited grace period can be stalled by a nohz_full CPU looping in kernel context. This possibility is currently handled by some carefully crafted checks in rcu_read_unlock_special() that enlist help from ksoftirqd when permitted by the scheduler. However, it is exactly these checks that require the scheduler avoid holding any of its rq or pi locks across rcu_read_unlock() without also having held them across the entire RCU read-side critical section.
It would therefore be very nice if expedited grace periods could handle nohz_full CPUs looping in kernel context without such checks. This commit therefore adds code to the expedited grace period's wait and cleanup code that forces the scheduler-clock interrupt on for CPUs that fail to quickly supply a quiescent state. "Quickly" is currently a hard-coded single-jiffy delay.
---
This is needed to avoid rcu_read_unlock_special() needing to enter the scheduler for the benefit of expedited grace periods on nohz_full CPUs, thus enabling more of Lai Jiangshan's patchset.
include/linux/tick.h +4 -2
@@ -109,9 +109,10 @@ enum tick_dep_bits {TICK_DEP_BIT_PERF_EVENTS =1 ,TICK_DEP_BIT_SCHED =2 ,TICK_DEP_BIT_CLOCK_UNSTABLE =3 , - TICK_DEP_BIT_RCU = 4 +TICK_DEP_BIT_RCU =4 , + TICK_DEP_BIT_RCU_EXP = 5 }; -#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU +#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU_EXP#define TICK_DEP_MASK_NONE 0#define TICK_DEP_MASK_POSIX_TIMER (1 << TICK_DEP_BIT_POSIX_TIMER)
@@ -119,6 +120,7 @@ enum tick_dep_bits {#define TICK_DEP_MASK_SCHED (1 << TICK_DEP_BIT_SCHED)#define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)#define TICK_DEP_MASK_RCU (1 << TICK_DEP_BIT_RCU) +#define TICK_DEP_MASK_RCU_EXP (1 << TICK_DEP_BIT_RCU_EXP)#ifdef CONFIG_NO_HZ_COMMON extern bool tick_nohz_enabled;
kernel/rcu/tree.h +1 -0
@@ -181,6 +181,7 @@ struct rcu_data {bool rcu_need_heavy_qs;/* GP old, so heavy quiescent state! */ bool rcu_urgent_qs;/* GP old need light quiescent state. */ bool rcu_forced_tick;/* Forced tick to provide QS. */ +bool rcu_forced_tick_exp;/* ... provide QS to expedited GP. */ #ifdef CONFIG_RCU_FAST_NO_HZ unsigned long last_accelerate;/* Last jiffy CBs were accelerated. */ unsigned long last_advance_all;/* Last jiffy CBs were all advanced. */
kernel/rcu/tree_exp.h +45 -7
86 lines not on this page · raw message
Quoted 9 lines · Paul E. McKenney
rcu: Enable tick for
nohz_fullCPUs slow to provide expedited QSAn expedited grace period can be stalled by a
nohz_fullCPU looping in kernel context. This possibility is currently handled by some carefully crafted checks inrcu_read_unlock_special()that enlist help from ksoftirqd when permitted by the scheduler. However, it is exactly these checks that require the scheduler avoid holding any of its rq or pi locks acrossrcu_read_unlock()without also having held them across the entire RCU read-side critical section.
Just got a bit more free to look into the topic of RCU-related scheduler deadlocks I was previously looking into (And I will continue working on the WIP patch for detecting bad scenarios).
Quoted 2 lines
It would therefore be very nice if expedited grace periods could handle
nohz_fullCPUs looping in kernel context without such checks.
Could you clarify 'without such checks'? Are you referring to these checks in _unlock_special()?
if (irqs_were_disabled && use_softirq &&
(in_interrupt() ||
(exp && !t->rcu_read_unlock_special.b.deferred_qs))) {
// Using softirq, safe to awaken, and we get
// no help from enabling irqs, unlike bh/preempt.
raise_softirq_irqoff(RCU_SOFTIRQ);I am sorry but I did not fully understand this patch motivation but -- you are just saying you want EXP GPs to be handled quickly on nohz_full CPUs, right? Are we not turning the tick on already in rcu_nmi_enter_common() for handling slow EXP GPs as well? And if not, can we do it from there instead?
Quoted 5 lines
This commit therefore adds code to the expedited grace period's wait and cleanup code that forces the scheduler-clock interrupt on for CPUs that fail to quickly supply a quiescent state. "Quickly" is currently a hard-coded single-jiffy delay.
Quoted 7 lines
Signed-off-by Paul E. McKenney <[email protected]>---
This is needed to avoid
rcu_read_unlock_special()needing to enter the scheduler for the benefit of expedited grace periods onnohz_fullCPUs, thus enabling more of Lai Jiangshan's patchset.
Still need to go through Lai's patches :-(. Will be looking to make more time for the same.
thanks!
- Joel
Quoted 131 lines
include/linux/tick.h +4
-2@@ -109,9 +109,10 @@ enum tick_dep_bits {TICK_DEP_BIT_PERF_EVENTS =1 ,TICK_DEP_BIT_SCHED =2 ,TICK_DEP_BIT_CLOCK_UNSTABLE =3 , - TICK_DEP_BIT_RCU = 4 +TICK_DEP_BIT_RCU =4 , + TICK_DEP_BIT_RCU_EXP = 5 }; -#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU +#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU_EXP#define TICK_DEP_MASK_NONE 0#define TICK_DEP_MASK_POSIX_TIMER (1 << TICK_DEP_BIT_POSIX_TIMER)@@ -119,6 +120,7 @@ enum tick_dep_bits {#define TICK_DEP_MASK_SCHED (1 << TICK_DEP_BIT_SCHED)#define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)#define TICK_DEP_MASK_RCU (1 << TICK_DEP_BIT_RCU) +#define TICK_DEP_MASK_RCU_EXP (1 << TICK_DEP_BIT_RCU_EXP)#ifdef CONFIG_NO_HZ_COMMON extern bool tick_nohz_enabled;kernel/rcu/tree.h +1
-0@@ -181,6 +181,7 @@ struct rcu_data {bool rcu_need_heavy_qs;/* GP old, so heavy quiescent state! */ bool rcu_urgent_qs;/* GP old need light quiescent state. */ bool rcu_forced_tick;/* Forced tick to provide QS. */ +bool rcu_forced_tick_exp;/* ... provide QS to expedited GP. */ #ifdef CONFIG_RCU_FAST_NO_HZ unsigned long last_accelerate;/* Last jiffy CBs were accelerated. */ unsigned long last_advance_all;/* Last jiffy CBs were all advanced. */ kernel/rcu/tree_exp.h +45
-7@@ -230,7 +230,9 @@ static void __maybe_unused rcu_report_exp_rnp(struct rcu_node *rnp, bool wake)static void rcu_report_exp_cpu_mult (struct rcu_node *rnp,unsigned long mask,bool wake) { +int cpu;unsigned long flags; +struct rcu_data *rdp;raw_spin_lock_irqsave_rcu_node (rnp, flags);if (!(rnp->expmask & mask)) {@@ -238,6 +240,13 @@ static void rcu_report_exp_cpu_mult(struct rcu_node *rnp,return ; }WRITE_ONCE (rnp->expmask, rnp->expmask & ~mask); +for_each_leaf_node_cpu_mask (rnp, cpu, mask) { + rdp =per_cpu_ptr (&rcu_data, cpu); +if (!IS_ENABLED (CONFIG_NO_HZ_FULL ) || !rdp->rcu_forced_tick_exp) +continue ; + rdp->rcu_forced_tick_exp = false; +tick_dep_clear_cpu (cpu,TICK_DEP_BIT_RCU_EXP ); + }__rcu_report_exp_rnp (rnp, wake, flags);/* Releases rnp->lock. */ }@@ -450,6 +459,26 @@ static void sync_rcu_exp_select_cpus(void) }/* +* Wait for the expedited grace period to elapse, within time limit. +* If the time limit is exceeded without the grace period elapsing, +* return false, otherwise return true. +*/ +static bool synchronize_rcu_expedited_wait_once (long tlimit) +{ +int t; +struct rcu_node *rnp_root =rcu_get_root (); + + t =swait_event_timeout_exclusive (rcu_state.expedited_wq, +sync_rcu_exp_done_unlocked (rnp_root), + tlimit); +// Workqueues should not be signaled. +if (t >0 ||sync_rcu_exp_done_unlocked (rnp_root)) +return true; +WARN_ON (t <0 );/* workqueues should not be signaled. */ +return false; +} + +/* * Wait for the expedited grace period to elapse, issuing any needed * RCU CPU stall warnings along the way. */ @@ -460,22 +489,31 @@ static void synchronize_rcu_expedited_wait(void)unsigned long jiffies_start;unsigned long mask;int ndetected; +struct rcu_data *rdp;struct rcu_node *rnp;struct rcu_node *rnp_root =rcu_get_root (); -int ret;trace_rcu_exp_grace_period (rcu_state.name,rcu_exp_gp_seq_endval (),TPS ("startwait" )); jiffies_stall =rcu_jiffies_till_stall_check (); jiffies_start = jiffies; +if (IS_ENABLED (CONFIG_NO_HZ_FULL )) { +if (synchronize_rcu_expedited_wait_once (1 )) +return ; +rcu_for_each_leaf_node (rnp) { +for_each_leaf_node_cpu_mask (rnp, cpu, rnp->expmask) { + rdp =per_cpu_ptr (&rcu_data, cpu); +if (rdp->rcu_forced_tick_exp) +continue ; + rdp->rcu_forced_tick_exp = true; +tick_dep_set_cpu (cpu,TICK_DEP_BIT_RCU_EXP ); + } + } +WARN_ON_ONCE (1 ); + }for (;;) { - ret =swait_event_timeout_exclusive ( - rcu_state.expedited_wq, -sync_rcu_exp_done_unlocked (rnp_root), - jiffies_stall); -if (ret >0 ||sync_rcu_exp_done_unlocked (rnp_root)) +if (synchronize_rcu_expedited_wait_once (jiffies_stall))return ; -WARN_ON (ret <0 );/* workqueues should not be signaled. */ if (rcu_cpu_stall_suppress)continue ;panic_on_rcu_stall ();
Paul E. McKenney
maintainerQuoted 27 lines · Joel Fernandes
Quoted 9 lines · Paul E. McKenney
rcu: Enable tick for
nohz_fullCPUs slow to provide expedited QSAn expedited grace period can be stalled by a
nohz_fullCPU looping in kernel context. This possibility is currently handled by some carefully crafted checks inrcu_read_unlock_special()that enlist help from ksoftirqd when permitted by the scheduler. However, it is exactly these checks that require the scheduler avoid holding any of its rq or pi locks acrossrcu_read_unlock()without also having held them across the entire RCU read-side critical section.Just got a bit more free to look into the topic of RCU-related scheduler deadlocks I was previously looking into (And I will continue working on the WIP patch for detecting bad scenarios).
Quoted 2 lines
It would therefore be very nice if expedited grace periods could handle
nohz_fullCPUs looping in kernel context without such checks.Could you clarify 'without such checks'? Are you referring to these checks in
_unlock_special()?if (irqs_were_disabled && use_softirq && (in_interrupt() || (exp && !t->rcu_read_unlock_special.b.deferred_qs))) { // Using softirq, safe to awaken, and we get // no help from enabling irqs, unlike bh/preempt. raise_softirq_irqoff(RCU_SOFTIRQ);
Quoted 4 lines
I am sorry but I did not fully understand this patch motivation but -- you are just saying you want EXP GPs to be handled quickly on
nohz_fullCPUs, right? Are we not turning the tick on already inrcu_nmi_enter_common()for handling slow EXP GPs as well? And if not, can we do it from there instead?
Currently, if the scheduler holds one of its rq or pi locks across rcu_read_unlock(), it must do so across the entire RCU read-side critical section. Although this odd requirement isn't a flaming disaster, it would be good to lift if feasible. According to Lai's analysis (which seems correct to me), the only remaining need for this requirement is the raise_softirq_irqoff() you call out above. If expedited grace periods can turn on the tick for holdout nohz_full CPUs, it seems to me that the above check can be changed so as to avoid invoking raise_softirq_irqoff() except in cases where it doesn't go into the scheduler. This could be done by removing the portion of the "if" condition following the "||".
Quoted 7 lines
Quoted 7 lines
This commit therefore adds code to the expedited grace period's wait and cleanup code that forces the scheduler-clock interrupt on for CPUs that fail to quickly supply a quiescent state. "Quickly" is currently a hard-coded single-jiffy delay.
Signed-off-by Paul E. McKenney <[email protected]>
Quoted 8 lines
Quoted 5 lines
---
This is needed to avoid
rcu_read_unlock_special()needing to enter the scheduler for the benefit of expedited grace periods onnohz_fullCPUs, thus enabling more of Lai Jiangshan's patchset.Still need to go through Lai's patches :-(. Will be looking to make more time for the same.
It would indeed be good to get another set of eyes on this, as it is a bit on the subtle side!
Thanx, Paul
Quoted 136 lines
thanks!
- Joel
Quoted 131 lines
include/linux/tick.h +4
-2@@ -109,9 +109,10 @@ enum tick_dep_bits {TICK_DEP_BIT_PERF_EVENTS =1 ,TICK_DEP_BIT_SCHED =2 ,TICK_DEP_BIT_CLOCK_UNSTABLE =3 , - TICK_DEP_BIT_RCU = 4 +TICK_DEP_BIT_RCU =4 , + TICK_DEP_BIT_RCU_EXP = 5 }; -#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU +#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU_EXP#define TICK_DEP_MASK_NONE 0#define TICK_DEP_MASK_POSIX_TIMER (1 << TICK_DEP_BIT_POSIX_TIMER)@@ -119,6 +120,7 @@ enum tick_dep_bits {#define TICK_DEP_MASK_SCHED (1 << TICK_DEP_BIT_SCHED)#define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)#define TICK_DEP_MASK_RCU (1 << TICK_DEP_BIT_RCU) +#define TICK_DEP_MASK_RCU_EXP (1 << TICK_DEP_BIT_RCU_EXP)#ifdef CONFIG_NO_HZ_COMMON extern bool tick_nohz_enabled;kernel/rcu/tree.h +1
-0@@ -181,6 +181,7 @@ struct rcu_data {bool rcu_need_heavy_qs;/* GP old, so heavy quiescent state! */ bool rcu_urgent_qs;/* GP old need light quiescent state. */ bool rcu_forced_tick;/* Forced tick to provide QS. */ +bool rcu_forced_tick_exp;/* ... provide QS to expedited GP. */ #ifdef CONFIG_RCU_FAST_NO_HZ unsigned long last_accelerate;/* Last jiffy CBs were accelerated. */ unsigned long last_advance_all;/* Last jiffy CBs were all advanced. */ kernel/rcu/tree_exp.h +45
-7@@ -230,7 +230,9 @@ static void __maybe_unused rcu_report_exp_rnp(struct rcu_node *rnp, bool wake)static void rcu_report_exp_cpu_mult (struct rcu_node *rnp,unsigned long mask,bool wake) { +int cpu;unsigned long flags; +struct rcu_data *rdp;raw_spin_lock_irqsave_rcu_node (rnp, flags);if (!(rnp->expmask & mask)) {@@ -238,6 +240,13 @@ static void rcu_report_exp_cpu_mult(struct rcu_node *rnp,return ; }WRITE_ONCE (rnp->expmask, rnp->expmask & ~mask); +for_each_leaf_node_cpu_mask (rnp, cpu, mask) { + rdp =per_cpu_ptr (&rcu_data, cpu); +if (!IS_ENABLED (CONFIG_NO_HZ_FULL ) || !rdp->rcu_forced_tick_exp) +continue ; + rdp->rcu_forced_tick_exp = false; +tick_dep_clear_cpu (cpu,TICK_DEP_BIT_RCU_EXP ); + }__rcu_report_exp_rnp (rnp, wake, flags);/* Releases rnp->lock. */ }@@ -450,6 +459,26 @@ static void sync_rcu_exp_select_cpus(void) }/* +* Wait for the expedited grace period to elapse, within time limit. +* If the time limit is exceeded without the grace period elapsing, +* return false, otherwise return true. +*/ +static bool synchronize_rcu_expedited_wait_once (long tlimit) +{ +int t; +struct rcu_node *rnp_root =rcu_get_root (); + + t =swait_event_timeout_exclusive (rcu_state.expedited_wq, +sync_rcu_exp_done_unlocked (rnp_root), + tlimit); +// Workqueues should not be signaled. +if (t >0 ||sync_rcu_exp_done_unlocked (rnp_root)) +return true; +WARN_ON (t <0 );/* workqueues should not be signaled. */ +return false; +} + +/* * Wait for the expedited grace period to elapse, issuing any needed * RCU CPU stall warnings along the way. */ @@ -460,22 +489,31 @@ static void synchronize_rcu_expedited_wait(void)unsigned long jiffies_start;unsigned long mask;int ndetected; +struct rcu_data *rdp;struct rcu_node *rnp;struct rcu_node *rnp_root =rcu_get_root (); -int ret;trace_rcu_exp_grace_period (rcu_state.name,rcu_exp_gp_seq_endval (),TPS ("startwait" )); jiffies_stall =rcu_jiffies_till_stall_check (); jiffies_start = jiffies; +if (IS_ENABLED (CONFIG_NO_HZ_FULL )) { +if (synchronize_rcu_expedited_wait_once (1 )) +return ; +rcu_for_each_leaf_node (rnp) { +for_each_leaf_node_cpu_mask (rnp, cpu, rnp->expmask) { + rdp =per_cpu_ptr (&rcu_data, cpu); +if (rdp->rcu_forced_tick_exp) +continue ; + rdp->rcu_forced_tick_exp = true; +tick_dep_set_cpu (cpu,TICK_DEP_BIT_RCU_EXP ); + } + } +WARN_ON_ONCE (1 ); + }for (;;) { - ret =swait_event_timeout_exclusive ( - rcu_state.expedited_wq, -sync_rcu_exp_done_unlocked (rnp_root), - jiffies_stall); -if (ret >0 ||sync_rcu_exp_done_unlocked (rnp_root)) +if (synchronize_rcu_expedited_wait_once (jiffies_stall))return ; -WARN_ON (ret <0 );/* workqueues should not be signaled. */ if (rcu_cpu_stall_suppress)continue ;panic_on_rcu_stall ();