Lazy-preemption-related updates
Paul E. McKenney
maintainercover Lazy-preemption-related updates
Hello!
This series makes adjustments to RCU to better handle lazy preemption:
1. fix header guard for rcu_all_qs(), courtesy of Ankur Arora.
2. rename PREEMPT_AUTO to PREEMPT_LAZY, courtesy of Ankur Arora.
3. update __cond_resched comment about RCU quiescent states,
courtesy of Ankur Arora.
4. handle unstable rdp in rcu_read_unlock_strict(), courtesy of
Ankur Arora.
5. handle quiescent states for PREEMPT_RCU=n, PREEMPT_COUNT=y,
courtesy of Ankur Arora.
6. provide quiescent states, courtesy of Ankur Arora.
7. limit PREEMPT_RCU configurations, courtesy of Ankur Arora.
8. Make scenario TREE10 build CONFIG_PREEMPT_LAZY=y.
9. Make scenario TREE07 build CONFIG_PREEMPT_LAZY=y.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/rcupdate.h | 2 - b/include/linux/rcutree.h | 2 - b/include/linux/srcutiny.h | 2 - b/kernel/rcu/Kconfig | 2 - b/kernel/rcu/srcutiny.c | 14 +++---- b/kernel/rcu/tree_plugin.h | 11 +++++ b/kernel/sched/core.c | 4 +- b/kernel/trace/trace_osnoise.c | 32 +++++++--------- b/tools/testing/selftests/rcutorture/configs/rcu/TREE07 | 3 + b/tools/testing/selftests/rcutorture/configs/rcu/TREE10 | 3 + kernel/rcu/Kconfig | 2 - kernel/rcu/tree_plugin.h | 11 +++-- 12 files changed, 51 insertions(+), 37 deletions(-)
Paul E. McKenney
maintainerPATCH 1/9 rcu: fix header guard for rcu_all_qs()
From: Ankur Arora <[email protected]>
rcu_all_qs() is defined for !CONFIG_PREEMPT_RCU but the declaration is conditioned on CONFIG_PREEMPTION.
With CONFIG_PREEMPT_LAZY, CONFIG_PREEMPTION=y does not imply CONFIG_PREEMPT_RCU=y.
Decouple the two.
--- include/linux/rcutree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/rcutree.h +1 -1
@@ -103,7 +103,7 @@ extern int rcu_scheduler_active;void rcu_end_inkernel_boot (void );bool rcu_inkernel_boot_has_ended (void );bool rcu_is_watching (void ); -#ifndef CONFIG_PREEMPTION +#ifndef CONFIG_PREEMPT_RCU void rcu_all_qs (void );#endif
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 2/9 rcu: rename PREEMPT_AUTO to PREEMPT_LAZY
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
Replace mentions of PREEMPT_AUTO with PREEMPT_LAZY.
Also, since PREMPT_LAZY implies PREEMPTION, we can reduce the TASKS_RCU selection criteria from:
NEED_TASKS_RCU && (PREEMPTION || PREEMPT_AUTO)
to:
NEED_TASKS_RCU && PREEMPTION
--- include/linux/srcutiny.h | 2 +- kernel/rcu/Kconfig | 2 +- kernel/rcu/srcutiny.c | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-)
include/linux/srcutiny.h +1 -1
@@ -64,7 +64,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp) {int idx; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY idx = ((READ_ONCE (ssp->srcu_idx) +1 ) &0x2 ) >>1 ;WRITE_ONCE (ssp->srcu_lock_nesting[idx],READ_ONCE (ssp->srcu_lock_nesting[idx]) +1 );preempt_enable ();
kernel/rcu/Kconfig +1 -1
@@ -91,7 +91,7 @@ config NEED_TASKS_RCU config TASKS_RCU bool - default NEED_TASKS_RCU && (PREEMPTION || PREEMPT_AUTO) + default NEED_TASKS_RCU && PREEMPTION select IRQ_WORK config FORCE_TASKS_RUDE_RCU
kernel/rcu/srcutiny.c +7 -7
@@ -98,7 +98,7 @@ void __srcu_read_unlock(struct srcu_struct *ssp, int idx) {int newval; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY newval =READ_ONCE (ssp->srcu_lock_nesting[idx]) -1 ;WRITE_ONCE (ssp->srcu_lock_nesting[idx], newval);preempt_enable ();
@@ -120,7 +120,7 @@ void srcu_drive_gp(struct work_struct *wp)struct srcu_struct *ssp; ssp = container_of(wp, struct srcu_struct, srcu_work); -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY if (ssp->srcu_gp_running ||ULONG_CMP_GE (ssp->srcu_idx,READ_ONCE (ssp->srcu_idx_max))) {preempt_enable ();return ;/* Already running or nothing to do. */
@@ -138,7 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)WRITE_ONCE (ssp->srcu_gp_waiting, true);/* srcu_read_unlock() wakes! */ preempt_enable ();swait_event_exclusive (ssp->srcu_wq, !READ_ONCE (ssp->srcu_lock_nesting[idx])); -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY WRITE_ONCE (ssp->srcu_gp_waiting, false);/* srcu_read_unlock() cheap. */ WRITE_ONCE (ssp->srcu_idx, ssp->srcu_idx +1 );preempt_enable ();
@@ -159,7 +159,7 @@ void srcu_drive_gp(struct work_struct *wp)* at interrupt level, but the ->srcu_gp_running checks will * straighten that out. */ -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY WRITE_ONCE (ssp->srcu_gp_running, false); idx =ULONG_CMP_LT (ssp->srcu_idx,READ_ONCE (ssp->srcu_idx_max));preempt_enable ();
@@ -172,7 +172,7 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) {unsigned long cookie; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY cookie =get_state_synchronize_srcu (ssp);if (ULONG_CMP_GE (READ_ONCE (ssp->srcu_idx_max), cookie)) {preempt_enable ();
@@ -199,7 +199,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, rhp->func = func; rhp->next =NULL ; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY local_irq_save (flags); *ssp->srcu_cb_tail = rhp; ssp->srcu_cb_tail = &rhp->next;
@@ -261,7 +261,7 @@ unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) {unsigned long ret; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY ret =get_state_synchronize_srcu (ssp);srcu_gp_start_if_needed (ssp);preempt_enable ();
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 3/9 sched: update __cond_resched comment about RCU quiescent states
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
Update comment in __cond_resched() clarifying how urgently needed quiescent state are provided.
--- kernel/sched/core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
kernel/sched/core.c +3 -1
@@ -7281,7 +7281,7 @@ int __sched __cond_resched(void)return 1 ; }/* -* In preemptible kernels, ->rcu_read_lock_nesting tells the tick +* In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick * whether the current CPU is in an RCU read-side critical section, * so the tick can report quiescent states even for CPUs looping * in kernel context. In contrast, in non-preemptible kernels,
@@ -7290,6 +7290,8 @@ int __sched __cond_resched(void)* RCU quiescent state. Therefore, the following code causes * cond_resched() to report a quiescent state, but only when RCU * is in urgent need of one. +* A third case, preemptible, but non-PREEMPT_RCU provides for +* urgently needed quiescent states via rcu_flavor_sched_clock_irq(). */ #ifndef CONFIG_PREEMPT_RCU rcu_all_qs ();
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 4/9 rcu: handle unstable rdp in rcu_read_unlock_strict()
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
rcu_read_unlock_strict() can be called with preemption enabled which can make for an unstable rdp and a racy norm value.
Fix this by dropping the preempt-count in __rcu_read_unlock() after the call to rcu_read_unlock_strict(), adjusting the preempt-count check appropriately.
--- include/linux/rcupdate.h | 2 +- kernel/rcu/tree_plugin.h | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
include/linux/rcupdate.h +1 -1
@@ -95,9 +95,9 @@ static inline void __rcu_read_lock(void)static inline void __rcu_read_unlock (void ) { -preempt_enable ();if (IS_ENABLED (CONFIG_RCU_STRICT_GRACE_PERIOD ))rcu_read_unlock_strict (); +preempt_enable (); }static inline int rcu_preempt_depth (void )
kernel/rcu/tree_plugin.h +10 -1
@@ -833,8 +833,17 @@ void rcu_read_unlock_strict(void) {struct rcu_data *rdp; -if (irqs_disabled () ||preempt_count () || !rcu_state.gp_kthread) +if (irqs_disabled () ||in_atomic_preempt_off () || !rcu_state.gp_kthread)return ; + +/* +* rcu_report_qs_rdp() can only be invoked with a stable rdp and +* from the local CPU. +* +* The in_atomic_preempt_off() check ensures that we come here holding +* the last preempt_count (which will get dropped once we return to +* __rcu_read_unlock(). +*/ rdp =this_cpu_ptr (&rcu_data); rdp->cpu_no_qs.b.norm = false;rcu_report_qs_rdp (rdp);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 5/9 rcu: handle quiescent states for PREEMPT_RCU=n, PREEMPT_COUNT=y
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
With PREEMPT_RCU=n, cond_resched() provides urgently needed quiescent states for read-side critical sections via rcu_all_qs(). One reason why this was needed: lacking preempt-count, the tick handler has no way of knowing whether it is executing in a read-side critical section or not.
With (PREEMPT_LAZY=y, PREEMPT_DYNAMIC=n), we get (PREEMPT_COUNT=y, PREEMPT_RCU=n). In this configuration cond_resched() is a stub and does not provide quiescent states via rcu_all_qs(). (PREEMPT_RCU=y provides this information via rcu_read_unlock() and its nesting counter.)
So, use the availability of preempt_count() to report quiescent states in rcu_flavor_sched_clock_irq().
--- kernel/rcu/tree_plugin.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
kernel/rcu/tree_plugin.h +7 -4
@@ -984,13 +984,16 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)*/ static void rcu_flavor_sched_clock_irq (int user) { -if (user ||rcu_is_cpu_rrupt_from_idle ()) { +if (user ||rcu_is_cpu_rrupt_from_idle () || + (IS_ENABLED (CONFIG_PREEMPT_COUNT ) && + (preempt_count () ==HARDIRQ_OFFSET ))) {/* * Get here if this CPU took its interrupt from user -* mode or from the idle loop, and if this is not a -* nested interrupt. In this case, the CPU is in -* a quiescent state, so note it. +* mode, from the idle loop without this being a nested +* interrupt, or while not holding the task preempt count +* (with PREEMPT_COUNT=y). In this case, the CPU is in a +* quiescent state, so note it. * * No memory barrier is required here because rcu_qs() * references only CPU-local variables that other CPUs
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 6/9 osnoise: provide quiescent states
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
To reduce RCU noise for nohz_full configurations, osnoise depends on cond_resched() providing quiescent states for PREEMPT_RCU=n configurations. For PREEMPT_RCU=y configurations -- where cond_resched() is a stub -- we do this by directly calling rcu_momentary_eqs().
With (PREEMPT_LAZY=y, PREEMPT_DYNAMIC=n), however, we have a configuration with (PREEMPTION=y, PREEMPT_RCU=n) where neither of the above can help.
Handle that by providing an explicit quiescent state here for all configurations.
As mentioned above this is not needed for non-stubbed cond_resched(), but, providing a quiescent state here just pulls in one that a future cond_resched() would provide, so doesn't cause any extra work for this configuration.
--- kernel/trace/trace_osnoise.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-)
kernel/trace/trace_osnoise.c +15 -17
@@ -1531,27 +1531,25 @@ static int run_osnoise(void)/* * In some cases, notably when running on a nohz_full CPU with -* a stopped tick PREEMPT_RCU has no way to account for QSs. -* This will eventually cause unwarranted noise as PREEMPT_RCU -* will force preemption as the means of ending the current -* grace period. We avoid this problem by calling -* rcu_momentary_eqs(), which performs a zero duration -* EQS allowing PREEMPT_RCU to end the current grace period. -* This call shouldn't be wrapped inside an RCU critical -* section. +* a stopped tick PREEMPT_RCU or PREEMPT_LAZY have no way to +* account for QSs. This will eventually cause unwarranted +* noise as RCU forces preemption as the means of ending the +* current grace period. We avoid this by calling +* rcu_momentary_eqs(), which performs a zero duration EQS +* allowing RCU to end the current grace period. This call +* shouldn't be wrapped inside an RCU critical section. * -* Note that in non PREEMPT_RCU kernels QSs are handled through -* cond_resched() +* Normally QSs for other cases are handled through cond_resched(). +* For simplicity, however, we call rcu_momentary_eqs() for all +* configurations here. */ -if (IS_ENABLED (CONFIG_PREEMPT_RCU )) { -if (!disable_irq) -local_irq_disable (); +if (!disable_irq) +local_irq_disable (); -rcu_momentary_eqs (); +rcu_momentary_eqs (); -if (!disable_irq) -local_irq_enable (); - } +if (!disable_irq) +local_irq_enable ();/* * For the non-preemptive kernel config: let threads runs, if
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 7/9 rcu: limit PREEMPT_RCU configurations
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
PREEMPT_LAZY can be enabled stand-alone or alongside PREEMPT_DYNAMIC which allows for dynamic switching of preemption models.
The choice of PREEMPT_RCU or not, however, is fixed at compile time.
Given that PREEMPT_RCU makes some trade-offs to optimize for latency as opposed to throughput, configurations with limited preemption might prefer the stronger forward-progress guarantees of PREEMPT_RCU=n.
Accordingly, explicitly limit PREEMPT_RCU=y to the latency oriented preemption models: PREEMPT, PREEMPT_RT, and the runtime configurable model PREEMPT_DYNAMIC.
This means the throughput oriented models, PREEMPT_NONE, PREEMPT_VOLUNTARY, and PREEMPT_LAZY will run with PREEMPT_RCU=n.
--- kernel/rcu/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/Kconfig +1 -1
@@ -18,7 +18,7 @@ config TREE_RCU config PREEMPT_RCU bool - default y if PREEMPTION + default y if (PREEMPT || PREEMPT_RT || PREEMPT_DYNAMIC) select TREE_RCU help This option selects the RCU implementation that is
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 8/9 rcutorture: Make scenario TREE10 build CONFIG_PREEMPT_LAZY=y
In reply to Paul E. McKenneyThis commit tests lazy preemption by causing the TREE10 rcutorture scenario to build its kernel with CONFIG_PREEMPT_LAZY=y.
--- tools/testing/selftests/rcutorture/configs/rcu/TREE10 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
tools/testing/selftests/rcutorture/configs/rcu/TREE10 +2 -1
@@ -1,6 +1,7 @@ CONFIG_SMP=y CONFIG_NR_CPUS=74 -CONFIG_PREEMPT_NONE=y +CONFIG_PREEMPT_NONE=n +CONFIG_PREEMPT_LAZY=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n CONFIG_PREEMPT_DYNAMIC=n
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 9/9 rcutorture: Make scenario TREE07 build CONFIG_PREEMPT_LAZY=y
In reply to Paul E. McKenneyThis commit tests lazy preemption by causing the TREE07 rcutorture scenario to build its kernel with CONFIG_PREEMPT_LAZY=y.
--- tools/testing/selftests/rcutorture/configs/rcu/TREE07 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
tools/testing/selftests/rcutorture/configs/rcu/TREE07 +2 -1
@@ -1,7 +1,8 @@ CONFIG_SMP=y CONFIG_NR_CPUS=16 -CONFIG_PREEMPT_NONE=y +CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n +CONFIG_PREEMPT_LAZY=y CONFIG_PREEMPT=n CONFIG_PREEMPT_DYNAMIC=n #CHECK#CONFIG_TREE_RCU=y
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerHello!
This series makes adjustments to RCU to better handle lazy preemption:
1. fix header guard for rcu_all_qs(), courtesy of Ankur Arora.
2. rename PREEMPT_AUTO to PREEMPT_LAZY, courtesy of Ankur Arora.
3. update __cond_resched comment about RCU quiescent states,
courtesy of Ankur Arora.
4. handle unstable rdp in rcu_read_unlock_strict(), courtesy of
Ankur Arora.
5. handle quiescent states for PREEMPT_RCU=n, PREEMPT_COUNT=y,
courtesy of Ankur Arora.
6. provide quiescent states, courtesy of Ankur Arora.
7. limit PREEMPT_RCU configurations, courtesy of Ankur Arora.
8. Make scenario TREE10 build CONFIG_PREEMPT_LAZY=y.
9. Make scenario TREE07 build CONFIG_PREEMPT_LAZY=y.
Changes since v1:
o Reword commit-log line containing only "to:" to avoid confusing
tools that parse commit logs.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/rcupdate.h | 2 - b/include/linux/rcutree.h | 2 - b/include/linux/srcutiny.h | 2 - b/kernel/rcu/Kconfig | 2 - b/kernel/rcu/srcutiny.c | 14 +++---- b/kernel/rcu/tree_plugin.h | 11 +++++ b/kernel/sched/core.c | 4 +- b/kernel/trace/trace_osnoise.c | 32 +++++++--------- b/tools/testing/selftests/rcutorture/configs/rcu/TREE07 | 3 + b/tools/testing/selftests/rcutorture/configs/rcu/TREE10 | 3 + kernel/rcu/Kconfig | 2 - kernel/rcu/tree_plugin.h | 11 +++-- 12 files changed, 51 insertions(+), 37 deletions(-)
Paul E. McKenney
maintainerPATCH v2 1/1 1/9] rcu: fix header guard for rcu_all_qs()
From: Ankur Arora <[email protected]>
rcu_all_qs() is defined for !CONFIG_PREEMPT_RCU but the declaration is conditioned on CONFIG_PREEMPTION.
With CONFIG_PREEMPT_LAZY, CONFIG_PREEMPTION=y does not imply CONFIG_PREEMPT_RCU=y.
Decouple the two.
--- include/linux/rcutree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/rcutree.h +1 -1
@@ -103,7 +103,7 @@ extern int rcu_scheduler_active;void rcu_end_inkernel_boot (void );bool rcu_inkernel_boot_has_ended (void );bool rcu_is_watching (void ); -#ifndef CONFIG_PREEMPTION +#ifndef CONFIG_PREEMPT_RCU void rcu_all_qs (void );#endif
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 2/9] rcu: rename PREEMPT_AUTO to PREEMPT_LAZY
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
Replace mentions of PREEMPT_AUTO with PREEMPT_LAZY.
Also, since PREMPT_LAZY implies PREEMPTION, we can reduce the TASKS_RCU selection criteria from this:
NEED_TASKS_RCU && (PREEMPTION || PREEMPT_AUTO)
to this:
NEED_TASKS_RCU && PREEMPTION
--- include/linux/srcutiny.h | 2 +- kernel/rcu/Kconfig | 2 +- kernel/rcu/srcutiny.c | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-)
include/linux/srcutiny.h +1 -1
@@ -64,7 +64,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp) {int idx; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY idx = ((READ_ONCE (ssp->srcu_idx) +1 ) &0x2 ) >>1 ;WRITE_ONCE (ssp->srcu_lock_nesting[idx],READ_ONCE (ssp->srcu_lock_nesting[idx]) +1 );preempt_enable ();
kernel/rcu/Kconfig +1 -1
@@ -91,7 +91,7 @@ config NEED_TASKS_RCU config TASKS_RCU bool - default NEED_TASKS_RCU && (PREEMPTION || PREEMPT_AUTO) + default NEED_TASKS_RCU && PREEMPTION select IRQ_WORK config FORCE_TASKS_RUDE_RCU
kernel/rcu/srcutiny.c +7 -7
@@ -98,7 +98,7 @@ void __srcu_read_unlock(struct srcu_struct *ssp, int idx) {int newval; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY newval =READ_ONCE (ssp->srcu_lock_nesting[idx]) -1 ;WRITE_ONCE (ssp->srcu_lock_nesting[idx], newval);preempt_enable ();
@@ -120,7 +120,7 @@ void srcu_drive_gp(struct work_struct *wp)struct srcu_struct *ssp; ssp = container_of(wp, struct srcu_struct, srcu_work); -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY if (ssp->srcu_gp_running ||ULONG_CMP_GE (ssp->srcu_idx,READ_ONCE (ssp->srcu_idx_max))) {preempt_enable ();return ;/* Already running or nothing to do. */
@@ -138,7 +138,7 @@ void srcu_drive_gp(struct work_struct *wp)WRITE_ONCE (ssp->srcu_gp_waiting, true);/* srcu_read_unlock() wakes! */ preempt_enable ();swait_event_exclusive (ssp->srcu_wq, !READ_ONCE (ssp->srcu_lock_nesting[idx])); -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY WRITE_ONCE (ssp->srcu_gp_waiting, false);/* srcu_read_unlock() cheap. */ WRITE_ONCE (ssp->srcu_idx, ssp->srcu_idx +1 );preempt_enable ();
@@ -159,7 +159,7 @@ void srcu_drive_gp(struct work_struct *wp)* at interrupt level, but the ->srcu_gp_running checks will * straighten that out. */ -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY WRITE_ONCE (ssp->srcu_gp_running, false); idx =ULONG_CMP_LT (ssp->srcu_idx,READ_ONCE (ssp->srcu_idx_max));preempt_enable ();
@@ -172,7 +172,7 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) {unsigned long cookie; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY cookie =get_state_synchronize_srcu (ssp);if (ULONG_CMP_GE (READ_ONCE (ssp->srcu_idx_max), cookie)) {preempt_enable ();
@@ -199,7 +199,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, rhp->func = func; rhp->next =NULL ; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY local_irq_save (flags); *ssp->srcu_cb_tail = rhp; ssp->srcu_cb_tail = &rhp->next;
@@ -261,7 +261,7 @@ unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) {unsigned long ret; -preempt_disable ();// Needed for PREEMPT_AUTO +preempt_disable ();// Needed for PREEMPT_LAZY ret =get_state_synchronize_srcu (ssp);srcu_gp_start_if_needed (ssp);preempt_enable ();
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 3/9] sched: update __cond_resched comment about RCU quiescent states
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
Update comment in __cond_resched() clarifying how urgently needed quiescent state are provided.
--- kernel/sched/core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
kernel/sched/core.c +3 -1
@@ -7281,7 +7281,7 @@ int __sched __cond_resched(void)return 1 ; }/* -* In preemptible kernels, ->rcu_read_lock_nesting tells the tick +* In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick * whether the current CPU is in an RCU read-side critical section, * so the tick can report quiescent states even for CPUs looping * in kernel context. In contrast, in non-preemptible kernels,
@@ -7290,6 +7290,8 @@ int __sched __cond_resched(void)* RCU quiescent state. Therefore, the following code causes * cond_resched() to report a quiescent state, but only when RCU * is in urgent need of one. +* A third case, preemptible, but non-PREEMPT_RCU provides for +* urgently needed quiescent states via rcu_flavor_sched_clock_irq(). */ #ifndef CONFIG_PREEMPT_RCU rcu_all_qs ();
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 4/9] rcu: handle unstable rdp in rcu_read_unlock_strict()
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
rcu_read_unlock_strict() can be called with preemption enabled which can make for an unstable rdp and a racy norm value.
Fix this by dropping the preempt-count in __rcu_read_unlock() after the call to rcu_read_unlock_strict(), adjusting the preempt-count check appropriately.
--- include/linux/rcupdate.h | 2 +- kernel/rcu/tree_plugin.h | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
include/linux/rcupdate.h +1 -1
@@ -95,9 +95,9 @@ static inline void __rcu_read_lock(void)static inline void __rcu_read_unlock (void ) { -preempt_enable ();if (IS_ENABLED (CONFIG_RCU_STRICT_GRACE_PERIOD ))rcu_read_unlock_strict (); +preempt_enable (); }static inline int rcu_preempt_depth (void )
kernel/rcu/tree_plugin.h +10 -1
@@ -833,8 +833,17 @@ void rcu_read_unlock_strict(void) {struct rcu_data *rdp; -if (irqs_disabled () ||preempt_count () || !rcu_state.gp_kthread) +if (irqs_disabled () ||in_atomic_preempt_off () || !rcu_state.gp_kthread)return ; + +/* +* rcu_report_qs_rdp() can only be invoked with a stable rdp and +* from the local CPU. +* +* The in_atomic_preempt_off() check ensures that we come here holding +* the last preempt_count (which will get dropped once we return to +* __rcu_read_unlock(). +*/ rdp =this_cpu_ptr (&rcu_data); rdp->cpu_no_qs.b.norm = false;rcu_report_qs_rdp (rdp);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 5/9] rcu: handle quiescent states for PREEMPT_RCU=n, PREEMPT_COUNT=y
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
With PREEMPT_RCU=n, cond_resched() provides urgently needed quiescent states for read-side critical sections via rcu_all_qs(). One reason why this was needed: lacking preempt-count, the tick handler has no way of knowing whether it is executing in a read-side critical section or not.
With (PREEMPT_LAZY=y, PREEMPT_DYNAMIC=n), we get (PREEMPT_COUNT=y, PREEMPT_RCU=n). In this configuration cond_resched() is a stub and does not provide quiescent states via rcu_all_qs(). (PREEMPT_RCU=y provides this information via rcu_read_unlock() and its nesting counter.)
So, use the availability of preempt_count() to report quiescent states in rcu_flavor_sched_clock_irq().
--- kernel/rcu/tree_plugin.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
kernel/rcu/tree_plugin.h +7 -4
@@ -984,13 +984,16 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)*/ static void rcu_flavor_sched_clock_irq (int user) { -if (user ||rcu_is_cpu_rrupt_from_idle ()) { +if (user ||rcu_is_cpu_rrupt_from_idle () || + (IS_ENABLED (CONFIG_PREEMPT_COUNT ) && + (preempt_count () ==HARDIRQ_OFFSET ))) {/* * Get here if this CPU took its interrupt from user -* mode or from the idle loop, and if this is not a -* nested interrupt. In this case, the CPU is in -* a quiescent state, so note it. +* mode, from the idle loop without this being a nested +* interrupt, or while not holding the task preempt count +* (with PREEMPT_COUNT=y). In this case, the CPU is in a +* quiescent state, so note it. * * No memory barrier is required here because rcu_qs() * references only CPU-local variables that other CPUs
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 6/9] osnoise: provide quiescent states
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
To reduce RCU noise for nohz_full configurations, osnoise depends on cond_resched() providing quiescent states for PREEMPT_RCU=n configurations. For PREEMPT_RCU=y configurations -- where cond_resched() is a stub -- we do this by directly calling rcu_momentary_eqs().
With (PREEMPT_LAZY=y, PREEMPT_DYNAMIC=n), however, we have a configuration with (PREEMPTION=y, PREEMPT_RCU=n) where neither of the above can help.
Handle that by providing an explicit quiescent state here for all configurations.
As mentioned above this is not needed for non-stubbed cond_resched(), but, providing a quiescent state here just pulls in one that a future cond_resched() would provide, so doesn't cause any extra work for this configuration.
--- kernel/trace/trace_osnoise.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-)
kernel/trace/trace_osnoise.c +15 -17
@@ -1531,27 +1531,25 @@ static int run_osnoise(void)/* * In some cases, notably when running on a nohz_full CPU with -* a stopped tick PREEMPT_RCU has no way to account for QSs. -* This will eventually cause unwarranted noise as PREEMPT_RCU -* will force preemption as the means of ending the current -* grace period. We avoid this problem by calling -* rcu_momentary_eqs(), which performs a zero duration -* EQS allowing PREEMPT_RCU to end the current grace period. -* This call shouldn't be wrapped inside an RCU critical -* section. +* a stopped tick PREEMPT_RCU or PREEMPT_LAZY have no way to +* account for QSs. This will eventually cause unwarranted +* noise as RCU forces preemption as the means of ending the +* current grace period. We avoid this by calling +* rcu_momentary_eqs(), which performs a zero duration EQS +* allowing RCU to end the current grace period. This call +* shouldn't be wrapped inside an RCU critical section. * -* Note that in non PREEMPT_RCU kernels QSs are handled through -* cond_resched() +* Normally QSs for other cases are handled through cond_resched(). +* For simplicity, however, we call rcu_momentary_eqs() for all +* configurations here. */ -if (IS_ENABLED (CONFIG_PREEMPT_RCU )) { -if (!disable_irq) -local_irq_disable (); +if (!disable_irq) +local_irq_disable (); -rcu_momentary_eqs (); +rcu_momentary_eqs (); -if (!disable_irq) -local_irq_enable (); - } +if (!disable_irq) +local_irq_enable ();/* * For the non-preemptive kernel config: let threads runs, if
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 7/9] rcu: limit PREEMPT_RCU configurations
In reply to Paul E. McKenneyFrom: Ankur Arora <[email protected]>
PREEMPT_LAZY can be enabled stand-alone or alongside PREEMPT_DYNAMIC which allows for dynamic switching of preemption models.
The choice of PREEMPT_RCU or not, however, is fixed at compile time.
Given that PREEMPT_RCU makes some trade-offs to optimize for latency as opposed to throughput, configurations with limited preemption might prefer the stronger forward-progress guarantees of PREEMPT_RCU=n.
Accordingly, explicitly limit PREEMPT_RCU=y to the latency oriented preemption models: PREEMPT, PREEMPT_RT, and the runtime configurable model PREEMPT_DYNAMIC.
This means the throughput oriented models, PREEMPT_NONE, PREEMPT_VOLUNTARY, and PREEMPT_LAZY will run with PREEMPT_RCU=n.
--- kernel/rcu/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/Kconfig +1 -1
@@ -18,7 +18,7 @@ config TREE_RCU config PREEMPT_RCU bool - default y if PREEMPTION + default y if (PREEMPT || PREEMPT_RT || PREEMPT_DYNAMIC) select TREE_RCU help This option selects the RCU implementation that is
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 8/9] rcutorture: Make scenario TREE10 build CONFIG_PREEMPT_LAZY=y
In reply to Paul E. McKenneyThis commit tests lazy preemption by causing the TREE10 rcutorture scenario to build its kernel with CONFIG_PREEMPT_LAZY=y.
--- tools/testing/selftests/rcutorture/configs/rcu/TREE10 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
tools/testing/selftests/rcutorture/configs/rcu/TREE10 +2 -1
@@ -1,6 +1,7 @@ CONFIG_SMP=y CONFIG_NR_CPUS=74 -CONFIG_PREEMPT_NONE=y +CONFIG_PREEMPT_NONE=n +CONFIG_PREEMPT_LAZY=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n CONFIG_PREEMPT_DYNAMIC=n
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 9/9] rcutorture: Make scenario TREE07 build CONFIG_PREEMPT_LAZY=y
In reply to Paul E. McKenneyThis commit tests lazy preemption by causing the TREE07 rcutorture scenario to build its kernel with CONFIG_PREEMPT_LAZY=y.
--- tools/testing/selftests/rcutorture/configs/rcu/TREE07 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
tools/testing/selftests/rcutorture/configs/rcu/TREE07 +2 -1
@@ -1,7 +1,8 @@ CONFIG_SMP=y CONFIG_NR_CPUS=16 -CONFIG_PREEMPT_NONE=y +CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n +CONFIG_PREEMPT_LAZY=y CONFIG_PREEMPT=n CONFIG_PREEMPT_DYNAMIC=n #CHECK#CONFIG_TREE_RCU=y
Signature · 1 line
2.40.1