Response-time changes for 3.15
cover Response-time changes for 3.15
Hello!
This series contains minor response-time improvements.
1. Create trivial always-true version of rcu_is_nocb_cpu() for
CONFIG_RCU_NOCB_CPU_ALL=y kernels.
2. Create trivial always-false version of rcu_needs_cpu() for
CONFIG_RCU_NOCB_CPU_ALL=y kernels.
3. Don't bother doing CONFIG_RCU_FAST_NO_HZ work at idle entry
and exit time for CONFIG_RCU_NOCB_CPU_ALL=y kernels.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/rcupdate.h | 4 ++++ b/include/linux/rcutiny.h | 6 ------ b/include/linux/rcutree.h | 2 ++ b/kernel/rcu/tree.c | 2 +- b/kernel/rcu/tree_plugin.h | 2 ++ include/linux/rcupdate.h | 9 +++++++++ kernel/rcu/tree_plugin.h | 11 +++++++++-- 7 files changed, 27 insertions(+), 9 deletions(-)
PATCH 1/3 rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL
From: "Paul E. McKenney" <[email protected]>
If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_is_nocb_cpu() will always return true, however, the current version nevertheless checks rcu_nocb_mask. This commit therefore creates a static inline implementation of rcu_is_nocb_cpu() that unconditionally returns true when CONFIG_RCU_NOCB_CPU_ALL=y.
--- include/linux/rcupdate.h | 4 ++++ kernel/rcu/tree_plugin.h | 2 ++ 2 files changed, 6 insertions(+)
include/linux/rcupdate.h +4 -0
@@ -1016,7 +1016,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))#ifdef CONFIG_RCU_NOCB_CPU +#ifdef CONFIG_RCU_NOCB_CPU_ALL +static inline bool rcu_is_nocb_cpu (int cpu) {return true; } +#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */ bool rcu_is_nocb_cpu (int cpu); +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */ #else static inline bool rcu_is_nocb_cpu (int cpu) {return false; }#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
kernel/rcu/tree_plugin.h +2 -0
@@ -2101,6 +2101,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_waitqueue_head (&rnp->nocb_gp_wq[1 ]); } +#ifndef CONFIG_RCU_NOCB_CPU_ALL /* Is the specified CPU a no-CPUs CPU? */ bool rcu_is_nocb_cpu (int cpu) {
@@ -2108,6 +2109,7 @@ bool rcu_is_nocb_cpu(int cpu)return cpumask_test_cpu (cpu, rcu_nocb_mask); return false; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Enqueue the specified string of rcu_head structures onto the specified
Signature · 1 line
1.8.1.5
PATCH 2/3 rcu: Optimize rcu_needs_cpu() for RCU_NOCB_CPU_ALL
From: "Paul E. McKenney" <[email protected]>
If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_needs_cpu() will always return false, however, the current version nevertheless checks for RCU callbacks. This commit therefore creates a static inline implementation of rcu_needs_cpu() that unconditionally returns false when CONFIG_RCU_NOCB_CPU_ALL=y.
--- include/linux/rcupdate.h | 9 +++++++++ include/linux/rcutiny.h | 6 ------ include/linux/rcutree.h | 2 ++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 4 ++++ 5 files changed, 16 insertions(+), 7 deletions(-)
include/linux/rcupdate.h +9 -0
@@ -1015,6 +1015,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)#define kfree_rcu (ptr, rcu_head) \ __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) +#if defined(CONFIG_TINY_RCU ) || defined(CONFIG_RCU_NOCB_CPU_ALL ) + +static inline int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) +{ + *delta_jiffies =ULONG_MAX ; +return 0 ; +} +#endif /* #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) */ +#ifdef CONFIG_RCU_NOCB_CPU #ifdef CONFIG_RCU_NOCB_CPU_ALL static inline bool rcu_is_nocb_cpu (int cpu) {return true; }
include/linux/rcutiny.h +0 -6
@@ -68,12 +68,6 @@ static inline void kfree_call_rcu(struct rcu_head *head,call_rcu (head, func); } -static inline int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) -{ - *delta_jiffies =ULONG_MAX ; -return 0 ; -} -static inline void rcu_note_context_switch (int cpu) {rcu_sched_qs (cpu);
include/linux/rcutree.h +2 -0
@@ -31,7 +31,9 @@#define __LINUX_RCUTREE_Hvoid rcu_note_context_switch (int cpu); +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies); +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ void rcu_cpu_stall_reset (void );/*
kernel/rcu/tree.c +1 -1
@@ -2880,7 +2880,7 @@ static int rcu_pending(int cpu)* non-NULL, store an indication of whether all callbacks are lazy. * (If there are no callbacks, all of them are deemed to be lazy.) */ -static int rcu_cpu_has_callbacks (int cpu,bool *all_lazy) +static int __maybe_unused rcu_cpu_has_callbacks (int cpu,bool *all_lazy) {bool al = true;bool hc = false;
kernel/rcu/tree_plugin.h +4 -0
@@ -1586,11 +1586,13 @@ static void rcu_prepare_kthreads(int cpu)* Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs * any flavor of RCU. */ +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) { *delta_jiffies =ULONG_MAX ;return rcu_cpu_has_callbacks (cpu,NULL ); } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
@@ -1696,6 +1698,7 @@ static bool rcu_try_advance_all_cbs(void)* * The caller must have disabled interrupts. */ +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *dj) {struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);
@@ -1726,6 +1729,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj) }return 0 ; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Prepare a CPU for idle from an RCU perspective. The first major task
Signature · 1 line
1.8.1.5
Re: [PATCH tip/core/rcu 2/3] rcu: Optimize rcu_needs_cpu() for RCU_NOCB_CPU_ALL
Quoted 9 lines · Paul E. McKenney
From: "Paul E. McKenney" <[email protected]>
If
CONFIG_RCU_NOCB_CPU_ALL=y, thenrcu_needs_cpu()will always return false, however, the current version nevertheless checks for RCU callbacks. This commit therefore creates a static inline implementation ofrcu_needs_cpu()that unconditionally returns false whenCONFIG_RCU_NOCB_CPU_ALL=y.Signed-off-by Paul E. McKenney <[email protected]>
Quoted 109 lines
--- include/linux/rcupdate.h | 9 +++++++++ include/linux/rcutiny.h | 6 ------ include/linux/rcutree.h | 2 ++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 4 ++++ 5 files changed, 16 insertions(+), 7 deletions(-)include/linux/rcupdate.h +9
-0@@ -1015,6 +1015,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)#define kfree_rcu (ptr, rcu_head) \ __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) +#if defined(CONFIG_TINY_RCU ) || defined(CONFIG_RCU_NOCB_CPU_ALL ) + +static inline int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) +{ + *delta_jiffies =ULONG_MAX ; +return 0 ; +} +#endif /* #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) */ +#ifdef CONFIG_RCU_NOCB_CPU #ifdef CONFIG_RCU_NOCB_CPU_ALL static inline bool rcu_is_nocb_cpu (int cpu) {return true; }include/linux/rcutiny.h +0
-6@@ -68,12 +68,6 @@ static inline void kfree_call_rcu(struct rcu_head *head,call_rcu (head, func); } -static inline int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) -{ - *delta_jiffies =ULONG_MAX ; -return 0 ; -} -static inline void rcu_note_context_switch (int cpu) {rcu_sched_qs (cpu);include/linux/rcutree.h +2
-0@@ -31,7 +31,9 @@#define __LINUX_RCUTREE_Hvoid rcu_note_context_switch (int cpu); +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies); +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ void rcu_cpu_stall_reset (void );/* kernel/rcu/tree.c +1
-1@@ -2880,7 +2880,7 @@ static int rcu_pending(int cpu)* non-NULL, store an indication of whether all callbacks are lazy. * (If there are no callbacks, all of them are deemed to be lazy.) */ -static int rcu_cpu_has_callbacks (int cpu,bool *all_lazy) +static int __maybe_unused rcu_cpu_has_callbacks (int cpu,bool *all_lazy) {bool al = true;bool hc = false;kernel/rcu/tree_plugin.h +4
-0@@ -1586,11 +1586,13 @@ static void rcu_prepare_kthreads(int cpu)* Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs * any flavor of RCU. */ +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) { *delta_jiffies =ULONG_MAX ;return rcu_cpu_has_callbacks (cpu,NULL ); } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up @@ -1696,6 +1698,7 @@ static bool rcu_try_advance_all_cbs(void)* * The caller must have disabled interrupts. */ +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *dj) {struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);@@ -1726,6 +1729,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj) }return 0 ; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Prepare a CPU for idle from an RCU perspective. The first major task Signature · 1 line
1.8.1.5
PATCH 3/3 rcu: Optimize RCU_FAST_NO_HZ for RCU_NOCB_CPU_ALL
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
If CONFIG_RCU_NOCB_CPU_ALL=y, then no CPU will ever have RCU callbacks because these callbacks will instead be handled by the rcuo kthreads. However, the current version of RCU_FAST_NO_HZ nevertheless checks for RCU callbacks. This commit therefore creates static inline implementations of rcu_prepare_for_idle() and rcu_cleanup_after_idle() that are no-ops when CONFIG_RCU_NOCB_CPU_ALL=y.
--- kernel/rcu/tree_plugin.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
kernel/rcu/tree_plugin.h +5 -2
@@ -1658,7 +1658,7 @@ extern int tick_nohz_active;* only if it has been awhile since the last time we did so. Afterwards, * if there are any callbacks ready for immediate invocation, return true. */ -static bool rcu_try_advance_all_cbs (void ) +static bool __maybe_unused rcu_try_advance_all_cbs (void ) {bool cbs_ready = false;struct rcu_data *rdp;
@@ -1743,6 +1743,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)*/ static void rcu_prepare_for_idle (int cpu) { +#ifndef CONFIG_RCU_NOCB_CPU_ALL struct rcu_data *rdp;struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);struct rcu_node *rnp;
@@ -1794,6 +1795,7 @@ static void rcu_prepare_for_idle(int cpu)rcu_accelerate_cbs (rsp, rnp, rdp); raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ }/*
@@ -1803,11 +1805,12 @@ static void rcu_prepare_for_idle(int cpu)*/ static void rcu_cleanup_after_idle (int cpu) { - +#ifndef CONFIG_RCU_NOCB_CPU_ALL if (rcu_is_nocb_cpu (cpu))return ;if (rcu_try_advance_all_cbs ())invoke_rcu_core (); +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ }/*
Signature · 1 line
1.8.1.5
Re: [PATCH tip/core/rcu 3/3] rcu: Optimize RCU_FAST_NO_HZ for RCU_NOCB_CPU_ALL
Quoted 8 lines · Paul E. McKenney
From: "Paul E. McKenney" <[email protected]>
If
CONFIG_RCU_NOCB_CPU_ALL=y, then no CPU will ever have RCU callbacks because these callbacks will instead be handled by the rcuo kthreads. However, the current version ofRCU_FAST_NO_HZnevertheless checks for RCU callbacks. This commit therefore creates static inline implementations ofrcu_prepare_for_idle()andrcu_cleanup_after_idle()that are no-ops whenCONFIG_RCU_NOCB_CPU_ALL=y.
"creates static inline implementations" isn't quite accurate; they're static, and in the header, and the compiler is likely to inline the no-op versions, but they're not actually marked as inline. The commit message should just say that it turns the implementations into no-ops.
Quoted 1 line
Signed-off-by Paul E. McKenney <[email protected]>
Quoted 50 lines
--- kernel/rcu/tree_plugin.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)kernel/rcu/tree_plugin.h +5
-2@@ -1658,7 +1658,7 @@ extern int tick_nohz_active;* only if it has been awhile since the last time we did so. Afterwards, * if there are any callbacks ready for immediate invocation, return true. */ -static bool rcu_try_advance_all_cbs (void ) +static bool __maybe_unused rcu_try_advance_all_cbs (void ) {bool cbs_ready = false;struct rcu_data *rdp;@@ -1743,6 +1743,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)*/ static void rcu_prepare_for_idle (int cpu) { +#ifndef CONFIG_RCU_NOCB_CPU_ALL struct rcu_data *rdp;struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);struct rcu_node *rnp;@@ -1794,6 +1795,7 @@ static void rcu_prepare_for_idle(int cpu)rcu_accelerate_cbs (rsp, rnp, rdp); raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ }/* @@ -1803,11 +1805,12 @@ static void rcu_prepare_for_idle(int cpu)*/ static void rcu_cleanup_after_idle (int cpu) { - +#ifndef CONFIG_RCU_NOCB_CPU_ALL if (rcu_is_nocb_cpu (cpu))return ;if (rcu_try_advance_all_cbs ())invoke_rcu_core (); +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ }/* Signature · 1 line
1.8.1.5
Re: [PATCH tip/core/rcu 1/3] rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL
In reply to Paul E. McKenneyQuoted 9 lines · Paul E. McKenney
From: "Paul E. McKenney" <[email protected]>
If
CONFIG_RCU_NOCB_CPU_ALL=y, thenrcu_is_nocb_cpu()will always return true, however, the current version nevertheless checksrcu_nocb_mask. This commit therefore creates a static inline implementation ofrcu_is_nocb_cpu()that unconditionally returns true whenCONFIG_RCU_NOCB_CPU_ALL=y.Signed-off-by Paul E. McKenney <[email protected]>
One suggestion below. With or without that change:
Quoted 10 lines
include/linux/rcupdate.h | 4 ++++ kernel/rcu/tree_plugin.h | 2 ++ 2 files changed, 6 insertions(+)include/linux/rcupdate.h +0
-0@@ -1016,7 +1016,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)__kfree_rcu (&((ptr)->rcu_head), offsetof(typeof (*(ptr )), rcu_head))
include/linux/rcupdate.h static inline notrace void rcu_read_unlock_sched_notrace(void) · line 1026
Quoted 10 lines
include/linux/rcupdate.h +4
-0#ifdef CONFIG_RCU_NOCB_CPU +#ifdef CONFIG_RCU_NOCB_CPU_ALL +static inline bool rcu_is_nocb_cpu (int cpu) {return true; } +#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */ bool rcu_is_nocb_cpu (int cpu); +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */ #else static inline bool rcu_is_nocb_cpu (int cpu) {return false; }#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
This seem clearer if written without the nesting:
#if defined(CONFIG_RCU_NOCB_CPU_ALL)
static inline bool rcu_is_nocb_cpu(int cpu) { return true; } #elif defined(CONFIG_RCU_NOCB_CPU)
bool rcu_is_nocb_cpu(int cpu);
#else
static inline bool rcu_is_nocb_cpu(int cpu) { return false; } #endif
That clearly distinguishes the three cases: all, some, none.
Quoted 23 lines
kernel/rcu/tree_plugin.h +2
-0@@ -2101,6 +2101,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_waitqueue_head (&rnp->nocb_gp_wq[1 ]); } +#ifndef CONFIG_RCU_NOCB_CPU_ALL /* Is the specified CPU a no-CPUs CPU? */ bool rcu_is_nocb_cpu (int cpu) {@@ -2108,6 +2109,7 @@ bool rcu_is_nocb_cpu(int cpu)return cpumask_test_cpu (cpu, rcu_nocb_mask); return false; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Enqueue the specified string of rcu_head structures onto the specified Signature · 1 line
1.8.1.5
Re: [PATCH tip/core/rcu 1/3] rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL
Quoted 9 lines · Josh Triplett
Quoted 8 lines · Paul E. McKenney
From: "Paul E. McKenney" <[email protected]>
If
CONFIG_RCU_NOCB_CPU_ALL=y, thenrcu_is_nocb_cpu()will always return true, however, the current version nevertheless checksrcu_nocb_mask. This commit therefore creates a static inline implementation ofrcu_is_nocb_cpu()that unconditionally returns true whenCONFIG_RCU_NOCB_CPU_ALL=y.
Quoted 4 lines
Quoted 1 line
Signed-off-by Paul E. McKenney <[email protected]>One suggestion below. With or without that change:
Reviewed-by Josh Triplett <[email protected]>
I like the change -- applied both it and your Reviewed-by.
Thanx, Paul
Quoted 57 lines
Quoted 10 lines
include/linux/rcupdate.h | 4 ++++ kernel/rcu/tree_plugin.h | 2 ++ 2 files changed, 6 insertions(+)include/linux/rcupdate.h +0
-0@@ -1016,7 +1016,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)__kfree_rcu (&((ptr)->rcu_head), offsetof(typeof (*(ptr )), rcu_head))include/linux/rcupdate.h static inline notrace void rcu_read_unlock_sched_notrace(void) · line 1026
Quoted 10 lines
include/linux/rcupdate.h +4
-0#ifdef CONFIG_RCU_NOCB_CPU +#ifdef CONFIG_RCU_NOCB_CPU_ALL +static inline bool rcu_is_nocb_cpu (int cpu) {return true; } +#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */ bool rcu_is_nocb_cpu (int cpu); +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */ #else static inline bool rcu_is_nocb_cpu (int cpu) {return false; }#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ This seem clearer if written without the nesting:
#if
defined(CONFIG_RCU_NOCB_CPU_ALL)
static inline boolrcu_is_nocb_cpu(int cpu){ return true; } #elifdefined(CONFIG_RCU_NOCB_CPU)
boolrcu_is_nocb_cpu(int cpu);
#else
static inline boolrcu_is_nocb_cpu(int cpu){ return false; } #endifThat clearly distinguishes the three cases: all, some, none.
Quoted 23 lines
kernel/rcu/tree_plugin.h +2
-0@@ -2101,6 +2101,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_waitqueue_head (&rnp->nocb_gp_wq[1 ]); } +#ifndef CONFIG_RCU_NOCB_CPU_ALL /* Is the specified CPU a no-CPUs CPU? */ bool rcu_is_nocb_cpu (int cpu) {@@ -2108,6 +2109,7 @@ bool rcu_is_nocb_cpu(int cpu)return cpumask_test_cpu (cpu, rcu_nocb_mask); return false; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Enqueue the specified string of rcu_head structures onto the specified Signature · 1 line
1.8.1.5