RT latency optimizations
cover RT latency optimizations
Hello!
This series provides some minor latency optimizations for kernels that build with CONFIG_RCU_NOCB_CPU_ALL=y:
1. Create a static inline implementation of rcu_is_nocb_cpu() that
unconditionally returns true when CONFIG_RCU_NOCB_CPU_ALL=y.
2. Create a static inline implementation of rcu_needs_cpu() that
unconditionally returns false when CONFIG_RCU_NOCB_CPU_ALL=y.
3. Create static inline implementations of rcu_prepare_for_idle()
and rcu_cleanup_after_idle() that are no-ops when CONFIG_RCU_NOCB_CPU_ALL=y.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/rcupdate.h | 13 +++++++++++++ b/include/linux/rcutiny.h | 6 ------ b/include/linux/rcutree.h | 2 ++ b/kernel/rcu/tree.c | 2 +- b/kernel/rcu/tree_plugin.h | 13 +++++++++++-- 5 files changed, 27 insertions(+), 9 deletions(-)
PATCH 3/3 Optimize RCU_FAST_NO_HZ for RCU_NOCB_CPU_ALL
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
If CONFIG_RCU_NOCB_CPU_ALL=y, then no CPU will ever have RCU callbacks because these callbacks will instead be handled by the rcuo kthreads. However, the current version of RCU_FAST_NO_HZ nevertheless checks for RCU callbacks. This commit therefore creates static inline implementations of rcu_prepare_for_idle() and rcu_cleanup_after_idle() that are no-ops when CONFIG_RCU_NOCB_CPU_ALL=y.
--- kernel/rcu/tree_plugin.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
kernel/rcu/tree_plugin.h +5 -2
@@ -1647,7 +1647,7 @@ extern int tick_nohz_enabled;* only if it has been awhile since the last time we did so. Afterwards, * if there are any callbacks ready for immediate invocation, return true. */ -static bool rcu_try_advance_all_cbs (void ) +static bool __maybe_unused rcu_try_advance_all_cbs (void ) {bool cbs_ready = false;struct rcu_data *rdp;
@@ -1732,6 +1732,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)*/ static void rcu_prepare_for_idle (int cpu) { +#ifndef CONFIG_RCU_NOCB_CPU_ALL struct rcu_data *rdp;struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);struct rcu_node *rnp;
@@ -1782,6 +1783,7 @@ static void rcu_prepare_for_idle(int cpu)rcu_accelerate_cbs (rsp, rnp, rdp); raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ }/*
@@ -1791,11 +1793,12 @@ static void rcu_prepare_for_idle(int cpu)*/ static void rcu_cleanup_after_idle (int cpu) { - +#ifndef CONFIG_RCU_NOCB_CPU_ALL if (rcu_is_nocb_cpu (cpu))return ;if (rcu_try_advance_all_cbs ())invoke_rcu_core (); +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ }/*
Signature · 1 line
1.8.1.5
PATCH 2/3 rcu: Optimize rcu_needs_cpu() for RCU_NOCB_CPU_ALL
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_needs_cpu() will always return false, however, the current version nevertheless checks for RCU callbacks. This commit therefore creates a static inline implementation of rcu_needs_cpu() that unconditionally returns false when CONFIG_RCU_NOCB_CPU_ALL=y.
--- include/linux/rcupdate.h | 9 +++++++++ include/linux/rcutiny.h | 6 ------ include/linux/rcutree.h | 2 ++ kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 4 ++++ 5 files changed, 16 insertions(+), 7 deletions(-)
include/linux/rcupdate.h +9 -0
@@ -1027,6 +1027,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)#define kfree_rcu (ptr, rcu_head) \ __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) +#if defined(CONFIG_TINY_RCU ) || defined(CONFIG_RCU_NOCB_CPU_ALL ) + +static inline int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) +{ + *delta_jiffies =ULONG_MAX ; +return 0 ; +} +#endif /* #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) */ +#ifdef CONFIG_RCU_NOCB_CPU #ifdef CONFIG_RCU_NOCB_CPU_ALL static inline bool rcu_is_nocb_cpu (int cpu) {return true; }
include/linux/rcutiny.h +0 -6
@@ -68,12 +68,6 @@ static inline void kfree_call_rcu(struct rcu_head *head,call_rcu (head, func); } -static inline int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) -{ - *delta_jiffies =ULONG_MAX ; -return 0 ; -} -static inline void rcu_note_context_switch (int cpu) {rcu_sched_qs (cpu);
include/linux/rcutree.h +2 -0
@@ -31,7 +31,9 @@#define __LINUX_RCUTREE_Hvoid rcu_note_context_switch (int cpu); +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies); +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ void rcu_cpu_stall_reset (void );/*
kernel/rcu/tree.c +1 -1
@@ -2851,7 +2851,7 @@ static int rcu_pending(int cpu)* non-NULL, store an indication of whether all callbacks are lazy. * (If there are no callbacks, all of them are deemed to be lazy.) */ -static int rcu_cpu_has_callbacks (int cpu,bool *all_lazy) +static int __maybe_unused rcu_cpu_has_callbacks (int cpu,bool *all_lazy) {bool al = true;bool hc = false;
kernel/rcu/tree_plugin.h +4 -0
@@ -1575,11 +1575,13 @@ static void rcu_prepare_kthreads(int cpu)* Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs * any flavor of RCU. */ +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *delta_jiffies) { *delta_jiffies =ULONG_MAX ;return rcu_cpu_has_callbacks (cpu,NULL ); } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
@@ -1685,6 +1687,7 @@ static bool rcu_try_advance_all_cbs(void)* * The caller must have disabled interrupts. */ +#ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu (int cpu,unsigned long *dj) {struct rcu_dynticks *rdtp = &per_cpu (rcu_dynticks, cpu);
@@ -1715,6 +1718,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj) }return 0 ; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Prepare a CPU for idle from an RCU perspective. The first major task
Signature · 1 line
1.8.1.5
PATCH 1/3 rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_is_nocb_cpu() will always return true, however, the current version nevertheless checks rcu_nocb_mask. This commit therefore creates a static inline implementation of rcu_is_nocb_cpu() that unconditionally returns true when CONFIG_RCU_NOCB_CPU_ALL=y.
--- include/linux/rcupdate.h | 4 ++++ kernel/rcu/tree_plugin.h | 2 ++ 2 files changed, 6 insertions(+)
include/linux/rcupdate.h +4 -0
@@ -1028,7 +1028,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))#ifdef CONFIG_RCU_NOCB_CPU +#ifdef CONFIG_RCU_NOCB_CPU_ALL +static inline bool rcu_is_nocb_cpu (int cpu) {return true; } +#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */ bool rcu_is_nocb_cpu (int cpu); +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */ #else static inline bool rcu_is_nocb_cpu (int cpu) {return false; }#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
kernel/rcu/tree_plugin.h +2 -0
@@ -2089,6 +2089,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_waitqueue_head (&rnp->nocb_gp_wq[1 ]); } +#ifndef CONFIG_RCU_NOCB_CPU_ALL /* Is the specified CPU a no-CPUs CPU? */ bool rcu_is_nocb_cpu (int cpu) {
@@ -2096,6 +2097,7 @@ bool rcu_is_nocb_cpu(int cpu)return cpumask_test_cpu (cpu, rcu_nocb_mask); return false; } +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ /* * Enqueue the specified string of rcu_head structures onto the specified
Signature · 1 line
1.8.1.5