rcu: Make call_rcu() lazy only when CONFIG_RCU_LAZY is enabled
PATCH v3 1/1 rcu: Make call_rcu() lazy only when CONFIG_RCU_LAZY is enabled
Currently, regardless of whether the CONFIG_RCU_LAZY is enabled, invoke the call_rcu() is always lazy, it also means that when CONFIG_RCU_LAZY is disabled, invoke the call_rcu_flush() is also lazy. therefore, this commit make call_rcu() lazy only when CONFIG_RCU_LAZY is enabled.
---
v1->v2->v3:
Use IS_ENABLED(CONFIG_RCU_LAZY) to the existing function of the same name.
kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/tree.c +1 -1
@@ -2887,7 +2887,7 @@ EXPORT_SYMBOL_GPL(call_rcu_flush);*/ void call_rcu (struct rcu_head *head,rcu_callback_t func) { -return __call_rcu_common (head, func, true); +return __call_rcu_common (head, func,IS_ENABLED (CONFIG_RCU_LAZY )); }EXPORT_SYMBOL_GPL (call_rcu);
Signature · 1 line
2.25.1
Quoted 9 lines · Zqiang
Currently, regardless of whether the
CONFIG_RCU_LAZYis enabled, invoke thecall_rcu()is always lazy, it also means that whenCONFIG_RCU_LAZYis disabled, invoke thecall_rcu_flush()is also lazy. therefore, this commit makecall_rcu()lazy only whenCONFIG_RCU_LAZYis enabled.
Quoted 4 lines
Signed-off-by Zqiang <[email protected]>---
v1->v2->v3:
UseIS_ENABLED(CONFIG_RCU_LAZY)to the existing function of the same name.
Best,
- J
Quoted 19 lines
kernel/rcu/tree.c| 2 +-1 file changed, 1 insertion(+), 1 deletion(-)kernel/rcu/tree.c +1
-1@@ -2887,7 +2887,7 @@ EXPORT_SYMBOL_GPL(call_rcu_flush);*/ void call_rcu (struct rcu_head *head,rcu_callback_t func) { -return __call_rcu_common (head, func, true); +return __call_rcu_common (head, func,IS_ENABLED (CONFIG_RCU_LAZY )); }EXPORT_SYMBOL_GPL (call_rcu);Signature · 1 line
2.25.1
Quoted 7 lines · Zqiang
Currently, regardless of whether the
CONFIG_RCU_LAZYis enabled, invoke thecall_rcu()is always lazy, it also means that whenCONFIG_RCU_LAZYis disabled, invoke thecall_rcu_flush()is also lazy. therefore, this commit makecall_rcu()lazy only whenCONFIG_RCU_LAZYis enabled.Signed-off-by Zqiang <[email protected]>
Again, good eyes, and thank you!
I folded this into the original with attribution as shown below.
Thanx, Paul
------------------------------------------------------------------------
commit 65d030d037b06fc01b1670a9a0662b5d85ceb850
Author: Joel Fernandes (Google) <[email protected]> Date: Sun Oct 16 16:22:54 2022 +0000
rcu: Make call_rcu() lazy to save power
Implement timer-based RCU callback batching (also known as lazy callbacks). With this we save about 5-10% of power consumed due to RCU requests that happen when system is lightly loaded or idle.
By default, all async callbacks (queued via call_rcu) are marked lazy. An alternate API call_rcu_flush() is provided for the few users, for example synchronize_rcu(), that need the old behavior.
The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as rcu_barrier() and hotplug), we re-use bypass lists which were originally introduced to address lock contention, to handle lazy CBs as well. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
[ paulmck: Fix formatting of inline call_rcu_lazy() definition. ] [ paulmck: Apply Zqiang feedback. ]
include/linux/rcupdate.h +9 -0
@@ -108,6 +108,15 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head,rcu_callback_t func) +{ +call_rcu (head, func); +} +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;
kernel/rcu/Kconfig +8 -0
@@ -311,4 +311,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure. +config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure, or callback list growing too big. + endmenu # "RCU Subsystem"
kernel/rcu/rcu.h +8 -0
@@ -474,6 +474,14 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; +#if defined(CONFIG_RCU_LAZY ) +unsigned long rcu_lazy_get_jiffies_till_flush (void ); +void rcu_lazy_set_jiffies_till_flush (unsigned long j); +#else +static inline unsigned long rcu_lazy_get_jiffies_till_flush (void ) {return 0 ; } +static inline void rcu_lazy_set_jiffies_till_flush (unsigned long j) { } +#endif +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq);
kernel/rcu/tiny.c +1 -1
@@ -44,7 +44,7 @@ static struct rcu_ctrlblk rcu_ctrlblk = {void rcu_barrier (void ) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush); }EXPORT_SYMBOL (rcu_barrier);
kernel/rcu/tree.c +83 -46
166 lines not on this page · raw message
kernel/rcu/tree.h +7 -4
@@ -263,14 +263,16 @@ struct rcu_data {unsigned long last_fqs_resched;/* Time of last rcu_resched(). */ unsigned long last_sched_clock;/* Jiffies of last rcu_sched_clock_irq(). */ +long lazy_len;/* Length of buffered lazy callbacks. */ int cpu; };/* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0#define RCU_NOCB_WAKE_BYPASS 1 -#define RCU_NOCB_WAKE 2 -#define RCU_NOCB_WAKE_FORCE 3 +#define RCU_NOCB_WAKE_LAZY 2 +#define RCU_NOCB_WAKE 3 +#define RCU_NOCB_WAKE_FORCE 4#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))/* For jiffies_till_first_fqs and */
@@ -441,9 +443,10 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp);static bool wake_nocb_gp (struct rcu_data *rdp,bool force);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j); +unsigned long j,bool lazy);static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags); +bool *was_alldone,unsigned long flags, +bool lazy);static void __call_rcu_nocb_wake (struct rcu_data *rdp,bool was_empty,unsigned long flags);static int rcu_nocb_need_deferred_wakeup (struct rcu_data *rdp,int level);
kernel/rcu/tree_exp.h +1 -1
@@ -937,7 +937,7 @@ void synchronize_rcu_expedited(void)/* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }
kernel/rcu/tree_nocb.h +129 -30
323 lines not on this page · raw message