rcu: call_rcu() power improvements
Subsystems (6)
v6 cover rcu: call_rcu() power improvements
v6 version of RCU lazy patches based on rcu/next branch.
To facilitate easier merge, I dropped tracing and other patches and just implemented the new changes. I will post the tracing patches later along with rcutop as I need to add new tracepoints that Frederic suggested.
Main changes:
1. rcu_barrier() wake up only for lazy bypass list.
2. Make all call_rcu() default-lazy and add call_rcu_flush() API.
3. Take care of some callers using call_rcu_flush() API.
I am seeing good performance and power with these patches on real ChromeOS x86 asymmetric hardware.
v5 cover letter is here:
https://lore.kernel.org/all/[email protected]/
Joel Fernandes (Google) (3):
rcu: Make call_rcu() lazy to save power
rcuscale: Add laziness and kfree tests
percpu-refcount: Use call_rcu_flush() for atomic switch
Vineeth Pillai (1):
rcu: shrinker for lazy rcu
include/linux/rcupdate.h | 7 ++ kernel/rcu/Kconfig | 8 ++ kernel/rcu/rcu.h | 8 ++ kernel/rcu/rcuscale.c | 65 ++++++++++- kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 133 ++++++++++++++-------- kernel/rcu/tree.h | 17 ++- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_nocb.h | 236 +++++++++++++++++++++++++++++++++------ lib/percpu-refcount.c | 3 +- 10 files changed, 395 insertions(+), 86 deletions(-)
--
2.37.3.998.g577e59143f-goog
PATCH v6 1/4 rcu: Make call_rcu() lazy to save power
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as rcu_barrier() and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:
[ Frederic Weisbec: Program the lazy timer only if WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ]
[ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make rcu_barrier() wake up only if main list empty. ]
[ Frederic Weisbec: Remove extra 'else if' branch in rcu_nocb_try_bypass(). ]
[ Joel: Fix issue where I was not resetting lazy_len after moving it to rdp ]
[ Paul/Thomas/Joel: Make call_rcu() default lazy so users don't mess up. ]
--- include/linux/rcupdate.h | 7 ++ kernel/rcu/Kconfig | 8 ++ kernel/rcu/rcu.h | 8 ++ kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 133 ++++++++++++++++++---------- kernel/rcu/tree.h | 17 +++- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_nocb.h | 184 ++++++++++++++++++++++++++++++++------- 8 files changed, 277 insertions(+), 84 deletions(-)
include/linux/rcupdate.h +7 -0
@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;
kernel/rcu/Kconfig +8 -0
@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure. +config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. + endmenu # "RCU Subsystem"
kernel/rcu/rcu.h +8 -0
@@ -474,6 +474,14 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; +#if defined(CONFIG_RCU_LAZY ) +unsigned long rcu_lazy_get_jiffies_till_flush (void ); +void rcu_lazy_set_jiffies_till_flush (unsigned long j); +#else +static inline unsigned long rcu_lazy_get_jiffies_till_flush (void ) {return 0 ; } +static inline void rcu_lazy_set_jiffies_till_flush (unsigned long j) { } +#endif +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq);
kernel/rcu/tiny.c +1 -1
@@ -44,7 +44,7 @@ static struct rcu_ctrlblk rcu_ctrlblk = {void rcu_barrier (void ) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush); }EXPORT_SYMBOL (rcu_barrier);
kernel/rcu/tree.c +87 -46
170 lines not on this page · raw message
kernel/rcu/tree.h +13 -4
@@ -263,14 +263,16 @@ struct rcu_data {unsigned long last_fqs_resched;/* Time of last rcu_resched(). */ unsigned long last_sched_clock;/* Jiffies of last rcu_sched_clock_irq(). */ +long lazy_len;/* Length of buffered lazy callbacks. */ int cpu; };/* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0#define RCU_NOCB_WAKE_BYPASS 1 -#define RCU_NOCB_WAKE 2 -#define RCU_NOCB_WAKE_FORCE 3 +#define RCU_NOCB_WAKE_LAZY 2 +#define RCU_NOCB_WAKE 3 +#define RCU_NOCB_WAKE_FORCE 4#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))/* For jiffies_till_first_fqs and */
@@ -439,10 +441,17 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp); + +#define FLUSH_BP_NONE 0 +/* Is the CB being enqueued after the flush, a lazy CB? */ +#define FLUSH_BP_LAZY BIT(0) +/* Wake up nocb-GP thread after flush? */ +#define FLUSH_BP_WAKE BIT(1)static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j); +unsigned long j,unsigned long flush_flags);static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags); +bool *was_alldone,unsigned long flags, +bool lazy);static void __call_rcu_nocb_wake (struct rcu_data *rdp,bool was_empty,unsigned long flags);static int rcu_nocb_need_deferred_wakeup (struct rcu_data *rdp,int level);
kernel/rcu/tree_exp.h +1 -1
@@ -937,7 +937,7 @@ void synchronize_rcu_expedited(void)/* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }
kernel/rcu/tree_nocb.h +152 -32
346 lines not on this page · raw message
Signature · 1 line
2.37.3.998.g577e59143f-goog
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 19 lines · Joel Fernandes (Google)
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ]
Quoted 8 lines
[ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]Suggested-by Paul McKenney <[email protected]>Signed-off-by Joel Fernandes (Google) <[email protected]>
I am going to put these on a local branch for testing, but a few comments and questions interspersed below.
Thanx, Paul
Quoted 153 lines
--- include/linux/rcupdate.h | 7 ++ kernel/rcu/Kconfig | 8 ++ kernel/rcu/rcu.h | 8 ++ kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 133 ++++++++++++++++++---------- kernel/rcu/tree.h | 17 +++- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_nocb.h | 184 ++++++++++++++++++++++++++++++++------- 8 files changed, 277 insertions(+), 84 deletions(-)include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +8
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure. +config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. + endmenu # "RCU Subsystem"kernel/rcu/rcu.h +8
-0@@ -474,6 +474,14 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; +#if defined(CONFIG_RCU_LAZY ) +unsigned long rcu_lazy_get_jiffies_till_flush (void ); +void rcu_lazy_set_jiffies_till_flush (unsigned long j); +#else +static inline unsigned long rcu_lazy_get_jiffies_till_flush (void ) {return 0 ; } +static inline void rcu_lazy_set_jiffies_till_flush (unsigned long j) { } +#endif +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq);kernel/rcu/tiny.c +1
-1@@ -44,7 +44,7 @@ static struct rcu_ctrlblk rcu_ctrlblk = {void rcu_barrier (void ) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush); }EXPORT_SYMBOL (rcu_barrier);kernel/rcu/tree.c +9
-43@@ -2728,47 +2728,8 @@ static void check_cb_ovld(struct rcu_data *rdp)raw_spin_unlock_rcu_node (rnp); } -/** -* call_rcu() - Queue an RCU callback for invocation after a grace period. -* @head: structure to be used for queueing the RCU updates. -* @func: actual callback function to be invoked after the grace period -* -* The callback function will be invoked some time after a full grace -* period elapses, in other words after all pre-existing RCU read-side -* critical sections have completed. However, the callback function -* might well execute concurrently with RCU read-side critical sections -* that started after call_rcu() was invoked. -* -* RCU read-side critical sections are delimited by rcu_read_lock() -* and rcu_read_unlock(), and may be nested. In addition, but only in -* v5.0 and later, regions of code across which interrupts, preemption, -* or softirqs have been disabled also serve as RCU read-side critical -* sections. This includes hardware interrupt handlers, softirq handlers, -* and NMI handlers. -* -* Note that all CPUs must agree that the grace period extended beyond -* all pre-existing RCU read-side critical section. On systems with more -* than one CPU, this means that when "func()" is invoked, each CPU is -* guaranteed to have executed a full memory barrier since the end of its -* last RCU read-side critical section whose beginning preceded the call -* to call_rcu(). It also means that each CPU executing an RCU read-side -* critical section that continues beyond the start of "func()" must have -* executed a memory barrier after the call_rcu() but before the beginning -* of that RCU read-side critical section. Note that these guarantees -* include CPUs that are offline, idle, or executing in user mode, as -* well as CPUs that are executing in the kernel. -* -* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the -* resulting RCU callback function "func()", then both CPU A and CPU B are -* guaranteed to execute a full memory barrier during the time interval -* between the call to call_rcu() and the invocation of "func()" -- even -* if CPU A and CPU B are the same CPU (but again only if the system has -* more than one CPU). -* -* Implementation of these memory-ordering guarantees is described here: -* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. -*/ -void call_rcu (struct rcu_head *head,rcu_callback_t func) +static void +__call_rcu_common (struct rcu_head *head,rcu_callback_t func,bool lazy) {static atomic_t doublefrees;unsigned long flags;@@ -2809,7 +2770,7 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) }check_cb_ovld (rdp); -if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags)) +if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags, lazy))return ;// Enqueued onto ->nocb_bypass, so just leave. // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock. rcu_segcblist_enqueue (&rdp->cblist, head);@@ -2831,8 +2792,84 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func)local_irq_restore (flags); } } -EXPORT_SYMBOL_GPL (call_rcu); +#ifdef CONFIG_RCU_LAZY +/** +* call_rcu_flush() - Queue RCU callback for invocation after grace period, and +* flush all lazy callbacks (including the new one) to the main ->cblist while +* doing so. +*
kernel/rcu/tree.c void call_rcu(struct rcu_head *head, rcu_callback_t func) · line 2811
Quoted 10 lines
kernel/rcu/tree.c +10
-0+* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side +* critical sections have completed. +* +* Use this API instead of call_rcu() if you don't mind the callback being +* invoked after very long periods of time on systems without memory pressure +* and on systems which are lightly loaded or mostly idle.
This comment is backwards, right? Shouldn't it say something like "Use this API instead of call_rcu() if you don't mind burning extra power..."?
Quoted 100 lines
kernel/rcu/tree.c +68
-3+* +* Other than the extra delay in callbacks being invoked, this function is +* identical to, and reuses call_rcu()'s logic. Refer to call_rcu() for more +* details about memory ordering and other functionality. +*/ +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, false); +} +EXPORT_SYMBOL_GPL (call_rcu_flush); +#endif + +/** +* call_rcu() - Queue an RCU callback for invocation after a grace period. +* By default the callbacks are 'lazy' and are kept hidden from the main +* ->cblist to prevent starting of grace periods too soon. +* If you desire grace periods to start very soon, use call_rcu_flush(). +* +* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side +* critical sections have completed. However, the callback function +* might well execute concurrently with RCU read-side critical sections +* that started after call_rcu() was invoked. +* +* RCU read-side critical sections are delimited by rcu_read_lock() +* and rcu_read_unlock(), and may be nested. In addition, but only in +* v5.0 and later, regions of code across which interrupts, preemption, +* or softirqs have been disabled also serve as RCU read-side critical +* sections. This includes hardware interrupt handlers, softirq handlers, +* and NMI handlers. +* +* Note that all CPUs must agree that the grace period extended beyond +* all pre-existing RCU read-side critical section. On systems with more +* than one CPU, this means that when "func()" is invoked, each CPU is +* guaranteed to have executed a full memory barrier since the end of its +* last RCU read-side critical section whose beginning preceded the call +* to call_rcu(). It also means that each CPU executing an RCU read-side +* critical section that continues beyond the start of "func()" must have +* executed a memory barrier after the call_rcu() but before the beginning +* of that RCU read-side critical section. Note that these guarantees +* include CPUs that are offline, idle, or executing in user mode, as +* well as CPUs that are executing in the kernel. +* +* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the +* resulting RCU callback function "func()", then both CPU A and CPU B are +* guaranteed to execute a full memory barrier during the time interval +* between the call to call_rcu() and the invocation of "func()" -- even +* if CPU A and CPU B are the same CPU (but again only if the system has +* more than one CPU). +* +* Implementation of these memory-ordering guarantees is described here: +* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. +*/ +void call_rcu (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, true); +} +EXPORT_SYMBOL_GPL (call_rcu);/* Maximum number of jiffies to wait before draining a batch. */ #define KFREE_DRAIN_JIFFIES (5 * HZ)@@ -3507,7 +3544,7 @@ void synchronize_rcu(void)if (rcu_gp_is_expedited ())synchronize_rcu_expedited ();else -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -4323,7 +4364,7 @@ void rcutree_migrate_callbacks(int cpu) my_rdp =this_cpu_ptr (&rcu_data); my_rnp = my_rdp->mynode;rcu_nocb_lock (my_rdp);/* irqs already disabled. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies,FLUSH_BP_NONE ));raw_spin_lock_rcu_node (my_rnp);/* irqs already disabled. */ /* Leverage recent GPs and set GP for new callbacks. */ needwake = rcu_advance_cbs(my_rnp, rdp) ||kernel/rcu/tree.h +0
-0
kernel/rcu/tree.h struct rcu_data { · line 266
Quoted 4 lines
kernel/rcu/tree.h +1
-0@@ -263,14 +263,16 @@ struct rcu_data {unsigned long last_fqs_resched;/* Time of last rcu_resched(). */ unsigned long last_sched_clock;/* Jiffies of last rcu_sched_clock_irq(). */ +long lazy_len;/* Length of buffered lazy callbacks. */
Do we ever actually care about the length as opposed to whether or not all the bypass callbacks are lazy? If not, a "some_nonlazy" boolean would be initialed to zero and ORed with the non-laziness of the added callback. Or, if there was a test anyway, simply set to 1 in the presence of a non-lazy callback. And as now, gets zeroed when the bypass is flushed.
This might shorten a few lines of code.
Quoted 85 lines
kernel/rcu/tree.h +12
-4int cpu; };/* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0#define RCU_NOCB_WAKE_BYPASS 1 -#define RCU_NOCB_WAKE 2 -#define RCU_NOCB_WAKE_FORCE 3 +#define RCU_NOCB_WAKE_LAZY 2 +#define RCU_NOCB_WAKE 3 +#define RCU_NOCB_WAKE_FORCE 4#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))/* For jiffies_till_first_fqs and */ @@ -439,10 +441,17 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp); + +#define FLUSH_BP_NONE 0 +/* Is the CB being enqueued after the flush, a lazy CB? */ +#define FLUSH_BP_LAZY BIT(0) +/* Wake up nocb-GP thread after flush? */ +#define FLUSH_BP_WAKE BIT(1)static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j); +unsigned long j,unsigned long flush_flags);static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags); +bool *was_alldone,unsigned long flags, +bool lazy);static void __call_rcu_nocb_wake (struct rcu_data *rdp,bool was_empty,unsigned long flags);static int rcu_nocb_need_deferred_wakeup (struct rcu_data *rdp,int level);kernel/rcu/tree_exp.h +1
-1@@ -937,7 +937,7 @@ void synchronize_rcu_expedited(void)/* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }kernel/rcu/tree_nocb.h +25
-0@@ -256,6 +256,31 @@ static bool wake_nocb_gp(struct rcu_data *rdp, bool force)return __wake_nocb_gp (rdp_gp, rdp, force, flags); } +/* +* LAZY_FLUSH_JIFFIES decides the maximum amount of time that +* can elapse before lazy callbacks are flushed. Lazy callbacks +* could be flushed much earlier for a number of other reasons +* however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are +* left unsubmitted to RCU after those many jiffies. +*/ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush =LAZY_FLUSH_JIFFIES ; + +#ifdef CONFIG_RCU_LAZY +// To be called only from test code. +void rcu_lazy_set_jiffies_till_flush (unsigned long jif) +{ + jiffies_till_flush = jif; +} +EXPORT_SYMBOL (rcu_lazy_set_jiffies_till_flush); + +unsigned long rcu_lazy_get_jiffies_till_flush (void ) +{ +return jiffies_till_flush; +} +EXPORT_SYMBOL (rcu_lazy_get_jiffies_till_flush); +#endif +/* * Arrange to wake the GP kthread for this NOCB group at some future * time when it is safe to do so. @@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);
kernel/rcu/tree_nocb.h static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 10 lines
kernel/rcu/tree_nocb.h +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {
Please leave the "&&" on the previous line and line up the "READ_ONCE(" with the "waketype". That makes it easier to tell the condition from the following code.
Quoted 25 lines
kernel/rcu/tree_nocb.h +8
-1+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -293,12 +322,16 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* proves to be initially empty, just return false because the no-CB GP * kthread may need to be awakened in this case. * +* Return true if there was something to be flushed and it succeeded, otherwise +* false. +* * Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +bool lazy = flush_flags &FLUSH_BP_LAZY ;WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));rcu_lockdep_assert_cblist_protected (rdp);
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 349
Quoted 8 lines
kernel/rcu/tree_nocb.h +4
-1@@ -310,7 +343,20 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,/* Note: ->cblist.len already accounts for ->nocb_bypass contents. */ if (rhp)rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); + +/* +* If the new CB requested was a lazy one, queue it onto the main +* ->cblist so we can take advantage of a sooner grade period.
"take advantage of a grace period that will happen regardless."?
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 353
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+*/ +if (lazy && rhp) { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); +rcu_cblist_enqueue (&rcl, rhp);
Would it makes sense to enqueue rhp onto ->nocb_bypass first, NULL out rhp, then let the rcu_cblist_flush_enqueue() be common code? Or did this function grow a later use of rhp that I missed?
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 357
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );
This WRITE_ONCE() can be dropped out of the "if" statement, correct?
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
Quoted 6 lines
kernel/rcu/tree_nocb.h +2
-0+ } +rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);rcu_nocb_bypass_unlock (rdp);
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 380
Quoted 10 lines
kernel/rcu/tree_nocb.h +5
-1@@ -326,13 +372,33 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) { +bool ret; +bool was_alldone = false; +bool bypass_all_lazy = false; +long bypass_ncbs;
Alphabetical order by variable name, please. (Yes, I know that this is strange, but what can I say?)
Quoted 86 lines
kernel/rcu/tree_nocb.h +30
-7+if (!rcu_rdp_is_offloaded (rdp))return true;rcu_lockdep_assert_cblist_protected (rdp);rcu_nocb_bypass_lock (rdp); -return rcu_nocb_do_flush_bypass (rdp, rhp, j); + +if (flush_flags &FLUSH_BP_WAKE ) { + was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + bypass_all_lazy = bypass_ncbs && (bypass_ncbs == rdp->lazy_len); + } + + ret =rcu_nocb_do_flush_bypass (rdp, rhp, j, flush_flags); + +// Wake up the nocb GP thread if needed. GP thread could be sleeping +// while waiting for lazy timer to expire (otherwise rcu_barrier may +// end up waiting for the duration of the lazy timer). +if (flush_flags &FLUSH_BP_WAKE && was_alldone && bypass_all_lazy) +wake_nocb_gp (rdp, false); + +return ret; }/* @@ -345,7 +411,7 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)if (!rcu_rdp_is_offloaded (rdp) || !rcu_nocb_bypass_trylock (rdp))return ; -WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j)); +WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE )); }/* @@ -367,12 +433,14 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)* there is only one CPU in operation. */ static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags, +bool lazy) {unsigned long c;unsigned long cur_gp_seq;unsigned long j = jiffies;long ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); +bool bypass_is_lazy = (ncbs ==READ_ONCE (rdp->lazy_len));lockdep_assert_irqs_disabled ();@@ -417,25 +485,30 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// If there hasn't yet been all that many ->cblist enqueues // this jiffy, tell the caller to enqueue onto ->cblist. But flush // ->nocb_bypass first. -if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { +// Lazy CBs throttle this back and do immediate bypass queuing. +if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist);if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" )); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j)); + +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE ));WARN_ON_ONCE (rcu_cblist_n_cbs (&rdp->nocb_bypass));return false;// Caller must enqueue the callback. }// If ->nocb_bypass has been used too long or is too full, // flush ->nocb_bypass to ->cblist. -if ((ncbs && j !=READ_ONCE (rdp->nocb_bypass_first)) || +if ((ncbs && !bypass_is_lazy && j !=READ_ONCE (rdp->nocb_bypass_first)) || + (ncbs && bypass_is_lazy && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush))) || ncbs >= qhimark) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); -if (!rcu_nocb_flush_bypass (rdp, rhp, j)) { +if (!rcu_nocb_flush_bypass (rdp, rhp, j, + lazy ?FLUSH_BP_LAZY :FLUSH_BP_NONE )) {if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" ));
kernel/rcu/tree_nocb.h static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 541
Quoted 9 lines
kernel/rcu/tree_nocb.h +3
-0@@ -460,16 +533,29 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// We need to use the bypass. rcu_nocb_wait_contended (rdp);rcu_nocb_bypass_lock (rdp); + ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass);rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ rcu_cblist_enqueue (&rdp->nocb_bypass, rhp); + +if (IS_ENABLED (CONFIG_RCU_LAZY ) && lazy)
Won't !IS_ENABLED(CONFIG_RCU_LAZY) mean that lazy cannot be set? Why do we need to check both? Or are you going for dead code? If so, shouldn't there be IS_ENABLED(CONFIG_RCU_LAZY) checks above as well?
Me, I am not convinced that the dead code would buy you much. In fact, the compiler might well be propagating the constants on its own.
Ah! The reason the compiler cannot figure this out is because you put the switch into rcu.h. If you instead always export the call_rcu_flush() definition, and check IS_ENABLED(CONFIG_RCU_LAZY) at the beginning of call_rcu(), the compiler should have the information that it needs to do this for you.
Quoted 22 lines
kernel/rcu/tree_nocb.h +11
-1+WRITE_ONCE (rdp->lazy_len, rdp->lazy_len +1 ); +if (!ncbs) {WRITE_ONCE (rdp->nocb_bypass_first, j);trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstBQ" )); } +rcu_nocb_bypass_unlock (rdp);smp_mb ();/* Order enqueue before wake. */ -if (ncbs) { + +// A wake up of the grace period kthread or timer adjustment needs to +// be done only if: +// 1. Bypass list was fully empty before (this is the first bypass list entry). +// Or, both the below conditions are met: +// 1. Bypass list had only lazy CBs before. +// 2. The new CB is non-lazy. +if (ncbs && (!bypass_is_lazy || lazy)) {local_irq_restore (flags); }else {// No-CBs GP kthread might be indefinitely asleep, if so, wake.
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 589
Quoted 6 lines
kernel/rcu/tree_nocb.h +1
-1@@ -499,7 +585,7 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, {unsigned long cur_gp_seq;unsigned long j; -long len; +long len, lazy_len, bypass_len;struct task_struct *t;
Again, alphabetical please, strange though that might seem.
Quoted 20 lines
kernel/rcu/tree_nocb.h +8
-1// If we are being polled or there is no kthread, just leave. @@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist); + bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" )); + }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);
kernel/rcu/tree_nocb.h static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · line 701
Quoted 7 lines
kernel/rcu/tree_nocb.h +2
-2@@ -604,8 +697,8 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)*/ static void nocb_gp_wait (struct rcu_data *my_rdp) { -bool bypass = false; -long bypass_ncbs; +bool bypass = false, lazy = false; +long bypass_ncbs, lazy_ncbs;
And ditto.
Quoted 55 lines
kernel/rcu/tree_nocb.h +23
-6int __maybe_unused cpu = my_rdp->cpu;unsigned long cur_gp_seq;unsigned long flags;@@ -640,24 +733,41 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)* won't be ignored for long. */ list_for_each_entry (rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { +bool flush_bypass = false; +trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("Check" ));rcu_nocb_lock_irqsave (rdp, flags);lockdep_assert_held (&rdp->nocb_lock); bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); -if (bypass_ncbs && + lazy_ncbs =READ_ONCE (rdp->lazy_len); + +if (bypass_ncbs && (lazy_ncbs == bypass_ncbs) && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush) || + bypass_ncbs >2 * qhimark)) { + flush_bypass = true; + }else if (bypass_ncbs && (lazy_ncbs != bypass_ncbs) && (time_after (j,READ_ONCE (rdp->nocb_bypass_first) +1 ) || bypass_ncbs >2 * qhimark)) { -// Bypass full or old, so flush it. - (void )rcu_nocb_try_flush_bypass (rdp, j); - bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + flush_bypass = true; }else if (!bypass_ncbs &&rcu_segcblist_empty (&rdp->cblist)) {rcu_nocb_unlock_irqrestore (rdp, flags);continue ;/* No callbacks here, try next. */ } + +if (flush_bypass) { +// Bypass full or old, so flush it. + (void )rcu_nocb_try_flush_bypass (rdp, j); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_ncbs =READ_ONCE (rdp->lazy_len); + } +if (bypass_ncbs) {trace_rcu_nocb_wake (rcu_state.name, rdp->cpu, -TPS ("Bypass" )); - bypass = true; + bypass_ncbs == lazy_ncbs ?TPS ("Lazy" ) :TPS ("Bypass" )); +if (bypass_ncbs == lazy_ncbs) + lazy = true; +else + bypass = true; } rnp = rdp->mynode;@@ -705,12 +815,21 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_gp = needwait_gp; my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq :0 ;
kernel/rcu/tree_nocb.h static void nocb_gp_wait(struct rcu_data *my_rdp) · line 822
Quoted 10 lines
kernel/rcu/tree_nocb.h +5
-5-if (bypass && !rcu_nocb_poll) { -// At least one child with non-empty ->nocb_bypass, so set -// timer in order to avoid stranding its callbacks. -wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , -TPS ("WakeBypassIsDeferred" )); +// At least one child with non-empty ->nocb_bypass, so set +// timer in order to avoid stranding its callbacks. +if (!rcu_nocb_poll) { +// If bypass list only has lazy CBs. Add a deferred +// lazy wake up.
One sentence rather than two.
Quoted 49 lines
kernel/rcu/tree_nocb.h +13
-3+if (lazy && !bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazyIsDeferred" )); +// Otherwise add a deferred bypass wake up. + }else if (bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , +TPS ("WakeBypassIsDeferred" )); + } } +if (rcu_nocb_poll) {/* Polling, so trace if first poll in the series. */ if (gotcbs)@@ -1036,7 +1155,7 @@ static long rcu_nocb_rdp_deoffload(void *arg)* return false, which means that future calls to rcu_nocb_try_bypass() * will refuse to put anything into the bypass. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_NONE ));/* * Start with invoking rcu_core() early. This way if the current thread * happens to preempt an ongoing call to rcu_core() in the middle, @@ -1278,6 +1397,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)raw_spin_lock_init (&rdp->nocb_gp_lock);timer_setup (&rdp->nocb_timer, do_nocb_deferred_wakeup_timer,0 );rcu_cblist_init (&rdp->nocb_bypass); +WRITE_ONCE (rdp->lazy_len,0 );mutex_init (&rdp->nocb_gp_kthread_mutex); }@@ -1559,13 +1679,13 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) }static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {return true; }static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags,bool lazy) {return false; }Signature · 1 line
2.37.3.998.g577e59143f-goog
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Hi Paul,
Let’s see whether my iPhone can handle replies ;-)
More below:
Quoted 28 lines · Paul E. McKenney
Quoted 25 lines · Joel Fernandes (Google)
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ][ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]
Quoted 7 lines
Quoted 2 lines
Suggested-by Paul McKenney <[email protected]>Signed-off-by Joel Fernandes (Google) <[email protected]>I am going to put these on a local branch for testing, but a few comments and questions interspersed below.
Thanx, Paul
Ok I replied below, thank you!
Quoted 157 lines
Quoted 157 lines
include/linux/rcupdate.h | 7 ++ kernel/rcu/Kconfig | 8 ++ kernel/rcu/rcu.h | 8 ++ kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 133 ++++++++++++++++++---------- kernel/rcu/tree.h | 17 +++- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_nocb.h | 184 ++++++++++++++++++++++++++++++++------- 8 files changed, 277 insertions(+), 84 deletions(-)include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +8
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure. +config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +endmenu # "RCU Subsystem"
kernel/rcu/rcu.h +8
-0@@ -474,6 +474,14 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; +#if defined(CONFIG_RCU_LAZY ) +unsigned long rcu_lazy_get_jiffies_till_flush (void ); +void rcu_lazy_set_jiffies_till_flush (unsigned long j); +#else +static inline unsigned long rcu_lazy_get_jiffies_till_flush (void ) {return 0 ; } +static inline void rcu_lazy_set_jiffies_till_flush (unsigned long j) { } +#endif +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq);kernel/rcu/tiny.c +1
-1@@ -44,7 +44,7 @@ static struct rcu_ctrlblk rcu_ctrlblk = {void rcu_barrier (void ) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush); }EXPORT_SYMBOL (rcu_barrier);kernel/rcu/tree.c +14
-43@@ -2728,47 +2728,8 @@ static void check_cb_ovld(struct rcu_data *rdp)raw_spin_unlock_rcu_node (rnp); } -/** -* call_rcu() - Queue an RCU callback for invocation after a grace period. -* @head: structure to be used for queueing the RCU updates. -* @func: actual callback function to be invoked after the grace period -* -* The callback function will be invoked some time after a full grace -* period elapses, in other words after all pre-existing RCU read-side -* critical sections have completed. However, the callback function -* might well execute concurrently with RCU read-side critical sections -* that started after call_rcu() was invoked. -* -* RCU read-side critical sections are delimited by rcu_read_lock() -* and rcu_read_unlock(), and may be nested. In addition, but only in -* v5.0 and later, regions of code across which interrupts, preemption, -* or softirqs have been disabled also serve as RCU read-side critical -* sections. This includes hardware interrupt handlers, softirq handlers, -* and NMI handlers. -* -* Note that all CPUs must agree that the grace period extended beyond -* all pre-existing RCU read-side critical section. On systems with more -* than one CPU, this means that when "func()" is invoked, each CPU is -* guaranteed to have executed a full memory barrier since the end of its -* last RCU read-side critical section whose beginning preceded the call -* to call_rcu(). It also means that each CPU executing an RCU read-side -* critical section that continues beyond the start of "func()" must have -* executed a memory barrier after the call_rcu() but before the beginning -* of that RCU read-side critical section. Note that these guarantees -* include CPUs that are offline, idle, or executing in user mode, as -* well as CPUs that are executing in the kernel. -* -* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the -* resulting RCU callback function "func()", then both CPU A and CPU B are -* guaranteed to execute a full memory barrier during the time interval -* between the call to call_rcu() and the invocation of "func()" -- even -* if CPU A and CPU B are the same CPU (but again only if the system has -* more than one CPU). -* -* Implementation of these memory-ordering guarantees is described here: -* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. -*/ -void call_rcu (struct rcu_head *head,rcu_callback_t func) +static void +__call_rcu_common (struct rcu_head *head,rcu_callback_t func,bool lazy) {static atomic_t doublefrees;unsigned long flags;@@ -2809,7 +2770,7 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) }check_cb_ovld (rdp); -if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags)) +if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags, lazy))return ;// Enqueued onto ->nocb_bypass, so just leave. // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock. rcu_segcblist_enqueue (&rdp->cblist, head);@@ -2831,8 +2792,84 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func)local_irq_restore (flags); } } -EXPORT_SYMBOL_GPL (call_rcu); +#ifdef CONFIG_RCU_LAZY +/** +* call_rcu_flush() - Queue RCU callback for invocation after grace period, and +* flush all lazy callbacks (including the new one) to the main ->cblist while +* doing so. +* +* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side
Quoted 8 lines
kernel/rcu/tree.c void call_rcu(struct rcu_head *head, rcu_callback_t func) · line 2811
Quoted 5 lines
kernel/rcu/tree.c +5
-0+* critical sections have completed. +* +* Use this API instead of call_rcu() if you don't mind the callback being +* invoked after very long periods of time on systems without memory pressure +* and on systems which are lightly loaded or mostly idle. This comment is backwards, right? Shouldn't it say something like "Use this API instead of
call_rcu()if you don't mind burning extra power..."?
Yes sorry my mistake :-(. It’s a stale comment from the rework and I’ll update it.
Quoted 104 lines
Quoted 104 lines
kernel/rcu/tree.c +68
-3+* +* Other than the extra delay in callbacks being invoked, this function is +* identical to, and reuses call_rcu()'s logic. Refer to call_rcu() for more +* details about memory ordering and other functionality. +*/ +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, false); +} +EXPORT_SYMBOL_GPL (call_rcu_flush); +#endif + +/** +* call_rcu() - Queue an RCU callback for invocation after a grace period. +* By default the callbacks are 'lazy' and are kept hidden from the main +* ->cblist to prevent starting of grace periods too soon. +* If you desire grace periods to start very soon, use call_rcu_flush(). +* +* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side +* critical sections have completed. However, the callback function +* might well execute concurrently with RCU read-side critical sections +* that started after call_rcu() was invoked. +* +* RCU read-side critical sections are delimited by rcu_read_lock() +* and rcu_read_unlock(), and may be nested. In addition, but only in +* v5.0 and later, regions of code across which interrupts, preemption, +* or softirqs have been disabled also serve as RCU read-side critical +* sections. This includes hardware interrupt handlers, softirq handlers, +* and NMI handlers. +* +* Note that all CPUs must agree that the grace period extended beyond +* all pre-existing RCU read-side critical section. On systems with more +* than one CPU, this means that when "func()" is invoked, each CPU is +* guaranteed to have executed a full memory barrier since the end of its +* last RCU read-side critical section whose beginning preceded the call +* to call_rcu(). It also means that each CPU executing an RCU read-side +* critical section that continues beyond the start of "func()" must have +* executed a memory barrier after the call_rcu() but before the beginning +* of that RCU read-side critical section. Note that these guarantees +* include CPUs that are offline, idle, or executing in user mode, as +* well as CPUs that are executing in the kernel. +* +* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the +* resulting RCU callback function "func()", then both CPU A and CPU B are +* guaranteed to execute a full memory barrier during the time interval +* between the call to call_rcu() and the invocation of "func()" -- even +* if CPU A and CPU B are the same CPU (but again only if the system has +* more than one CPU). +* +* Implementation of these memory-ordering guarantees is described here: +* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. +*/ +void call_rcu (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, true); +} +EXPORT_SYMBOL_GPL (call_rcu);/* Maximum number of jiffies to wait before draining a batch. */ #define KFREE_DRAIN_JIFFIES (5 * HZ)@@ -3507,7 +3544,7 @@ void synchronize_rcu(void)if (rcu_gp_is_expedited ())synchronize_rcu_expedited ();else -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -4323,7 +4364,7 @@ void rcutree_migrate_callbacks(int cpu) my_rdp =this_cpu_ptr (&rcu_data); my_rnp = my_rdp->mynode;rcu_nocb_lock (my_rdp);/* irqs already disabled. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies,FLUSH_BP_NONE ));raw_spin_lock_rcu_node (my_rnp);/* irqs already disabled. */ /* Leverage recent GPs and set GP for new callbacks. */ needwake = rcu_advance_cbs(my_rnp, rdp) ||kernel/rcu/tree.h +0
-0@@ -263,14 +263,16 @@ struct rcu_data {unsigned long last_fqs_resched;/* Time of last rcu_resched(). */ unsigned long last_sched_clock;/* Jiffies of last rcu_sched_clock_irq(). */
Quoted 7 lines
kernel/rcu/tree.h struct rcu_data { · line 266
Quoted 1 line
kernel/rcu/tree.h +1
-0+long lazy_len;/* Length of buffered lazy callbacks. */ Do we ever actually care about the length as opposed to whether or not all the bypass callbacks are lazy? If not, a "
some_nonlazy" boolean would be initialed to zero and ORed with the non-laziness of the added callback. Or, if there was a test anyway, simply set to 1 in the presence of a non-lazy callback. And as now, gets zeroed when the bypass is flushed.
We had discussed this before, and my point was we could use it for tracing and future extension as well. If it’s ok with you, I prefer to keep it this way than having to come back add the length later.
On a minor point as well, if you really want it this
way, I am afraid of changing this now since I tested this way for last several iterations and it’s easy to add a regression. So I prefer to keep it this way and then I can add more patches later on top if that’s Ok with you.
Quoted 89 lines
This might shorten a few lines of code.
Quoted 86 lines
kernel/rcu/tree.h +12
-4int cpu; };/* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0#define RCU_NOCB_WAKE_BYPASS 1 -#define RCU_NOCB_WAKE 2 -#define RCU_NOCB_WAKE_FORCE 3 +#define RCU_NOCB_WAKE_LAZY 2 +#define RCU_NOCB_WAKE 3 +#define RCU_NOCB_WAKE_FORCE 4#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))/* For jiffies_till_first_fqs and */ @@ -439,10 +441,17 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp); + +#define FLUSH_BP_NONE 0 +/* Is the CB being enqueued after the flush, a lazy CB? */ +#define FLUSH_BP_LAZY BIT(0) +/* Wake up nocb-GP thread after flush? */ +#define FLUSH_BP_WAKE BIT(1)static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j); +unsigned long j,unsigned long flush_flags);static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags); +bool *was_alldone,unsigned long flags, +bool lazy);static void __call_rcu_nocb_wake (struct rcu_data *rdp,bool was_empty,unsigned long flags);static int rcu_nocb_need_deferred_wakeup (struct rcu_data *rdp,int level);kernel/rcu/tree_exp.h +1
-1@@ -937,7 +937,7 @@ void synchronize_rcu_expedited(void)/* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }kernel/rcu/tree_nocb.h +25
-0@@ -256,6 +256,31 @@ static bool wake_nocb_gp(struct rcu_data *rdp, bool force)return __wake_nocb_gp (rdp_gp, rdp, force, flags); } +/* +* LAZY_FLUSH_JIFFIES decides the maximum amount of time that +* can elapse before lazy callbacks are flushed. Lazy callbacks +* could be flushed much earlier for a number of other reasons +* however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are +* left unsubmitted to RCU after those many jiffies. +*/ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush =LAZY_FLUSH_JIFFIES ; + +#ifdef CONFIG_RCU_LAZY +// To be called only from test code. +void rcu_lazy_set_jiffies_till_flush (unsigned long jif) +{ + jiffies_till_flush = jif; +} +EXPORT_SYMBOL (rcu_lazy_set_jiffies_till_flush); + +unsigned long rcu_lazy_get_jiffies_till_flush (void ) +{ +return jiffies_till_flush; +} +EXPORT_SYMBOL (rcu_lazy_get_jiffies_till_flush); +#endif +/* * Arrange to wake the GP kthread for this NOCB group at some future * time when it is safe to do so. @@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);
Quoted 13 lines
kernel/rcu/tree_nocb.h static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 9 lines
kernel/rcu/tree_nocb.h +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {Please leave the "&&" on the previous line and line up the "READ_ONCE(" with the "waketype". That makes it easier to tell the condition from the following code.
Will do!
Quoted 23 lines
Quoted 23 lines
kernel/rcu/tree_nocb.h +8
-1+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -293,12 +322,16 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* proves to be initially empty, just return false because the no-CB GP * kthread may need to be awakened in this case. * +* Return true if there was something to be flushed and it succeeded, otherwise +* false. +* * Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +bool lazy = flush_flags &FLUSH_BP_LAZY ;
Quoted 13 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 349
Quoted 11 lines
kernel/rcu/tree_nocb.h +4
-1WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));rcu_lockdep_assert_cblist_protected (rdp);@@ -310,7 +343,20 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,/* Note: ->cblist.len already accounts for ->nocb_bypass contents. */ if (rhp)rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); + +/* +* If the new CB requested was a lazy one, queue it onto the main +* ->cblist so we can take advantage of a sooner grade period. "take advantage of a grace period that will happen regardless."?
Sure will update.
Quoted 9 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 353
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0+*/ +if (lazy && rhp) { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); +rcu_cblist_enqueue (&rcl, rhp);Would it makes sense to enqueue rhp onto ->nocb_bypass first, NULL out rhp, then let the
rcu_cblist_flush_enqueue()be common code? Or did this function grow a later use of rhp that I missed?
No that could be done, but it prefer to keep it this
way because rhp is a function parameter and I
prefer not to modify those since it could add a bug in future where rhp passed by user is now NULL for some reason, half way through the function.
Quoted 7 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 357
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?
Yes will update.
Quoted 3 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
Quoted 12 lines
Quoted 12 lines
kernel/rcu/tree_nocb.h +3
-1+ } +rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);rcu_nocb_bypass_unlock (rdp);@@ -326,13 +372,33 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags)
Quoted 8 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 380
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0{ +bool ret; +bool was_alldone = false; +bool bypass_all_lazy = false; +long bypass_ncbs;Alphabetical order by variable name, please. (Yes, I know that this is strange, but what can I say?)
Sure.
Quoted 101 lines
Quoted 87 lines
kernel/rcu/tree_nocb.h +30
-7+if (!rcu_rdp_is_offloaded (rdp))return true;rcu_lockdep_assert_cblist_protected (rdp);rcu_nocb_bypass_lock (rdp); -return rcu_nocb_do_flush_bypass (rdp, rhp, j); + +if (flush_flags &FLUSH_BP_WAKE ) { + was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + bypass_all_lazy = bypass_ncbs && (bypass_ncbs == rdp->lazy_len); + } + + ret =rcu_nocb_do_flush_bypass (rdp, rhp, j, flush_flags); + +// Wake up the nocb GP thread if needed. GP thread could be sleeping +// while waiting for lazy timer to expire (otherwise rcu_barrier may +// end up waiting for the duration of the lazy timer). +if (flush_flags &FLUSH_BP_WAKE && was_alldone && bypass_all_lazy) +wake_nocb_gp (rdp, false); + +return ret; }/* @@ -345,7 +411,7 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)if (!rcu_rdp_is_offloaded (rdp) || !rcu_nocb_bypass_trylock (rdp))return ; -WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j)); +WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE )); }/* @@ -367,12 +433,14 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)* there is only one CPU in operation. */ static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags, +bool lazy) {unsigned long c;unsigned long cur_gp_seq;unsigned long j = jiffies;long ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); +bool bypass_is_lazy = (ncbs ==READ_ONCE (rdp->lazy_len));lockdep_assert_irqs_disabled ();@@ -417,25 +485,30 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// If there hasn't yet been all that many ->cblist enqueues // this jiffy, tell the caller to enqueue onto ->cblist. But flush // ->nocb_bypass first. -if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { +// Lazy CBs throttle this back and do immediate bypass queuing. +if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist);if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" )); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j)); + +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE ));WARN_ON_ONCE (rcu_cblist_n_cbs (&rdp->nocb_bypass));return false;// Caller must enqueue the callback. }// If ->nocb_bypass has been used too long or is too full, // flush ->nocb_bypass to ->cblist. -if ((ncbs && j !=READ_ONCE (rdp->nocb_bypass_first)) || +if ((ncbs && !bypass_is_lazy && j !=READ_ONCE (rdp->nocb_bypass_first)) || + (ncbs && bypass_is_lazy && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush))) || ncbs >= qhimark) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); -if (!rcu_nocb_flush_bypass (rdp, rhp, j)) { +if (!rcu_nocb_flush_bypass (rdp, rhp, j, + lazy ?FLUSH_BP_LAZY :FLUSH_BP_NONE )) {if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" ));kernel/rcu/tree_nocb.h static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 541
Quoted 9 lines
kernel/rcu/tree_nocb.h +3
-0@@ -460,16 +533,29 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// We need to use the bypass. rcu_nocb_wait_contended (rdp);rcu_nocb_bypass_lock (rdp); + ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass);rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ rcu_cblist_enqueue (&rdp->nocb_bypass, rhp); + +if (IS_ENABLED (CONFIG_RCU_LAZY ) && lazy)Won't !IS_ENABLED(CONFIG_RCU_LAZY) mean that lazy cannot be set? Why do we need to check both? Or are you going for dead code? If so, shouldn't there be
IS_ENABLED(CONFIG_RCU_LAZY)checks above as well?
Quoted 8 lines
Me, I am not convinced that the dead code would buy you much. In fact, the compiler might well be propagating the constants on its own.
Ah! The reason the compiler cannot figure this out is because you put the switch into
rcu.h. If you instead always export thecall_rcu_flush()definition, and checkIS_ENABLED(CONFIG_RCU_LAZY)at the beginning ofcall_rcu(), the compiler should have the information that it needs to do this for you.
Ah ok, I will try to do it this way.
Quoted 23 lines
Quoted 23 lines
kernel/rcu/tree_nocb.h +11
-1+WRITE_ONCE (rdp->lazy_len, rdp->lazy_len +1 ); +if (!ncbs) {WRITE_ONCE (rdp->nocb_bypass_first, j);trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstBQ" )); } +rcu_nocb_bypass_unlock (rdp);smp_mb ();/* Order enqueue before wake. */ -if (ncbs) { + +// A wake up of the grace period kthread or timer adjustment needs to +// be done only if: +// 1. Bypass list was fully empty before (this is the first bypass list entry). +// Or, both the below conditions are met: +// 1. Bypass list had only lazy CBs before. +// 2. The new CB is non-lazy. +if (ncbs && (!bypass_is_lazy || lazy)) {local_irq_restore (flags); }else {// No-CBs GP kthread might be indefinitely asleep, if so, wake. @@ -499,7 +585,7 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
Quoted 8 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 589
Quoted 6 lines
kernel/rcu/tree_nocb.h +1
-1{unsigned long cur_gp_seq;unsigned long j; -long len; +long len, lazy_len, bypass_len;struct task_struct *t;Again, alphabetical please, strange though that might seem.
Yes sure
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_nocb.h +8
-1// If we are being polled or there is no kthread, just leave. @@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist); + bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" )); + }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);@@ -604,8 +697,8 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)*/
Quoted 8 lines
kernel/rcu/tree_nocb.h static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · line 701
Quoted 6 lines
kernel/rcu/tree_nocb.h +2
-2static void nocb_gp_wait (struct rcu_data *my_rdp) { -bool bypass = false; -long bypass_ncbs; +bool bypass = false, lazy = false; +long bypass_ncbs, lazy_ncbs;And ditto.
Ok
Quoted 56 lines
Quoted 56 lines
kernel/rcu/tree_nocb.h +23
-6int __maybe_unused cpu = my_rdp->cpu;unsigned long cur_gp_seq;unsigned long flags;@@ -640,24 +733,41 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)* won't be ignored for long. */ list_for_each_entry (rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { +bool flush_bypass = false; +trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("Check" ));rcu_nocb_lock_irqsave (rdp, flags);lockdep_assert_held (&rdp->nocb_lock); bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); -if (bypass_ncbs && + lazy_ncbs =READ_ONCE (rdp->lazy_len); + +if (bypass_ncbs && (lazy_ncbs == bypass_ncbs) && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush) || + bypass_ncbs >2 * qhimark)) { + flush_bypass = true; + }else if (bypass_ncbs && (lazy_ncbs != bypass_ncbs) && (time_after (j,READ_ONCE (rdp->nocb_bypass_first) +1 ) || bypass_ncbs >2 * qhimark)) { -// Bypass full or old, so flush it. - (void )rcu_nocb_try_flush_bypass (rdp, j); - bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + flush_bypass = true; }else if (!bypass_ncbs &&rcu_segcblist_empty (&rdp->cblist)) {rcu_nocb_unlock_irqrestore (rdp, flags);continue ;/* No callbacks here, try next. */ } + +if (flush_bypass) { +// Bypass full or old, so flush it. + (void )rcu_nocb_try_flush_bypass (rdp, j); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_ncbs =READ_ONCE (rdp->lazy_len); + } +if (bypass_ncbs) {trace_rcu_nocb_wake (rcu_state.name, rdp->cpu, -TPS ("Bypass" )); - bypass = true; + bypass_ncbs == lazy_ncbs ?TPS ("Lazy" ) :TPS ("Bypass" )); +if (bypass_ncbs == lazy_ncbs) + lazy = true; +else + bypass = true; } rnp = rdp->mynode;@@ -705,12 +815,21 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_gp = needwait_gp; my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq :0 ;
Quoted 12 lines
kernel/rcu/tree_nocb.h static void nocb_gp_wait(struct rcu_data *my_rdp) · line 822
Quoted 10 lines
kernel/rcu/tree_nocb.h +5
-5-if (bypass && !rcu_nocb_poll) { -// At least one child with non-empty ->nocb_bypass, so set -// timer in order to avoid stranding its callbacks. -wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , -TPS ("WakeBypassIsDeferred" )); +// At least one child with non-empty ->nocb_bypass, so set +// timer in order to avoid stranding its callbacks. +if (!rcu_nocb_poll) { +// If bypass list only has lazy CBs. Add a deferred +// lazy wake up. One sentence rather than two.
Ok
Quoted 50 lines
Quoted 50 lines
kernel/rcu/tree_nocb.h +13
-3+if (lazy && !bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazyIsDeferred" )); +// Otherwise add a deferred bypass wake up. + }else if (bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , +TPS ("WakeBypassIsDeferred" )); + } } +if (rcu_nocb_poll) {/* Polling, so trace if first poll in the series. */ if (gotcbs)@@ -1036,7 +1155,7 @@ static long rcu_nocb_rdp_deoffload(void *arg)* return false, which means that future calls to rcu_nocb_try_bypass() * will refuse to put anything into the bypass. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_NONE ));/* * Start with invoking rcu_core() early. This way if the current thread * happens to preempt an ongoing call to rcu_core() in the middle, @@ -1278,6 +1397,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)raw_spin_lock_init (&rdp->nocb_gp_lock);timer_setup (&rdp->nocb_timer, do_nocb_deferred_wakeup_timer,0 );rcu_cblist_init (&rdp->nocb_bypass); +WRITE_ONCE (rdp->lazy_len,0 );mutex_init (&rdp->nocb_gp_kthread_mutex); }@@ -1559,13 +1679,13 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) }static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {return true; }static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags,bool lazy) {return false; }Signature · 1 line
2.37.3.998.g577e59143f-goog
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 2 lines · Joel Fernandes
Hi Paul,
Let’s see whether my iPhone can handle replies ;-)
Now let's see if it can handle replies to replies! Are you using a keyboard, or small-screen finger gestures?
Quoted 206 lines
More below:
Quoted 28 lines · Paul E. McKenney
Quoted 25 lines · Joel Fernandes (Google)
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ][ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]Quoted 7 lines
Quoted 2 lines
Suggested-by Paul McKenney <[email protected]>Signed-off-by Joel Fernandes (Google) <[email protected]>I am going to put these on a local branch for testing, but a few comments and questions interspersed below.
Thanx, Paul
Ok I replied below, thank you!
Quoted 157 lines
Quoted 157 lines
include/linux/rcupdate.h | 7 ++ kernel/rcu/Kconfig | 8 ++ kernel/rcu/rcu.h | 8 ++ kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 133 ++++++++++++++++++---------- kernel/rcu/tree.h | 17 +++- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_nocb.h | 184 ++++++++++++++++++++++++++++++++------- 8 files changed, 277 insertions(+), 84 deletions(-)include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +8
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure. +config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +endmenu # "RCU Subsystem"
kernel/rcu/rcu.h +8
-0@@ -474,6 +474,14 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; +#if defined(CONFIG_RCU_LAZY ) +unsigned long rcu_lazy_get_jiffies_till_flush (void ); +void rcu_lazy_set_jiffies_till_flush (unsigned long j); +#else +static inline unsigned long rcu_lazy_get_jiffies_till_flush (void ) {return 0 ; } +static inline void rcu_lazy_set_jiffies_till_flush (unsigned long j) { } +#endif +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq);kernel/rcu/tiny.c +1
-1@@ -44,7 +44,7 @@ static struct rcu_ctrlblk rcu_ctrlblk = {void rcu_barrier (void ) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush); }EXPORT_SYMBOL (rcu_barrier);kernel/rcu/tree.c +14
-43@@ -2728,47 +2728,8 @@ static void check_cb_ovld(struct rcu_data *rdp)raw_spin_unlock_rcu_node (rnp); } -/** -* call_rcu() - Queue an RCU callback for invocation after a grace period. -* @head: structure to be used for queueing the RCU updates. -* @func: actual callback function to be invoked after the grace period -* -* The callback function will be invoked some time after a full grace -* period elapses, in other words after all pre-existing RCU read-side -* critical sections have completed. However, the callback function -* might well execute concurrently with RCU read-side critical sections -* that started after call_rcu() was invoked. -* -* RCU read-side critical sections are delimited by rcu_read_lock() -* and rcu_read_unlock(), and may be nested. In addition, but only in -* v5.0 and later, regions of code across which interrupts, preemption, -* or softirqs have been disabled also serve as RCU read-side critical -* sections. This includes hardware interrupt handlers, softirq handlers, -* and NMI handlers. -* -* Note that all CPUs must agree that the grace period extended beyond -* all pre-existing RCU read-side critical section. On systems with more -* than one CPU, this means that when "func()" is invoked, each CPU is -* guaranteed to have executed a full memory barrier since the end of its -* last RCU read-side critical section whose beginning preceded the call -* to call_rcu(). It also means that each CPU executing an RCU read-side -* critical section that continues beyond the start of "func()" must have -* executed a memory barrier after the call_rcu() but before the beginning -* of that RCU read-side critical section. Note that these guarantees -* include CPUs that are offline, idle, or executing in user mode, as -* well as CPUs that are executing in the kernel. -* -* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the -* resulting RCU callback function "func()", then both CPU A and CPU B are -* guaranteed to execute a full memory barrier during the time interval -* between the call to call_rcu() and the invocation of "func()" -- even -* if CPU A and CPU B are the same CPU (but again only if the system has -* more than one CPU). -* -* Implementation of these memory-ordering guarantees is described here: -* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. -*/ -void call_rcu (struct rcu_head *head,rcu_callback_t func) +static void +__call_rcu_common (struct rcu_head *head,rcu_callback_t func,bool lazy) {static atomic_t doublefrees;unsigned long flags;@@ -2809,7 +2770,7 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) }check_cb_ovld (rdp); -if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags)) +if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags, lazy))return ;// Enqueued onto ->nocb_bypass, so just leave. // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock. rcu_segcblist_enqueue (&rdp->cblist, head);@@ -2831,8 +2792,84 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func)local_irq_restore (flags); } } -EXPORT_SYMBOL_GPL (call_rcu); +#ifdef CONFIG_RCU_LAZY +/** +* call_rcu_flush() - Queue RCU callback for invocation after grace period, and +* flush all lazy callbacks (including the new one) to the main ->cblist while +* doing so. +* +* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side Quoted 8 lines
kernel/rcu/tree.c void call_rcu(struct rcu_head *head, rcu_callback_t func) · line 2811
Quoted 5 lines
kernel/rcu/tree.c +5
-0+* critical sections have completed. +* +* Use this API instead of call_rcu() if you don't mind the callback being +* invoked after very long periods of time on systems without memory pressure +* and on systems which are lightly loaded or mostly idle. This comment is backwards, right? Shouldn't it say something like "Use this API instead of
call_rcu()if you don't mind burning extra power..."?
Quoted 1 line
Yes sorry my mistake :-(. It’s a stale comment from the rework and I’ll update it.
Very good, thank you!
Quoted 116 lines
Quoted 103 lines
Quoted 103 lines
kernel/rcu/tree.c +68
-3+* +* Other than the extra delay in callbacks being invoked, this function is +* identical to, and reuses call_rcu()'s logic. Refer to call_rcu() for more +* details about memory ordering and other functionality. +*/ +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, false); +} +EXPORT_SYMBOL_GPL (call_rcu_flush); +#endif + +/** +* call_rcu() - Queue an RCU callback for invocation after a grace period. +* By default the callbacks are 'lazy' and are kept hidden from the main +* ->cblist to prevent starting of grace periods too soon. +* If you desire grace periods to start very soon, use call_rcu_flush(). +* +* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side +* critical sections have completed. However, the callback function +* might well execute concurrently with RCU read-side critical sections +* that started after call_rcu() was invoked. +* +* RCU read-side critical sections are delimited by rcu_read_lock() +* and rcu_read_unlock(), and may be nested. In addition, but only in +* v5.0 and later, regions of code across which interrupts, preemption, +* or softirqs have been disabled also serve as RCU read-side critical +* sections. This includes hardware interrupt handlers, softirq handlers, +* and NMI handlers. +* +* Note that all CPUs must agree that the grace period extended beyond +* all pre-existing RCU read-side critical section. On systems with more +* than one CPU, this means that when "func()" is invoked, each CPU is +* guaranteed to have executed a full memory barrier since the end of its +* last RCU read-side critical section whose beginning preceded the call +* to call_rcu(). It also means that each CPU executing an RCU read-side +* critical section that continues beyond the start of "func()" must have +* executed a memory barrier after the call_rcu() but before the beginning +* of that RCU read-side critical section. Note that these guarantees +* include CPUs that are offline, idle, or executing in user mode, as +* well as CPUs that are executing in the kernel. +* +* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the +* resulting RCU callback function "func()", then both CPU A and CPU B are +* guaranteed to execute a full memory barrier during the time interval +* between the call to call_rcu() and the invocation of "func()" -- even +* if CPU A and CPU B are the same CPU (but again only if the system has +* more than one CPU). +* +* Implementation of these memory-ordering guarantees is described here: +* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. +*/ +void call_rcu (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, true); +} +EXPORT_SYMBOL_GPL (call_rcu);/* Maximum number of jiffies to wait before draining a batch. */ #define KFREE_DRAIN_JIFFIES (5 * HZ)@@ -3507,7 +3544,7 @@ void synchronize_rcu(void)if (rcu_gp_is_expedited ())synchronize_rcu_expedited ();else -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -4323,7 +4364,7 @@ void rcutree_migrate_callbacks(int cpu) my_rdp =this_cpu_ptr (&rcu_data); my_rnp = my_rdp->mynode;rcu_nocb_lock (my_rdp);/* irqs already disabled. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies,FLUSH_BP_NONE ));raw_spin_lock_rcu_node (my_rnp);/* irqs already disabled. */ /* Leverage recent GPs and set GP for new callbacks. */ needwake = rcu_advance_cbs(my_rnp, rdp) ||kernel/rcu/tree.h +0
-0@@ -263,14 +263,16 @@ struct rcu_data {unsigned long last_fqs_resched;/* Time of last rcu_resched(). */ unsigned long last_sched_clock;/* Jiffies of last rcu_sched_clock_irq(). */ Quoted 7 lines
kernel/rcu/tree.h struct rcu_data { · line 266
Quoted 1 line
kernel/rcu/tree.h +1
-0+long lazy_len;/* Length of buffered lazy callbacks. */ Do we ever actually care about the length as opposed to whether or not all the bypass callbacks are lazy? If not, a "
some_nonlazy" boolean would be initialed to zero and ORed with the non-laziness of the added callback. Or, if there was a test anyway, simply set to 1 in the presence of a non-lazy callback. And as now, gets zeroed when the bypass is flushed.We had discussed this before, and my point was we could use it for tracing and future extension as well. If it’s ok with you, I prefer to keep it this way than having to come back add the length later.
Quoted 6 lines
On a minor point as well, if you really want it this
way, I am afraid of changing this now since I tested this way for last several iterations and it’s easy to add a regression. So I prefer to keep it this way and then I can add more patches later on top if that’s Ok with you.
No, you are right, the debug information might be helpful.
But hey, at least I am consistent!
Quoted 153 lines
Quoted 88 lines
This might shorten a few lines of code.
Quoted 86 lines
kernel/rcu/tree.h +12
-4int cpu; };/* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0#define RCU_NOCB_WAKE_BYPASS 1 -#define RCU_NOCB_WAKE 2 -#define RCU_NOCB_WAKE_FORCE 3 +#define RCU_NOCB_WAKE_LAZY 2 +#define RCU_NOCB_WAKE 3 +#define RCU_NOCB_WAKE_FORCE 4#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))/* For jiffies_till_first_fqs and */ @@ -439,10 +441,17 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp); + +#define FLUSH_BP_NONE 0 +/* Is the CB being enqueued after the flush, a lazy CB? */ +#define FLUSH_BP_LAZY BIT(0) +/* Wake up nocb-GP thread after flush? */ +#define FLUSH_BP_WAKE BIT(1)static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j); +unsigned long j,unsigned long flush_flags);static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags); +bool *was_alldone,unsigned long flags, +bool lazy);static void __call_rcu_nocb_wake (struct rcu_data *rdp,bool was_empty,unsigned long flags);static int rcu_nocb_need_deferred_wakeup (struct rcu_data *rdp,int level);kernel/rcu/tree_exp.h +1
-1@@ -937,7 +937,7 @@ void synchronize_rcu_expedited(void)/* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }kernel/rcu/tree_nocb.h +25
-0@@ -256,6 +256,31 @@ static bool wake_nocb_gp(struct rcu_data *rdp, bool force)return __wake_nocb_gp (rdp_gp, rdp, force, flags); } +/* +* LAZY_FLUSH_JIFFIES decides the maximum amount of time that +* can elapse before lazy callbacks are flushed. Lazy callbacks +* could be flushed much earlier for a number of other reasons +* however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are +* left unsubmitted to RCU after those many jiffies. +*/ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush =LAZY_FLUSH_JIFFIES ; + +#ifdef CONFIG_RCU_LAZY +// To be called only from test code. +void rcu_lazy_set_jiffies_till_flush (unsigned long jif) +{ + jiffies_till_flush = jif; +} +EXPORT_SYMBOL (rcu_lazy_set_jiffies_till_flush); + +unsigned long rcu_lazy_get_jiffies_till_flush (void ) +{ +return jiffies_till_flush; +} +EXPORT_SYMBOL (rcu_lazy_get_jiffies_till_flush); +#endif +/* * Arrange to wake the GP kthread for this NOCB group at some future * time when it is safe to do so. @@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);Quoted 13 lines
kernel/rcu/tree_nocb.h static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 9 lines
kernel/rcu/tree_nocb.h +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {Please leave the "&&" on the previous line and line up the "READ_ONCE(" with the "waketype". That makes it easier to tell the condition from the following code.
Will do!
Quoted 23 lines
Quoted 23 lines
kernel/rcu/tree_nocb.h +8
-1+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -293,12 +322,16 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* proves to be initially empty, just return false because the no-CB GP * kthread may need to be awakened in this case. * +* Return true if there was something to be flushed and it succeeded, otherwise +* false. +* * Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +bool lazy = flush_flags &FLUSH_BP_LAZY ;Quoted 13 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 349
Quoted 11 lines
kernel/rcu/tree_nocb.h +4
-1WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));rcu_lockdep_assert_cblist_protected (rdp);@@ -310,7 +343,20 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,/* Note: ->cblist.len already accounts for ->nocb_bypass contents. */ if (rhp)rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); + +/* +* If the new CB requested was a lazy one, queue it onto the main +* ->cblist so we can take advantage of a sooner grade period. "take advantage of a grace period that will happen regardless."?
Sure will update.
Quoted 9 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 353
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0+*/ +if (lazy && rhp) { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); +rcu_cblist_enqueue (&rcl, rhp);Would it makes sense to enqueue rhp onto ->nocb_bypass first, NULL out rhp, then let the
rcu_cblist_flush_enqueue()be common code? Or did this function grow a later use of rhp that I missed?
Quoted 6 lines
No that could be done, but it prefer to keep it this
way because rhp is a function parameter and I
prefer not to modify those since it could add a bug in future where rhp passed by user is now NULL for some reason, half way through the function.
I agree that changing a function parameter is bad practice.
So the question becomes whether introducing a local would outweigh consolidating this code. Could you please at least give it a shot?
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 357
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
struct rcu_head *rhp = rhp_in;
And then:
if (lazy && rhp) {
rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
rhp = NULL;
}
rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);
WRITE_ONCE(rdp->lazy_len, 0);Or did I mess something up?
Quoted 20 lines
Quoted 11 lines
Quoted 11 lines
kernel/rcu/tree_nocb.h +3
-1+ } +rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);rcu_nocb_bypass_unlock (rdp);@@ -326,13 +372,33 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags)Quoted 8 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 380
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0{ +bool ret; +bool was_alldone = false; +bool bypass_all_lazy = false; +long bypass_ncbs;Alphabetical order by variable name, please. (Yes, I know that this is strange, but what can I say?)
Quoted 1 line
Sure.
Thank you!
Quoted 103 lines
Quoted 103 lines
Quoted 86 lines
kernel/rcu/tree_nocb.h +30
-7+if (!rcu_rdp_is_offloaded (rdp))return true;rcu_lockdep_assert_cblist_protected (rdp);rcu_nocb_bypass_lock (rdp); -return rcu_nocb_do_flush_bypass (rdp, rhp, j); + +if (flush_flags &FLUSH_BP_WAKE ) { + was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + bypass_all_lazy = bypass_ncbs && (bypass_ncbs == rdp->lazy_len); + } + + ret =rcu_nocb_do_flush_bypass (rdp, rhp, j, flush_flags); + +// Wake up the nocb GP thread if needed. GP thread could be sleeping +// while waiting for lazy timer to expire (otherwise rcu_barrier may +// end up waiting for the duration of the lazy timer). +if (flush_flags &FLUSH_BP_WAKE && was_alldone && bypass_all_lazy) +wake_nocb_gp (rdp, false); + +return ret; }/* @@ -345,7 +411,7 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)if (!rcu_rdp_is_offloaded (rdp) || !rcu_nocb_bypass_trylock (rdp))return ; -WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j)); +WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE )); }/* @@ -367,12 +433,14 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)* there is only one CPU in operation. */ static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags, +bool lazy) {unsigned long c;unsigned long cur_gp_seq;unsigned long j = jiffies;long ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); +bool bypass_is_lazy = (ncbs ==READ_ONCE (rdp->lazy_len));lockdep_assert_irqs_disabled ();@@ -417,25 +485,30 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// If there hasn't yet been all that many ->cblist enqueues // this jiffy, tell the caller to enqueue onto ->cblist. But flush // ->nocb_bypass first. -if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { +// Lazy CBs throttle this back and do immediate bypass queuing. +if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist);if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" )); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j)); + +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE ));WARN_ON_ONCE (rcu_cblist_n_cbs (&rdp->nocb_bypass));return false;// Caller must enqueue the callback. }// If ->nocb_bypass has been used too long or is too full, // flush ->nocb_bypass to ->cblist. -if ((ncbs && j !=READ_ONCE (rdp->nocb_bypass_first)) || +if ((ncbs && !bypass_is_lazy && j !=READ_ONCE (rdp->nocb_bypass_first)) || + (ncbs && bypass_is_lazy && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush))) || ncbs >= qhimark) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); -if (!rcu_nocb_flush_bypass (rdp, rhp, j)) { +if (!rcu_nocb_flush_bypass (rdp, rhp, j, + lazy ?FLUSH_BP_LAZY :FLUSH_BP_NONE )) {if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" ));kernel/rcu/tree_nocb.h static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 541
Quoted 9 lines
kernel/rcu/tree_nocb.h +3
-0@@ -460,16 +533,29 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// We need to use the bypass. rcu_nocb_wait_contended (rdp);rcu_nocb_bypass_lock (rdp); + ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass);rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ rcu_cblist_enqueue (&rdp->nocb_bypass, rhp); + +if (IS_ENABLED (CONFIG_RCU_LAZY ) && lazy)Won't !IS_ENABLED(CONFIG_RCU_LAZY) mean that lazy cannot be set? Why do we need to check both? Or are you going for dead code? If so, shouldn't there be
IS_ENABLED(CONFIG_RCU_LAZY)checks above as well?Me, I am not convinced that the dead code would buy you much. In fact, the compiler might well be propagating the constants on its own.
Quoted 7 lines
Quoted 5 lines
Ah! The reason the compiler cannot figure this out is because you put the switch into
rcu.h. If you instead always export thecall_rcu_flush()definition, and checkIS_ENABLED(CONFIG_RCU_LAZY)at the beginning ofcall_rcu(), the compiler should have the information that it needs to do this for you.Ah ok, I will try to do it this way.
Very good, thank you, for this and for the rest below.
Thanx, Paul
Quoted 187 lines
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_nocb.h +11
-1+WRITE_ONCE (rdp->lazy_len, rdp->lazy_len +1 ); +if (!ncbs) {WRITE_ONCE (rdp->nocb_bypass_first, j);trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstBQ" )); } +rcu_nocb_bypass_unlock (rdp);smp_mb ();/* Order enqueue before wake. */ -if (ncbs) { + +// A wake up of the grace period kthread or timer adjustment needs to +// be done only if: +// 1. Bypass list was fully empty before (this is the first bypass list entry). +// Or, both the below conditions are met: +// 1. Bypass list had only lazy CBs before. +// 2. The new CB is non-lazy. +if (ncbs && (!bypass_is_lazy || lazy)) {local_irq_restore (flags); }else {// No-CBs GP kthread might be indefinitely asleep, if so, wake. @@ -499,7 +585,7 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,Quoted 8 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 589
Quoted 6 lines
kernel/rcu/tree_nocb.h +1
-1{unsigned long cur_gp_seq;unsigned long j; -long len; +long len, lazy_len, bypass_len;struct task_struct *t;Again, alphabetical please, strange though that might seem.
Yes sure
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_nocb.h +8
-1// If we are being polled or there is no kthread, just leave. @@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist); + bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" )); + }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);@@ -604,8 +697,8 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)*/ Quoted 8 lines
kernel/rcu/tree_nocb.h static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · line 701
Quoted 6 lines
kernel/rcu/tree_nocb.h +2
-2static void nocb_gp_wait (struct rcu_data *my_rdp) { -bool bypass = false; -long bypass_ncbs; +bool bypass = false, lazy = false; +long bypass_ncbs, lazy_ncbs;And ditto.
Ok
Quoted 56 lines
Quoted 56 lines
kernel/rcu/tree_nocb.h +23
-6int __maybe_unused cpu = my_rdp->cpu;unsigned long cur_gp_seq;unsigned long flags;@@ -640,24 +733,41 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)* won't be ignored for long. */ list_for_each_entry (rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { +bool flush_bypass = false; +trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("Check" ));rcu_nocb_lock_irqsave (rdp, flags);lockdep_assert_held (&rdp->nocb_lock); bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); -if (bypass_ncbs && + lazy_ncbs =READ_ONCE (rdp->lazy_len); + +if (bypass_ncbs && (lazy_ncbs == bypass_ncbs) && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush) || + bypass_ncbs >2 * qhimark)) { + flush_bypass = true; + }else if (bypass_ncbs && (lazy_ncbs != bypass_ncbs) && (time_after (j,READ_ONCE (rdp->nocb_bypass_first) +1 ) || bypass_ncbs >2 * qhimark)) { -// Bypass full or old, so flush it. - (void )rcu_nocb_try_flush_bypass (rdp, j); - bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + flush_bypass = true; }else if (!bypass_ncbs &&rcu_segcblist_empty (&rdp->cblist)) {rcu_nocb_unlock_irqrestore (rdp, flags);continue ;/* No callbacks here, try next. */ } + +if (flush_bypass) { +// Bypass full or old, so flush it. + (void )rcu_nocb_try_flush_bypass (rdp, j); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_ncbs =READ_ONCE (rdp->lazy_len); + } +if (bypass_ncbs) {trace_rcu_nocb_wake (rcu_state.name, rdp->cpu, -TPS ("Bypass" )); - bypass = true; + bypass_ncbs == lazy_ncbs ?TPS ("Lazy" ) :TPS ("Bypass" )); +if (bypass_ncbs == lazy_ncbs) + lazy = true; +else + bypass = true; } rnp = rdp->mynode;@@ -705,12 +815,21 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_gp = needwait_gp; my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq :0 ;Quoted 12 lines
kernel/rcu/tree_nocb.h static void nocb_gp_wait(struct rcu_data *my_rdp) · line 822
Quoted 10 lines
kernel/rcu/tree_nocb.h +5
-5-if (bypass && !rcu_nocb_poll) { -// At least one child with non-empty ->nocb_bypass, so set -// timer in order to avoid stranding its callbacks. -wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , -TPS ("WakeBypassIsDeferred" )); +// At least one child with non-empty ->nocb_bypass, so set +// timer in order to avoid stranding its callbacks. +if (!rcu_nocb_poll) { +// If bypass list only has lazy CBs. Add a deferred +// lazy wake up. One sentence rather than two.
Ok
Quoted 50 lines
Quoted 50 lines
kernel/rcu/tree_nocb.h +13
-3+if (lazy && !bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazyIsDeferred" )); +// Otherwise add a deferred bypass wake up. + }else if (bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , +TPS ("WakeBypassIsDeferred" )); + } } +if (rcu_nocb_poll) {/* Polling, so trace if first poll in the series. */ if (gotcbs)@@ -1036,7 +1155,7 @@ static long rcu_nocb_rdp_deoffload(void *arg)* return false, which means that future calls to rcu_nocb_try_bypass() * will refuse to put anything into the bypass. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_NONE ));/* * Start with invoking rcu_core() early. This way if the current thread * happens to preempt an ongoing call to rcu_core() in the middle, @@ -1278,6 +1397,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)raw_spin_lock_init (&rdp->nocb_gp_lock);timer_setup (&rdp->nocb_timer, do_nocb_deferred_wakeup_timer,0 );rcu_cblist_init (&rdp->nocb_bypass); +WRITE_ONCE (rdp->lazy_len,0 );mutex_init (&rdp->nocb_gp_kthread_mutex); }@@ -1559,13 +1679,13 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) }static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {return true; }static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags,bool lazy) {return false; }Signature · 1 line
2.37.3.998.g577e59143f-goog
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 10 lines · Paul E. McKenney
Quoted 2 lines · Joel Fernandes
Hi Paul,
Let’s see whether my iPhone can handle replies ;-)Now let's see if it can handle replies to replies! Are you using a keyboard, or small-screen finger gestures?
Haha here it comes. It’s the usual onscreen keyboard which has finger gesture support and is quite fast to type with ;-). It’s also connected to my apple watch so I can reply from there but I’ll take it one step a time ;-)
Quoted 206 lines
Quoted 206 lines
More below:
Quoted 28 lines
Quoted 1 line · Paul E. McKenney
Quoted 25 lines · Joel Fernandes (Google)
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ][ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]Quoted 7 lines
Quoted 2 lines
Suggested-by Paul McKenney <[email protected]>Signed-off-by Joel Fernandes (Google) <[email protected]>I am going to put these on a local branch for testing, but a few comments and questions interspersed below.
Thanx, Paul
Ok I replied below, thank you!
Quoted 165 lines
Quoted 152 lines
include/linux/rcupdate.h | 7 ++ kernel/rcu/Kconfig | 8 ++ kernel/rcu/rcu.h | 8 ++ kernel/rcu/tiny.c | 2 +- kernel/rcu/tree.c | 133 ++++++++++++++++++---------- kernel/rcu/tree.h | 17 +++- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_nocb.h | 184 ++++++++++++++++++++++++++++++++------- 8 files changed, 277 insertions(+), 84 deletions(-)include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +8
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure. +config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +endmenu # "RCU Subsystem"
kernel/rcu/rcu.h +8
-0@@ -474,6 +474,14 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; +#if defined(CONFIG_RCU_LAZY ) +unsigned long rcu_lazy_get_jiffies_till_flush (void ); +void rcu_lazy_set_jiffies_till_flush (unsigned long j); +#else +static inline unsigned long rcu_lazy_get_jiffies_till_flush (void ) {return 0 ; } +static inline void rcu_lazy_set_jiffies_till_flush (unsigned long j) { } +#endif +#if defined(CONFIG_TREE_RCU )void rcutorture_get_gp_data (enum rcutorture_type test_type,int *flags,unsigned long *gp_seq);kernel/rcu/tiny.c +1
-1@@ -44,7 +44,7 @@ static struct rcu_ctrlblk rcu_ctrlblk = {void rcu_barrier (void ) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush); }EXPORT_SYMBOL (rcu_barrier);kernel/rcu/tree.c +9
-43@@ -2728,47 +2728,8 @@ static void check_cb_ovld(struct rcu_data *rdp)raw_spin_unlock_rcu_node (rnp); } -/** -* call_rcu() - Queue an RCU callback for invocation after a grace period. -* @head: structure to be used for queueing the RCU updates. -* @func: actual callback function to be invoked after the grace period -* -* The callback function will be invoked some time after a full grace -* period elapses, in other words after all pre-existing RCU read-side -* critical sections have completed. However, the callback function -* might well execute concurrently with RCU read-side critical sections -* that started after call_rcu() was invoked. -* -* RCU read-side critical sections are delimited by rcu_read_lock() -* and rcu_read_unlock(), and may be nested. In addition, but only in -* v5.0 and later, regions of code across which interrupts, preemption, -* or softirqs have been disabled also serve as RCU read-side critical -* sections. This includes hardware interrupt handlers, softirq handlers, -* and NMI handlers. -* -* Note that all CPUs must agree that the grace period extended beyond -* all pre-existing RCU read-side critical section. On systems with more -* than one CPU, this means that when "func()" is invoked, each CPU is -* guaranteed to have executed a full memory barrier since the end of its -* last RCU read-side critical section whose beginning preceded the call -* to call_rcu(). It also means that each CPU executing an RCU read-side -* critical section that continues beyond the start of "func()" must have -* executed a memory barrier after the call_rcu() but before the beginning -* of that RCU read-side critical section. Note that these guarantees -* include CPUs that are offline, idle, or executing in user mode, as -* well as CPUs that are executing in the kernel. -* -* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the -* resulting RCU callback function "func()", then both CPU A and CPU B are -* guaranteed to execute a full memory barrier during the time interval -* between the call to call_rcu() and the invocation of "func()" -- even -* if CPU A and CPU B are the same CPU (but again only if the system has -* more than one CPU). -* -* Implementation of these memory-ordering guarantees is described here: -* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. -*/ -void call_rcu (struct rcu_head *head,rcu_callback_t func) +static void +__call_rcu_common (struct rcu_head *head,rcu_callback_t func,bool lazy) {static atomic_t doublefrees;unsigned long flags;@@ -2809,7 +2770,7 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) }check_cb_ovld (rdp); -if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags)) +if (rcu_nocb_try_bypass (rdp, head, &was_alldone, flags, lazy))return ;// Enqueued onto ->nocb_bypass, so just leave. // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock. rcu_segcblist_enqueue (&rdp->cblist, head);@@ -2831,8 +2792,84 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func)local_irq_restore (flags); } } -EXPORT_SYMBOL_GPL (call_rcu); +#ifdef CONFIG_RCU_LAZY +/** +* call_rcu_flush() - Queue RCU callback for invocation after grace period, and +* flush all lazy callbacks (including the new one) to the main ->cblist while +* doing so. +* kernel/rcu/tree.c void call_rcu(struct rcu_head *head, rcu_callback_t func) · line 2811
Quoted 10 lines
kernel/rcu/tree.c +10
-0+* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side +* critical sections have completed. +* +* Use this API instead of call_rcu() if you don't mind the callback being +* invoked after very long periods of time on systems without memory pressure +* and on systems which are lightly loaded or mostly idle. This comment is backwards, right? Shouldn't it say something like "Use this API instead of
call_rcu()if you don't mind burning extra power..."?
Quoted 3 lines
Quoted 1 line
Yes sorry my mistake :-(. It’s a stale comment from the rework and I’ll update it.
Very good, thank you!
Anything for you!
Quoted 124 lines
Quoted 117 lines
Quoted 104 lines
Quoted 104 lines
kernel/rcu/tree.c +68
-3+* +* Other than the extra delay in callbacks being invoked, this function is +* identical to, and reuses call_rcu()'s logic. Refer to call_rcu() for more +* details about memory ordering and other functionality. +*/ +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, false); +} +EXPORT_SYMBOL_GPL (call_rcu_flush); +#endif + +/** +* call_rcu() - Queue an RCU callback for invocation after a grace period. +* By default the callbacks are 'lazy' and are kept hidden from the main +* ->cblist to prevent starting of grace periods too soon. +* If you desire grace periods to start very soon, use call_rcu_flush(). +* +* @head: structure to be used for queueing the RCU updates. +* @func: actual callback function to be invoked after the grace period +* +* The callback function will be invoked some time after a full grace +* period elapses, in other words after all pre-existing RCU read-side +* critical sections have completed. However, the callback function +* might well execute concurrently with RCU read-side critical sections +* that started after call_rcu() was invoked. +* +* RCU read-side critical sections are delimited by rcu_read_lock() +* and rcu_read_unlock(), and may be nested. In addition, but only in +* v5.0 and later, regions of code across which interrupts, preemption, +* or softirqs have been disabled also serve as RCU read-side critical +* sections. This includes hardware interrupt handlers, softirq handlers, +* and NMI handlers. +* +* Note that all CPUs must agree that the grace period extended beyond +* all pre-existing RCU read-side critical section. On systems with more +* than one CPU, this means that when "func()" is invoked, each CPU is +* guaranteed to have executed a full memory barrier since the end of its +* last RCU read-side critical section whose beginning preceded the call +* to call_rcu(). It also means that each CPU executing an RCU read-side +* critical section that continues beyond the start of "func()" must have +* executed a memory barrier after the call_rcu() but before the beginning +* of that RCU read-side critical section. Note that these guarantees +* include CPUs that are offline, idle, or executing in user mode, as +* well as CPUs that are executing in the kernel. +* +* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the +* resulting RCU callback function "func()", then both CPU A and CPU B are +* guaranteed to execute a full memory barrier during the time interval +* between the call to call_rcu() and the invocation of "func()" -- even +* if CPU A and CPU B are the same CPU (but again only if the system has +* more than one CPU). +* +* Implementation of these memory-ordering guarantees is described here: +* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst. +*/ +void call_rcu (struct rcu_head *head,rcu_callback_t func) +{ +return __call_rcu_common (head, func, true); +} +EXPORT_SYMBOL_GPL (call_rcu);/* Maximum number of jiffies to wait before draining a batch. */ #define KFREE_DRAIN_JIFFIES (5 * HZ)@@ -3507,7 +3544,7 @@ void synchronize_rcu(void)if (rcu_gp_is_expedited ())synchronize_rcu_expedited ();else -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -4323,7 +4364,7 @@ void rcutree_migrate_callbacks(int cpu) my_rdp =this_cpu_ptr (&rcu_data); my_rnp = my_rdp->mynode;rcu_nocb_lock (my_rdp);/* irqs already disabled. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (my_rdp,NULL , jiffies,FLUSH_BP_NONE ));raw_spin_lock_rcu_node (my_rnp);/* irqs already disabled. */ /* Leverage recent GPs and set GP for new callbacks. */ needwake = rcu_advance_cbs(my_rnp, rdp) ||kernel/rcu/tree.h +0
-0@@ -263,14 +263,16 @@ struct rcu_data {unsigned long last_fqs_resched;/* Time of last rcu_resched(). */ unsigned long last_sched_clock;/* Jiffies of last rcu_sched_clock_irq(). */ Quoted 7 lines
kernel/rcu/tree.h struct rcu_data { · line 266
Quoted 1 line
kernel/rcu/tree.h +1
-0+long lazy_len;/* Length of buffered lazy callbacks. */ Do we ever actually care about the length as opposed to whether or not all the bypass callbacks are lazy? If not, a "
some_nonlazy" boolean would be initialed to zero and ORed with the non-laziness of the added callback. Or, if there was a test anyway, simply set to 1 in the presence of a non-lazy callback. And as now, gets zeroed when the bypass is flushed.We had discussed this before, and my point was we could use it for tracing and future extension as well. If it’s ok with you, I prefer to keep it this way than having to come back add the length later.
Quoted 6 lines
On a minor point as well, if you really want it this way, I am afraid of changing this now since I tested this way for last several iterations and it’s easy to add a regression. So I prefer to keep it this way and then I can add more patches later on top if that’s Ok with you.
Quoted 3 lines
No, you are right, the debug information might be helpful.
But hey, at least I am consistent!
Haha ;-)
Quoted 161 lines
Quoted 154 lines
Quoted 89 lines
This might shorten a few lines of code.
Quoted 86 lines
kernel/rcu/tree.h +12
-4int cpu; };/* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0#define RCU_NOCB_WAKE_BYPASS 1 -#define RCU_NOCB_WAKE 2 -#define RCU_NOCB_WAKE_FORCE 3 +#define RCU_NOCB_WAKE_LAZY 2 +#define RCU_NOCB_WAKE 3 +#define RCU_NOCB_WAKE_FORCE 4#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500))/* For jiffies_till_first_fqs and */ @@ -439,10 +441,17 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp); + +#define FLUSH_BP_NONE 0 +/* Is the CB being enqueued after the flush, a lazy CB? */ +#define FLUSH_BP_LAZY BIT(0) +/* Wake up nocb-GP thread after flush? */ +#define FLUSH_BP_WAKE BIT(1)static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j); +unsigned long j,unsigned long flush_flags);static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags); +bool *was_alldone,unsigned long flags, +bool lazy);static void __call_rcu_nocb_wake (struct rcu_data *rdp,bool was_empty,unsigned long flags);static int rcu_nocb_need_deferred_wakeup (struct rcu_data *rdp,int level);kernel/rcu/tree_exp.h +1
-1@@ -937,7 +937,7 @@ void synchronize_rcu_expedited(void)/* If expedited grace periods are prohibited, fall back to normal. */ if (rcu_gp_is_normal ()) { -wait_rcu_gp (call_rcu); +wait_rcu_gp (call_rcu_flush);return ; }kernel/rcu/tree_nocb.h +25
-0@@ -256,6 +256,31 @@ static bool wake_nocb_gp(struct rcu_data *rdp, bool force)return __wake_nocb_gp (rdp_gp, rdp, force, flags); } +/* +* LAZY_FLUSH_JIFFIES decides the maximum amount of time that +* can elapse before lazy callbacks are flushed. Lazy callbacks +* could be flushed much earlier for a number of other reasons +* however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are +* left unsubmitted to RCU after those many jiffies. +*/ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush =LAZY_FLUSH_JIFFIES ; + +#ifdef CONFIG_RCU_LAZY +// To be called only from test code. +void rcu_lazy_set_jiffies_till_flush (unsigned long jif) +{ + jiffies_till_flush = jif; +} +EXPORT_SYMBOL (rcu_lazy_set_jiffies_till_flush); + +unsigned long rcu_lazy_get_jiffies_till_flush (void ) +{ +return jiffies_till_flush; +} +EXPORT_SYMBOL (rcu_lazy_get_jiffies_till_flush); +#endif +/* * Arrange to wake the GP kthread for this NOCB group at some future * time when it is safe to do so. @@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);Quoted 13 lines
kernel/rcu/tree_nocb.h static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 9 lines
kernel/rcu/tree_nocb.h +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {Please leave the "&&" on the previous line and line up the "READ_ONCE(" with the "waketype". That makes it easier to tell the condition from the following code.
Will do!
Quoted 23 lines
Quoted 23 lines
kernel/rcu/tree_nocb.h +8
-1+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -293,12 +322,16 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* proves to be initially empty, just return false because the no-CB GP * kthread may need to be awakened in this case. * +* Return true if there was something to be flushed and it succeeded, otherwise +* false. +* * Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +bool lazy = flush_flags &FLUSH_BP_LAZY ;Quoted 13 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 349
Quoted 11 lines
kernel/rcu/tree_nocb.h +4
-1WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));rcu_lockdep_assert_cblist_protected (rdp);@@ -310,7 +343,20 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,/* Note: ->cblist.len already accounts for ->nocb_bypass contents. */ if (rhp)rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); + +/* +* If the new CB requested was a lazy one, queue it onto the main +* ->cblist so we can take advantage of a sooner grade period. "take advantage of a grace period that will happen regardless."?
Sure will update.
Quoted 9 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 353
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0+*/ +if (lazy && rhp) { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); +rcu_cblist_enqueue (&rcl, rhp);Would it makes sense to enqueue rhp onto ->nocb_bypass first, NULL out rhp, then let the
rcu_cblist_flush_enqueue()be common code? Or did this function grow a later use of rhp that I missed?Quoted 6 lines
No that could be done, but it prefer to keep it this way because rhp is a function parameter and I prefer not to modify those since it could add a bug in future where rhp passed by user is now NULL for some reason, half way through the function.
Quoted 4 lines
I agree that changing a function parameter is bad practice.
So the question becomes whether introducing a local would outweigh consolidating this code. Could you please at least give it a shot?
Yes for sure I will try this. Was thinking the same.
Quoted 23 lines
Quoted 9 lines
Quoted 7 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 357
Quoted 5 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
struct rcu_head *rhp = rhp_in;And then:
Quoted 8 lines
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL; } rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);Or did I mess something up?
Yes you are spot on. It seems more shorter, I’ll do it this way!
Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
Quoted 7 lines
kernel/rcu/tree_nocb.h +2
-0+ } +rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);rcu_nocb_bypass_unlock (rdp);kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 380
Quoted 10 lines
kernel/rcu/tree_nocb.h +5
-1@@ -326,13 +372,33 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* Note that this function always returns true if rhp is NULL. */ static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) { +bool ret; +bool was_alldone = false; +bool bypass_all_lazy = false; +long bypass_ncbs;Alphabetical order by variable name, please. (Yes, I know that this is strange, but what can I say?)
Quoted 3 lines
Quoted 1 line
Sure.
Thank you!
My pleasure!
Quoted 110 lines
Quoted 110 lines
Quoted 109 lines
Quoted 87 lines
kernel/rcu/tree_nocb.h +30
-7+if (!rcu_rdp_is_offloaded (rdp))return true;rcu_lockdep_assert_cblist_protected (rdp);rcu_nocb_bypass_lock (rdp); -return rcu_nocb_do_flush_bypass (rdp, rhp, j); + +if (flush_flags &FLUSH_BP_WAKE ) { + was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + bypass_all_lazy = bypass_ncbs && (bypass_ncbs == rdp->lazy_len); + } + + ret =rcu_nocb_do_flush_bypass (rdp, rhp, j, flush_flags); + +// Wake up the nocb GP thread if needed. GP thread could be sleeping +// while waiting for lazy timer to expire (otherwise rcu_barrier may +// end up waiting for the duration of the lazy timer). +if (flush_flags &FLUSH_BP_WAKE && was_alldone && bypass_all_lazy) +wake_nocb_gp (rdp, false); + +return ret; }/* @@ -345,7 +411,7 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)if (!rcu_rdp_is_offloaded (rdp) || !rcu_nocb_bypass_trylock (rdp))return ; -WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j)); +WARN_ON_ONCE (!rcu_nocb_do_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE )); }/* @@ -367,12 +433,14 @@ static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)* there is only one CPU in operation. */ static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags, +bool lazy) {unsigned long c;unsigned long cur_gp_seq;unsigned long j = jiffies;long ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); +bool bypass_is_lazy = (ncbs ==READ_ONCE (rdp->lazy_len));lockdep_assert_irqs_disabled ();@@ -417,25 +485,30 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// If there hasn't yet been all that many ->cblist enqueues // this jiffy, tell the caller to enqueue onto ->cblist. But flush // ->nocb_bypass first. -if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { +// Lazy CBs throttle this back and do immediate bypass queuing. +if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist);if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" )); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j)); + +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , j,FLUSH_BP_NONE ));WARN_ON_ONCE (rcu_cblist_n_cbs (&rdp->nocb_bypass));return false;// Caller must enqueue the callback. }// If ->nocb_bypass has been used too long or is too full, // flush ->nocb_bypass to ->cblist. -if ((ncbs && j !=READ_ONCE (rdp->nocb_bypass_first)) || +if ((ncbs && !bypass_is_lazy && j !=READ_ONCE (rdp->nocb_bypass_first)) || + (ncbs && bypass_is_lazy && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush))) || ncbs >= qhimark) {rcu_nocb_lock (rdp); *was_alldone = !rcu_segcblist_pend_cbs (&rdp->cblist); -if (!rcu_nocb_flush_bypass (rdp, rhp, j)) { +if (!rcu_nocb_flush_bypass (rdp, rhp, j, + lazy ?FLUSH_BP_LAZY :FLUSH_BP_NONE )) {if (*was_alldone)trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstQ" ));kernel/rcu/tree_nocb.h static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 541
Quoted 9 lines
kernel/rcu/tree_nocb.h +3
-0@@ -460,16 +533,29 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,// We need to use the bypass. rcu_nocb_wait_contended (rdp);rcu_nocb_bypass_lock (rdp); + ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass);rcu_segcblist_inc_len (&rdp->cblist);/* Must precede enqueue. */ rcu_cblist_enqueue (&rdp->nocb_bypass, rhp); + +if (IS_ENABLED (CONFIG_RCU_LAZY ) && lazy)Won't !IS_ENABLED(CONFIG_RCU_LAZY) mean that lazy cannot be set? Why do we need to check both? Or are you going for dead code? If so, shouldn't there be
IS_ENABLED(CONFIG_RCU_LAZY)checks above as well?Me, I am not convinced that the dead code would buy you much. In fact, the compiler might well be propagating the constants on its own.
Ah! The reason the compiler cannot figure this out is because you put the switch into
rcu.h. If you instead always export thecall_rcu_flush()definition, and checkIS_ENABLED(CONFIG_RCU_LAZY)at the beginning ofcall_rcu(), the compiler should have the information that it needs to do this for you.
Quoted 3 lines
Quoted 1 line
Ah ok, I will try to do it this way.
Very good, thank you, for this and for the rest below.
Glad we could do this review, thanks.
- Joel
Quoted 190 lines
Thanx, Paul
Quoted 187 lines
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_nocb.h +11
-1+WRITE_ONCE (rdp->lazy_len, rdp->lazy_len +1 ); +if (!ncbs) {WRITE_ONCE (rdp->nocb_bypass_first, j);trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("FirstBQ" )); } +rcu_nocb_bypass_unlock (rdp);smp_mb ();/* Order enqueue before wake. */ -if (ncbs) { + +// A wake up of the grace period kthread or timer adjustment needs to +// be done only if: +// 1. Bypass list was fully empty before (this is the first bypass list entry). +// Or, both the below conditions are met: +// 1. Bypass list had only lazy CBs before. +// 2. The new CB is non-lazy. +if (ncbs && (!bypass_is_lazy || lazy)) {local_irq_restore (flags); }else {// No-CBs GP kthread might be indefinitely asleep, if so, wake. @@ -499,7 +585,7 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,Quoted 8 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 589
Quoted 6 lines
kernel/rcu/tree_nocb.h +1
-1{unsigned long cur_gp_seq;unsigned long j; -long len; +long len, lazy_len, bypass_len;struct task_struct *t;Again, alphabetical please, strange though that might seem.
Yes sure
Quoted 22 lines
Quoted 22 lines
kernel/rcu/tree_nocb.h +8
-1// If we are being polled or there is no kthread, just leave. @@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist); + bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" )); + }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);@@ -604,8 +697,8 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)*/ Quoted 8 lines
kernel/rcu/tree_nocb.h static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · line 701
Quoted 6 lines
kernel/rcu/tree_nocb.h +2
-2static void nocb_gp_wait (struct rcu_data *my_rdp) { -bool bypass = false; -long bypass_ncbs; +bool bypass = false, lazy = false; +long bypass_ncbs, lazy_ncbs;And ditto.
Ok
Quoted 56 lines
Quoted 56 lines
kernel/rcu/tree_nocb.h +23
-6int __maybe_unused cpu = my_rdp->cpu;unsigned long cur_gp_seq;unsigned long flags;@@ -640,24 +733,41 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)* won't be ignored for long. */ list_for_each_entry (rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { +bool flush_bypass = false; +trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("Check" ));rcu_nocb_lock_irqsave (rdp, flags);lockdep_assert_held (&rdp->nocb_lock); bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); -if (bypass_ncbs && + lazy_ncbs =READ_ONCE (rdp->lazy_len); + +if (bypass_ncbs && (lazy_ncbs == bypass_ncbs) && + (time_after (j,READ_ONCE (rdp->nocb_bypass_first) + jiffies_till_flush) || + bypass_ncbs >2 * qhimark)) { + flush_bypass = true; + }else if (bypass_ncbs && (lazy_ncbs != bypass_ncbs) && (time_after (j,READ_ONCE (rdp->nocb_bypass_first) +1 ) || bypass_ncbs >2 * qhimark)) { -// Bypass full or old, so flush it. - (void )rcu_nocb_try_flush_bypass (rdp, j); - bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + flush_bypass = true; }else if (!bypass_ncbs &&rcu_segcblist_empty (&rdp->cblist)) {rcu_nocb_unlock_irqrestore (rdp, flags);continue ;/* No callbacks here, try next. */ } + +if (flush_bypass) { +// Bypass full or old, so flush it. + (void )rcu_nocb_try_flush_bypass (rdp, j); + bypass_ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_ncbs =READ_ONCE (rdp->lazy_len); + } +if (bypass_ncbs) {trace_rcu_nocb_wake (rcu_state.name, rdp->cpu, -TPS ("Bypass" )); - bypass = true; + bypass_ncbs == lazy_ncbs ?TPS ("Lazy" ) :TPS ("Bypass" )); +if (bypass_ncbs == lazy_ncbs) + lazy = true; +else + bypass = true; } rnp = rdp->mynode;@@ -705,12 +815,21 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_gp = needwait_gp; my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq :0 ;Quoted 12 lines
kernel/rcu/tree_nocb.h static void nocb_gp_wait(struct rcu_data *my_rdp) · line 822
Quoted 10 lines
kernel/rcu/tree_nocb.h +5
-5-if (bypass && !rcu_nocb_poll) { -// At least one child with non-empty ->nocb_bypass, so set -// timer in order to avoid stranding its callbacks. -wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , -TPS ("WakeBypassIsDeferred" )); +// At least one child with non-empty ->nocb_bypass, so set +// timer in order to avoid stranding its callbacks. +if (!rcu_nocb_poll) { +// If bypass list only has lazy CBs. Add a deferred +// lazy wake up. One sentence rather than two.
Ok
Quoted 50 lines
Quoted 50 lines
kernel/rcu/tree_nocb.h +13
-3+if (lazy && !bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazyIsDeferred" )); +// Otherwise add a deferred bypass wake up. + }else if (bypass) { +wake_nocb_gp_defer (my_rdp,RCU_NOCB_WAKE_BYPASS , +TPS ("WakeBypassIsDeferred" )); + } } +if (rcu_nocb_poll) {/* Polling, so trace if first poll in the series. */ if (gotcbs)@@ -1036,7 +1155,7 @@ static long rcu_nocb_rdp_deoffload(void *arg)* return false, which means that future calls to rcu_nocb_try_bypass() * will refuse to put anything into the bypass. */ -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_NONE ));/* * Start with invoking rcu_core() early. This way if the current thread * happens to preempt an ongoing call to rcu_core() in the middle, @@ -1278,6 +1397,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)raw_spin_lock_init (&rdp->nocb_gp_lock);timer_setup (&rdp->nocb_timer, do_nocb_deferred_wakeup_timer,0 );rcu_cblist_init (&rdp->nocb_bypass); +WRITE_ONCE (rdp->lazy_len,0 );mutex_init (&rdp->nocb_gp_kthread_mutex); }@@ -1559,13 +1679,13 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) }static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -unsigned long j) +unsigned long j,unsigned long flush_flags) {return true; }static bool rcu_nocb_try_bypass (struct rcu_data *rdp,struct rcu_head *rhp, -bool *was_alldone,unsigned long flags) +bool *was_alldone,unsigned long flags,bool lazy) {return false; }Signature · 1 line
2.37.3.998.g577e59143f-goog
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Paul E. McKenneyHi Paul,
Back to Mutt for this one ;-)
Replies below:
On Sat, Sep 24, 2022 at 02:11:32PM -0700, Paul E. McKenney wrote:
[...]
Quoted 39 lines
Quoted 9 lines
Quoted 8 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+ */ + if (lazy && rhp) { + rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, NULL); + rcu_cblist_enqueue(&rcl, rhp);Would it makes sense to enqueue rhp onto ->nocb_bypass first, NULL out rhp, then let the
rcu_cblist_flush_enqueue()be common code? Or did this function grow a later use of rhp that I missed?Quoted 6 lines
No that could be done, but it prefer to keep it this
way because rhp is a function parameter and I
prefer not to modify those since it could add a bug in future where rhp passed by user is now NULL for some reason, half way through the function.I agree that changing a function parameter is bad practice.
So the question becomes whether introducing a local would outweigh consolidating this code. Could you please at least give it a shot?
Quoted 8 lines
Quoted 6 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+ WRITE_ONCE(rdp->lazy_len, 0); + } else { + rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); + WRITE_ONCE(rdp->lazy_len, 0);This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.
else
1. flush bypass CBs on to main list
2. queue new CB on to bypass list.
Quoted 5 lines
} rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);Or did I mess something up?
So the rcu_cblist_flush_enqueue() has to happen before the rcu_cblist_enqueue() to preserve the ordering of flushing into the main list, and queuing on to the main list for the "if". Where as in your snip, the order is reversed.
If I consolidate it then, it looks like the following. However, it is a bit more unreadable. I could instead just take the WRITE_ONCE out of both if/else and move it to after the if/else, that would be cleanest. Does that sound good to you? Thanks!
---8<-----------------------
kernel/rcu/tree_nocb.h +9 -9
@@ -327,10 +327,11 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* * Note that this function always returns true if rhp is NULL. */ -static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, +static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp_in,unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +struct rcu_head *rhp = rhp_in;bool lazy = flush_flags &FLUSH_BP_LAZY ;WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));
@@ -348,14 +349,13 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* If the new CB requested was a lazy one, queue it onto the main * ->cblist so we can take advantage of a sooner grade period. */ -if (lazy && rhp) { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); -rcu_cblist_enqueue (&rcl, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - }else { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - } +if (lazy && rhp) + rhp =NULL ; +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +if (lazy && rhp_in) +rcu_cblist_enqueue (&rcl, rhp_in); + +WRITE_ONCE (rdp->lazy_len,0 );rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 3 lines · Joel Fernandes
Hi Paul,
Back to Mutt for this one ;-)
As they used to say in my youth, "Get a horse!" ;-)
Quoted 51 lines
Replies below:
On Sat, Sep 24, 2022 at 02:11:32PM -0700, Paul E. McKenney wrote:
[...]Quoted 39 lines
Quoted 9 lines
Quoted 8 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+ */ + if (lazy && rhp) { + rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, NULL); + rcu_cblist_enqueue(&rcl, rhp);Would it makes sense to enqueue rhp onto ->nocb_bypass first, NULL out rhp, then let the
rcu_cblist_flush_enqueue()be common code? Or did this function grow a later use of rhp that I missed?Quoted 6 lines
No that could be done, but it prefer to keep it this
way because rhp is a function parameter and I
prefer not to modify those since it could add a bug in future where rhp passed by user is now NULL for some reason, half way through the function.I agree that changing a function parameter is bad practice.
So the question becomes whether introducing a local would outweigh consolidating this code. Could you please at least give it a shot?
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 358
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;
Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.
And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Or am I on the wrong branch of this "if" statement?
Quoted 8 lines
else
1. flush bypass CBs on to main list
2. queue new CB on to bypass list.Quoted 4 lines
} rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);
Quoted 6 lines
Quoted 1 line
Or did I mess something up?
So the
rcu_cblist_flush_enqueue()has to happen before thercu_cblist_enqueue()to preserve the ordering of flushing into the main list, and queuing on to the main list for the "if". Where as in your snip, the order is reversed.
Did I pick the correct branch of the "if" statement above? Or were you instead talking about the "else" clause?
I would have been more worried about getting cblist->len right.
Quoted 4 lines
If I consolidate it then, it looks like the following. However, it is a bit more unreadable. I could instead just take the
WRITE_ONCEout of both if/else and move it to after the if/else, that would be cleanest. Does that sound good to you? Thanks!
Let's first figure out whether or not we are talking past one another. ;-)
Thanx, Paul
Quoted 41 lines
---8<-----------------------
kernel/rcu/tree_nocb.h +9
-9@@ -327,10 +327,11 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* * Note that this function always returns true if rhp is NULL. */ -static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, +static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp_in,unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +struct rcu_head *rhp = rhp_in;bool lazy = flush_flags &FLUSH_BP_LAZY ;WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));@@ -348,14 +349,13 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* If the new CB requested was a lazy one, queue it onto the main * ->cblist so we can take advantage of a sooner grade period. */ -if (lazy && rhp) { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); -rcu_cblist_enqueue (&rcl, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - }else { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - } +if (lazy && rhp) + rhp =NULL ; +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +if (lazy && rhp_in) +rcu_cblist_enqueue (&rcl, rhp_in); + +WRITE_ONCE (rdp->lazy_len,0 );rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Hi Paul,
On Mon, Sep 26, 2022 at 10:42:40AM -0700, Paul E. McKenney wrote:
[..]
Quoted 33 lines
Quoted 26 lines
Quoted 18 lines
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 358
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.
Quoted 6 lines
And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Or am I on the wrong branch of this "if" statement?
But we have to flush first, and then queue the new one. Otherwise wouldn't the callbacks be invoked out of order? Or did I miss something?
Quoted 15 lines
Quoted 8 lines
else
1. flush bypass CBs on to main list
2. queue new CB on to bypass list.Quoted 4 lines
} rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);Quoted 6 lines
Quoted 1 line
Or did I mess something up?
So the
rcu_cblist_flush_enqueue()has to happen before thercu_cblist_enqueue()to preserve the ordering of flushing into the main list, and queuing on to the main list for the "if". Where as in your snip, the order is reversed.
Quoted 4 lines
Did I pick the correct branch of the "if" statement above? Or were you instead talking about the "else" clause?
I would have been more worried about getting
cblist->lenright.
Hmm, I think my concern was more the ordering of callbacks, and moving the write to length should be Ok.
Quoted 6 lines
Quoted 4 lines
If I consolidate it then, it looks like the following. However, it is a bit more unreadable. I could instead just take the
WRITE_ONCEout of both if/else and move it to after the if/else, that would be cleanest. Does that sound good to you? Thanks!Let's first figure out whether or not we are talking past one another. ;-)
Haha yeah :-)
thanks,
- Joel
Quoted 44 lines
Thanx, Paul
Quoted 41 lines
---8<-----------------------
kernel/rcu/tree_nocb.h +9
-9@@ -327,10 +327,11 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* * Note that this function always returns true if rhp is NULL. */ -static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, +static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp_in,unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +struct rcu_head *rhp = rhp_in;bool lazy = flush_flags &FLUSH_BP_LAZY ;WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));@@ -348,14 +349,13 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,* If the new CB requested was a lazy one, queue it onto the main * ->cblist so we can take advantage of a sooner grade period. */ -if (lazy && rhp) { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); -rcu_cblist_enqueue (&rcl, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - }else { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - } +if (lazy && rhp) + rhp =NULL ; +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +if (lazy && rhp_in) +rcu_cblist_enqueue (&rcl, rhp_in); + +WRITE_ONCE (rdp->lazy_len,0 );rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 42 lines · Joel Fernandes
Hi Paul,
On Mon, Sep 26, 2022 at 10:42:40AM -0700, Paul E. McKenney wrote:
[..]Quoted 38 lines
Quoted 26 lines
Quoted 18 lines
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 358
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Quoted 4 lines
Quoted 1 line
Or am I on the wrong branch of this "if" statement?
But we have to flush first, and then queue the new one. Otherwise wouldn't the callbacks be invoked out of order? Or did I miss something?
I don't think so...
We want the new callback to be last, right? One way to do that is to flush the bypass, then queue the new callback onto ->cblist. Another way to do that is to enqueue the new callback onto the end of the bypass, then flush the bypass. Why wouldn't these result in the same order?
Quoted 15 lines
Quoted 15 lines
Quoted 14 lines
else
1. flush bypass CBs on to main list
2. queue new CB on to bypass list.Quoted 5 lines
} rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);Or did I mess something up?
So the
rcu_cblist_flush_enqueue()has to happen before thercu_cblist_enqueue()to preserve the ordering of flushing into the main list, and queuing on to the main list for the "if". Where as in your snip, the order is reversed.
Quoted 7 lines
Quoted 4 lines
Did I pick the correct branch of the "if" statement above? Or were you instead talking about the "else" clause?
I would have been more worried about getting
cblist->lenright.Hmm, I think my concern was more the ordering of callbacks, and moving the write to length should be Ok.
OK, sounds good to me! ;-)
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
If I consolidate it then, it looks like the following. However, it is a bit more unreadable. I could instead just take the
WRITE_ONCEout of both if/else and move it to after the if/else, that would be cleanest. Does that sound good to you? Thanks!Let's first figure out whether or not we are talking past one another. ;-)
Haha yeah :-)
So were we? ;-)
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 52 lines · Paul E. McKenney
Quoted 42 lines · Joel Fernandes
Hi Paul,
On Mon, Sep 26, 2022 at 10:42:40AM -0700, Paul E. McKenney wrote:
[..]Quoted 38 lines
Quoted 26 lines
Quoted 18 lines
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 358
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Quoted 4 lines
Quoted 1 line
Or am I on the wrong branch of this "if" statement?
But we have to flush first, and then queue the new one. Otherwise wouldn't the callbacks be invoked out of order? Or did I miss something?
Quoted 6 lines
I don't think so...
We want the new callback to be last, right? One way to do that is to flush the bypass, then queue the new callback onto ->cblist. Another way to do that is to enqueue the new callback onto the end of the bypass, then flush the bypass. Why wouldn't these result in the same order?
Yes you are right, sorry. I was fixated on the main list. Both your snippet and my patch will be equivalent then. However I find your snippet a bit confusing, as in it is not immediately obvious - why would we queue something on to a list, if we were about to flush it. But any way, it does make it a clever piece of code in some sense and I am ok with doing it this way ;-)
Thanks,
- Joel
Quoted 38 lines
Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
else
1. flush bypass CBs on to main list
2. queue new CB on to bypass list.Quoted 5 lines
} rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);Or did I mess something up?
So the
rcu_cblist_flush_enqueue()has to happen before thercu_cblist_enqueue()to preserve the ordering of flushing into the main list, and queuing on to the main list for the "if". Where as in your snip, the order is reversed.Quoted 7 lines
Quoted 4 lines
Did I pick the correct branch of the "if" statement above? Or were you instead talking about the "else" clause?
I would have been more worried about getting
cblist->lenright.Hmm, I think my concern was more the ordering of callbacks, and moving the write to length should be Ok.
OK, sounds good to me! ;-)
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
If I consolidate it then, it looks like the following. However, it is a bit more unreadable. I could instead just take the
WRITE_ONCEout of both if/else and move it to after the if/else, that would be cleanest. Does that sound good to you? Thanks!Let's first figure out whether or not we are talking past one another. ;-)
Haha yeah :-)
So were we? ;-)
Thanx, Paul
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 52 lines · Joel Fernandes
Quoted 52 lines · Paul E. McKenney
Quoted 46 lines · Joel Fernandes
Hi Paul,
On Mon, Sep 26, 2022 at 10:42:40AM -0700, Paul E. McKenney wrote:
[..]Quoted 33 lines
Quoted 26 lines
Quoted 18 lines
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 358
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.Quoted 6 lines
And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Or am I on the wrong branch of this "if" statement?
But we have to flush first, and then queue the new one. Otherwise wouldn't the callbacks be invoked out of order? Or did I miss something?
Quoted 8 lines
Quoted 6 lines
I don't think so...
We want the new callback to be last, right? One way to do that is to flush the bypass, then queue the new callback onto ->cblist. Another way to do that is to enqueue the new callback onto the end of the bypass, then flush the bypass. Why wouldn't these result in the same order?
Yes you are right, sorry. I was fixated on the main list. Both your snippet and my patch will be equivalent then. However I find your snippet a bit confusing, as in it is not immediately obvious - why would we queue something on to a list, if we were about to flush it. But any way, it does make it a clever piece of code in some sense and I am ok with doing it this way ;-)
As long as the ->cblist.len comes out with the right value. ;-)
Thanx, Paul
Quoted 43 lines
Thanks,
- Joel
Quoted 38 lines
Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
else
1. flush bypass CBs on to main list
2. queue new CB on to bypass list.Quoted 5 lines
} rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); WRITE_ONCE(rdp->lazy_len, 0);Or did I mess something up?
So the
rcu_cblist_flush_enqueue()has to happen before thercu_cblist_enqueue()to preserve the ordering of flushing into the main list, and queuing on to the main list for the "if". Where as in your snip, the order is reversed.Quoted 7 lines
Quoted 4 lines
Did I pick the correct branch of the "if" statement above? Or were you instead talking about the "else" clause?
I would have been more worried about getting
cblist->lenright.Hmm, I think my concern was more the ordering of callbacks, and moving the write to length should be Ok.
OK, sounds good to me! ;-)
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
If I consolidate it then, it looks like the following. However, it is a bit more unreadable. I could instead just take the
WRITE_ONCEout of both if/else and move it to after the if/else, that would be cleanest. Does that sound good to you? Thanks!Let's first figure out whether or not we are talking past one another. ;-)
Haha yeah :-)
So were we? ;-)
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 55 lines · Paul E. McKenney
Quoted 54 lines · Joel Fernandes
Quoted 54 lines · Paul E. McKenney
Quoted 42 lines · Joel Fernandes
Hi Paul,
On Mon, Sep 26, 2022 at 10:42:40AM -0700, Paul E. McKenney wrote:
[..]Quoted 38 lines
Quoted 26 lines
Quoted 18 lines
Quoted 8 lines
Quoted 6 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+ WRITE_ONCE(rdp->lazy_len, 0); + } else { + rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); + WRITE_ONCE(rdp->lazy_len, 0);This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Quoted 4 lines
Quoted 1 line
Or am I on the wrong branch of this "if" statement?
But we have to flush first, and then queue the new one. Otherwise wouldn't the callbacks be invoked out of order? Or did I miss something?
I don't think so...
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
We want the new callback to be last, right? One way to do that is to flush the bypass, then queue the new callback onto ->cblist. Another way to do that is to enqueue the new callback onto the end of the bypass, then flush the bypass. Why wouldn't these result in the same order?
Yes you are right, sorry. I was fixated on the main list. Both your snippet and my patch will be equivalent then. However I find your snippet a bit confusing, as in it is not immediately obvious - why would we queue something on to a list, if we were about to flush it. But any way, it does make it a clever piece of code in some sense and I am ok with doing it this way ;-)
As long as the ->cblist.len comes out with the right value. ;-)
The ->cblist.len's value is not effected by your suggested change, because the bypass list's length is already accounted into the ->cblist.len, and for the new rhp, after rcu_nocb_do_flush_bypass() is called, it either ends up in the bypass list (if it is !lazy) or on the main cblist (if its lazy). So everything just works. Below is the change. If its OK with you though, I will put it in a separate commit just to be extra safe, since the code before it was well tested and I am still testing it.
Thanks.
---8<-----------------------
From: "Joel Fernandes (Google)" <[email protected]>
Subject: [PATCH] rcu: Refactor code a bit in rcu_nocb_do_flush_bypass()
This consolidates the code a bit and makes it cleaner. Functionally it is the same.
--- kernel/rcu/tree_nocb.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
kernel/rcu/tree_nocb.h +8 -8
@@ -327,10 +327,11 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* * Note that this function always returns true if rhp is NULL. */ -static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, +static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp_in,unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +struct rcu_head *rhp = rhp_in;bool lazy = flush_flags &FLUSH_BP_LAZY ;WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));
@@ -347,16 +348,15 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,/* * If the new CB requested was a lazy one, queue it onto the main * ->cblist so that we can take advantage of the grace-period that will -* happen regardless. +* happen regardless. But queue it onto the bypass list first so that +* the lazy CB is ordered with the existing CBs in the bypass list. */ if (lazy && rhp) { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); -rcu_cblist_enqueue (&rcl, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - }else { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); +rcu_cblist_enqueue (&rdp->nocb_bypass, rhp); + rhp =NULL ; } +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);
Signature · 1 line
2.38.0.rc1.362.ged0d419d3c-goog
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 65 lines · Joel Fernandes
Quoted 55 lines · Paul E. McKenney
Quoted 54 lines · Joel Fernandes
Quoted 54 lines · Paul E. McKenney
Quoted 42 lines · Joel Fernandes
Hi Paul,
On Mon, Sep 26, 2022 at 10:42:40AM -0700, Paul E. McKenney wrote:
[..]Quoted 38 lines
Quoted 26 lines
Quoted 18 lines
Quoted 8 lines
Quoted 6 lines
kernel/rcu/tree_nocb.h static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 359
Quoted 4 lines
kernel/rcu/tree_nocb.h +4
-0+WRITE_ONCE (rdp->lazy_len,0 ); + }else { +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );This
WRITE_ONCE()can be dropped out of the "if" statement, correct?Yes will update.
Thank you!
Quoted 4 lines
Quoted 2 lines
If so, this could be an "if" statement with two statements in its "then" clause, no "else" clause, and two statements following the "if" statement.
I don’t think we can get rid of the else part but I’ll see what it looks like.
In the function header, s/rhp/rhp_in/, then:
Quoted 7 lines
struct rcu_head *rhp = rhp_in;And then:
if (lazy && rhp) { rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); rhp = NULL;Quoted 6 lines
This enqueues on to the bypass list, where as if lazy && rhp, I want to queue the new rhp on to the main cblist. So the pseudo code in my patch is:
if (lazy and rhp) then
1. flush bypass CBs on to main list.
2. queue new CB on to main list.And the difference is here, correct? I enqueue to the bypass list, which is then flushed (in order) to the main list. In contrast, you flush the bypass list, then enqueue to the main list. Either way, the callback referenced by rhp ends up at the end of ->cblist.
Quoted 4 lines
Quoted 1 line
Or am I on the wrong branch of this "if" statement?
But we have to flush first, and then queue the new one. Otherwise wouldn't the callbacks be invoked out of order? Or did I miss something?
I don't think so...
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
We want the new callback to be last, right? One way to do that is to flush the bypass, then queue the new callback onto ->cblist. Another way to do that is to enqueue the new callback onto the end of the bypass, then flush the bypass. Why wouldn't these result in the same order?
Yes you are right, sorry. I was fixated on the main list. Both your snippet and my patch will be equivalent then. However I find your snippet a bit confusing, as in it is not immediately obvious - why would we queue something on to a list, if we were about to flush it. But any way, it does make it a clever piece of code in some sense and I am ok with doing it this way ;-)
As long as the ->cblist.len comes out with the right value. ;-)
Quoted 7 lines
The ->cblist.len's value is not effected by your suggested change, because the bypass list's length is already accounted into the ->cblist.len, and for the new rhp, after
rcu_nocb_do_flush_bypass()is called, it either ends up in the bypass list (if it is !lazy) or on the main cblist (if its lazy). So everything just works. Below is the change. If its OK with you though, I will put it in a separate commit just to be extra safe, since the code before it was well tested and I am still testing it.
Having this as a separate simplification commit is fine by me.
And thank you for digging into this!
Thanx, Paul
Quoted 59 lines
Thanks.
---8<-----------------------
From: "Joel Fernandes (Google)" <[email protected]>
Subject: [PATCH] rcu: Refactor code a bit inrcu_nocb_do_flush_bypass()This consolidates the code a bit and makes it cleaner. Functionally it is the same.
Reported-by Paul E. McKenney <[email protected]>Signed-off-by Joel Fernandes (Google) <[email protected]>--- kernel/rcu/tree_nocb.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)kernel/rcu/tree_nocb.h +8
-8@@ -327,10 +327,11 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,* * Note that this function always returns true if rhp is NULL. */ -static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp, +static bool rcu_nocb_do_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp_in,unsigned long j,unsigned long flush_flags) {struct rcu_cblist rcl; +struct rcu_head *rhp = rhp_in;bool lazy = flush_flags &FLUSH_BP_LAZY ;WARN_ON_ONCE (!rcu_rdp_is_offloaded (rdp));@@ -347,16 +348,15 @@ static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,/* * If the new CB requested was a lazy one, queue it onto the main * ->cblist so that we can take advantage of the grace-period that will -* happen regardless. +* happen regardless. But queue it onto the bypass list first so that +* the lazy CB is ordered with the existing CBs in the bypass list. */ if (lazy && rhp) { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass,NULL ); -rcu_cblist_enqueue (&rcl, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); - }else { -rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); -WRITE_ONCE (rdp->lazy_len,0 ); +rcu_cblist_enqueue (&rdp->nocb_bypass, rhp); + rhp =NULL ; } +rcu_cblist_flush_enqueue (&rcl, &rdp->nocb_bypass, rhp); +WRITE_ONCE (rdp->lazy_len,0 );rcu_segcblist_insert_pend_cbs (&rdp->cblist, &rcl);WRITE_ONCE (rdp->nocb_bypass_first, j);Signature · 1 line
2.38.0.rc1.362.ged0d419d3c-goog
Frederic Weisbecker
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Joel Fernandes (Google)kernel/rcu/tree.c static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 10 lines · Joel Fernandes (Google)
kernel/rcu/tree.c +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));
This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling rcu_segcblist_entrain() as well, with "Fixes: " tag.
And then FLUSH_BP_WAKE is probably not needed anymore.
Quoted 5 lines
kernel/rcu/tree.c +0
-0if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);
kernel/rcu/tree.c static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 10 lines
kernel/rcu/tree.c +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {
This can be a plain READ since ->nocb_defer_wakeup is only written under ->nocb_gp_lock.
Quoted 10 lines
kernel/rcu/tree_nocb.h +3
-0+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist);
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 609
Quoted 10 lines
kernel/rcu/tree_nocb.h +7
-1+ bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" ));
I'm trying to think of a case where rcu_nocb_try_bypass() returns false (queue to regular list) but then call_rcu() -> __call_rcu_nocb_wake() ends up seeing a lazy bypass queue even though we are queueing a non-lazy callback (should have flushed in this case).
Looks like it shouldn't happen, even with concurrent (de-offloading) but just in case, can we add:
WARN_ON_ONCE(lazy_len != len)
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 613
Quoted 4 lines
kernel/rcu/tree_nocb.h +1
-0+ }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);
Thanks.
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Hi Frederic, thanks for the response, replies below courtesy fruit company’s device:
Quoted 16 lines · Frederic Weisbecker
kernel/rcu/tree.c static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 10 lines · Joel Fernandes (Google)
kernel/rcu/tree.c +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling
rcu_segcblist_entrain()as well, with "Fixes: " tag.
I wanted to do that, however on discussion with Paul I thought of making this optimization only for all lazy bypass CBs. That makes it directly related this patch since the laziness notion is first introduced here. On the other hand I could make this change in a later patch since we are not super bisectable anyway courtesy of the last patch (which is not really an issue if the CONFIG is kept off during someone’s bisection.
Quoted 1 line
And then
FLUSH_BP_WAKEis probably not needed anymore.
It is needed as the API is in tree_nocb.h and we have to have that handle the details of laziness there rather than tree.c. We could add new apis to get rid of flag but it’s cleaner (and Paul seemed to be ok with it).
Quoted 6 lines
Quoted 6 lines
kernel/rcu/tree.c +0
-0if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);
Quoted 11 lines
kernel/rcu/tree_nocb.h static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 9 lines
kernel/rcu/tree_nocb.h +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {This can be a plain READ since ->nocb_defer_wakeup is only written under ->nocb_gp_lock.
Yes makes sense, will do.
Quoted 26 lines
Quoted 10 lines
kernel/rcu/tree_nocb.h +3
-0+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist);kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 609
Quoted 10 lines
kernel/rcu/tree_nocb.h +7
-1+ bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" ));I'm trying to think of a case where
rcu_nocb_try_bypass()returns false (queue to regular list) but thencall_rcu()->__call_rcu_nocb_wake()ends up seeing a lazy bypass queue even though we are queueing a non-lazy callback (should have flushed in this case).
Quoted 2 lines
Looks like it shouldn't happen, even with concurrent (de-offloading) but just in case, can we add:
Yes I also feel this couldn’t happen because irq is off and nocb lock is held throughout the calls to the above 2 functions. Unless I missed the race you’re describing?
Quoted 2 lines
WARN_ON_ONCE(lazy_len != len)But this condition can be true even in normal circumstances? len also contains DONE CBs which are ready to be invoked. Or did I miss something?
Thanks,
- Joel
Quoted 7 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 613
Quoted 5 lines
kernel/rcu/tree_nocb.h +1
-0+ }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);Thanks.
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 24 lines · Joel Fernandes
Hi Frederic, thanks for the response, replies below courtesy fruit company’s device:
Quoted 16 lines
kernel/rcu/tree.c static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 13 lines · Frederic Weisbecker
On Thu, Sep 22, 2022 at 10:01:01PM +0000, Joel Fernandes (Google) wrote:
kernel/rcu/tree.c +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); -WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies)); +/* +* Flush the bypass list, but also wake up the GP thread as otherwise +* bypass/lazy CBs maynot be noticed, and can cause real long delays! +*/ +WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies,FLUSH_BP_WAKE ));This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling
rcu_segcblist_entrain()as well, with "Fixes: " tag.
Quoted 9 lines
I wanted to do that, however on discussion with Paul I thought of making this optimization only for all lazy bypass CBs. That makes it directly related this patch since the laziness notion is first introduced here. On the other hand I could make this change in a later patch since we are not super bisectable anyway courtesy of the last patch (which is not really an issue if the CONFIG is kept off during someone’s bisection.
Or are we saying it’s worth doing the wake up for rcu barrier even for regular bypass CB? That’d save 2 jiffies on rcu barrier. If we agree it’s needed, then yes splitting the patch makes sense.
Please let me know your opinions, thanks,
- Joel
Quoted 82 lines
Quoted 2 lines
And then
FLUSH_BP_WAKEis probably not needed anymore.It is needed as the API is in
tree_nocb.hand we have to have that handle the details of laziness there rather thantree.c. We could add new apis to get rid of flag but it’s cleaner (and Paul seemed to be ok with it).Quoted 6 lines
Quoted 6 lines
kernel/rcu/tree.c +0
-0if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count); }else {@@ -269,10 +294,14 @@ static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,raw_spin_lock_irqsave (&rdp_gp->nocb_gp_lock, flags);Quoted 11 lines
kernel/rcu/tree_nocb.h static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, · line 301
Quoted 9 lines
kernel/rcu/tree_nocb.h +4
-3/* -* Bypass wakeup overrides previous deferments. In case -* of callback storm, no need to wake up too early. +* Bypass wakeup overrides previous deferments. In case of +* callback storm, no need to wake up too early. */ -if (waketype ==RCU_NOCB_WAKE_BYPASS ) { +if (waketype ==RCU_NOCB_WAKE_LAZY + &&READ_ONCE (rdp->nocb_defer_wakeup) ==RCU_NOCB_WAKE_NOT ) {This can be a plain READ since ->nocb_defer_wakeup is only written under ->nocb_gp_lock.
Yes makes sense, will do.
Quoted 26 lines
Quoted 10 lines
kernel/rcu/tree_nocb.h +3
-0+mod_timer (&rdp_gp->nocb_timer, jiffies + jiffies_till_flush); +WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); + }else if (waketype ==RCU_NOCB_WAKE_BYPASS ) {mod_timer (&rdp_gp->nocb_timer, jiffies +2 );WRITE_ONCE (rdp_gp->nocb_defer_wakeup, waketype); }else {@@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist);kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 609
Quoted 10 lines
kernel/rcu/tree_nocb.h +7
-1+ bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" ));I'm trying to think of a case where
rcu_nocb_try_bypass()returns false (queue to regular list) but thencall_rcu()->__call_rcu_nocb_wake()ends up seeing a lazy bypass queue even though we are queueing a non-lazy callback (should have flushed in this case).Quoted 2 lines
Looks like it shouldn't happen, even with concurrent (de-offloading) but just in case, can we add:
Yes I also feel this couldn’t happen because irq is off and nocb lock is held throughout the calls to the above 2 functions. Unless I missed the race you’re describing?
Quoted 2 lines
WARN_ON_ONCE(lazy_len != len)But this condition can be true even in normal circumstances? len also contains DONE CBs which are ready to be invoked. Or did I miss something?
Thanks,
- Joel
Quoted 7 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 613
Quoted 5 lines
kernel/rcu/tree_nocb.h +1
-0+ }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);Thanks.
Frederic Weisbecker
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 34 lines · Joel Fernandes
Quoted 24 lines · Joel Fernandes
Hi Frederic, thanks for the response, replies below courtesy fruit company’s device:
Quoted 16 lines
Review static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 13 lines · Frederic Weisbecker
On Thu, Sep 22, 2022 at 10:01:01PM +0000, Joel Fernandes (Google) wrote:
(quoted diff) +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback; debug_rcu_head_queue(&rdp->barrier_head); rcu_nocb_lock(rdp); - WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); + /* + * Flush the bypass list, but also wake up the GP thread as otherwise + * bypass/lazy CBs maynot be noticed, and can cause real long delays! + */ + WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, FLUSH_BP_WAKE));This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling
rcu_segcblist_entrain()as well, with "Fixes: " tag.Quoted 9 lines
I wanted to do that, however on discussion with Paul I thought of making this optimization only for all lazy bypass CBs. That makes it directly related this patch since the laziness notion is first introduced here. On the other hand I could make this change in a later patch since we are not super bisectable anyway courtesy of the last patch (which is not really an issue if the CONFIG is kept off during someone’s bisection.
Quoted 5 lines
Or are we saying it’s worth doing the wake up for rcu barrier even for regular bypass CB? That’d save 2 jiffies on rcu barrier. If we agree it’s needed, then yes splitting the patch makes sense.
Please let me know your opinions, thanks,
- Joel
Sure, I mean since we are fixing the buggy rcu_barrier_entrain() anyway, let's just fix bypass as well. Such as in the following (untested):
kernel/rcu/tree.c +9 -0
@@ -3834,6 +3834,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) {unsigned long gseq =READ_ONCE (rcu_state.barrier_sequence);unsigned long lseq =READ_ONCE (rdp->barrier_seq_snap); +bool wake_nocb = false; +bool was_alldone = false;lockdep_assert_held (&rcu_state.barrier_lock);if (rcu_seq_state (lseq) || !rcu_seq_state (gseq) ||rcu_seq_ctr (lseq) !=rcu_seq_ctr (gseq))
@@ -3842,6 +3844,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); +if (rcu_rdp_is_offloaded (rdp) && !rcu_segcblist_pend_cbs (&rdp->cblist)) + was_alldone = true;WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count);
@@ -3849,7 +3853,12 @@ static void rcu_barrier_entrain(struct rcu_data *rdp)debug_rcu_head_unqueue (&rdp->barrier_head);rcu_barrier_trace (TPS ("IRQNQ" ),-1 , rcu_state.barrier_sequence); } +if (was_alldone &&rcu_segcblist_pend_cbs (&rdp->cblist)) + wake_nocb = true;rcu_nocb_unlock (rdp); +if (wake_nocb) +wake_nocb_gp (rdp, false); +smp_store_release (&rdp->barrier_seq_snap, gseq); }
kernel/rcu/tree.h +1 -0
@@ -439,6 +439,7 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp); +static bool wake_nocb_gp (struct rcu_data *rdp,bool force);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j); static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
kernel/rcu/tree_nocb.h +4 -0
@@ -1600,6 +1600,10 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) { } +static bool wake_nocb_gp (struct rcu_data *rdp,bool force) +{ +} +static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j) {
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 39 lines · Frederic Weisbecker
Quoted 38 lines · Joel Fernandes
Quoted 24 lines · Joel Fernandes
Hi Frederic, thanks for the response, replies below courtesy fruit company’s device:
Quoted 16 lines
Review static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 13 lines · Frederic Weisbecker
On Thu, Sep 22, 2022 at 10:01:01PM +0000, Joel Fernandes (Google) wrote:
(quoted diff) +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback; debug_rcu_head_queue(&rdp->barrier_head); rcu_nocb_lock(rdp); - WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); + /* + * Flush the bypass list, but also wake up the GP thread as otherwise + * bypass/lazy CBs maynot be noticed, and can cause real long delays! + */ + WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, FLUSH_BP_WAKE));This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling
rcu_segcblist_entrain()as well, with "Fixes: " tag.Quoted 9 lines
I wanted to do that, however on discussion with Paul I thought of making this optimization only for all lazy bypass CBs. That makes it directly related this patch since the laziness notion is first introduced here. On the other hand I could make this change in a later patch since we are not super bisectable anyway courtesy of the last patch (which is not really an issue if the CONFIG is kept off during someone’s bisection.
Or are we saying it’s worth doing the wake up for rcu barrier even for regular bypass CB? That’d save 2 jiffies on rcu barrier. If we agree it’s needed, then yes splitting the patch makes sense.
Quoted 6 lines
Quoted 3 lines
Please let me know your opinions, thanks,
- Joel
Sure, I mean since we are fixing the buggy
rcu_barrier_entrain()anyway, let's just fix bypass as well. Such as in the following (untested):
Got it. This sounds good to me, and will simplify the code a bit more for sure.
I guess a question for Paul - are you Ok with rcu_barrier() causing wake ups if the bypass list has any non-lazy CBs as well? That should be OK, IMO.
Quoted 24 lines
kernel/rcu/tree.c +4
-0@@ -3834,6 +3834,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) {unsigned long gseq =READ_ONCE (rcu_state.barrier_sequence);unsigned long lseq =READ_ONCE (rdp->barrier_seq_snap); +bool wake_nocb = false; +bool was_alldone = false;lockdep_assert_held (&rcu_state.barrier_lock);if (rcu_seq_state (lseq) || !rcu_seq_state (gseq) ||rcu_seq_ctr (lseq) !=rcu_seq_ctr (gseq))@@ -3842,6 +3844,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); +if (rcu_rdp_is_offloaded (rdp) && !rcu_segcblist_pend_cbs (&rdp->cblist)) + was_alldone = true;WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count);
kernel/rcu/tree.c static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3861
Quoted 9 lines
kernel/rcu/tree.c +5
-0@@ -3849,7 +3853,12 @@ static void rcu_barrier_entrain(struct rcu_data *rdp)debug_rcu_head_unqueue (&rdp->barrier_head);rcu_barrier_trace (TPS ("IRQNQ" ),-1 , rcu_state.barrier_sequence); } +if (was_alldone &&rcu_segcblist_pend_cbs (&rdp->cblist)) + wake_nocb = true;rcu_nocb_unlock (rdp); +if (wake_nocb) +wake_nocb_gp (rdp, false); +
Thanks for the code snippet, I like how you are checking if the bypass list is empty, without actually checking it ;-)
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 44 lines · Joel Fernandes
Quoted 43 lines · Frederic Weisbecker
Quoted 41 lines · Joel Fernandes
Quoted 24 lines · Joel Fernandes
Hi Frederic, thanks for the response, replies below courtesy fruit company’s device:
Quoted 16 lines
Review static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 13 lines · Frederic Weisbecker
On Thu, Sep 22, 2022 at 10:01:01PM +0000, Joel Fernandes (Google) wrote:
(quoted diff) +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback; debug_rcu_head_queue(&rdp->barrier_head); rcu_nocb_lock(rdp); - WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); + /* + * Flush the bypass list, but also wake up the GP thread as otherwise + * bypass/lazy CBs maynot be noticed, and can cause real long delays! + */ + WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, FLUSH_BP_WAKE));This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling
rcu_segcblist_entrain()as well, with "Fixes: " tag.Quoted 9 lines
I wanted to do that, however on discussion with Paul I thought of making this optimization only for all lazy bypass CBs. That makes it directly related this patch since the laziness notion is first introduced here. On the other hand I could make this change in a later patch since we are not super bisectable anyway courtesy of the last patch (which is not really an issue if the CONFIG is kept off during someone’s bisection.
Or are we saying it’s worth doing the wake up for rcu barrier even for regular bypass CB? That’d save 2 jiffies on rcu barrier. If we agree it’s needed, then yes splitting the patch makes sense.
Please let me know your opinions, thanks,
- Joel
Quoted 7 lines
Quoted 2 lines
Sure, I mean since we are fixing the buggy
rcu_barrier_entrain()anyway, let's just fix bypass as well. Such as in the following (untested):Got it. This sounds good to me, and will simplify the code a bit more for sure.
I guess a question for Paul - are you Ok with
rcu_barrier()causing wake ups if the bypass list has any non-lazy CBs as well? That should be OK, IMO.
In theory, I am OK with it. In practice, you are the guys with the hardware that can measure power consumption, not me! ;-)
Quoted 27 lines
Quoted 27 lines
kernel/rcu/tree.c +5
-0@@ -3834,6 +3834,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) {unsigned long gseq =READ_ONCE (rcu_state.barrier_sequence);unsigned long lseq =READ_ONCE (rdp->barrier_seq_snap); +bool wake_nocb = false; +bool was_alldone = false;lockdep_assert_held (&rcu_state.barrier_lock);if (rcu_seq_state (lseq) || !rcu_seq_state (gseq) ||rcu_seq_ctr (lseq) !=rcu_seq_ctr (gseq))@@ -3842,6 +3844,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); +if (rcu_rdp_is_offloaded (rdp) && !rcu_segcblist_pend_cbs (&rdp->cblist)) + was_alldone = true;WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count);@@ -3849,7 +3853,12 @@ static void rcu_barrier_entrain(struct rcu_data *rdp)debug_rcu_head_unqueue (&rdp->barrier_head);rcu_barrier_trace (TPS ("IRQNQ" ),-1 , rcu_state.barrier_sequence); } +if (was_alldone &&rcu_segcblist_pend_cbs (&rdp->cblist))
Quoted 8 lines
kernel/rcu/tree.c static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3861
Quoted 5 lines
kernel/rcu/tree.c +4
-0+ wake_nocb = true;rcu_nocb_unlock (rdp); +if (wake_nocb) +wake_nocb_gp (rdp, false); +Thanks for the code snippet, I like how you are checking if the bypass list is empty, without actually checking it ;-)
That certainly is consistent with the RCU philosophy. :-)
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 52 lines · Paul E. McKenney
Quoted 47 lines · Joel Fernandes
Quoted 46 lines · Frederic Weisbecker
Quoted 34 lines · Joel Fernandes
Quoted 24 lines · Joel Fernandes
Hi Frederic, thanks for the response, replies below courtesy fruit company’s device:
Quoted 16 lines
Review static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3946
Quoted 13 lines · Frederic Weisbecker
On Thu, Sep 22, 2022 at 10:01:01PM +0000, Joel Fernandes (Google) wrote:
(quoted diff) +5
-1@@ -3902,7 +3939,11 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback; debug_rcu_head_queue(&rdp->barrier_head); rcu_nocb_lock(rdp); - WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); + /* + * Flush the bypass list, but also wake up the GP thread as otherwise + * bypass/lazy CBs maynot be noticed, and can cause real long delays! + */ + WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, FLUSH_BP_WAKE));This fixes an issue that goes beyond lazy implementation. It should be done in a separate patch, handling
rcu_segcblist_entrain()as well, with "Fixes: " tag.Quoted 9 lines
I wanted to do that, however on discussion with Paul I thought of making this optimization only for all lazy bypass CBs. That makes it directly related this patch since the laziness notion is first introduced here. On the other hand I could make this change in a later patch since we are not super bisectable anyway courtesy of the last patch (which is not really an issue if the CONFIG is kept off during someone’s bisection.
Quoted 7 lines
Or are we saying it’s worth doing the wake up for rcu barrier even for regular bypass CB? That’d save 2 jiffies on rcu barrier. If we agree it’s needed, then yes splitting the patch makes sense.
Please let me know your opinions, thanks,
- Joel
Sure, I mean since we are fixing the buggy
rcu_barrier_entrain()anyway, let's just fix bypass as well. Such as in the following (untested):
Quoted 7 lines
Quoted 4 lines
Got it. This sounds good to me, and will simplify the code a bit more for sure.
I guess a question for Paul - are you Ok with
rcu_barrier()causing wake ups if the bypass list has any non-lazy CBs as well? That should be OK, IMO.In theory, I am OK with it. In practice, you are the guys with the hardware that can measure power consumption, not me! ;-)
Ok I’ll do it this way and I’ll add Frederic’s Suggested-by tag. About power, I have already measured and it has no effect on power that I could find.
Thanks!
- Joel
Quoted 40 lines
Quoted 28 lines
Quoted 28 lines
kernel/rcu/tree.c +5
-0@@ -3834,6 +3834,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) {unsigned long gseq =READ_ONCE (rcu_state.barrier_sequence);unsigned long lseq =READ_ONCE (rdp->barrier_seq_snap); +bool wake_nocb = false; +bool was_alldone = false;lockdep_assert_held (&rcu_state.barrier_lock);if (rcu_seq_state (lseq) || !rcu_seq_state (gseq) ||rcu_seq_ctr (lseq) !=rcu_seq_ctr (gseq))@@ -3842,6 +3844,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) rdp->barrier_head.func = rcu_barrier_callback;debug_rcu_head_queue (&rdp->barrier_head);rcu_nocb_lock (rdp); +if (rcu_rdp_is_offloaded (rdp) && !rcu_segcblist_pend_cbs (&rdp->cblist)) + was_alldone = true;WARN_ON_ONCE (!rcu_nocb_flush_bypass (rdp,NULL , jiffies));if (rcu_segcblist_entrain (&rdp->cblist, &rdp->barrier_head)) {atomic_inc (&rcu_state.barrier_cpu_count);@@ -3849,7 +3853,12 @@ static void rcu_barrier_entrain(struct rcu_data *rdp)debug_rcu_head_unqueue (&rdp->barrier_head);rcu_barrier_trace (TPS ("IRQNQ" ),-1 , rcu_state.barrier_sequence); } +if (was_alldone &&rcu_segcblist_pend_cbs (&rdp->cblist))Quoted 8 lines
kernel/rcu/tree.c static void rcu_barrier_entrain(struct rcu_data *rdp) · line 3861
Quoted 5 lines
kernel/rcu/tree.c +4
-0+ wake_nocb = true;rcu_nocb_unlock (rdp); +if (wake_nocb) +wake_nocb_gp (rdp, false); +Thanks for the code snippet, I like how you are checking if the bypass list is empty, without actually checking it ;-)
That certainly is consistent with the RCU philosophy. :-)
Thanx, Paul
Frederic Weisbecker
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Joel FernandesQuoted 7 lines · Joel Fernandes
Quoted 1 line
And then
FLUSH_BP_WAKEis probably not needed anymore.It is needed as the API is in
tree_nocb.hand we have to have that handle the details of laziness there rather thantree.c. We could add new apis to get rid of flag but it’s cleaner (and Paul seemed to be ok with it).
If the wake up is handled outside the flush function, as in the diff I just posted, there is no more user left of FLUSH_BP_WAKE, IIRC...
Quoted 20 lines
Quoted 20 lines
Quoted 4 lines
kernel/rcu/tree_nocb.h +0
-0@@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist);kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 609
Quoted 10 lines
kernel/rcu/tree_nocb.h +7
-1+ bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" ));I'm trying to think of a case where
rcu_nocb_try_bypass()returns false (queue to regular list) but thencall_rcu()->__call_rcu_nocb_wake()ends up seeing a lazy bypass queue even though we are queueing a non-lazy callback (should have flushed in this case).
Quoted 7 lines
Quoted 2 lines
Looks like it shouldn't happen, even with concurrent (de-offloading) but just in case, can we add:
Yes I also feel this couldn’t happen because irq is off and nocb lock is held throughout the calls to the above 2 functions. Unless I missed the race you’re describing?
At least I can't find any either...
Quoted 8 lines
Quoted 3 lines
WARN_ON_ONCE(lazy_len != len)But this condition can be true even in normal circumstances? len also contains DONE CBs which are ready to be invoked. Or did I miss something?
Duh, good point, nevermind then :-)
Thanks.
Quoted 12 lines
Thanks,
- Joel
Quoted 7 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 613
Quoted 5 lines
kernel/rcu/tree_nocb.h +1
-0+ }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);Thanks.
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 11 lines · Frederic Weisbecker
Quoted 7 lines · Joel Fernandes
Quoted 1 line
And then
FLUSH_BP_WAKEis probably not needed anymore.It is needed as the API is in
tree_nocb.hand we have to have that handle the details of laziness there rather thantree.c. We could add new apis to get rid of flag but it’s cleaner (and Paul seemed to be ok with it).If the wake up is handled outside the flush function, as in the diff I just posted, there is no more user left of
FLUSH_BP_WAKE, IIRC...
To get rid of FLUSH_BP_WAKE, we might need to pull some rcu_data fields out from under #ifdef in order to allow them to be accessed by common code. Which might be a good tradeoff, as the size of rcu_data has not been a concern. Plus the increase in size would be quite small.
Thanx, Paul
Quoted 55 lines
Quoted 20 lines
Quoted 20 lines
Quoted 4 lines
kernel/rcu/tree_nocb.h +0
-0@@ -512,9 +598,16 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, }// Need to actually to a wakeup. len =rcu_segcblist_n_cbs (&rdp->cblist);kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 609
Quoted 10 lines
kernel/rcu/tree_nocb.h +7
-1+ bypass_len =rcu_cblist_n_cbs (&rdp->nocb_bypass); + lazy_len =READ_ONCE (rdp->lazy_len);if (was_alldone) { rdp->qlen_last_fqs_check = len; -if (!irqs_disabled_flags (flags)) { +// Only lazy CBs in bypass list +if (lazy_len && bypass_len == lazy_len) { +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp_defer (rdp,RCU_NOCB_WAKE_LAZY , +TPS ("WakeLazy" ));I'm trying to think of a case where
rcu_nocb_try_bypass()returns false (queue to regular list) but thencall_rcu()->__call_rcu_nocb_wake()ends up seeing a lazy bypass queue even though we are queueing a non-lazy callback (should have flushed in this case).Quoted 7 lines
Quoted 2 lines
Looks like it shouldn't happen, even with concurrent (de-offloading) but just in case, can we add:
Yes I also feel this couldn’t happen because irq is off and nocb lock is held throughout the calls to the above 2 functions. Unless I missed the race you’re describing?
At least I can't find any either...
Quoted 8 lines
Quoted 3 lines
WARN_ON_ONCE(lazy_len != len)But this condition can be true even in normal circumstances? len also contains DONE CBs which are ready to be invoked. Or did I miss something?
Duh, good point, nevermind then :-)
Thanks.
Quoted 12 lines
Thanks,
- Joel
Quoted 7 lines
kernel/rcu/tree_nocb.h static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, · line 613
Quoted 5 lines
kernel/rcu/tree_nocb.h +1
-0+ }else if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */ rcu_nocb_unlock_irqrestore (rdp, flags);wake_nocb_gp (rdp, false);Thanks.
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Joel Fernandes (Google)Quoted 17 lines
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
Quoted 8 lines
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ][ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]
I think it make sense to add some data to the commit message illustrating what this patch does.
From my side i gave a try of this patch on my setup. Some data:
<snip>
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_v6.script | sort -nk 6 | grep rcu name: rcuop/23 pid: 184 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/26 pid: 206 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/29 pid: 227 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/2 pid: 35 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/34 pid: 263 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/35 pid: 270 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/36 pid: 277 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/37 pid: 284 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/38 pid: 291 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/49 pid: 370 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/59 pid: 441 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/63 pid: 469 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/0 pid: 16 woken-up 2 interval: min 8034 max 8034 avg 4017 name: rcuog/24 pid: 191 woken-up 2 interval: min 7941 max 7941 avg 3970 name: rcuog/32 pid: 248 woken-up 2 interval: min 7542 max 7542 avg 3771 name: rcuog/48 pid: 362 woken-up 2 interval: min 8065 max 8065 avg 4032 name: rcuog/56 pid: 419 woken-up 2 interval: min 8076 max 8076 avg 4038 name: rcuop/21 pid: 170 woken-up 2 interval: min 13311438 max 13311438 avg 6655719 name: rcuog/16 pid: 134 woken-up 4 interval: min 8029 max 13303387 avg 3329863 name: rcuop/9 pid: 85 woken-up 4 interval: min 10007570 max 10007586 avg 7505684 name: rcuog/8 pid: 77 woken-up 8 interval: min 6240 max 10001242 avg 3753622 name: rcu_preempt pid: 15 woken-up 18 interval: min 6058 max 9999713 avg 2140788 name: test_rcu/0 pid: 1411 woken-up 10003 interval: min 165 max 19072 avg 4275 root@pc638:/home/urezki/rcu_v6#
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_default.script | sort -nk 6 | grep rcu name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/51 pid: 384 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/32 pid: 248 woken-up 2 interval: min 11927 max 11927 avg 5963 name: rcuop/63 pid: 469 woken-up 2 interval: min 23963 max 23963 avg 11981 name: rcuog/56 pid: 419 woken-up 3 interval: min 11132 max 23967 avg 11699 name: rcuop/50 pid: 377 woken-up 3 interval: min 8057 max 4944344 avg 1650800 name: rcuog/48 pid: 362 woken-up 8 interval: min 2712 max 37430015 avg 5298801 name: rcuop/16 pid: 135 woken-up 4790 interval: min 7340 max 16649 avg 8843 name: rcuog/16 pid: 134 woken-up 4792 interval: min 7368 max 16644 avg 8844 name: rcu_preempt pid: 15 woken-up 5302 interval: min 26 max 12179 avg 7994 name: test_rcu/0 pid: 1353 woken-up 10003 interval: min 169 max 18508 avg 4236 root@pc638:/home/urezki/rcu_v6# <snip>
so it is obvious that the patch does the job.
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.
kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +
Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Also you do not need to do:
#ifdef LAZY
...
#else
...
#endif
kernel/rcu/Kconfig
Quoted 10 lines
kernel/rcu/Kconfig +9
-0+/* + * LAZY_FLUSH_JIFFIES decides the maximum amount of time that + * can elapse before lazy callbacks are flushed. Lazy callbacks + * could be flushed much earlier for a number of other reasons + * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are + * left unsubmitted to RCU after those many jiffies. + */ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush = LAZY_FLUSH_JIFFIES;
Make it configurable? I do not think you actually need 10 seconds here. Reducing it will reduce a possibility to hit a low memory condition. 1 second would be far enough i think.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Hi Vlad,
Quoted 19 lines · Uladzislau Rezki
Quoted 19 lines
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ]
Quoted 8 lines
Quoted 6 lines
[ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]I think it make sense to add some data to the commit message illustrating what this patch does.
Sure, will do!
Quoted 45 lines
From my side i gave a try of this patch on my setup. Some data:
<snip>
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_v6.script | sort -nk 6 | grep rcu name: rcuop/23 pid: 184 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/26 pid: 206 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/29 pid: 227 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/2 pid: 35 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/34 pid: 263 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/35 pid: 270 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/36 pid: 277 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/37 pid: 284 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/38 pid: 291 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/49 pid: 370 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/59 pid: 441 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/63 pid: 469 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/0 pid: 16 woken-up 2 interval: min 8034 max 8034 avg 4017 name: rcuog/24 pid: 191 woken-up 2 interval: min 7941 max 7941 avg 3970 name: rcuog/32 pid: 248 woken-up 2 interval: min 7542 max 7542 avg 3771 name: rcuog/48 pid: 362 woken-up 2 interval: min 8065 max 8065 avg 4032 name: rcuog/56 pid: 419 woken-up 2 interval: min 8076 max 8076 avg 4038 name: rcuop/21 pid: 170 woken-up 2 interval: min 13311438 max 13311438 avg 6655719 name: rcuog/16 pid: 134 woken-up 4 interval: min 8029 max 13303387 avg 3329863 name: rcuop/9 pid: 85 woken-up 4 interval: min 10007570 max 10007586 avg 7505684 name: rcuog/8 pid: 77 woken-up 8 interval: min 6240 max 10001242 avg 3753622 name: rcu_preempt pid: 15 woken-up 18 interval: min 6058 max 9999713 avg 2140788 name: test_rcu/0 pid: 1411 woken-up 10003 interval: min 165 max 19072 avg 4275 root@pc638:/home/urezki/rcu_v6#root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_default.script | sort -nk 6 | grep rcu name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/51 pid: 384 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/32 pid: 248 woken-up 2 interval: min 11927 max 11927 avg 5963 name: rcuop/63 pid: 469 woken-up 2 interval: min 23963 max 23963 avg 11981 name: rcuog/56 pid: 419 woken-up 3 interval: min 11132 max 23967 avg 11699 name: rcuop/50 pid: 377 woken-up 3 interval: min 8057 max 4944344 avg 1650800 name: rcuog/48 pid: 362 woken-up 8 interval: min 2712 max 37430015 avg 5298801 name: rcuop/16 pid: 135 woken-up 4790 interval: min 7340 max 16649 avg 8843 name: rcuog/16 pid: 134 woken-up 4792 interval: min 7368 max 16644 avg 8844 name: rcu_preempt pid: 15 woken-up 5302 interval: min 26 max 12179 avg 7994 name: test_rcu/0 pid: 1353 woken-up 10003 interval: min 169 max 18508 avg 4236 root@pc638:/home/urezki/rcu_v6# <snip>
Quoted 1 line
so it is obvious that the patch does the job.
Thanks a lot for testing!
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.
Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off(); panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off(); panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
Quoted 6 lines
...
#else
...
#endifQuoted 1 line
Quoted 12 lines
kernel/rcu/Kconfig
Quoted 9 lines
kernel/rcu/Kconfig +9
-0+/* + * LAZY_FLUSH_JIFFIES decides the maximum amount of time that + * can elapse before lazy callbacks are flushed. Lazy callbacks + * could be flushed much earlier for a number of other reasons + * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are + * left unsubmitted to RCU after those many jiffies. + */ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush = LAZY_FLUSH_JIFFIES;Make it configurable? I do not think you actually need 10 seconds here. Reducing it will reduce a possibility to hit a low memory condition. 1 second would be far enough i think.
Hmm, I can make the delay configurable but for now I'll keep this as default as all of our power testing has been done with that and I don't want risk losing the optimization.
Honestly, I am not worried too about memory pressure as we have a shrinker which triggers flushes on the slightest hint of memory pressure. If it is not handling it properly, then we need to fix the shrinker.
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 103 lines · Joel Fernandes
Hi Vlad,
Quoted 19 lines · Uladzislau Rezki
Quoted 19 lines
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ]Quoted 8 lines
Quoted 6 lines
[ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]I think it make sense to add some data to the commit message illustrating what this patch does.
Sure, will do!
Quoted 45 lines
From my side i gave a try of this patch on my setup. Some data:
<snip>
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_v6.script | sort -nk 6 | grep rcu name: rcuop/23 pid: 184 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/26 pid: 206 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/29 pid: 227 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/2 pid: 35 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/34 pid: 263 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/35 pid: 270 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/36 pid: 277 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/37 pid: 284 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/38 pid: 291 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/49 pid: 370 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/59 pid: 441 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/63 pid: 469 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/0 pid: 16 woken-up 2 interval: min 8034 max 8034 avg 4017 name: rcuog/24 pid: 191 woken-up 2 interval: min 7941 max 7941 avg 3970 name: rcuog/32 pid: 248 woken-up 2 interval: min 7542 max 7542 avg 3771 name: rcuog/48 pid: 362 woken-up 2 interval: min 8065 max 8065 avg 4032 name: rcuog/56 pid: 419 woken-up 2 interval: min 8076 max 8076 avg 4038 name: rcuop/21 pid: 170 woken-up 2 interval: min 13311438 max 13311438 avg 6655719 name: rcuog/16 pid: 134 woken-up 4 interval: min 8029 max 13303387 avg 3329863 name: rcuop/9 pid: 85 woken-up 4 interval: min 10007570 max 10007586 avg 7505684 name: rcuog/8 pid: 77 woken-up 8 interval: min 6240 max 10001242 avg 3753622 name: rcu_preempt pid: 15 woken-up 18 interval: min 6058 max 9999713 avg 2140788 name: test_rcu/0 pid: 1411 woken-up 10003 interval: min 165 max 19072 avg 4275 root@pc638:/home/urezki/rcu_v6#root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_default.script | sort -nk 6 | grep rcu name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/51 pid: 384 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/32 pid: 248 woken-up 2 interval: min 11927 max 11927 avg 5963 name: rcuop/63 pid: 469 woken-up 2 interval: min 23963 max 23963 avg 11981 name: rcuog/56 pid: 419 woken-up 3 interval: min 11132 max 23967 avg 11699 name: rcuop/50 pid: 377 woken-up 3 interval: min 8057 max 4944344 avg 1650800 name: rcuog/48 pid: 362 woken-up 8 interval: min 2712 max 37430015 avg 5298801 name: rcuop/16 pid: 135 woken-up 4790 interval: min 7340 max 16649 avg 8843 name: rcuog/16 pid: 134 woken-up 4792 interval: min 7368 max 16644 avg 8844 name: rcu_preempt pid: 15 woken-up 5302 interval: min 26 max 12179 avg 7994 name: test_rcu/0 pid: 1353 woken-up 10003 interval: min 169 max 18508 avg 4236 root@pc638:/home/urezki/rcu_v6# <snip>Quoted 1 line
so it is obvious that the patch does the job.
Thanks a lot for testing!
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.
We do seem to be in need of some way to quickly and easily locate the callback that needed to be _flush() due to a wakeup.
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.
Or is there a better way to do this?
Thanx, Paul
Quoted 77 lines
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
Quoted 6 lines
...
#else
...
#endifQuoted 1 line
Quoted 12 lines
kernel/rcu/Kconfig
Quoted 9 lines
kernel/rcu/Kconfig +9
-0+/* + * LAZY_FLUSH_JIFFIES decides the maximum amount of time that + * can elapse before lazy callbacks are flushed. Lazy callbacks + * could be flushed much earlier for a number of other reasons + * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are + * left unsubmitted to RCU after those many jiffies. + */ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush = LAZY_FLUSH_JIFFIES;Make it configurable? I do not think you actually need 10 seconds here. Reducing it will reduce a possibility to hit a low memory condition. 1 second would be far enough i think.
Hmm, I can make the delay configurable but for now I'll keep this as default as all of our power testing has been done with that and I don't want risk losing the optimization.
Honestly, I am not worried too about memory pressure as we have a shrinker which triggers flushes on the slightest hint of memory pressure. If it is not handling it properly, then we need to fix the shrinker.
thanks,
- Joel
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 116 lines · Paul E. McKenney
Quoted 103 lines · Joel Fernandes
Hi Vlad,
Quoted 19 lines · Uladzislau Rezki
Quoted 19 lines
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ]Quoted 8 lines
Quoted 6 lines
[ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]I think it make sense to add some data to the commit message illustrating what this patch does.
Sure, will do!
Quoted 45 lines
From my side i gave a try of this patch on my setup. Some data:
<snip>
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_v6.script | sort -nk 6 | grep rcu name: rcuop/23 pid: 184 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/26 pid: 206 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/29 pid: 227 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/2 pid: 35 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/34 pid: 263 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/35 pid: 270 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/36 pid: 277 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/37 pid: 284 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/38 pid: 291 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/49 pid: 370 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/59 pid: 441 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/63 pid: 469 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/0 pid: 16 woken-up 2 interval: min 8034 max 8034 avg 4017 name: rcuog/24 pid: 191 woken-up 2 interval: min 7941 max 7941 avg 3970 name: rcuog/32 pid: 248 woken-up 2 interval: min 7542 max 7542 avg 3771 name: rcuog/48 pid: 362 woken-up 2 interval: min 8065 max 8065 avg 4032 name: rcuog/56 pid: 419 woken-up 2 interval: min 8076 max 8076 avg 4038 name: rcuop/21 pid: 170 woken-up 2 interval: min 13311438 max 13311438 avg 6655719 name: rcuog/16 pid: 134 woken-up 4 interval: min 8029 max 13303387 avg 3329863 name: rcuop/9 pid: 85 woken-up 4 interval: min 10007570 max 10007586 avg 7505684 name: rcuog/8 pid: 77 woken-up 8 interval: min 6240 max 10001242 avg 3753622 name: rcu_preempt pid: 15 woken-up 18 interval: min 6058 max 9999713 avg 2140788 name: test_rcu/0 pid: 1411 woken-up 10003 interval: min 165 max 19072 avg 4275 root@pc638:/home/urezki/rcu_v6#root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_default.script | sort -nk 6 | grep rcu name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/51 pid: 384 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/32 pid: 248 woken-up 2 interval: min 11927 max 11927 avg 5963 name: rcuop/63 pid: 469 woken-up 2 interval: min 23963 max 23963 avg 11981 name: rcuog/56 pid: 419 woken-up 3 interval: min 11132 max 23967 avg 11699 name: rcuop/50 pid: 377 woken-up 3 interval: min 8057 max 4944344 avg 1650800 name: rcuog/48 pid: 362 woken-up 8 interval: min 2712 max 37430015 avg 5298801 name: rcuop/16 pid: 135 woken-up 4790 interval: min 7340 max 16649 avg 8843 name: rcuog/16 pid: 134 woken-up 4792 interval: min 7368 max 16644 avg 8844 name: rcu_preempt pid: 15 woken-up 5302 interval: min 26 max 12179 avg 7994 name: test_rcu/0 pid: 1353 woken-up 10003 interval: min 169 max 18508 avg 4236 root@pc638:/home/urezki/rcu_v6# <snip>Quoted 1 line
so it is obvious that the patch does the job.
Thanks a lot for testing!
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.
<snip>
kernel/workqueue.c +1 -1
@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }
<snip>
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]
Quoted 49 lines
Quoted 33 lines
Quoted 21 lines
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }
Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy call_rcu() callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a new trace_end_invoke_callback to fire after the callback is invoked). Thoughts?
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 69 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]Quoted 49 lines
Quoted 33 lines
Quoted 21 lines
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Quoted 5 lines
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?
Could you look for wakeups invoked between trace_rcu_batch_start() and trace_rcu_batch_end() that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.
Note that you would need to check for wakeups from interrupt handlers even with the extra trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 80 lines · Paul E. McKenney
Quoted 69 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]Quoted 49 lines
Quoted 33 lines
Quoted 21 lines
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Quoted 5 lines
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.
Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.
True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Thanks,
- Joel
Quoted 2 lines
Thanx, Paul
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 80 lines · Joel Fernandes
Quoted 80 lines · Paul E. McKenney
Quoted 74 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]Quoted 49 lines
Quoted 33 lines
Quoted 21 lines
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?
Quoted 5 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like spin_lock_bh().
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 89 lines · Paul E. McKenney
Quoted 80 lines · Joel Fernandes
Quoted 80 lines · Paul E. McKenney
Quoted 74 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]Quoted 49 lines
Quoted 33 lines
Quoted 21 lines
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 5 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().
Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag in rcu_head to mark CBs as lazy.
2. Add a trace_rcu_invoke_callback_end() trace point.
Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both trace_rcu_invoke_callback_start() and trace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.
4. Put an in-kernel probe on trace_rcu_sched_wakeup().
Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.
#3 actually does not even need probes if we can directly call the functions from the rcu_do_batch() function.
I'll work on it in the morning and also look into Vlad's config.
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 113 lines · Joel Fernandes
Quoted 89 lines · Paul E. McKenney
Quoted 80 lines · Joel Fernandes
Quoted 80 lines · Paul E. McKenney
Quoted 74 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]Quoted 49 lines
Quoted 33 lines
Quoted 21 lines
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 5 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.
Quoted 8 lines
4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.
This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Quoted 1 line
I'll work on it in the morning and also look into Vlad's config.
Sounds good!
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
On Mon, Sep 26, 2022 at 08:22:46PM -0700, Paul E. McKenney wrote:
[..]
Quoted 76 lines
Quoted 76 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
Quoted 36 lines
Quoted 13 lines
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 7 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.
Quoted 5 lines
Quoted 2 lines
#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Maybe we can borrow the least-significant bit of rhp->func to mark laziness? Then it can be production as long as we're ok with the trace_sched_wakeup probe.
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 81 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 08:22:46PM -0700, Paul E. McKenney wrote:
[..]Quoted 79 lines
Quoted 78 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
Quoted 36 lines
Quoted 13 lines
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 7 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.
Quoted 6 lines
Quoted 2 lines
This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Maybe we can borrow the least-significant bit of
rhp->functo mark laziness? Then it can be production as long as we're ok with thetrace_sched_wakeupprobe.
Last time I tried this, there were architectures that could have odd-valued function addresses. Maybe this is no longer the case?
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 85 lines · Paul E. McKenney
Quoted 84 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 08:22:46PM -0700, Paul E. McKenney wrote:
[..]Quoted 81 lines
Quoted 70 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
Quoted 36 lines
Quoted 13 lines
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 7 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.Quoted 8 lines
4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Quoted 6 lines
Quoted 3 lines
Maybe we can borrow the least-significant bit of
rhp->functo mark laziness? Then it can be production as long as we're ok with thetrace_sched_wakeupprobe.Last time I tried this, there were architectures that could have odd-valued function addresses. Maybe this is no longer the case?
Oh ok! If this happens, maybe we can just make it depend on x86-64 assuming x86-64 does not have pointer oddness. We can also add a warning for if the function address is odd before setting the bit.
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 90 lines · Joel Fernandes
Quoted 89 lines · Paul E. McKenney
Quoted 87 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 08:22:46PM -0700, Paul E. McKenney wrote:
[..]Quoted 76 lines
Quoted 76 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
Quoted 36 lines
Quoted 13 lines
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 7 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.Quoted 5 lines
Quoted 2 lines
#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Maybe we can borrow the least-significant bit of
rhp->functo mark laziness? Then it can be production as long as we're ok with thetrace_sched_wakeupprobe.
Quoted 6 lines
Quoted 2 lines
Last time I tried this, there were architectures that could have odd-valued function addresses. Maybe this is no longer the case?
Oh ok! If this happens, maybe we can just make it depend on x86-64 assuming x86-64 does not have pointer oddness. We can also add a warning for if the function address is odd before setting the bit.
Let me rephrase this... ;-)
Given that this used to not work and still might not work, let's see if we can find some other way to debug this. Unless and until it can be demonstrated that there is no supported compiler that will generated odd-valued function addresses on any supported architecture.
Plus there was a time that x86 did odd-valued pointer addresses. The instruction set is plenty fine with this, so it would have to be a compiler and assembly-language convention to avoid it.
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 100 lines · Paul E. McKenney
Quoted 90 lines · Joel Fernandes
Quoted 89 lines · Paul E. McKenney
Quoted 87 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 08:22:46PM -0700, Paul E. McKenney wrote:
[..]Quoted 76 lines
Quoted 76 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
Quoted 36 lines
Quoted 13 lines
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 7 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.Quoted 5 lines
Quoted 2 lines
#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Maybe we can borrow the least-significant bit of
rhp->functo mark laziness? Then it can be production as long as we're ok with thetrace_sched_wakeupprobe.Quoted 6 lines
Quoted 2 lines
Last time I tried this, there were architectures that could have odd-valued function addresses. Maybe this is no longer the case?
Oh ok! If this happens, maybe we can just make it depend on x86-64 assuming x86-64 does not have pointer oddness. We can also add a warning for if the function address is odd before setting the bit.
Let me rephrase this... ;-)
Quoted 8 lines
Given that this used to not work and still might not work, let's see if we can find some other way to debug this. Unless and until it can be demonstrated that there is no supported compiler that will generated odd-valued function addresses on any supported architecture.
Plus there was a time that x86 did odd-valued pointer addresses. The instruction set is plenty fine with this, so it would have to be a compiler and assembly-language convention to avoid it.
Ok, so then I am not sure how to make it work in production at the moment. I could track the lazy callbacks in a hashtable but then that's overhead.
Or, I could focus on trying Vlad's config and figure out what's going on and keep the auto-debug for later.
On another thought, this is the sort of thing that should be doable via Daniel Bristot's runtime verification framework, as its a classical "see if these traces look right" issue which should be teachable to a computer with a few rules.
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 110 lines · Joel Fernandes
Quoted 100 lines · Paul E. McKenney
Quoted 90 lines · Joel Fernandes
Quoted 89 lines · Paul E. McKenney
Quoted 87 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 08:22:46PM -0700, Paul E. McKenney wrote:
[..]Quoted 76 lines
Quoted 76 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
Quoted 36 lines
Quoted 13 lines
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }Quoted 3 lines
?
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Ah, but at least its progress, thanks. Could you send me a patch to include in the next revision with details of this?
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such
callback functions? We might not want -all- callbacks that do wakeups
to use call_rcu_flush(), but knowing which are which should speed up
slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all lazy
call_rcu()callbacks that do a wake up. We could use the trace hooks to do it in kernel I think. I can talk to Steve to get ideas on how to do that but I think it can be done purely from trace events (we might need a newtrace_end_invoke_callbackto fire after the callback is invoked). Thoughts?Quoted 7 lines
Quoted 3 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 9 lines
Quoted 5 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but not eliminated.True! Since this is a debugging option, can we not just disable interrupts across callback invocation?
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().Sorry if my last email bounced. Looks like my iPhone betrayed me this once ;)
I was thinking something like this:
1. Put a flag inrcu_headto mark CBs as lazy.
2. Add atrace_rcu_invoke_callback_end()trace point.Both #1 and #2 can be a debug CONFIG option. #2 can be a tracepoint and not exposed if needed.
3. Put an in-kernel probe on both
trace_rcu_invoke_callback_start()andtrace_rcu_invoke_callback_end(). In the start probe, set a per-task flag if the current CB is lazy. In the end probe, clear it.4. Put an in-kernel probe on
trace_rcu_sched_wakeup().Splat in the wake up probe if:
1. Hard IRQs are on.
2. The per-cpu flag is set.Quoted 5 lines
Quoted 2 lines
#3 actually does not even need probes if we can directly call the functions from the
rcu_do_batch()function.This is fine for an experiment or a debugging session, but a solution based totally on instrumentation would be better for production use.
Maybe we can borrow the least-significant bit of
rhp->functo mark laziness? Then it can be production as long as we're ok with thetrace_sched_wakeupprobe.Quoted 6 lines
Quoted 2 lines
Last time I tried this, there were architectures that could have odd-valued function addresses. Maybe this is no longer the case?
Oh ok! If this happens, maybe we can just make it depend on x86-64 assuming x86-64 does not have pointer oddness. We can also add a warning for if the function address is odd before setting the bit.
Let me rephrase this... ;-)
Quoted 8 lines
Given that this used to not work and still might not work, let's see if we can find some other way to debug this. Unless and until it can be demonstrated that there is no supported compiler that will generated odd-valued function addresses on any supported architecture.
Plus there was a time that x86 did odd-valued pointer addresses. The instruction set is plenty fine with this, so it would have to be a compiler and assembly-language convention to avoid it.
Quoted 5 lines
Ok, so then I am not sure how to make it work in production at the moment. I could track the lazy callbacks in a hashtable but then that's overhead.
Or, I could focus on trying Vlad's config and figure out what's going on and keep the auto-debug for later.
For one thing, experience with manual debugging might inform later auto-debugging efforts.
Quoted 3 lines
On another thought, this is the sort of thing that should be doable via Daniel Bristot's runtime verification framework, as its a classical "see if these traces look right" issue which should be teachable to a computer with a few rules.
Worth a shot! Failing that, there is always BPF. ;-)
Thanx, Paul
A message that is not in the archive (sent off-list or not yet received): CAEXW_YRpAjvmBPzRA-hRQpuaDuZUzfndLb3q+e3BUyWprg5wkQ@mail.gmail.com
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 128 lines · Joel Fernandes
Quoted 126 lines · Paul E. McKenney
Quoted 3 lines · Joel Fernandes
Quoted 3 lines
On Sep 26, 2022, at 6:32 PM, Paul E. McKenney <[email protected]>
Quoted 8 lines · wrote:
Quoted 8 lines
Quoted 6 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 09:32:44PM +0200, Uladzislau Rezki wrote:
[...]Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 NetworkConnection
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
[ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rwxa/form2 tray
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
[ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executablestack
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
[ 104.170142] EXT4-fs (sda1): mounted filesystem with ordereddata mode. Quota mode: none.
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
[ 104.340125] systemd[1]: systemd 241 running in system mode.(+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP
(quoted diff) +1
-1+GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid)Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
[ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still
users which must
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
be waiting for "RCU" in a sync way.
I was wondering if you can compare boot logs and see which
timestamp does the
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
slow down start from. That way, we can narrow down the callback.
Also another
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk thatseems off in
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
your comparison of good vs bad. For example, if "crng init done"
timestamp is
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
off, put the "
tracing_off();panic();" there. Then grab the serialconsole
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
output to see what were the last callbacks that was queued/invoked.
We do seem to be in need of some way to quickly and easily locate
the
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
kernel/workqueue.c bool queue_rcu_work(struct workqueue_struct
Quoted 8 lines
Quoted 2 lines
callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +0
-0@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct*wq, struct
rcu_work*rwork)Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
kernel/workqueue.c bool queue_rcu_work(struct workqueue_struct · line 1771
Quoted 2 lines
kernel/workqueue.c +0
-0if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT,
work_data_bits(work))) {Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
kernel/workqueue.c +1
-1rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }?
But it does not fully solve my boot-up issue. Will debug tomorrow
further.
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Ah, but at least its progress, thanks. Could you send me a patch to
include
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
in the next revision with details of this?
Quoted 1 line
Quoted 1 line
Might one more proactive approach be to use Coccinelle to locate
such
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
callback functions? We might not want -all- callbacks that do
wakeups
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that
measures
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
a boot time and if needed does some profiling it would be good.
Otherwise it
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
is a manual debugging mainly, IMHO.
Paul, What about using a default-off kernel CONFIG that splats on all
lazy
Quoted 1 line
Quoted 1 line
Quoted 1 line
call_rcu()callbacks that do a wake up. We could use the trace hooksto do it
Quoted 1 line
Quoted 1 line
Quoted 1 line
in kernel I think. I can talk to Steve to get ideas on how to do that
but I
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
think it can be done purely from trace events (we might need a new
trace_end_invoke_callbackto fire after the callback is invoked).Thoughts?
Quoted 6 lines
Quoted 4 lines
Could you look for wakeups invoked between
trace_rcu_batch_start()andtrace_rcu_batch_end()that are not from interrupt context? This would of course need to be associated with a task rather than a CPU.Yes this sounds good, but we also need to know if the callbacks are lazy
or not since wake-up is ok from a non lazy one. I think I’ll need a table to track that at queuing time.
Agreed.
Quoted 3 lines
Quoted 3 lines
Note that you would need to check for wakeups from interrupt handlers even with the extra
trace_end_invoke_callback(). The window where an interrupt handler could do a wakeup would be reduced, but noteliminated.
Quoted 2 lines
True! Since this is a debugging option, can we not just disable
interrupts across callback invocation?
Quoted 7 lines
Quoted 2 lines
Not without terminally annoying lockdep, at least for any RCU callbacks doing things like
spin_lock_bh().The easy fix for that is adding “depends on !LOCKDEP” to the
Kconfig;-) just kidding. Hmm I think I can just look at the preempt flags and determine if wake up happened in hard Irq context, and ignore those instances.
Or instrument/trace a few carefully chosen context-tracking functions.
Thanx, Paul
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Uladzislau RezkiQuoted 133 lines · Uladzislau Rezki
Quoted 116 lines · Paul E. McKenney
Quoted 103 lines · Joel Fernandes
Hi Vlad,
Quoted 19 lines · Uladzislau Rezki
Quoted 19 lines
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ]Quoted 8 lines
Quoted 6 lines
[ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]I think it make sense to add some data to the commit message illustrating what this patch does.
Sure, will do!
Quoted 45 lines
From my side i gave a try of this patch on my setup. Some data:
<snip>
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_v6.script | sort -nk 6 | grep rcu name: rcuop/23 pid: 184 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/26 pid: 206 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/29 pid: 227 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/2 pid: 35 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/34 pid: 263 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/35 pid: 270 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/36 pid: 277 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/37 pid: 284 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/38 pid: 291 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/49 pid: 370 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/59 pid: 441 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/63 pid: 469 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/0 pid: 16 woken-up 2 interval: min 8034 max 8034 avg 4017 name: rcuog/24 pid: 191 woken-up 2 interval: min 7941 max 7941 avg 3970 name: rcuog/32 pid: 248 woken-up 2 interval: min 7542 max 7542 avg 3771 name: rcuog/48 pid: 362 woken-up 2 interval: min 8065 max 8065 avg 4032 name: rcuog/56 pid: 419 woken-up 2 interval: min 8076 max 8076 avg 4038 name: rcuop/21 pid: 170 woken-up 2 interval: min 13311438 max 13311438 avg 6655719 name: rcuog/16 pid: 134 woken-up 4 interval: min 8029 max 13303387 avg 3329863 name: rcuop/9 pid: 85 woken-up 4 interval: min 10007570 max 10007586 avg 7505684 name: rcuog/8 pid: 77 woken-up 8 interval: min 6240 max 10001242 avg 3753622 name: rcu_preempt pid: 15 woken-up 18 interval: min 6058 max 9999713 avg 2140788 name: test_rcu/0 pid: 1411 woken-up 10003 interval: min 165 max 19072 avg 4275 root@pc638:/home/urezki/rcu_v6#root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_default.script | sort -nk 6 | grep rcu name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/51 pid: 384 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/32 pid: 248 woken-up 2 interval: min 11927 max 11927 avg 5963 name: rcuop/63 pid: 469 woken-up 2 interval: min 23963 max 23963 avg 11981 name: rcuog/56 pid: 419 woken-up 3 interval: min 11132 max 23967 avg 11699 name: rcuop/50 pid: 377 woken-up 3 interval: min 8057 max 4944344 avg 1650800 name: rcuog/48 pid: 362 woken-up 8 interval: min 2712 max 37430015 avg 5298801 name: rcuop/16 pid: 135 woken-up 4790 interval: min 7340 max 16649 avg 8843 name: rcuog/16 pid: 134 woken-up 4792 interval: min 7368 max 16644 avg 8844 name: rcu_preempt pid: 15 woken-up 5302 interval: min 26 max 12179 avg 7994 name: test_rcu/0 pid: 1353 woken-up 10003 interval: min 169 max 18508 avg 4236 root@pc638:/home/urezki/rcu_v6# <snip>Quoted 1 line
so it is obvious that the patch does the job.
Thanks a lot for testing!
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.We do seem to be in need of some way to quickly and easily locate the callback that needed to be
_flush()due to a wakeup.<snip>
kernel/workqueue.c +1
-1@@ -1771,7 +1771,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)if (!test_and_set_bit (WORK_STRUCT_PENDING_BIT ,work_data_bits (work))) { rwork->wq = wq; -call_rcu (&rwork->rcu, rcu_work_rcufn); +call_rcu_flush (&rwork->rcu, rcu_work_rcufn);return true; }
Quoted 1 line
?
This one does seem like a good candidate. Someday there might need to be a queue_rcu_work_flush() vs. queue_rcu_work(), but that needs to be proven.
Quoted 1 line
But it does not fully solve my boot-up issue. Will debug tomorrow further.
Sounds good!
Quoted 10 lines
Quoted 7 lines
Might one more proactive approach be to use Coccinelle to locate such callback functions? We might not want -all- callbacks that do wakeups to use
call_rcu_flush(), but knowing which are which should speed up slow-boot debugging by quite a bit.Or is there a better way to do this?
I am not sure what Coccinelle is. If we had something automated that measures a boot time and if needed does some profiling it would be good. Otherwise it is a manual debugging mainly, IMHO.
Coccinelle is sort of like a variant of the "sed" command that understands C syntax. It is useful for searching for patterns in Linux-kernel source code.
But if you are able to easily find the call_rcu() invocations that are slowing things down, maybe it is not needed. For now, anyway.
Thanx, Paul
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Joel FernandesQuoted 151 lines · Joel Fernandes
Hi Vlad,
Quoted 19 lines · Uladzislau Rezki
Quoted 19 lines
Implement timer-based RCU lazy callback batching. The batch is flushed whenever a certain amount of time has passed, or the batch on a particular CPU grows too big. Also memory pressure will flush it in a future patch.
To handle several corner cases automagically (such as
rcu_barrier()and hotplug), we re-use bypass lists to handle lazy CBs. The bypass list length has the lazy CB length included in it. A separate lazy CB length counter is also introduced to keep track of the number of lazy CBs.
v5->v6:[ Frederic Weisbec: Program the lazy timer only if
WAKE_NOT, since other
deferral levels wake much earlier so for those it is not needed. ][ Frederic Weisbec: Use flush flags to keep bypass API code clean. ]
[ Frederic Weisbec: Make
rcu_barrier()wake up only if main list empty. ]Quoted 8 lines
Quoted 6 lines
[ Frederic Weisbec: Remove extra 'else if' branch in
rcu_nocb_try_bypass(). ][ Joel: Fix issue where I was not resetting
lazy_lenafter moving it to rdp ][ Paul/Thomas/Joel: Make
call_rcu()default lazy so users don't mess up. ]I think it make sense to add some data to the commit message illustrating what this patch does.
Sure, will do!
Quoted 45 lines
From my side i gave a try of this patch on my setup. Some data:
<snip>
root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_v6.script | sort -nk 6 | grep rcu name: rcuop/23 pid: 184 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/26 pid: 206 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/29 pid: 227 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/2 pid: 35 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/34 pid: 263 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/35 pid: 270 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/36 pid: 277 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/37 pid: 284 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/38 pid: 291 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/49 pid: 370 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/59 pid: 441 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/63 pid: 469 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/0 pid: 16 woken-up 2 interval: min 8034 max 8034 avg 4017 name: rcuog/24 pid: 191 woken-up 2 interval: min 7941 max 7941 avg 3970 name: rcuog/32 pid: 248 woken-up 2 interval: min 7542 max 7542 avg 3771 name: rcuog/48 pid: 362 woken-up 2 interval: min 8065 max 8065 avg 4032 name: rcuog/56 pid: 419 woken-up 2 interval: min 8076 max 8076 avg 4038 name: rcuop/21 pid: 170 woken-up 2 interval: min 13311438 max 13311438 avg 6655719 name: rcuog/16 pid: 134 woken-up 4 interval: min 8029 max 13303387 avg 3329863 name: rcuop/9 pid: 85 woken-up 4 interval: min 10007570 max 10007586 avg 7505684 name: rcuog/8 pid: 77 woken-up 8 interval: min 6240 max 10001242 avg 3753622 name: rcu_preempt pid: 15 woken-up 18 interval: min 6058 max 9999713 avg 2140788 name: test_rcu/0 pid: 1411 woken-up 10003 interval: min 165 max 19072 avg 4275 root@pc638:/home/urezki/rcu_v6#root@pc638:/home/urezki/rcu_v6# ./perf_script_parser ./perf_default.script | sort -nk 6 | grep rcu name: rcuop/33 pid: 256 woken-up 1 interval: min 0 max 0 avg 0 name: rcuop/51 pid: 384 woken-up 1 interval: min 0 max 0 avg 0 name: rcuog/32 pid: 248 woken-up 2 interval: min 11927 max 11927 avg 5963 name: rcuop/63 pid: 469 woken-up 2 interval: min 23963 max 23963 avg 11981 name: rcuog/56 pid: 419 woken-up 3 interval: min 11132 max 23967 avg 11699 name: rcuop/50 pid: 377 woken-up 3 interval: min 8057 max 4944344 avg 1650800 name: rcuog/48 pid: 362 woken-up 8 interval: min 2712 max 37430015 avg 5298801 name: rcuop/16 pid: 135 woken-up 4790 interval: min 7340 max 16649 avg 8843 name: rcuog/16 pid: 134 woken-up 4792 interval: min 7368 max 16644 avg 8844 name: rcu_preempt pid: 15 woken-up 5302 interval: min 26 max 12179 avg 7994 name: test_rcu/0 pid: 1353 woken-up 10003 interval: min 169 max 18508 avg 4236 root@pc638:/home/urezki/rcu_v6# <snip>Quoted 1 line
so it is obvious that the patch does the job.
Thanks a lot for testing!
Quoted 16 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.Quoted 4 lines
<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 7 lines
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
config RCU_LAZY
we might run into issues related to run-time switching though.
Quoted 19 lines
Quoted 6 lines
...
#else
...
#endifQuoted 1 line
Quoted 12 lines
kernel/rcu/Kconfig
Quoted 9 lines
kernel/rcu/Kconfig +9
-0+/* + * LAZY_FLUSH_JIFFIES decides the maximum amount of time that + * can elapse before lazy callbacks are flushed. Lazy callbacks + * could be flushed much earlier for a number of other reasons + * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are + * left unsubmitted to RCU after those many jiffies. + */ +#define LAZY_FLUSH_JIFFIES (10 * HZ) +static unsigned long jiffies_till_flush = LAZY_FLUSH_JIFFIES;Make it configurable? I do not think you actually need 10 seconds here. Reducing it will reduce a possibility to hit a low memory condition. 1 second would be far enough i think.
Quoted 4 lines
Hmm, I can make the delay configurable but for now I'll keep this as default as all of our power testing has been done with that and I don't want risk losing the optimization.
Fine to me. Later on is OK.
Quoted 5 lines
Honestly, I am not worried too about memory pressure as we have a shrinker which triggers flushes on the slightest hint of memory pressure. If it is not handling it properly, then we need to fix the shrinker.
Will not speculate here since i have not tested this patch enough with different time out and a low mem. condition.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]
Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Quoted 43 lines
Quoted 43 lines
Quoted 42 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Also you do not need to do:
#ifdef LAZY
Quoted 8 lines
Quoted 3 lines
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
config
RCU_LAZYwe might run into issues related to run-time switching though.
When we started off, Paul said he wanted it kernel CONFIGurable. I will defer to Paul on a decision for that. I prefer kernel CONFIG so people don't forget to pass a boot param.
thanks,
- Joel
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 85 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Quoted 48 lines
Quoted 39 lines
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 7 lines
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
Quoted 7 lines
Quoted 3 lines
config
RCU_LAZYwe might run into issues related to run-time switching though.
When we started off, Paul said he wanted it kernel CONFIGurable. I will defer to Paul on a decision for that. I prefer kernel CONFIG so people don't forget to pass a boot param.
I am fine with a kernel boot parameter for this one. You guys were the ones preferring Kconfig options. ;-)
But in that case, the CONFIG_RCU_NOCB_CPU would come into play to handle the case where there is no bypass.
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 92 lines · Paul E. McKenney
Quoted 87 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Quoted 50 lines
Quoted 39 lines
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 7 lines
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
config
RCU_LAZY
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
we might run into issues related to run-time switching though.
When we started off, Paul said he wanted it kernel CONFIGurable. I will defer to Paul on a decision for that. I prefer kernel CONFIG so people don't forget to pass a boot param.
I am fine with a kernel boot parameter for this one. You guys were the ones preferring
Kconfigoptions. ;-)
Yes I still prefer that.. ;-)
Quoted 2 lines
But in that case, the
CONFIG_RCU_NOCB_CPUwould come into play to handle the case where there is no bypass.
If you don’t mind, let’s do both like we did for NOCB_CPU_ALL. In which case, Vlad since this was your suggestion, would you be so kind to send a patch adding a boot parameter on top of the series? ;-). I’ll include it in the next version. I’d suggest keep the boot param default off and add a CONFIG option that forces the boot param to be turned on.
Thanks,
- Joel
Quoted 2 lines
Thanx, Paul
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 101 lines · Joel Fernandes
Quoted 92 lines · Paul E. McKenney
Quoted 87 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Quoted 50 lines
Quoted 39 lines
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 7 lines
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
config
RCU_LAZYQuoted 8 lines
Quoted 5 lines
Quoted 1 line
we might run into issues related to run-time switching though.
When we started off, Paul said he wanted it kernel CONFIGurable. I will defer to Paul on a decision for that. I prefer kernel CONFIG so people don't forget to pass a boot param.
I am fine with a kernel boot parameter for this one. You guys were the ones preferring
Kconfigoptions. ;-)
Quoted 6 lines
Yes I still prefer that.. ;-)
Quoted 2 lines
But in that case, the
CONFIG_RCU_NOCB_CPUwould come into play to handle the case where there is no bypass.If you don’t mind, let’s do both like we did for
NOCB_CPU_ALL. In which case, Vlad since this was your suggestion, would you be so kind to send a patch adding a boot parameter on top of the series? ;-). I’ll include it in the next version. I’d suggest keep the boot param default off and add a CONFIG option that forces the boot param to be turned on.
NOCB_CPU_ALL? If you are thinking in terms of laziness/flushing being done on a per-CPU basis among the rcu_nocbs CPUs, that sounds like something for later.
Are you thinking in terms of Kconfig options that allow: (1) No laziness.
(2) Laziness on all rcu_nocbs CPUs, but only if specified by a boot parameter. (3) Laziness on all rcu_nocbs CPUs regardless of boot parameter. I could get behind that.
Thanx, Paul
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
On Mon, Sep 26, 2022 at 04:57:55PM -0700, Paul E. McKenney wrote:
[..]
Quoted 70 lines
Quoted 61 lines
Quoted 50 lines
Quoted 50 lines
Quoted 50 lines
Quoted 39 lines
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 7 lines
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
config
RCU_LAZYQuoted 8 lines
Quoted 5 lines
Quoted 1 line
we might run into issues related to run-time switching though.
When we started off, Paul said he wanted it kernel CONFIGurable. I will defer to Paul on a decision for that. I prefer kernel CONFIG so people don't forget to pass a boot param.
I am fine with a kernel boot parameter for this one. You guys were the ones preferring
Kconfigoptions. ;-)Yes I still prefer that.. ;-)
Quoted 8 lines
Quoted 2 lines
But in that case, the
CONFIG_RCU_NOCB_CPUwould come into play to handle the case where there is no bypass.If you don’t mind, let’s do both like we did for
NOCB_CPU_ALL. In which case, Vlad since this was your suggestion, would you be so kind to send a patch adding a boot parameter on top of the series? ;-). I’ll include it in the next version. I’d suggest keep the boot param default off and add a CONFIG option that forces the boot param to be turned on.
Quoted 3 lines
NOCB_CPU_ALL? If you are thinking in terms of laziness/flushing being done on a per-CPU basis among thercu_nocbsCPUs, that sounds like something for later.
Oh, no, I was just trying to bring that up as an example of making boot parameters and CONFIG options for the same thing.
Quoted 4 lines
Are you thinking in terms of
Kconfigoptions that allow: (1) No laziness.
(2) Laziness on allrcu_nocbsCPUs, but only if specified by a boot parameter. (3) Laziness on allrcu_nocbsCPUs regardless of boot parameter. I could get behind that.
Sure agreed, or we could just make it CONFIG_RCU_LAZY_DEFAULT=y and if boot param is specified, override the CONFIG. That will be the simplest and least confusing IMO.
thanks :)
- Joel
Quoted 1 line
Paul E. McKenney
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 79 lines · Joel Fernandes
On Mon, Sep 26, 2022 at 04:57:55PM -0700, Paul E. McKenney wrote:
[..]Quoted 70 lines
Quoted 61 lines
Quoted 50 lines
Quoted 50 lines
Quoted 50 lines
Quoted 39 lines
Quoted 39 lines
Quoted 25 lines
include/linux/rcupdate.h +7
-0@@ -108,6 +108,13 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_RCU_LAZY +void call_rcu_flush (struct rcu_head *head,rcu_callback_t func); +#else +static inline void call_rcu_flush (struct rcu_head *head, +rcu_callback_t func) {call_rcu (head, func); } +#endif +/* Internal to kernel */ void rcu_init (void );extern int rcu_scheduler_active;kernel/rcu/Kconfig +0
-0@@ -314,4 +314,12 @@ config TASKS_TRACE_RCU_READ_MB Say N here if you hate read-side memory barriers. Take the default if you are unsure.kernel/rcu/Kconfig config TASKS_TRACE_RCU_READ_MB · line 325
Quoted 10 lines
kernel/rcu/Kconfig +8
-0+config RCU_LAZY + bool "RCU callback lazy invocation functionality" + depends on RCU_NOCB_CPU + default n + help + To save power, batch RCU callbacks and flush after delay, memory + pressure or callback list growing too big. +Do you think you need this kernel option? Can we just consider and make it a run-time configurable? For example much more users will give it a try, so it will increase a coverage. By default it can be off.
Quoted 7 lines
Quoted 3 lines
Also you do not need to do:
#ifdef LAZY
How does the "LAZY" macro end up being runtime-configurable? That's static / compile time. Did I miss something?
I am talking about removing if:
config
RCU_LAZYQuoted 8 lines
Quoted 5 lines
Quoted 1 line
we might run into issues related to run-time switching though.
When we started off, Paul said he wanted it kernel CONFIGurable. I will defer to Paul on a decision for that. I prefer kernel CONFIG so people don't forget to pass a boot param.
I am fine with a kernel boot parameter for this one. You guys were the ones preferring
Kconfigoptions. ;-)Yes I still prefer that.. ;-)
Quoted 8 lines
Quoted 2 lines
But in that case, the
CONFIG_RCU_NOCB_CPUwould come into play to handle the case where there is no bypass.If you don’t mind, let’s do both like we did for
NOCB_CPU_ALL. In which case, Vlad since this was your suggestion, would you be so kind to send a patch adding a boot parameter on top of the series? ;-). I’ll include it in the next version. I’d suggest keep the boot param default off and add a CONFIG option that forces the boot param to be turned on.Quoted 3 lines
NOCB_CPU_ALL? If you are thinking in terms of laziness/flushing being done on a per-CPU basis among thercu_nocbsCPUs, that sounds like something for later.Oh, no, I was just trying to bring that up as an example of making boot parameters and CONFIG options for the same thing.
Quoted 8 lines
Quoted 4 lines
Are you thinking in terms of
Kconfigoptions that allow: (1) No laziness.
(2) Laziness on allrcu_nocbsCPUs, but only if specified by a boot parameter. (3) Laziness on allrcu_nocbsCPUs regardless of boot parameter. I could get behind that.Sure agreed, or we could just make it
CONFIG_RCU_LAZY_DEFAULT=yand if boot param is specified, override the CONFIG. That will be the simplest and least confusing IMO.
If CONFIG_RCU_LAZY_DEFAULT=n, what (if anything) does the boot parameter do?
Not criticizing, not yet, anyway. ;-)
Thanx, Paul
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Joel FernandesQuoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.
Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0
<snip>
15 seconds delay between two prints. I have logged all call_rcu() users between those two prints:
<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched
# | / _---=> hardirq/softirq
# || / _--=> preempt-depth
# ||| / _-=> migrate-disable
# |||| / delay
# TASK-PID CPU# ||||| TIMESTAMP FUNCTION
# | | | ||||| | |
systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection
systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0
rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0
kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0
<...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0
<...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0
kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0
rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0
rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0
kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0
<...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0
kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0
kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0
kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0
<...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0
kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0
kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0
<...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0
<...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0
kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0
<...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0
<...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0
kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0
<...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0
<...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0
kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0
kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0
kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0
<...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0
kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0
kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0
<...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0
kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0
<...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0
<...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0
kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0
kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0
<...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0
<...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0
kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0
rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0
systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2
systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0
systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0
<snip>
__dentry_kill() function and after 4 seconds there is another one queue_rcu_work(). I have checked the __dentry_kill() if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.
Second delay:
<snip>
rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0
<snip>
10 seconds. But please note that it could be that there were not any callbacks queued during 10 seconds.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 231 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0<snip>
15 seconds delay between two prints. I have logged all
call_rcu()users between those two prints:<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched# | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0 rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0 <...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0 <...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0 kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0 rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0 kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0 <...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0 kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0 kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0 kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0 <...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0 kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0 kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0 <...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0 <...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0 kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0 <...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0 <...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0 kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0 <...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0 <...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0 kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0 kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0 kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0 <...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0 kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0 kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0 <...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0 kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0 <...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0 <...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0 kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0 kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0 <...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0 <...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0 kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0 rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0 systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
Quoted 8 lines
<snip>
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0<snip>
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.
Can you log rcu_barrier() as well? It could be that the print is just a side effect of something else that is not being printed.
If you follow the steps I shared, we should be able to get a full log of RCU traces.
Quoted 9 lines
Second delay:
<snip>
rcuop/5-56 [058] b.... 7.105814:__call_rcu_common: ->0x0:file_free_rcu+0x32/0x50<-0x0
rcuop/20-163 [023] b.... 17.345648:__call_rcu_common: ->0x0:file_free_rcu+0x32/0x50<-0x0
<snip>10 seconds. But please note that it could be that there were not any callbacks queued during 10 seconds.
True, there are things other than call_rcu() like rcu_barrier() that can slow down the boot path. We fixed that and it is not an issue over here, but I am not sure if you are triggering it some how.
thanks,
- Joel
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 237 lines · Joel Fernandes
Quoted 236 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0<snip>
15 seconds delay between two prints. I have logged all
call_rcu()users between those two prints:<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched# | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0 rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0 <...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0 <...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0 kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0 rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0 kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0 <...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0 kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0 kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0 kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0 <...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0 kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0 kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0 <...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0 <...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0 kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0 <...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0 <...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0 kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0 <...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0 <...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0 kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0 kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0 kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0 <...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0 kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0 kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0 <...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0 kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0 <...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0 <...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0 kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0 kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0 <...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0 <...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0 kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0 rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0 systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0<snip>
Quoted 7 lines
Quoted 3 lines
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.Can you log
rcu_barrier()as well? It could be that the print is just a side effect of something else that is not being printed.
It has nothing to do with rcu_barrier() in my case. Also i have checked the synchronize_rcu() it also works as expected, i.e. it is not a blocking reason.
Have you tried my config?
--
Uladzislau Rezki
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 245 lines · Uladzislau Rezki
Quoted 237 lines · Joel Fernandes
Quoted 236 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0<snip>
15 seconds delay between two prints. I have logged all
call_rcu()users between those two prints:<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched# | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0 rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0 <...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0 <...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0 kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0 rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0 kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0 <...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0 kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0 kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0 kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0 <...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0 kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0 kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0 <...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0 <...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0 kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0 <...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0 <...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0 kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0 <...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0 <...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0 kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0 kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0 kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0 <...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0 kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0 kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0 <...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0 kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0 <...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0 <...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0 kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0 kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0 <...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0 <...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0 kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0 rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0 systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0<snip>
Quoted 7 lines
Quoted 3 lines
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.Can you log
rcu_barrier()as well? It could be that the print is just a side effect of something else that is not being printed.
Quoted 8 lines
It has nothing to do with
rcu_barrier()in my case. Also i have checked thesynchronize_rcu()it also works as expected, i.e. it is not a blocking reason.Have you tried my config?
--
Uladzislau Rezki
OK. Seems one place i have spot:
<snip>
[ 7.074847] calling init_sr+0x0/0x1000 [sr_mod] @ 668 [ 22.422808] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.422815] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.664590] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 32.664642] initcall init_sr+0x0/0x1000 [sr_mod] returned 0 after 25589786 usecs
<snip>
--
Uladzislau Rezki
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 262 lines · Uladzislau Rezki
Quoted 245 lines · Uladzislau Rezki
Quoted 237 lines · Joel Fernandes
Quoted 236 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0<snip>
15 seconds delay between two prints. I have logged all
call_rcu()users between those two prints:<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched# | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0 rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0 <...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0 <...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0 kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0 rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0 kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0 <...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0 kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0 kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0 kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0 <...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0 kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0 kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0 <...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0 <...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0 kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0 <...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0 <...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0 kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0 <...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0 <...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0 kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0 kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0 kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0 <...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0 kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0 kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0 <...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0 kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0 <...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0 <...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0 kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0 kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0 <...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0 <...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0 kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0 rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0 systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0<snip>
Quoted 7 lines
Quoted 3 lines
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.Can you log
rcu_barrier()as well? It could be that the print is just a side effect of something else that is not being printed.Quoted 8 lines
It has nothing to do with
rcu_barrier()in my case. Also i have checked thesynchronize_rcu()it also works as expected, i.e. it is not a blocking reason.Have you tried my config?
--
Uladzislau RezkiOK. Seems one place i have spot:
<snip>
[ 7.074847] calling init_sr+0x0/0x1000 [sr_mod] @ 668 [ 22.422808] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.422815] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.664590] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 32.664642] initcall init_sr+0x0/0x1000 [sr_mod] returned 0 after 25589786 usecs
Quoted 4 lines
<snip>
--
Uladzislau Rezki
OK. Found the boot up issue. In my case i had 120 seconds delay:
<snip>
drivers/scsi/scsi_error.c +1 -1
@@ -312,7 +312,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)* Ensure that all tasks observe the host state change before the * host_failed change. */ -call_rcu (&scmd->rcu, scsi_eh_inc_host_failed); +call_rcu_flush (&scmd->rcu, scsi_eh_inc_host_failed); }/**
<snip>
After this change the boot-up time settles back to normal 4 seconds.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 269 lines · Uladzislau Rezki
Quoted 264 lines · Uladzislau Rezki
Quoted 246 lines · Uladzislau Rezki
Quoted 237 lines · Joel Fernandes
Quoted 236 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0<snip>
15 seconds delay between two prints. I have logged all
call_rcu()users between those two prints:<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched# | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0 rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0 <...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0 <...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0 kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0 rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0 kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0 <...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0 kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0 kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0 kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0 <...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0 kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0 kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0 <...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0 <...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0 kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0 <...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0 <...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0 kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0 <...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0 <...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0 kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0 kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0 kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0 <...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0 kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0 kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0 <...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0 kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0 <...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0 <...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0 kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0 kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0 <...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0 <...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0 kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0 rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0 systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024:__call_rcu_common: ->0x0:__dentry_kill+0x140/0x180<-0x2
kworker/0:3-546 [000] d..1. 6.329516:__call_rcu_common: ->0x0:queue_rcu_work+0x2b/0x40<-0xffff8c70effe40a0
<snip>Quoted 7 lines
Quoted 3 lines
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.Can you log
rcu_barrier()as well? It could be that the print is just a side effect of something else that is not being printed.Quoted 8 lines
It has nothing to do with
rcu_barrier()in my case. Also i have checked thesynchronize_rcu()it also works as expected, i.e. it is not a blocking reason.Have you tried my config?
--
Uladzislau RezkiOK. Seems one place i have spot:
<snip>
[ 7.074847] calling init_sr+0x0/0x1000 [sr_mod] @ 668 [ 22.422808] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.422815] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.664590] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 32.664642] initcall init_sr+0x0/0x1000 [sr_mod] returned 0 after 25589786 usecs<snip>
Quoted 4 lines
Quoted 2 lines
--
Uladzislau RezkiOK. Found the boot up issue. In my case i had 120 seconds delay:
Wow, nice work.
drivers/scsi/scsi_error.c void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · line 315
Quoted 11 lines
<snip>
drivers/scsi/scsi_error.c +1
-1@@ -312,7 +312,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)* Ensure that all tasks observe the host state change before the * host_failed change. */ -call_rcu (&scmd->rcu, scsi_eh_inc_host_failed); +call_rcu_flush (&scmd->rcu, scsi_eh_inc_host_failed);
Great! Thanks. I’ll include this and the other one you converted in the next revision.
Thanks,
- Joel
Quoted 9 lines
drivers/scsi/scsi_error.c +0
-0}/** After this change the boot-up time settles back to normal 4 seconds.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 280 lines · Joel Fernandes
Quoted 267 lines · Uladzislau Rezki
Quoted 264 lines · Uladzislau Rezki
Quoted 246 lines · Uladzislau Rezki
Quoted 237 lines · Joel Fernandes
Quoted 236 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Hi Vlad,
On Mon, Sep 26, 2022 at 09:39:23PM +0200, Uladzislau Rezki wrote:
[...]Quoted 21 lines
Quoted 21 lines
Quoted 20 lines
On my KVM machine the boot time is affected:
<snip>
[ 2.273406] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 11.945283] e1000 0000:00:03.0 ens3: renamed from eth0 [ 22.165198] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.165206] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.406981] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 104.115418] process '/usr/bin/fstype' started with executable stack [ 104.170142] EXT4-fs (sda1): mounted filesystem with ordered data mode. Quota mode: none. [ 104.340125] systemd[1]: systemd 241 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid) [ 104.340193] systemd[1]: Detected virtualization kvm. [ 104.340196] systemd[1]: Detected architecture x86-64. [ 104.359032] systemd[1]: Set hostname to <pc638>. [ 105.740109] random: crng init done [ 105.741267] systemd[1]: Reached target Remote File Systems.<snip>
2 - 11 and second delay is between 32 - 104. So there are still users which must be waiting for "RCU" in a sync way.
Quoted 8 lines
Quoted 8 lines
I was wondering if you can compare boot logs and see which timestamp does the slow down start from. That way, we can narrow down the callback. Also another idea is, add "trace_event=rcu:rcu_callback,rcu:rcu_invoke_callback
ftrace_dump_on_oops" to the boot params, and then manually call "tracing_off();panic();" from the code at the first printk that seems off in your comparison of good vs bad. For example, if "crng init done" timestamp is off, put the "tracing_off();panic();" there. Then grab the serial console output to see what were the last callbacks that was queued/invoked.Quoted 3 lines
Would you be willing to try these steps? Meanwhile I will try on my side as well with the .config you sent me in another email.
Not exactly those steps. But see below:
<snip>
[ 2.291319] e1000 0000:00:03.0 eth0: Intel(R) PRO/1000 Network Connection [ 17.302946] e1000 0000:00:03.0 ens3: renamed from eth0<snip>
15 seconds delay between two prints. I have logged all
call_rcu()users between those two prints:<snip>
# tracer: nop
#
# entries-in-buffer/entries-written: 166/166 #P:64
#
# _-----=> irqs-off/BH-disabled
# / _----=> need-resched# | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / _-=> migrate-disable # |||| / delay # TASK-PID CPU# ||||| TIMESTAMP FUNCTION # | | | ||||| | | systemd-udevd-669 [002] ..... 2.338739: e1000_probe: Intel(R) PRO/1000 Network Connection systemd-udevd-665 [061] ..... 2.338952: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 2.338962: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338965: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338968: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 2.338987: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.338989: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.338999: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 2.339002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0 rcuop/0-17 [000] b.... 6.337320: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 kworker/38:1-744 [038] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f97e40a0 <...>-739 [035] d..1. 6.841479: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8be40a0 <...>-732 [021] d..1. 6.841486: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f53e40a0 kworker/36:1-740 [036] d..1. 6.841487: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f8fe40a0 rcuop/21-170 [023] b.... 6.849276: __call_rcu_common: -> 0x0: exit_creds+0x63/0x70 <- 0x0 rcuop/38-291 [052] b.... 6.849950: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/38-291 [052] b.... 6.849957: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 kworker/5:1-712 [005] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f13e40a0 kworker/19:1-727 [019] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f4be40a0 <...>-719 [007] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1be40a0 kworker/13:1-721 [013] d..1. 7.097392: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f33e40a0 kworker/52:1-756 [052] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcfe40a0 kworker/29:1-611 [029] d..1. 7.097395: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f73e40a0 <...>-754 [049] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc3e40a0 kworker/12:1-726 [012] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f2fe40a0 kworker/53:1-710 [053] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd3e40a0 <...>-762 [061] d..1. 7.097405: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ff3e40a0 <...>-757 [054] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fd7e40a0 kworker/25:1-537 [025] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f63e40a0 <...>-714 [004] d..1. 7.097408: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0fe40a0 <...>-749 [044] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fafe40a0 kworker/51:1-755 [051] d..1. 7.097413: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fcbe40a0 <...>-764 [063] d..1. 7.097415: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70ffbe40a0 <...>-753 [045] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fb3e40a0 kworker/43:1-748 [043] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fabe40a0 kworker/41:1-747 [041] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa3e40a0 kworker/57:1-760 [057] d..1. 7.097416: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe3e40a0 <...>-720 [008] d..1. 7.097418: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f1fe40a0 kworker/58:1-759 [058] d..1. 7.097421: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fe7e40a0 kworker/16:1-728 [016] d..1. 7.097424: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f3fe40a0 <...>-722 [010] d..1. 7.097427: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f27e40a0 kworker/22:1-733 [022] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f57e40a0 <...>-731 [026] d..1. 7.097432: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f67e40a0 <...>-752 [048] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fbfe40a0 kworker/18:0-147 [018] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f47e40a0 kworker/39:1-745 [039] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f9be40a0 <...>-716 [003] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70f0be40a0 <...>-703 [050] d..1. 7.097437: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fc7e40a0 kworker/42:1-746 [042] d..1. 7.097444: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70fa7e40a0 rcuop/13-113 [013] b.... 7.105592: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/13-113 [013] b.... 7.105595: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105608: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/10-92 [040] b.... 7.105610: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/16-135 [023] b.... 7.105613: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105636: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/8-78 [039] b.... 7.105640: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105651: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/12-106 [040] b.... 7.105652: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105727: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/19-156 [000] b.... 7.105730: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105808: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/5-56 [058] b.... 7.105814: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345648: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/20-163 [023] b.... 17.345655: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345675: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/14-120 [013] b.... 17.345681: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345714: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/6-63 [013] b.... 17.345715: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345753: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/9-85 [000] b.... 17.345758: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345775: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345776: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/17-142 [000] b.... 17.345777: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345810: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/11-99 [000] b.... 17.345811: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345832: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345834: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/1-28 [000] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345835: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 rcuop/15-127 [013] b.... 17.345837: __call_rcu_common: -> 0x0: file_free_rcu+0x32/0x50 <- 0x0 systemd-udevd-633 [035] ..... 17.346591: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346609: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346659: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-633 [035] ..... 17.346666: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-669 [002] ..... 17.347573: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 kworker/2:2-769 [002] ..... 17.347659: __call_rcu_common: -> 0x0: __wait_rcu_gp+0xff/0x120 <- 0x0 systemd-udevd-675 [012] ..... 17.347981: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.348037: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348098: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348117: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348120: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348156: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348166: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348176: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.348179: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348186: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348197: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348200: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348231: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348240: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348250: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348259: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348262: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348305: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348317: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348332: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348336: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348394: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348403: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348406: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348503: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348531: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.348535: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348536: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.348563: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.348575: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348628: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348704: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348828: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348884: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348904: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348954: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348983: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.348993: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349002: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349014: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349024: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349026: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349119: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349182: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349243: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349430: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349462: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349472: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349483: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349486: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349583: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349632: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349666: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349699: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349727: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349733: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349739: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-645 [053] ..... 17.349742: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349765: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-645 [053] ..... 17.349766: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349780: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-665 [061] ..... 17.349800: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349815: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349829: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024:__call_rcu_common: ->0x0:__dentry_kill+0x140/0x180<-0x2
kworker/0:3-546 [000] d..1. 6.329516:__call_rcu_common: ->0x0:queue_rcu_work+0x2b/0x40<-0xffff8c70effe40a0
<snip>Quoted 7 lines
Quoted 3 lines
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.Can you log
rcu_barrier()as well? It could be that the print is just a side effect of something else that is not being printed.Quoted 8 lines
It has nothing to do with
rcu_barrier()in my case. Also i have checked thesynchronize_rcu()it also works as expected, i.e. it is not a blocking reason.Have you tried my config?
--
Uladzislau RezkiOK. Seems one place i have spot:
<snip>
[ 7.074847] calling init_sr+0x0/0x1000 [sr_mod] @ 668 [ 22.422808] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray [ 22.422815] cdrom: Uniform CD-ROM driver Revision: 3.20 [ 32.664590] sr 1:0:0:0: Attached scsi CD-ROM sr0 [ 32.664642] initcall init_sr+0x0/0x1000 [sr_mod] returned 0 after 25589786 usecs<snip>
Quoted 4 lines
Quoted 2 lines
--
Uladzislau RezkiOK. Found the boot up issue. In my case i had 120 seconds delay:
Wow, nice work.
Quoted 13 lines
drivers/scsi/scsi_error.c void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · line 315
Quoted 11 lines
<snip>
drivers/scsi/scsi_error.c +1
-1@@ -312,7 +312,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)* Ensure that all tasks observe the host state change before the * host_failed change. */ -call_rcu (&scmd->rcu, scsi_eh_inc_host_failed); +call_rcu_flush (&scmd->rcu, scsi_eh_inc_host_failed);Great! Thanks. I’ll include this and the other one you converted in the next revision.
By the way, any chance you could check android as well, just to rule out any trouble markers? ChromeOS and your Linux distro are doing well on boot so that’s a good sign.
(Also let’s start trimming emails before Steven starts sending out nastygrams ;-)).
Thanks,
- Joel
Quoted 14 lines
Thanks,
- Joel
Quoted 9 lines
drivers/scsi/scsi_error.c +0
-0}/** After this change the boot-up time settles back to normal 4 seconds.
--
Uladzislau Rezki
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 19 lines
Quoted 5 lines
Quoted 2 lines
OK. Found the boot up issue. In my case i had 120 seconds delay:
Wow, nice work.
Quoted 13 lines
drivers/scsi/scsi_error.c void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · line 315
Quoted 11 lines
<snip>
drivers/scsi/scsi_error.c +1
-1@@ -312,7 +312,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)* Ensure that all tasks observe the host state change before the * host_failed change. */ -call_rcu (&scmd->rcu, scsi_eh_inc_host_failed); +call_rcu_flush (&scmd->rcu, scsi_eh_inc_host_failed);Great! Thanks. I’ll include this and the other one you converted in the next revision.
Quoted 2 lines
By the way, any chance you could check android as well, just to rule out any trouble markers? ChromeOS and your Linux distro are doing well on boot so that’s a good sign.
I will check v6 on Android. I will get back shortly.
Quoted 2 lines
(Also let’s start trimming emails before Steven starts sending out nastygrams ;-)).
Done :)
--
Uladzislau Rezki
Uladzislau Rezki
maintainerRe: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Quoted 19 lines
Quoted 19 lines
Quoted 18 lines
Quoted 2 lines
OK. Found the boot up issue. In my case i had 120 seconds delay:
Wow, nice work.
drivers/scsi/scsi_error.c void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · line 315
Quoted 11 lines
<snip>
drivers/scsi/scsi_error.c +1
-1@@ -312,7 +312,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)* Ensure that all tasks observe the host state change before the * host_failed change. */ -call_rcu (&scmd->rcu, scsi_eh_inc_host_failed); +call_rcu_flush (&scmd->rcu, scsi_eh_inc_host_failed);Great! Thanks. I’ll include this and the other one you converted in the next revision.
Quoted 4 lines
Quoted 2 lines
By the way, any chance you could check android as well, just to rule out any trouble markers? ChromeOS and your Linux distro are doing well on boot so that’s a good sign.
I will check v6 on Android. I will get back shortly.
OK. Works and boots well. I do not see any issues with Android so far.
--
Uladzislau Rezki
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
Hi Vlad,
Quoted 20 lines · Uladzislau Rezki
Quoted 20 lines
Quoted 20 lines
Quoted 19 lines
Quoted 3 lines
OK. Found the boot up issue. In my case i had 120 seconds delay:
Wow, nice work.
drivers/scsi/scsi_error.c void scsi_eh_scmd_add(struct scsi_cmnd *scmd) · line 315
Quoted 11 lines
<snip>
drivers/scsi/scsi_error.c +1
-1@@ -312,7 +312,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)* Ensure that all tasks observe the host state change before the * host_failed change. */ -call_rcu (&scmd->rcu, scsi_eh_inc_host_failed); +call_rcu_flush (&scmd->rcu, scsi_eh_inc_host_failed);Great! Thanks. I’ll include this and the other one you converted in the next revision.
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
By the way, any chance you could check android as well, just to rule out any trouble markers? ChromeOS and your Linux distro are doing well on boot so that’s a good sign.
I will check v6 on Android. I will get back shortly.
OK. Works and boots well. I do not see any issues with Android so far.
That's great news and thank you so much for testing Android and all your efforts on this work!
- Joel
Re: [PATCH v6 1/4] rcu: Make call_rcu() lazy to save power
In reply to Uladzislau RezkiOn Tue, Sep 27, 2022 at 04:59:44PM +0200, Uladzislau Rezki wrote:
[...]
Quoted 25 lines
Quoted 25 lines
Quoted 24 lines
systemd-udevd-642 [050] ..... 17.349832: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349834: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349835: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-675 [012] ..... 17.349853: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-642 [050] ..... 17.349861: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 systemd-udevd-675 [012] ..... 17.349873: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.349879: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350007: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350011: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350080: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350175: __call_rcu_common: -> ____fput+0x0/0x10: task_work_run+0x5c/0x90 <- 0x0 systemd-udevd-665 [061] ..... 17.350362: dev_change_name: --> renamed from eth0<snip>
First delay:
<snip>
systemd-udevd-645 [053] ..... 2.339024: __call_rcu_common: -> 0x0: __dentry_kill+0x140/0x180 <- 0x2 kworker/0:3-546 [000] d..1. 6.329516: __call_rcu_common: -> 0x0: queue_rcu_work+0x2b/0x40 <- 0xffff8c70effe40a0<snip>
__dentry_kill()function and after 4 seconds there is another onequeue_rcu_work(). I have checked the__dentry_kill()if it can do any sync talk with RCU but from the first glance i do not see anything critical. But more attention is required.
Quoted 8 lines
Quoted 3 lines
Can you log
rcu_barrier()as well? It could be that the print is just a side effect of something else that is not being printed.It has nothing to do with
rcu_barrier()in my case. Also i have checked thesynchronize_rcu()it also works as expected, i.e. it is not a blocking reason.Have you tried my config?
Yes I am in the process of trying it.
thanks,
- Joel
PATCH v6 2/4 rcu: shrinker for lazy rcu
In reply to Joel Fernandes (Google)From: Vineeth Pillai <[email protected]>
The shrinker is used to speed up the free'ing of memory potentially held by RCU lazy callbacks. RCU kernel module test cases show this to be effective. Test is introduced in a later patch.
--- kernel/rcu/tree_nocb.h | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+)
kernel/rcu/tree_nocb.h +52 -0
@@ -1332,6 +1332,55 @@ int rcu_nocb_cpu_offload(int cpu) }EXPORT_SYMBOL_GPL (rcu_nocb_cpu_offload); +static unsigned long +lazy_rcu_shrink_count (struct shrinker *shrink,struct shrink_control *sc) +{ +int cpu; +unsigned long count =0 ; + +/* Snapshot count of all CPUs */ +for_each_possible_cpu (cpu) { +struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); + + count +=READ_ONCE (rdp->lazy_len); + } + +return count ? count :SHRINK_EMPTY ; +} + +static unsigned long +lazy_rcu_shrink_scan (struct shrinker *shrink,struct shrink_control *sc) +{ +int cpu; +unsigned long flags; +unsigned long count =0 ; + +/* Snapshot count of all CPUs */ +for_each_possible_cpu (cpu) { +struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); +int _count =READ_ONCE (rdp->lazy_len); + +if (_count ==0 ) +continue ; +rcu_nocb_lock_irqsave (rdp, flags); +WRITE_ONCE (rdp->lazy_len,0 ); +rcu_nocb_unlock_irqrestore (rdp, flags); +wake_nocb_gp (rdp, false); + sc->nr_to_scan -= _count; + count += _count; +if (sc->nr_to_scan <=0 ) +break ; + } +return count ? count :SHRINK_STOP ; +} + +static struct shrinker lazy_rcu_shrinker = { + .count_objects = lazy_rcu_shrink_count, + .scan_objects = lazy_rcu_shrink_scan, + .batch =0 , + .seeks =DEFAULT_SEEKS , +}; +void __init rcu_init_nohz (void ) {int cpu;
@@ -1362,6 +1411,9 @@ void __init rcu_init_nohz(void)if (!rcu_state.nocb_is_setup)return ; +if (register_shrinker (&lazy_rcu_shrinker,"rcu-lazy" )) +pr_err ("Failed to register lazy_rcu shrinker!\n" ); +if (!cpumask_subset (rcu_nocb_mask, cpu_possible_mask)) {pr_info ("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n" ); cpumask_and(rcu_nocb_mask, cpu_possible_mask,
Signature · 1 line
2.37.3.998.g577e59143f-goog
PATCH v6 3/4 rcuscale: Add laziness and kfree tests
In reply to Joel Fernandes (Google)We add 2 tests to rcuscale, first one is a startup test to check whether we are not too lazy or too hard working. Two, emulate kfree_rcu() itself to use call_rcu() and check memory pressure. In my testing, the new call_rcu() does well to keep memory pressure under control, similar to kfree_rcu().
--- kernel/rcu/rcuscale.c | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-)
kernel/rcu/rcuscale.c +64 -1
@@ -95,6 +95,7 @@ torture_param(int, verbose, 1, "Enable verbose debugging printk()s");torture_param (int, writer_holdoff,0 ,"Holdoff (us) between GPs, zero to disable" );torture_param (int, kfree_rcu_test,0 ,"Do we run a kfree_rcu() scale test?" );torture_param (int, kfree_mult,1 ,"Multiple of kfree_obj size to allocate." ); +torture_param (int, kfree_by_call_rcu,0 ,"Use call_rcu() to emulate kfree_rcu()?" );static char *scale_type ="rcu" ;module_param (scale_type, charp,0444 );
@@ -659,6 +660,14 @@ struct kfree_obj {struct rcu_head rh; }; +/* Used if doing RCU-kfree'ing via call_rcu(). */ +static void kfree_call_rcu (struct rcu_head *rh) +{ + struct kfree_obj *obj = container_of(rh, struct kfree_obj, rh); + +kfree (obj); +} +static int kfree_scale_thread (void *arg) {
@@ -696,6 +705,11 @@ kfree_scale_thread(void *arg)if (!alloc_ptr)return -ENOMEM ; +if (kfree_by_call_rcu) { +call_rcu (&(alloc_ptr->rh), kfree_call_rcu); +continue ; + } +// By default kfree_rcu_test_single and kfree_rcu_test_double are // initialized to false. If both have the same value (false or true) // both are randomly tested, otherwise only the one with value true
@@ -767,11 +781,58 @@ kfree_scale_shutdown(void *arg)return -EINVAL ; } +// Used if doing RCU-kfree'ing via call_rcu(). +static unsigned long jiffies_at_lazy_cb; +static struct rcu_head lazy_test1_rh; +static int rcu_lazy_test1_cb_called; +static void call_rcu_lazy_test1 (struct rcu_head *rh) +{ + jiffies_at_lazy_cb = jiffies; +WRITE_ONCE (rcu_lazy_test1_cb_called,1 ); +} +static int __init kfree_scale_init (void ) {long i;int firsterr =0 ; +unsigned long orig_jif, jif_start; + +// Also, do a quick self-test to ensure laziness is as much as +// expected. +if (kfree_by_call_rcu && !IS_ENABLED (CONFIG_RCU_LAZY )) { +pr_alert ("CONFIG_RCU_LAZY is disabled, falling back to kfree_rcu() " +"for delayed RCU kfree'ing\n" ); + kfree_by_call_rcu =0 ; + } + +if (kfree_by_call_rcu) { +/* do a test to check the timeout. */ + orig_jif =rcu_lazy_get_jiffies_till_flush (); + +rcu_lazy_set_jiffies_till_flush (2 *HZ ); +rcu_barrier (); + + jif_start = jiffies; + jiffies_at_lazy_cb =0 ; +call_rcu (&lazy_test1_rh, call_rcu_lazy_test1); + +smp_cond_load_relaxed (&rcu_lazy_test1_cb_called,VAL ==1 ); + +rcu_lazy_set_jiffies_till_flush (orig_jif); + +if (WARN_ON_ONCE (jiffies_at_lazy_cb - jif_start <2 *HZ )) { +pr_alert ("ERROR: call_rcu() CBs are not being lazy as expected!\n" ); +WARN_ON_ONCE (1 ); +return -1 ; + } + +if (WARN_ON_ONCE (jiffies_at_lazy_cb - jif_start >3 *HZ )) { +pr_alert ("ERROR: call_rcu() CBs are being too lazy!\n" ); +WARN_ON_ONCE (1 ); +return -1 ; + } + } kfree_nrealthreads =compute_real (kfree_nthreads);/* Start up the kthreads. */
@@ -784,7 +845,9 @@ kfree_scale_init(void)schedule_timeout_uninterruptible (1 ); } -pr_alert ("kfree object size=%zu\n" , kfree_mult *sizeof (struct kfree_obj )); +pr_alert ("kfree object size=%zu, kfree_by_call_rcu=%d\n" , + kfree_mult *sizeof (struct kfree_obj ), + kfree_by_call_rcu); kfree_reader_tasks =kcalloc (kfree_nrealthreads,sizeof (kfree_reader_tasks[0 ]),GFP_KERNEL );
Signature · 1 line
2.37.3.998.g577e59143f-goog
PATCH v6 4/4 percpu-refcount: Use call_rcu_flush() for atomic switch
In reply to Joel Fernandes (Google)call_rcu() changes to save power will slow down percpu refcounter per-CPU to atomic switch path. The primitive uses RCU when switching to atomic mode.
The enqueued async callback wakes up waiters waiting in the percpu_ref_switch_waitq. This will slow down the per-CPU refcount users such as blk_pre_runtime_suspend().
Use the call_rcu_flush() API instead which reverts to the old behavior.
--- lib/percpu-refcount.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
lib/percpu-refcount.c +2 -1
@@ -230,7 +230,8 @@ static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, percpu_ref_noop_confirm_switch;percpu_ref_get (ref);/* put after confirmation */ -call_rcu (&ref->data->rcu, percpu_ref_switch_to_atomic_rcu); +call_rcu_flush (&ref->data->rcu, + percpu_ref_switch_to_atomic_rcu); }static void __percpu_ref_switch_to_percpu (struct percpu_ref *ref)
Signature · 1 line
2.37.3.998.g577e59143f-goog