RCU CPU stall-warning changes for v5.2
cover RCU CPU stall-warning changes for v5.2
Hello!
This series is primarily code movement for RCU CPU stall warnings. If I am having a hard time finding the various scattered pieces of this code, it is in need of consolidation!
1-3. Move RCU CPU stall-warning code into kernel/rcu/tree_stall.h.
4. Inline RCU task stall-warning helper functions.
5. Move rcu_print_task_exp_stall() to tree_exp.h.
6. Inline RCU stall-warning info helper functions.
7. Move FAST_NO_HZ stall-warning code to tree_stall.h.
8. Organize functions in tree_stall.h.
9. Move irq-disabled stall-warning checking to tree_stall.h.
10. Move forward-progress checkers into tree_stall.h
11. Fix nohz status in stall warning, courtesy of Neeraj Upadhyay.
Thanx, Paul
------------------------------------------------------------------------
rcu.h | 1 tree.c | 479 ----------------------------- tree.h | 18 - tree_exp.h | 32 + tree_plugin.h | 213 ------------ tree_stall.h | 951 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- update.c | 59 --- 7 files changed, 875 insertions(+), 878 deletions(-)
PATCH 4/11 rcu: Inline RCU task stall-warning helper functions
The rcu_print_detail_task_stall(), rcu_print_task_stall_begin(), and rcu_print_task_stall_end() functions were defined to allow long-gone Kconfig options to provide an abbreviated RCU CPU stall warning printout. This commit saves a few lines of code by inlining them into their sole callers.
While in the area, a useless call of rcu_print_detail_task_stall_rnp() on the root rcu_node structure was eliminated. If there is only one rcu_node structure, its tasks get printed twice, but if there are more, the root rcu_node structure is guaranteed to have an empty list of blocked tasks, hence the uselessness. (Long ago, root rcu_node structures with non-empty ->blkd_tasks lists could happen, but no longer.)
--- kernel/rcu/tree.h | 1 - kernel/rcu/tree_stall.h | 36 +++++++----------------------------- 2 files changed, 7 insertions(+), 30 deletions(-)
kernel/rcu/tree.h +0 -1
@@ -445,7 +445,6 @@ static void rcu_dynticks_task_enter(void);static void rcu_dynticks_task_exit (void );/* Forward declarations for tree_stall.h */ -static void rcu_print_detail_task_stall (void );static int rcu_print_task_stall (struct rcu_node *rnp);static void record_gp_stall_check_time (void );static void check_cpu_stall (struct rcu_data *rdp);
kernel/rcu/tree_stall.h +7 -29
@@ -94,30 +94,6 @@ static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)raw_spin_unlock_irqrestore_rcu_node (rnp, flags); } -/* -* Dump detailed information for all tasks blocking the current RCU -* grace period. -*/ -static void rcu_print_detail_task_stall (void ) -{ -struct rcu_node *rnp =rcu_get_root (); - -rcu_print_detail_task_stall_rnp (rnp); -rcu_for_each_leaf_node (rnp) -rcu_print_detail_task_stall_rnp (rnp); -} - -static void rcu_print_task_stall_begin (struct rcu_node *rnp) -{ -pr_err ("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):" , - rnp->level, rnp->grplo, rnp->grphi); -} - -static void rcu_print_task_stall_end (void ) -{ -pr_cont ("\n" ); -} -/* * Scan the current list of tasks blocked within RCU read-side critical * sections, printing out the tid of each.
@@ -129,14 +105,15 @@ static int rcu_print_task_stall(struct rcu_node *rnp)if (!rcu_preempt_blocked_readers_cgp (rnp))return 0 ; -rcu_print_task_stall_begin (rnp); +pr_err ("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):" , + rnp->level, rnp->grplo, rnp->grphi); t =list_entry (rnp->gp_tasks->prev, struct task_struct, rcu_node_entry);list_for_each_entry_continue (t, &rnp->blkd_tasks, rcu_node_entry) {pr_cont (" P%d" , t->pid); ndetected++; } -rcu_print_task_stall_end (); +pr_cont ("\n" );return ndetected; }
@@ -146,7 +123,7 @@ static int rcu_print_task_stall(struct rcu_node *rnp)* Because preemptible RCU does not exist, we never have to check for * tasks blocked within RCU read-side critical sections. */ -static void rcu_print_detail_task_stall (void ) +static void rcu_print_detail_task_stall_rnp (struct rcu_node *rnp) { }
@@ -253,7 +230,7 @@ static void print_other_cpu_stall(unsigned long gp_seq)unsigned long gpa;unsigned long j;int ndetected =0 ; -struct rcu_node *rnp =rcu_get_root (); +struct rcu_node *rnp;long totqlen =0 ;/* Kick and suppress, if so configured. */
@@ -291,7 +268,8 @@ static void print_other_cpu_stall(unsigned long gp_seq)rcu_dump_cpu_stacks ();/* Complain about tasks blocking the grace period. */ -rcu_print_detail_task_stall (); +rcu_for_each_leaf_node (rnp) +rcu_print_detail_task_stall_rnp (rnp); }else {if (rcu_seq_current (&rcu_state.gp_seq) != gp_seq) {pr_err ("INFO: Stall ended before state dump start\n" );
Signature · 1 line
2.17.1
PATCH 5/11 rcu: Move rcu_print_task_exp_stall() to tree_exp.h
In reply to Paul E. McKenneyBecause expedited CPU stall warnings are contained within the kernel/rcu/tree_exp.h file, rcu_print_task_exp_stall() should live there too. This commit carries out the required code motion.
--- kernel/rcu/tree_exp.h | 32 ++++++++++++++++++++++++++++++++ kernel/rcu/tree_plugin.h | 31 ------------------------------- 2 files changed, 32 insertions(+), 31 deletions(-)
kernel/rcu/tree_exp.h +32 -0
@@ -10,6 +10,7 @@#include <linux/lockdep.h> static void rcu_exp_handler (void *unused); +static int rcu_print_task_exp_stall (struct rcu_node *rnp);/* * Record the start of an expedited grace period.
@@ -670,6 +671,27 @@ static void sync_sched_exp_online_cleanup(int cpu) { } +/* +* Scan the current list of tasks blocked within RCU read-side critical +* sections, printing out the tid of each that is blocking the current +* expedited grace period. +*/ +static int rcu_print_task_exp_stall (struct rcu_node *rnp) +{ +struct task_struct *t; +int ndetected =0 ; + +if (!rnp->exp_tasks) +return 0 ; + t =list_entry (rnp->exp_tasks->prev, + struct task_struct, rcu_node_entry); +list_for_each_entry_continue (t, &rnp->blkd_tasks, rcu_node_entry) { +pr_cont (" P%d" , t->pid); + ndetected++; + } +return ndetected; +} +#else /* #ifdef CONFIG_PREEMPT_RCU */ /* Invoked on each online non-idle CPU for expedited quiescent state. */
@@ -709,6 +731,16 @@ static void sync_sched_exp_online_cleanup(int cpu) WARN_ON_ONCE(ret); } +/* +* Because preemptible RCU does not exist, we never have to check for +* tasks blocked within RCU read-side critical sections that are +* blocking the current expedited grace period. +*/ +static int rcu_print_task_exp_stall (struct rcu_node *rnp) +{ +return 0 ; +} +#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /**
kernel/rcu/tree_plugin.h +0 -31
@@ -642,27 +642,6 @@ static void rcu_read_unlock_special(struct task_struct *t)rcu_preempt_deferred_qs_irqrestore (t, flags); } -/* -* Scan the current list of tasks blocked within RCU read-side critical -* sections, printing out the tid of each that is blocking the current -* expedited grace period. -*/ -static int rcu_print_task_exp_stall (struct rcu_node *rnp) -{ -struct task_struct *t; -int ndetected =0 ; - -if (!rnp->exp_tasks) -return 0 ; - t =list_entry (rnp->exp_tasks->prev, - struct task_struct, rcu_node_entry); -list_for_each_entry_continue (t, &rnp->blkd_tasks, rcu_node_entry) { -pr_cont (" P%d" , t->pid); - ndetected++; - } -return ndetected; -} -/* * Check that the list of blocked tasks for the newly completed grace * period is in fact empty. It is a serious bug to complete a grace
@@ -906,16 +885,6 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) }static void rcu_preempt_deferred_qs (struct task_struct *t) { } -/* -* Because preemptible RCU does not exist, we never have to check for -* tasks blocked within RCU read-side critical sections that are -* blocking the current expedited grace period. -*/ -static int rcu_print_task_exp_stall (struct rcu_node *rnp) -{ -return 0 ; -} -/* * Because there is no preemptible RCU, there can be no readers blocked, * so there is no need to check for blocked tasks. So check only for
Signature · 1 line
2.17.1
PATCH 8/11 rcu: Organize functions in tree_stall.h
In reply to Paul E. McKenneyThis commit does only code movement, removal of now-unneeded forward declarations, and addition of comments. It organizes the functions that implement RCU CPU stall warnings for normal grace periods into three categories:
1. Control of RCU CPU stall warnings, including computing timeouts.
2. Interaction of stall warnings with grace periods.
3. Actual printing of the RCU CPU stall-warning messages.
--- kernel/rcu/tree.h | 1 - kernel/rcu/tree_stall.h | 180 ++++++++++++++++++++++------------------ 2 files changed, 97 insertions(+), 84 deletions(-)
kernel/rcu/tree.h +0 -1
@@ -442,6 +442,5 @@ static void rcu_dynticks_task_enter(void);static void rcu_dynticks_task_exit (void );/* Forward declarations for tree_stall.h */ -static int rcu_print_task_stall (struct rcu_node *rnp);static void record_gp_stall_check_time (void );static void check_cpu_stall (struct rcu_data *rdp);
kernel/rcu/tree_stall.h +97 -83
232 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 7/11 rcu: Move FAST_NO_HZ stall-warning code to tree_stall.h
In reply to Paul E. McKenneyThis commit further consolidates the stall-warning code by moving print_cpu_stall_info() and its helper functions along with zero_cpu_stall_ticks() to kernel/rcu/tree_stall.h.
--- kernel/rcu/tree.h | 1 - kernel/rcu/tree_plugin.h | 80 ---------------------------------------- kernel/rcu/tree_stall.h | 80 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 81 deletions(-)
kernel/rcu/tree.h +0 -1
@@ -416,7 +416,6 @@ static void rcu_prepare_for_idle(void);static bool rcu_preempt_has_tasks (struct rcu_node *rnp);static bool rcu_preempt_need_deferred_qs (struct task_struct *t);static void rcu_preempt_deferred_qs (struct task_struct *t); -static void print_cpu_stall_info (int cpu);static void zero_cpu_stall_ticks (struct rcu_data *rdp);static bool rcu_nocb_cpu_needs_barrier (int cpu);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);
kernel/rcu/tree_plugin.h +0 -80
86 lines not on this page · raw message
kernel/rcu/tree_stall.h +80 -0
86 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 2/11 rcu: Move RCU CPU stall-warning code out of tree_plugin.h
In reply to Paul E. McKenneyThe RCU CPU stall-warning code for normal grace periods is currently scattered across two files, due to earlier Tiny RCU support for RCU CPU stall warnings and for old Kconfig options that have long since been retired. Given that it is hard for the lead RCU maintainer to find relevant stall-warning code, it would be good to consolidate it. This commit continues this process by moving stall-warning code from kernel/rcu/tree_plugin.c to a new kernel/rcu/tree_stall.h file.
--- kernel/rcu/tree_plugin.h | 90 ------------------------------------- kernel/rcu/tree_stall.h | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 90 deletions(-)
kernel/rcu/tree_plugin.h +0 -90
@@ -642,79 +642,6 @@ static void rcu_read_unlock_special(struct task_struct *t)rcu_preempt_deferred_qs_irqrestore (t, flags); } -/* -* Dump detailed information for all tasks blocking the current RCU -* grace period on the specified rcu_node structure. -*/ -static void rcu_print_detail_task_stall_rnp (struct rcu_node *rnp) -{ -unsigned long flags; -struct task_struct *t; - -raw_spin_lock_irqsave_rcu_node (rnp, flags); -if (!rcu_preempt_blocked_readers_cgp (rnp)) { -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -return ; - } - t =list_entry (rnp->gp_tasks->prev, - struct task_struct, rcu_node_entry); -list_for_each_entry_continue (t, &rnp->blkd_tasks, rcu_node_entry) { -/* -* We could be printing a lot while holding a spinlock. -* Avoid triggering hard lockup. -*/ -touch_nmi_watchdog (); -sched_show_task (t); - } -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -} - -/* -* Dump detailed information for all tasks blocking the current RCU -* grace period. -*/ -static void rcu_print_detail_task_stall (void ) -{ -struct rcu_node *rnp =rcu_get_root (); - -rcu_print_detail_task_stall_rnp (rnp); -rcu_for_each_leaf_node (rnp) -rcu_print_detail_task_stall_rnp (rnp); -} - -static void rcu_print_task_stall_begin (struct rcu_node *rnp) -{ -pr_err ("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):" , - rnp->level, rnp->grplo, rnp->grphi); -} - -static void rcu_print_task_stall_end (void ) -{ -pr_cont ("\n" ); -} - -/* -* Scan the current list of tasks blocked within RCU read-side critical -* sections, printing out the tid of each. -*/ -static int rcu_print_task_stall (struct rcu_node *rnp) -{ -struct task_struct *t; -int ndetected =0 ; - -if (!rcu_preempt_blocked_readers_cgp (rnp)) -return 0 ; -rcu_print_task_stall_begin (rnp); - t =list_entry (rnp->gp_tasks->prev, - struct task_struct, rcu_node_entry); -list_for_each_entry_continue (t, &rnp->blkd_tasks, rcu_node_entry) { -pr_cont (" P%d" , t->pid); - ndetected++; - } -rcu_print_task_stall_end (); -return ndetected; -} -/* * Scan the current list of tasks blocked within RCU read-side critical * sections, printing out the tid of each that is blocking the current
@@ -979,23 +906,6 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) }static void rcu_preempt_deferred_qs (struct task_struct *t) { } -/* -* Because preemptible RCU does not exist, we never have to check for -* tasks blocked within RCU read-side critical sections. -*/ -static void rcu_print_detail_task_stall (void ) -{ -} - -/* -* Because preemptible RCU does not exist, we never have to check for -* tasks blocked within RCU read-side critical sections. -*/ -static int rcu_print_task_stall (struct rcu_node *rnp) -{ -return 0 ; -} -/* * Because preemptible RCU does not exist, we never have to check for * tasks blocked within RCU read-side critical sections that are
kernel/rcu/tree_stall.h +95 -0
98 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 3/11 rcu: Move RCU CPU stall-warning code out of tree.c
In reply to Paul E. McKenneyThis commit completes the process of consolidating the code for RCU CPU stall warnings for normal grace periods by moving the remaining such code from kernel/rcu/tree.c to kernel/rcu/tree_stall.h.
--- kernel/rcu/tree.c | 291 --------------------------------------- kernel/rcu/tree.h | 10 +- kernel/rcu/tree_stall.h | 292 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 299 insertions(+), 294 deletions(-)
kernel/rcu/tree.c +0 -291
@@ -102,8 +102,6 @@ int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;/* Number of rcu_nodes at specified level. */ int num_rcu_lvl[] =NUM_RCU_LVL_INIT ;int rcu_num_nodes__read_mostly =NUM_RCU_NODES ;/* Total # rcu_nodes in use. */ -/* panic() on RCU Stall sysctl. */ -int sysctl_panic_on_rcu_stall__read_mostly ;/* Commandeer a sysrq key to dump RCU's tree. */ static bool sysrq_rcu;module_param (sysrq_rcu, bool,0444 );
@@ -1167,295 +1165,6 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)return 0 ; } -static void record_gp_stall_check_time (void ) -{ -unsigned long j = jiffies; -unsigned long j1; - - rcu_state.gp_start = j; - j1 =rcu_jiffies_till_stall_check (); -/* Record ->gp_start before ->jiffies_stall. */ -smp_store_release (&rcu_state.jiffies_stall, j + j1);/* ^^^ */ - rcu_state.jiffies_resched = j + j1 /2 ; - rcu_state.n_force_qs_gpstart =READ_ONCE (rcu_state.n_force_qs); -} - -/* -* Complain about starvation of grace-period kthread. -*/ -static void rcu_check_gp_kthread_starvation (void ) -{ -struct task_struct *gpk = rcu_state.gp_kthread; -unsigned long j; - - j = jiffies -READ_ONCE (rcu_state.gp_activity); -if (j >2 *HZ ) { -pr_err ("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n" , - rcu_state.name, j, - (long )rcu_seq_current (&rcu_state.gp_seq), -READ_ONCE (rcu_state.gp_flags), -gp_state_getname (rcu_state.gp_state), rcu_state.gp_state, - gpk ? gpk->state : ~0 , gpk ?task_cpu (gpk) :-1 ); -if (gpk) { -pr_err ("RCU grace-period kthread stack dump:\n" ); -sched_show_task (gpk); -wake_up_process (gpk); - } - } -} - -/* -* Dump stacks of all tasks running on stalled CPUs. First try using -* NMIs, but fall back to manual remote stack tracing on architectures -* that don't support NMI-based stack dumps. The NMI-triggered stack -* traces are more accurate because they are printed by the target CPU. -*/ -static void rcu_dump_cpu_stacks (void ) -{ -int cpu; -unsigned long flags; -struct rcu_node *rnp; - -rcu_for_each_leaf_node (rnp) { -raw_spin_lock_irqsave_rcu_node (rnp, flags); -for_each_leaf_node_possible_cpu (rnp, cpu) -if (rnp->qsmask &leaf_node_cpu_bit (rnp, cpu)) -if (!trigger_single_cpu_backtrace (cpu)) -dump_cpu_task (cpu); -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); - } -} - -/* -* If too much time has passed in the current grace period, and if -* so configured, go kick the relevant kthreads. -*/ -static void rcu_stall_kick_kthreads (void ) -{ -unsigned long j; - -if (!rcu_kick_kthreads) -return ; - j =READ_ONCE (rcu_state.jiffies_kick_kthreads); -if (time_after (jiffies, j) && rcu_state.gp_kthread && - (rcu_gp_in_progress () ||READ_ONCE (rcu_state.gp_flags))) { -WARN_ONCE (1 ,"Kicking %s grace-period kthread\n" , - rcu_state.name); -rcu_ftrace_dump (DUMP_ALL ); -wake_up_process (rcu_state.gp_kthread); -WRITE_ONCE (rcu_state.jiffies_kick_kthreads, j +HZ ); - } -} - -static void panic_on_rcu_stall (void ) -{ -if (sysctl_panic_on_rcu_stall) -panic ("RCU Stall\n" ); -} - -static void print_other_cpu_stall (unsigned long gp_seq) -{ -int cpu; -unsigned long flags; -unsigned long gpa; -unsigned long j; -int ndetected =0 ; -struct rcu_node *rnp =rcu_get_root (); -long totqlen =0 ; - -/* Kick and suppress, if so configured. */ -rcu_stall_kick_kthreads (); -if (rcu_cpu_stall_suppress) -return ; - -/* -* OK, time to rat on our buddy... -* See Documentation/RCU/stallwarn.txt for info on how to debug -* RCU CPU stall warnings. -*/ -pr_err ("INFO: %s detected stalls on CPUs/tasks:" , rcu_state.name); -print_cpu_stall_info_begin (); -rcu_for_each_leaf_node (rnp) { -raw_spin_lock_irqsave_rcu_node (rnp, flags); - ndetected +=rcu_print_task_stall (rnp); -if (rnp->qsmask !=0 ) { -for_each_leaf_node_possible_cpu (rnp, cpu) -if (rnp->qsmask &leaf_node_cpu_bit (rnp, cpu)) { -print_cpu_stall_info (cpu); - ndetected++; - } - } -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); - } - -print_cpu_stall_info_end (); -for_each_possible_cpu (cpu) - totqlen +=rcu_get_n_cbs_cpu (cpu); -pr_cont ("(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n" , -smp_processor_id (), (long )(jiffies - rcu_state.gp_start), - (long )rcu_seq_current (&rcu_state.gp_seq), totqlen); -if (ndetected) { -rcu_dump_cpu_stacks (); - -/* Complain about tasks blocking the grace period. */ -rcu_print_detail_task_stall (); - }else { -if (rcu_seq_current (&rcu_state.gp_seq) != gp_seq) { -pr_err ("INFO: Stall ended before state dump start\n" ); - }else { - j = jiffies; - gpa =READ_ONCE (rcu_state.gp_activity); -pr_err ("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n" , - rcu_state.name, j - gpa, j, gpa, -READ_ONCE (jiffies_till_next_fqs), -rcu_get_root ()->qsmask); -/* In this case, the current CPU might be at fault. */ -sched_show_task (current); - } - } -/* Rewrite if needed in case of slow consoles. */ -if (ULONG_CMP_GE (jiffies,READ_ONCE (rcu_state.jiffies_stall))) -WRITE_ONCE (rcu_state.jiffies_stall, - jiffies +3 *rcu_jiffies_till_stall_check () +3 ); - -rcu_check_gp_kthread_starvation (); - -panic_on_rcu_stall (); - -rcu_force_quiescent_state ();/* Kick them all. */ -} - -static void print_cpu_stall (void ) -{ -int cpu; -unsigned long flags; -struct rcu_data *rdp =this_cpu_ptr (&rcu_data); -struct rcu_node *rnp =rcu_get_root (); -long totqlen =0 ; - -/* Kick and suppress, if so configured. */ -rcu_stall_kick_kthreads (); -if (rcu_cpu_stall_suppress) -return ; - -/* -* OK, time to rat on ourselves... -* See Documentation/RCU/stallwarn.txt for info on how to debug -* RCU CPU stall warnings. -*/ -pr_err ("INFO: %s self-detected stall on CPU" , rcu_state.name); -print_cpu_stall_info_begin (); -raw_spin_lock_irqsave_rcu_node (rdp->mynode, flags); -print_cpu_stall_info (smp_processor_id ()); -raw_spin_unlock_irqrestore_rcu_node (rdp->mynode, flags); -print_cpu_stall_info_end (); -for_each_possible_cpu (cpu) - totqlen +=rcu_get_n_cbs_cpu (cpu); -pr_cont (" (t=%lu jiffies g=%ld q=%lu)\n" , - jiffies - rcu_state.gp_start, - (long )rcu_seq_current (&rcu_state.gp_seq), totqlen); - -rcu_check_gp_kthread_starvation (); - -rcu_dump_cpu_stacks (); - -raw_spin_lock_irqsave_rcu_node (rnp, flags); -/* Rewrite if needed in case of slow consoles. */ -if (ULONG_CMP_GE (jiffies,READ_ONCE (rcu_state.jiffies_stall))) -WRITE_ONCE (rcu_state.jiffies_stall, - jiffies +3 *rcu_jiffies_till_stall_check () +3 ); -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); - -panic_on_rcu_stall (); - -/* -* Attempt to revive the RCU machinery by forcing a context switch. -* -* A context switch would normally allow the RCU state machine to make -* progress and it could be we're stuck in kernel space without context -* switches for an entirely unreasonable amount of time. -*/ -set_tsk_need_resched (current); -set_preempt_need_resched (); -} - -static void check_cpu_stall (struct rcu_data *rdp) -{ -unsigned long gs1; -unsigned long gs2; -unsigned long gps; -unsigned long j; -unsigned long jn; -unsigned long js; -struct rcu_node *rnp; - -if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) || - !rcu_gp_in_progress ()) -return ; -rcu_stall_kick_kthreads (); - j = jiffies; - -/* -* Lots of memory barriers to reject false positives. -* -* The idea is to pick up rcu_state.gp_seq, then -* rcu_state.jiffies_stall, then rcu_state.gp_start, and finally -* another copy of rcu_state.gp_seq. These values are updated in -* the opposite order with memory barriers (or equivalent) during -* grace-period initialization and cleanup. Now, a false positive -* can occur if we get an new value of rcu_state.gp_start and a old -* value of rcu_state.jiffies_stall. But given the memory barriers, -* the only way that this can happen is if one grace period ends -* and another starts between these two fetches. This is detected -* by comparing the second fetch of rcu_state.gp_seq with the -* previous fetch from rcu_state.gp_seq. -* -* Given this check, comparisons of jiffies, rcu_state.jiffies_stall, -* and rcu_state.gp_start suffice to forestall false positives. -*/ - gs1 =READ_ONCE (rcu_state.gp_seq); -smp_rmb ();/* Pick up ->gp_seq first... */ - js =READ_ONCE (rcu_state.jiffies_stall); -smp_rmb ();/* ...then ->jiffies_stall before the rest... */ - gps =READ_ONCE (rcu_state.gp_start); -smp_rmb ();/* ...and finally ->gp_start before ->gp_seq again. */ - gs2 =READ_ONCE (rcu_state.gp_seq); -if (gs1 != gs2 || -ULONG_CMP_LT (j, js) || -ULONG_CMP_GE (gps, js)) -return ;/* No stall or GP completed since entering function. */ - rnp = rdp->mynode; - jn = jiffies +3 *rcu_jiffies_till_stall_check () +3 ; -if (rcu_gp_in_progress () && - (READ_ONCE (rnp->qsmask) & rdp->grpmask) && -cmpxchg (&rcu_state.jiffies_stall, js, jn) == js) { - -/* We haven't checked in, so go dump stack. */ -print_cpu_stall (); - - }else if (rcu_gp_in_progress () && -ULONG_CMP_GE (j, js +RCU_STALL_RAT_DELAY ) && -cmpxchg (&rcu_state.jiffies_stall, js, jn) == js) { - -/* They had a few time units to dump stack, so complain. */ -print_other_cpu_stall (gs2); - } -} - -/** -* rcu_cpu_stall_reset - prevent further stall warnings in current grace period -* -* Set the stall-warning timeout way off into the future, thus preventing -* any RCU CPU stall-warning messages from appearing in the current set of -* RCU grace periods. -* -* The caller must disable hard irqs. -*/ -void rcu_cpu_stall_reset (void ) -{ -WRITE_ONCE (rcu_state.jiffies_stall, jiffies +ULONG_MAX /2 ); -} -/* Trace-event wrapper function for trace_rcu_future_grace_period. */ static void trace_rcu_this_gp (struct rcu_node *rnp,struct rcu_data *rdp,unsigned long gp_seq_req,const char *s)
kernel/rcu/tree.h +7 -3
@@ -393,15 +393,13 @@ static const char *tp_rcu_varname __used __tracepoint_string = rcu_name;int rcu_dynticks_snap (struct rcu_data *rdp); -/* Forward declarations for rcutree_plugin.h */ +/* Forward declarations for tree_plugin.h */ static void rcu_bootup_announce (void );static void rcu_qs (void );static int rcu_preempt_blocked_readers_cgp (struct rcu_node *rnp);#ifdef CONFIG_HOTPLUG_CPU static bool rcu_preempt_has_tasks (struct rcu_node *rnp);#endif /* #ifdef CONFIG_HOTPLUG_CPU */ -static void rcu_print_detail_task_stall (void ); -static int rcu_print_task_stall (struct rcu_node *rnp);static int rcu_print_task_exp_stall (struct rcu_node *rnp);static void rcu_preempt_check_blocked_tasks (struct rcu_node *rnp);static void rcu_flavor_sched_clock_irq (int user);
@@ -445,3 +443,9 @@ static void rcu_bind_gp_kthread(void);static bool rcu_nohz_full_cpu (void );static void rcu_dynticks_task_enter (void );static void rcu_dynticks_task_exit (void ); + +/* Forward declarations for tree_stall.h */ +static void rcu_print_detail_task_stall (void ); +static int rcu_print_task_stall (struct rcu_node *rnp); +static void record_gp_stall_check_time (void ); +static void check_cpu_stall (struct rcu_data *rdp);
kernel/rcu/tree_stall.h +292 -0
301 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 10/11 rcu: Move forward-progress checkers into tree_stall.h
In reply to Paul E. McKenneyThis commit further consolidates stall-warning functionality by moving forward-progress checkers into kernel/rcu/tree_stall.h, updating a comment or two while in the area. More specifically, this commit moves show_rcu_gp_kthreads(), rcu_check_gp_start_stall(), rcu_fwd_progress_check(), sysrq_rcu, sysrq_show_rcu(), sysrq_rcudump_op, and rcu_sysrq_init() from kernel/rcu/tree.c to kernel/rcu/tree_stall.h.
--- kernel/rcu/tree.c | 166 -------------------------------------- kernel/rcu/tree.h | 2 + kernel/rcu/tree_stall.h | 171 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 166 deletions(-)
kernel/rcu/tree.c +0 -166
@@ -102,9 +102,6 @@ int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;/* Number of rcu_nodes at specified level. */ int num_rcu_lvl[] =NUM_RCU_LVL_INIT ;int rcu_num_nodes__read_mostly =NUM_RCU_NODES ;/* Total # rcu_nodes in use. */ -/* Commandeer a sysrq key to dump RCU's tree. */ -static bool sysrq_rcu; -module_param (sysrq_rcu, bool,0444 );/* * The rcu_scheduler_active variable is initialized to the value
@@ -510,74 +507,6 @@ static const char *gp_state_getname(short gs)return gp_state_names[gs]; } -/* -* Show the state of the grace-period kthreads. -*/ -void show_rcu_gp_kthreads (void ) -{ -int cpu; -unsigned long j; -unsigned long ja; -unsigned long jr; -unsigned long jw; -struct rcu_data *rdp; -struct rcu_node *rnp; - - j = jiffies; - ja = j -READ_ONCE (rcu_state.gp_activity); - jr = j -READ_ONCE (rcu_state.gp_req_activity); - jw = j -READ_ONCE (rcu_state.gp_wake_time); -pr_info ("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n" , - rcu_state.name,gp_state_getname (rcu_state.gp_state), - rcu_state.gp_state, - rcu_state.gp_kthread ? rcu_state.gp_kthread->state :0x1ffffL , - ja, jr, jw, (long )READ_ONCE (rcu_state.gp_wake_seq), - (long )READ_ONCE (rcu_state.gp_seq), - (long )READ_ONCE (rcu_get_root ()->gp_seq_needed), -READ_ONCE (rcu_state.gp_flags)); -rcu_for_each_node_breadth_first (rnp) { -if (ULONG_CMP_GE (rcu_state.gp_seq, rnp->gp_seq_needed)) -continue ; -pr_info ("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n" , - rnp->grplo, rnp->grphi, (long )rnp->gp_seq, - (long )rnp->gp_seq_needed); -if (!rcu_is_leaf_node (rnp)) -continue ; -for_each_leaf_node_possible_cpu (rnp, cpu) { - rdp =per_cpu_ptr (&rcu_data, cpu); -if (rdp->gpwrap || -ULONG_CMP_GE (rcu_state.gp_seq, - rdp->gp_seq_needed)) -continue ; -pr_info ("\tcpu %d ->gp_seq_needed %ld\n" , - cpu, (long )rdp->gp_seq_needed); - } - } -/* sched_show_task(rcu_state.gp_kthread); */ -} -EXPORT_SYMBOL_GPL (show_rcu_gp_kthreads); - -/* Dump grace-period-request information due to commandeered sysrq. */ -static void sysrq_show_rcu (int key) -{ -show_rcu_gp_kthreads (); -} - -static struct sysrq_key_op sysrq_rcudump_op = { - .handler = sysrq_show_rcu, - .help_msg ="show-rcu(y)" , - .action_msg ="Show RCU tree" , - .enable_mask =SYSRQ_ENABLE_DUMP , -}; - -static int __init rcu_sysrq_init (void ) -{ -if (sysrq_rcu) -return register_sysrq_key ('y' , &sysrq_rcudump_op); -return 0 ; -} -early_initcall (rcu_sysrq_init); -/* * Send along grace-period-related data for rcutorture diagnostics. */
@@ -2323,101 +2252,6 @@ void rcu_force_quiescent_state(void) }EXPORT_SYMBOL_GPL (rcu_force_quiescent_state); -/* -* This function checks for grace-period requests that fail to motivate -* RCU to come out of its idle mode. -*/ -void -rcu_check_gp_start_stall (struct rcu_node *rnp,struct rcu_data *rdp, -const unsigned long gpssdelay) -{ -unsigned long flags; -unsigned long j; -struct rcu_node *rnp_root =rcu_get_root (); -static atomic_t warned =ATOMIC_INIT (0 ); - -if (!IS_ENABLED (CONFIG_PROVE_RCU ) ||rcu_gp_in_progress () || -ULONG_CMP_GE (rnp_root->gp_seq, rnp_root->gp_seq_needed)) -return ; - j = jiffies;/* Expensive access, and in common case don't get here. */ -if (time_before (j,READ_ONCE (rcu_state.gp_req_activity) + gpssdelay) || -time_before (j,READ_ONCE (rcu_state.gp_activity) + gpssdelay) || -atomic_read (&warned)) -return ; - -raw_spin_lock_irqsave_rcu_node (rnp, flags); - j = jiffies; -if (rcu_gp_in_progress () || -ULONG_CMP_GE (rnp_root->gp_seq, rnp_root->gp_seq_needed) || -time_before (j,READ_ONCE (rcu_state.gp_req_activity) + gpssdelay) || -time_before (j,READ_ONCE (rcu_state.gp_activity) + gpssdelay) || -atomic_read (&warned)) { -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -return ; - } -/* Hold onto the leaf lock to make others see warned==1. */ - -if (rnp_root != rnp) -raw_spin_lock_rcu_node (rnp_root);/* irqs already disabled. */ - j = jiffies; -if (rcu_gp_in_progress () || -ULONG_CMP_GE (rnp_root->gp_seq, rnp_root->gp_seq_needed) || -time_before (j, rcu_state.gp_req_activity + gpssdelay) || -time_before (j, rcu_state.gp_activity + gpssdelay) || -atomic_xchg (&warned,1 )) { -raw_spin_unlock_rcu_node (rnp_root);/* irqs remain disabled. */ -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -return ; - } -WARN_ON (1 ); -if (rnp_root != rnp) -raw_spin_unlock_rcu_node (rnp_root); -raw_spin_unlock_irqrestore_rcu_node (rnp, flags); -show_rcu_gp_kthreads (); -} - -/* -* Do a forward-progress check for rcutorture. This is normally invoked -* due to an OOM event. The argument "j" gives the time period during -* which rcutorture would like progress to have been made. -*/ -void rcu_fwd_progress_check (unsigned long j) -{ -unsigned long cbs; -int cpu; -unsigned long max_cbs =0 ; -int max_cpu =-1 ; -struct rcu_data *rdp; - -if (rcu_gp_in_progress ()) { -pr_info ("%s: GP age %lu jiffies\n" , - __func__, jiffies - rcu_state.gp_start); -show_rcu_gp_kthreads (); - }else { -pr_info ("%s: Last GP end %lu jiffies ago\n" , - __func__, jiffies - rcu_state.gp_end); -preempt_disable (); - rdp =this_cpu_ptr (&rcu_data); -rcu_check_gp_start_stall (rdp->mynode, rdp, j); -preempt_enable (); - } -for_each_possible_cpu (cpu) { - cbs =rcu_get_n_cbs_cpu (cpu); -if (!cbs) -continue ; -if (max_cpu <0 ) -pr_info ("%s: callbacks" , __func__); -pr_cont (" %d: %lu" , cpu, cbs); -if (cbs <= max_cbs) -continue ; - max_cbs = cbs; - max_cpu = cpu; - } -if (max_cpu >=0 ) -pr_cont ("\n" ); -} -EXPORT_SYMBOL_GPL (rcu_fwd_progress_check); -/* Perform RCU core processing work for the current CPU. */ static __latent_entropy void rcu_core(struct softirq_action *unused) {
kernel/rcu/tree.h +2 -0
@@ -445,3 +445,5 @@ static void rcu_dynticks_task_exit(void);static void record_gp_stall_check_time (void );static void rcu_iw_handler (struct irq_work *iwp);static void check_cpu_stall (struct rcu_data *rdp); +static void rcu_check_gp_start_stall (struct rcu_node *rnp,struct rcu_data *rdp, +const unsigned long gpssdelay);
kernel/rcu/tree_stall.h +171 -0
174 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 1/11 rcu: Move RCU CPU stall-warning code out of update.c
In reply to Paul E. McKenneyThe RCU CPU stall-warning code for normal grace periods is currently scattered across three files, due to earlier Tiny RCU support for RCU CPU stall warnings and for old Kconfig options that have long since been retired. Given that it is hard for the lead RCU maintainer to find relevant stall-warning code, it would be good to consolidate it. This commit starts this process by moving stall-warning code from kernel/rcu/update.c to a new kernel/rcu/tree_stall.h file.
Note that the definitions of rcu_cpu_stall_suppress and rcu_cpu_stall_timeout must remain in kernel/rcu/update.h to provide compatibility for kernel boot parameter lists.
--- kernel/rcu/rcu.h | 1 + kernel/rcu/tree.c | 1 + kernel/rcu/tree_stall.h | 63 +++++++++++++++++++++++++++++++++++++++++ kernel/rcu/update.c | 59 +------------------------------------- 4 files changed, 66 insertions(+), 58 deletions(-) create mode 100644 kernel/rcu/tree_stall.h
kernel/rcu/rcu.h +1 -0
@@ -233,6 +233,7 @@ static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)#ifdef CONFIG_RCU_STALL_COMMON extern int rcu_cpu_stall_suppress; +extern int rcu_cpu_stall_timeout;int rcu_jiffies_till_stall_check (void );#define rcu_ftrace_dump_stall_suppress () \
kernel/rcu/tree.c +1 -0
@@ -3858,5 +3858,6 @@ void __init rcu_init(void)srcu_init (); } +#include "tree_stall.h" #include "tree_exp.h" #include "tree_plugin.h"
kernel/rcu/tree_stall.h new file +63 -0
63 lines not on this page · raw message
kernel/rcu/update.c +1 -58
69 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 11/11 rcu: Fix nohz status in stall warning
In reply to Paul E. McKenneyFrom: Neeraj Upadhyay <[email protected]>
The Documentation/RCU/stallwarn.txt file says that stall warnings print "D" if dyntick-idle processing is enabled, but the code in print_cpu_stall_fast_no_hz() prints "." instead. This commit therefore reverses the sense of the test to make the code match the documentation.
--- kernel/rcu/tree_stall.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/tree_stall.h +1 -1
@@ -267,7 +267,7 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu) rdp->last_accelerate &0xffff , jiffies &0xffff ,".l" [rdp->all_lazy],".L" [!rcu_segcblist_n_nonlazy_cbs (&rdp->cblist)], - ".D"[!rdp->tick_nohz_enabled_snap]); + ".D"[!!rdp->tick_nohz_enabled_snap]); }#else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
Signature · 1 line
2.17.1
PATCH 6/11 rcu: Inline RCU stall-warning info helper functions
In reply to Paul E. McKenneyThe print_cpu_stall_info_begin() and print_cpu_stall_info_end() print a single character each onto the console, and are a holdover from a time when RCU CPU stall warning messages could be abbreviated using a long-gone Kconfig option. This commit therefore adds these single characters to already-printed strings in the calling functions, and then eliminates both print_cpu_stall_info_begin() and print_cpu_stall_info_end().
--- kernel/rcu/tree.h | 2 -- kernel/rcu/tree_plugin.h | 12 ------------ kernel/rcu/tree_stall.h | 12 ++++-------- 3 files changed, 4 insertions(+), 22 deletions(-)
kernel/rcu/tree.h +0 -2
@@ -416,9 +416,7 @@ static void rcu_prepare_for_idle(void);static bool rcu_preempt_has_tasks (struct rcu_node *rnp);static bool rcu_preempt_need_deferred_qs (struct task_struct *t);static void rcu_preempt_deferred_qs (struct task_struct *t); -static void print_cpu_stall_info_begin (void );static void print_cpu_stall_info (int cpu); -static void print_cpu_stall_info_end (void );static void zero_cpu_stall_ticks (struct rcu_data *rdp);static bool rcu_nocb_cpu_needs_barrier (int cpu);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);
kernel/rcu/tree_plugin.h +0 -12
@@ -1550,12 +1550,6 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu)#endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */ -/* Initiate the stall-info list. */ -static void print_cpu_stall_info_begin (void ) -{ -pr_cont ("\n" ); -} -/* * Print out diagnostic information for the specified stalled CPU. *
@@ -1606,12 +1600,6 @@ static void print_cpu_stall_info(int cpu) fast_no_hz); } -/* Terminate the stall-info list. */ -static void print_cpu_stall_info_end (void ) -{ -pr_err ("\t" ); -} -/* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */ static void zero_cpu_stall_ticks (struct rcu_data *rdp) {
kernel/rcu/tree_stall.h +4 -8
@@ -243,8 +243,7 @@ static void print_other_cpu_stall(unsigned long gp_seq)* See Documentation/RCU/stallwarn.txt for info on how to debug * RCU CPU stall warnings. */ -pr_err ("INFO: %s detected stalls on CPUs/tasks:" , rcu_state.name); -print_cpu_stall_info_begin (); +pr_err ("INFO: %s detected stalls on CPUs/tasks:\n" , rcu_state.name);rcu_for_each_leaf_node (rnp) {raw_spin_lock_irqsave_rcu_node (rnp, flags); ndetected +=rcu_print_task_stall (rnp);
@@ -258,10 +257,9 @@ static void print_other_cpu_stall(unsigned long gp_seq)raw_spin_unlock_irqrestore_rcu_node (rnp, flags); } -print_cpu_stall_info_end ();for_each_possible_cpu (cpu) totqlen +=rcu_get_n_cbs_cpu (cpu); -pr_cont ("(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n" , +pr_cont ("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n" ,smp_processor_id (), (long )(jiffies - rcu_state.gp_start), (long )rcu_seq_current (&rcu_state.gp_seq), totqlen);if (ndetected) {
@@ -314,15 +312,13 @@ static void print_cpu_stall(void)* See Documentation/RCU/stallwarn.txt for info on how to debug * RCU CPU stall warnings. */ -pr_err ("INFO: %s self-detected stall on CPU" , rcu_state.name); -print_cpu_stall_info_begin (); +pr_err ("INFO: %s self-detected stall on CPU\n" , rcu_state.name);raw_spin_lock_irqsave_rcu_node (rdp->mynode, flags);print_cpu_stall_info (smp_processor_id ());raw_spin_unlock_irqrestore_rcu_node (rdp->mynode, flags); -print_cpu_stall_info_end ();for_each_possible_cpu (cpu) totqlen +=rcu_get_n_cbs_cpu (cpu); -pr_cont (" (t=%lu jiffies g=%ld q=%lu)\n" , +pr_cont ("\t(t=%lu jiffies g=%ld q=%lu)\n" , jiffies - rcu_state.gp_start, (long )rcu_seq_current (&rcu_state.gp_seq), totqlen);
Signature · 1 line
2.17.1
PATCH 9/11 rcu: Move irq-disabled stall-warning checking to tree_stall.h
In reply to Paul E. McKenneyThe rcu_iw_handler() function's sole purpose in life is to indicate whether a stalled CPU had interrupts disabled, so it belongs in kernel/rcu/tree_stall.h. This commit therefore makes that move, clarifying its header comment while in the area.
--- kernel/rcu/tree.c | 21 --------------------- kernel/rcu/tree.h | 1 + kernel/rcu/tree_stall.h | 20 ++++++++++++++++++++ 3 files changed, 21 insertions(+), 21 deletions(-)
kernel/rcu/tree.c +0 -21
@@ -1031,27 +1031,6 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)return 0 ; } -/* -* Handler for the irq_work request posted when a grace period has -* gone on for too long, but not yet long enough for an RCU CPU -* stall warning. Set state appropriately, but just complain if -* there is unexpected state on entry. -*/ -static void rcu_iw_handler (struct irq_work *iwp) -{ -struct rcu_data *rdp; -struct rcu_node *rnp; - - rdp = container_of(iwp, struct rcu_data, rcu_iw); - rnp = rdp->mynode; -raw_spin_lock_rcu_node (rnp); -if (!WARN_ON_ONCE (!rdp->rcu_iw_pending)) { - rdp->rcu_iw_gp_seq = rnp->gp_seq; - rdp->rcu_iw_pending = false; - } -raw_spin_unlock_rcu_node (rnp); -} -/* * Return true if the specified CPU has passed through a quiescent * state by virtue of being in or having passed through an dynticks
kernel/rcu/tree.h +1 -0
@@ -443,4 +443,5 @@ static void rcu_dynticks_task_exit(void);/* Forward declarations for tree_stall.h */ static void record_gp_stall_check_time (void ); +static void rcu_iw_handler (struct irq_work *iwp);static void check_cpu_stall (struct rcu_data *rdp);
kernel/rcu/tree_stall.h +20 -0
@@ -139,6 +139,26 @@ static void rcu_stall_kick_kthreads(void) } } +/* +* Handler for the irq_work request posted about halfway into the RCU CPU +* stall timeout, and used to detect excessive irq disabling. Set state +* appropriately, but just complain if there is unexpected state on entry. +*/ +static void rcu_iw_handler (struct irq_work *iwp) +{ +struct rcu_data *rdp; +struct rcu_node *rnp; + + rdp = container_of(iwp, struct rcu_data, rcu_iw); + rnp = rdp->mynode; +raw_spin_lock_rcu_node (rnp); +if (!WARN_ON_ONCE (!rdp->rcu_iw_pending)) { + rdp->rcu_iw_gp_seq = rnp->gp_seq; + rdp->rcu_iw_pending = false; + } +raw_spin_unlock_rcu_node (rnp); +} +////////////////////////////////////////////////////////////////////////////// // // Printing RCU CPU stall warnings
Signature · 1 line
2.17.1