rcu: add support for dynamic ticks and preempt rcu
Subsystems (2)
PATCH 1/19 rcu: add support for dynamic ticks and preempt rcu
From: Steven Rostedt <[email protected]>
The PREEMPT-RCU can get stuck if a CPU goes idle and NO_HZ is set. The idle CPU will not progress the RCU through its grace period and a synchronize_rcu my get stuck. Without this patch I have a box that will not boot when PREEMPT_RCU and NO_HZ are set. That same box boots fine with this patch.
This patch comes from the -rt kernel where it has been tested for several months.
Signed-off-by Steven Rostedt <[email protected]>
Signed-off-by Paul E. McKenney <[email protected]>
Signed-off-by Ingo Molnar <[email protected]>
--- include/linux/hardirq.h | 10 ++ include/linux/rcuclassic.h | 3 include/linux/rcupreempt.h | 22 ++++ kernel/rcupreempt.c | 224 ++++++++++++++++++++++++++++++++++++++++++++- kernel/softirq.c | 1 kernel/time/tick-sched.c | 3 include/linux/hardirq.h | 10 ++ include/linux/rcuclassic.h | 3 include/linux/rcupreempt.h | 22 ++++ kernel/rcupreempt.c | 224 ++++++++++++++++++++++++++++++++++++++++++++- kernel/softirq.c | 1 kernel/time/tick-sched.c | 3 6 files changed, 259 insertions(+), 4 deletions(-)
include/linux/hardirq.h +10 -0
@@ -109,6 +109,14 @@ static inline void account_system_vtime( }#endif +#if defined(CONFIG_PREEMPT_RCU ) && defined(CONFIG_NO_HZ ) +extern void rcu_irq_enter (void ); +extern void rcu_irq_exit (void ); +#else +# define rcu_irq_enter () do { } while (0) +# define rcu_irq_exit () do { } while (0) +#endif /* CONFIG_PREEMPT_RCU */ +/* * It is safe to do non-atomic ops on ->hardirq_context, * because NMI handlers may not preempt and the ops are
@@ -117,6 +125,7 @@ static inline void account_system_vtime(*/ #define __irq_enter () \ do { \ + rcu_irq_enter(); \ account_system_vtime(current); \ add_preempt_count(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \
@@ -135,6 +144,7 @@ extern void irq_enter(void);trace_hardirq_exit (); \account_system_vtime (current); \sub_preempt_count (HARDIRQ_OFFSET ); \ +rcu_irq_exit (); \ }while (0 )/*
include/linux/rcuclassic.h +3 -0
@@ -160,5 +160,8 @@ extern void rcu_restart_cpu(int cpu);extern long rcu_batches_completed (void );extern long rcu_batches_completed_bh (void ); +#define rcu_enter_nohz () do { } while (0) +#define rcu_exit_nohz () do { } while (0) +#endif /* __KERNEL__ */ #endif /* __LINUX_RCUCLASSIC_H */
include/linux/rcupreempt.h +22 -0
@@ -82,5 +82,27 @@ extern struct rcupreempt_trace *rcupreemstruct softirq_action ; +#ifdef CONFIG_NO_HZ +DECLARE_PER_CPU (long, dynticks_progress_counter); + +static inline void rcu_enter_nohz (void ) +{ +__get_cpu_var (dynticks_progress_counter)++; +WARN_ON (__get_cpu_var (dynticks_progress_counter) &0x1 ); +mb (); +} + +static inline void rcu_exit_nohz (void ) +{ +mb (); +__get_cpu_var (dynticks_progress_counter)++; +WARN_ON (!(__get_cpu_var (dynticks_progress_counter) &0x1 )); +} + +#else /* CONFIG_NO_HZ */ +#define rcu_enter_nohz () do { } while (0) +#define rcu_exit_nohz () do { } while (0) +#endif /* CONFIG_NO_HZ */ +#endif /* __KERNEL__ */ #endif /* __LINUX_RCUPREEMPT_H */
kernel/rcupreempt.c +220 -4
262 lines not on this page · raw message
kernel/softirq.c +1 -0
@@ -313,6 +313,7 @@ void irq_exit(void)/* Make sure that timer wheel updates are propagated */ if (!in_interrupt () &&idle_cpu (smp_processor_id ()) && !need_resched ())tick_nohz_stop_sched_tick (); +rcu_irq_exit ();#endif preempt_enable_no_resched (); }
kernel/time/tick-sched.c +3 -0
@@ -282,6 +282,7 @@ void tick_nohz_stop_sched_tick(void) ts->idle_tick = ts->sched_timer.expires; ts->tick_stopped =1 ; ts->idle_jiffies = last_jiffies; +rcu_enter_nohz (); }/*
@@ -375,6 +376,8 @@ void tick_nohz_restart_sched_tick(void)return ; } +rcu_exit_nohz (); +/* Update jiffies first */ select_nohz_load_balancer (0 ); now =ktime_get ();