v6 expedited "big hammer" RCU grace periods
Subsystems (4)
RFC PATCH 1/1 v6 expedited "big hammer" RCU grace periods
Sixth cut of "big hammer" expedited RCU grace periods.
This uses per-CPU kthreads that are scheduled in parallel by a call to smp_call_function() by synchronize_sched_expedited(). The synchronize_rcu_expedited() and and synchronize_bh_expedited() primitives invoke synchronize_sched_expedited(), except for CONFIG_PREEMPT_RCU, where they instead invoke synchronize_rcu() and synchronize_rcu_bh(), respectively. This will be fixed in the future, after preemptable RCU is folded into the rcutree implementation.
As before, this does nothing to expedite callbacks already registered with call_rcu() or call_rcu_bh(), but there is no need to.
Passes 10 hours of rcutorture testing in parallel with a script that randomly offlines and onlines CPUs. Grace periods take about 44 microseconds on an 8-CPU Power machine, which I believe is good enough from a performance viewpoint (for those keeping track, yes, this is a slower machine than the one I used yesterday). Scalability may eventually need to be addressed in the smp_call_function() primitive and perhaps also in the scan through the CPUs that determines when all have completed.
This is not for inclusion -- although it seems to be a reasonable implementation, it is likely that better implementation could be obtained by leveraging either migration kthreads or workqueues, as suggested by Ingo and Lai. Next on the list!!!
Shortcomings:
o The per-CPU kthreads do not boost themselves to real-time
priority, and thus could be blocked by real-time processes. Use of real-time priority might also speed things up a bit.
o Does not address preemptable RCU.
o Does not leverage existing facilities (workqueues or migration
threads, as noted earlier).
Changes since v5:
o Fixed several embarrassing locking bugs, including those
noted by Ingo and Lai.
o Added a missing set of braces.
o Cut out the extra kthread, so that synchronize_sched_expedited()
directly calls smp_call_function() and waits for the quiescent states.
o Removed some debug code, but promoted one to production.
o Fix a compiler warning.
Changes since v4:
o Use per-CPU kthreads to force the quiescent states in parallel.
Changes since v3:
o Use a kthread that schedules itself on each CPU in turn to
force a grace period. The synchronize_rcu() primitive wakes up the kthread in order to avoid messing with affinity masks on user tasks.
o Tried a number of additional variations on the v3 approach, none
of which helped much.
Changes since v2:
o Use reschedule IPIs rather than a softirq.
Changes since v1:
o Added rcutorture support, and added exports required by
rcutorture.
o Added comment stating that smp_call_function() implies a
memory barrier, suggested by Mathieu.
o Added #include for delay.h.
---
include/linux/rcuclassic.h | 16 ++ include/linux/rcupdate.h | 25 ++-- include/linux/rcupreempt.h | 10 + include/linux/rcutree.h | 13 ++ kernel/rcupdate.c | 249 +++++++++++++++++++++++++++++++++++++++++++++ kernel/rcupreempt.c | 1 kernel/rcutorture.c | 200 +++++++++++++++++++----------------- 7 files changed, 408 insertions(+), 106 deletions(-)
include/linux/rcuclassic.h +15 -1
@@ -158,14 +158,28 @@ extern struct lockdep_map rcu_lock_map;#define call_rcu_sched (head, func) call_rcu(head, func) +static inline void synchronize_rcu_expedited (void ) +{ +synchronize_sched_expedited (); +} + +static inline void synchronize_rcu_bh_expedited (void ) +{ +synchronize_sched_expedited (); +} +extern void __rcu_init (void ); -#define rcu_init_sched () do { } while (0)extern void rcu_check_callbacks (int cpu,int user);extern void rcu_restart_cpu (int cpu);extern long rcu_batches_completed (void );extern long rcu_batches_completed_bh (void ); +static inline void rcu_init_sched (void ) +{ +synchronize_sched_expedited_init (); +} +#define rcu_enter_nohz () do { } while (0)#define rcu_exit_nohz () do { } while (0)
include/linux/rcupdate.h +13 -12
@@ -51,7 +51,19 @@ struct rcu_head {void (*func)(struct rcu_head *head); }; -/* Internal to kernel, but needed by rcupreempt.h. */ +/* Exported common interfaces */ +extern void synchronize_rcu (void ); +extern void rcu_barrier (void ); +extern void rcu_barrier_bh (void ); +extern void rcu_barrier_sched (void ); +extern void synchronize_sched_expedited (void ); +extern int sched_expedited_torture_stats (char *page); + +/* Internal to kernel */ +extern void rcu_init (void ); +extern void rcu_scheduler_starting (void ); +extern void synchronize_sched_expedited_init (void ); +extern int rcu_needs_cpu (int cpu);extern int rcu_scheduler_active;#if defined(CONFIG_CLASSIC_RCU )
@@ -259,15 +271,4 @@ extern void call_rcu(struct rcu_head *head,extern void call_rcu_bh (struct rcu_head *head,void (*func)(struct rcu_head *head)); -/* Exported common interfaces */ -extern void synchronize_rcu (void ); -extern void rcu_barrier (void ); -extern void rcu_barrier_bh (void ); -extern void rcu_barrier_sched (void ); - -/* Internal to kernel */ -extern void rcu_init (void ); -extern void rcu_scheduler_starting (void ); -extern int rcu_needs_cpu (int cpu); -#endif /* __LINUX_RCUPDATE_H */
include/linux/rcupreempt.h +10 -0
@@ -74,6 +74,16 @@ extern int rcu_needs_cpu(int cpu);extern void __synchronize_sched (void ); +static inline void synchronize_rcu_expedited (void ) +{ +synchronize_rcu ();/* Placeholder for new rcupreempt implementation. */ +} + +static inline void synchronize_rcu_bh_expedited (void ) +{ +synchronize_rcu ();/* Placeholder for new rcupreempt implementation. */ +} +extern void __rcu_init (void );extern void rcu_init_sched (void );extern void rcu_check_callbacks (int cpu,int user);
include/linux/rcutree.h +12 -1
@@ -279,8 +279,14 @@ static inline void __rcu_read_unlock_bh(void)#define call_rcu_sched (head, func) call_rcu(head, func) -static inline void rcu_init_sched (void ) +static inline void synchronize_rcu_expedited (void ) +{ +synchronize_sched_expedited (); +} + +static inline void synchronize_rcu_bh_expedited (void ) { +synchronize_sched_expedited (); }extern void __rcu_init (void );
@@ -290,6 +296,11 @@ extern void rcu_restart_cpu(int cpu);extern long rcu_batches_completed (void );extern long rcu_batches_completed_bh (void ); +static inline void rcu_init_sched (void ) +{ +synchronize_sched_expedited_init (); +} +#ifdef CONFIG_NO_HZ void rcu_enter_nohz (void );void rcu_exit_nohz (void );
kernel/rcupdate.c +249 -0
270 lines not on this page · raw message
kernel/rcupreempt.c +1 -0
@@ -1507,6 +1507,7 @@ void __init rcu_init_sched(void)NULL , "rcu_sched_grace_period");WARN_ON (IS_ERR (rcu_sched_grace_period_task)); +synchronize_sched_expedited_init (); }#ifdef CONFIG_RCU_TRACE
kernel/rcutorture.c +108 -92
292 lines not on this page · raw message