In-kernel programmatic grace-period expediting for v4.1
Subsystems (3)
cover In-kernel programmatic grace-period expediting for v4.1
Hello!
This series provides an in-kernel API to expedite and unexpedite normal RCU grace-period primitives such as synchronize_rcu(). It also provides a Kconfig parameter that implicitly expedites at boot time, along with a function that notes the end of in-kernel boot. This last function is intended to be invoked just before init is spawned.
1. Provide rcu_expedite_gp() and rcu_unexpedite_gp().
2. Add rcu_expedite_gp() and rcu_unexpedite_gp() to rcutorture.
3. Change open-coded references ot the rcu_expedited variable to
instead use the new rcu_gp_is_expedited() function.
4. Add a CONFIG_RCU_EXPEDITE_BOOT Kconfig parameter that emulates
a very early boot rcu_expedite_gp(). Also provide a new rcu_end_inkernel_boot() function that provides the corresponding rcu_unexpedite_gp() if CONFIG_RCU_EXPEDITE_BOOT.
5. Make consistent use of variables in rcu_torture_writer().
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/rcupdate.h | 21 ++++++++++++++++ b/init/Kconfig | 13 +++++++++ b/kernel/rcu/rcutorture.c | 27 +++++++++++++++++++- b/kernel/rcu/srcu.c | 2 - b/kernel/rcu/tree.c | 9 +++--- b/kernel/rcu/tree_plugin.h | 2 - b/kernel/rcu/update.c | 59 ++++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 125 insertions(+), 8 deletions(-)
PATCH 5/5 rcutorture: Make consistent use of variables
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
The "if" statement at the beginning of rcu_torture_writer() should use the same set of variables. In theory, this does not matter because the corresponding variables (gp_sync and gp_sync1) have the same value at this point in the code, but in practice such puzzles should be removed. This commit therefore makes the use of variables consistent.
--- kernel/rcu/rcutorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/rcutorture.c +1 -1
@@ -875,7 +875,7 @@ rcu_torture_writer(void *arg) torture_type);/* Initialize synctype[] array. If none set, take default. */ -if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync) +if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1) gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync) synctype[nsynctypes++] =RTWS_COND_GET ;
Signature · 1 line
1.8.1.5
PATCH 4/5 rcu: Add Kconfig option to expedite grace periods during boot
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
This commit adds a CONFIG_RCU_EXPEDITE_BOOT Kconfig parameter that emulates a very early boot rcu_expedite_gp(). A late-boot call to rcu_end_inkernel_boot() will provide the corresponding rcu_unexpedite_gp(). The late-boot call to rcu_end_inkernel_boot() should be made just before init is spawned.
According to Arjan:
Quoted 8 lines
To show the boot time, I'm using the timestamp of the "Write protecting" line, that's pretty much the last thing we print prior to ring 3 execution.
A kernel with default RCU behavior (inside KVM, only virtual devices) looks like this:
[ 0.038724] Write protecting the kernel read-only data: 10240k
Quoted 7 lines
a kernel with expedited RCU (using the command line option, so that I don't have to recompile between measurements and thus am completely oranges-to-oranges)
[ 0.031768] Write protecting the kernel read-only data: 10240kwhich, in percentage, is an 18% improvement.
--- include/linux/rcupdate.h | 1 + init/Kconfig | 13 +++++++++++++ kernel/rcu/update.c | 11 ++++++++++- 3 files changed, 24 insertions(+), 1 deletion(-)
include/linux/rcupdate.h +1 -0
@@ -278,6 +278,7 @@ static inline int rcu_preempt_depth(void)/* Internal to kernel */ void rcu_init (void ); +void rcu_end_inkernel_boot (void );void rcu_sched_qs (void );void rcu_bh_qs (void );void rcu_check_callbacks (int user);
init/Kconfig +13 -0
@@ -791,6 +791,19 @@ config RCU_NOCB_CPU_ALL endchoice +config RCU_EXPEDITE_BOOT + bool + default n + help + This option enables expedited grace periods at boot time, + as if rcu_expedite_gp() had been invoked early in boot. + The corresponding rcu_unexpedite_gp() is invoked from + rcu_end_inkernel_boot(), which is intended to be invoked + at the end of the kernel-only boot sequence, just before + init is exec'ed. + + Accept the default if unsure. + endmenu # "RCU Subsystem" config BUILD_BIN2C
kernel/rcu/update.c +10 -1
@@ -64,7 +64,8 @@ module_param(rcu_expedited, int, 0);#ifndef CONFIG_TINY_RCU -static atomic_t rcu_expedited_nesting; +static atomic_t rcu_expedited_nesting = +ATOMIC_INIT (IS_ENABLED (CONFIG_RCU_EXPEDITE_BOOT ) ?1 :0 );/* * Should normal grace-period primitives be expedited? Intended for
@@ -109,6 +110,14 @@ EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);#endif /* #ifndef CONFIG_TINY_RCU */ +/* +* Inform RCU of the end of the in-kernel boot sequence. +*/ +void rcu_end_inkernel_boot (void ) +{ +if (IS_ENABLED (CONFIG_RCU_EXPEDITE_BOOT )) +rcu_unexpedite_gp (); +}#ifdef CONFIG_PREEMPT_RCU
Signature · 1 line
1.8.1.5
PATCH 2/5 rcu: Add rcu_expedite_gp() and rcu_unexpedite_gp() to rcutorture
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
--- kernel/rcu/rcutorture.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
kernel/rcu/rcutorture.c +25 -0
@@ -853,6 +853,8 @@ rcu_torture_fqs(void *arg)static int rcu_torture_writer (void *arg) { +bool can_expedite = !rcu_gp_is_expedited (); +int expediting =0 ;unsigned long gp_snap;bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;bool gp_sync1 = gp_sync;
@@ -865,6 +867,12 @@ rcu_torture_writer(void *arg)int nsynctypes =0 ;VERBOSE_TOROUT_STRING ("rcu_torture_writer task started" ); +pr_alert ("%s" TORTURE_FLAG +" Grace periods expedited from boot/sysfs for %s,\n" , + torture_type, cur_ops->name); +pr_alert ("%s" TORTURE_FLAG +" Testing of dynamic grace-period expediting diabled.\n" , + torture_type);/* Initialize synctype[] array. If none set, take default. */ if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync)
@@ -949,9 +957,26 @@ rcu_torture_writer(void *arg) } }rcutorture_record_progress (++rcu_torture_current_version); +/* Cycle through nesting levels of rcu_expedite_gp() calls. */ +if (can_expedite && + !(torture_random (&rand) &0xff & (!!expediting -1 ))) { +WARN_ON_ONCE (expediting ==0 &&rcu_gp_is_expedited ()); +if (expediting >=0 ) +rcu_expedite_gp (); +else +rcu_unexpedite_gp (); +if (++expediting >3 ) + expediting = -expediting; + } rcu_torture_writer_state =RTWS_STUTTER ;stutter_wait ("rcu_torture_writer" ); }while (!torture_must_stop ()); +/* Reset expediting back to unexpedited. */ +if (expediting >0 ) + expediting = -expediting; +while (can_expedite && expediting++ <0 ) +rcu_unexpedite_gp (); +WARN_ON_ONCE (can_expedite &&rcu_gp_is_expedited ()); rcu_torture_writer_state =RTWS_STOPPING ;torture_kthread_stopping ("rcu_torture_writer" );return 0 ;
Signature · 1 line
1.8.1.5
PATCH 3/5 rcu: Update from rcu_expedited variable to rcu_gp_is_expedited()
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
This commit updates open-coded tests of the rcu_expedited variable to instead use rcu_gp_is_expedited().
--- kernel/rcu/srcu.c | 2 +- kernel/rcu/tree.c | 9 +++++---- kernel/rcu/tree_plugin.h | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-)
kernel/rcu/srcu.c +1 -1
@@ -507,7 +507,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)*/ void synchronize_srcu (struct srcu_struct *sp) { -__synchronize_srcu (sp, rcu_expedited +__synchronize_srcu (sp,rcu_gp_is_expedited () ?SYNCHRONIZE_SRCU_EXP_TRYCOUNT :SYNCHRONIZE_SRCU_TRYCOUNT ); }
kernel/rcu/tree.c +5 -4
@@ -2954,7 +2954,7 @@ void synchronize_sched(void) "Illegal synchronize_sched() in RCU-sched read-side critical section");if (rcu_blocking_is_gp ())return ; -if (rcu_expedited) +if (rcu_gp_is_expedited ())synchronize_sched_expedited ();else wait_rcu_gp (call_rcu_sched);
@@ -2981,7 +2981,7 @@ void synchronize_rcu_bh(void) "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");if (rcu_blocking_is_gp ())return ; -if (rcu_expedited) +if (rcu_gp_is_expedited ())synchronize_rcu_bh_expedited ();else wait_rcu_gp (call_rcu_bh);
@@ -3660,11 +3660,12 @@ static int rcu_pm_notify(struct notifier_block *self,case PM_HIBERNATION_PREPARE :case PM_SUSPEND_PREPARE :if (nr_cpu_ids <=256 )/* Expediting bad for large systems. */ - rcu_expedited =1 ; +rcu_expedite_gp ();break ;case PM_POST_HIBERNATION :case PM_POST_SUSPEND : - rcu_expedited =0 ; +if (nr_cpu_ids <=256 )/* Expediting bad for large systems. */ +rcu_unexpedite_gp ();break ;default :break ;
kernel/rcu/tree_plugin.h +1 -1
@@ -585,7 +585,7 @@ void synchronize_rcu(void) "Illegal synchronize_rcu() in RCU read-side critical section");if (!rcu_scheduler_active)return ; -if (rcu_expedited) +if (rcu_gp_is_expedited ())synchronize_rcu_expedited ();else wait_rcu_gp (call_rcu);
Signature · 1 line
1.8.1.5
PATCH 1/5 rcu: Provide rcu_expedite_gp() and rcu_unexpedite_gp()
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
Currently, expediting of normal synchronous grace-period primitives (synchronize_rcu() and friends) is controlled by the rcu_expedited() boot/sysfs parameter. This works well, but does not handle nesting. This commit therefore provides rcu_expedite_gp() to enable expediting and rcu_unexpedite_gp() to cancel a prior rcu_expedite_gp(), both of which support nesting.
--- include/linux/rcupdate.h | 20 ++++++++++++++++++++ kernel/rcu/update.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+)
include/linux/rcupdate.h +20 -0
@@ -48,6 +48,26 @@extern int rcu_expedited;/* for sysctl */ +#ifdef CONFIG_TINY_RCU +/* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */ +static inline bool rcu_gp_is_expedited (void )/* Internal RCU use. */ +{ +return false; +} + +static inline void rcu_expedite_gp (void ) +{ +} + +static inline void rcu_unexpedite_gp (void ) +{ +} +#else /* #ifdef CONFIG_TINY_RCU */ +bool rcu_gp_is_expedited (void );/* Internal RCU use. */ +void rcu_expedite_gp (void ); +void rcu_unexpedite_gp (void ); +#endif /* #else #ifdef CONFIG_TINY_RCU */ +enum rcutorture_type {RCU_FLAVOR ,RCU_BH_FLAVOR ,
kernel/rcu/update.c +48 -0
54 lines not on this page · raw message
Signature · 1 line
1.8.1.5