rcu,cleanup: move synchronize_sched_expedited() out of sched.c
PATCH v2 1/2 rcu,cleanup: move synchronize_sched_expedited() out of sched.c
The first version of synchronize_sched_expedited() use the migration code of the scheduler code, so it have to be implemented in sched.c
but now, the synchronize_sched_expedited() does not use such code, it is time to move it out of sched.c.
Different rcu implementation' synchronize_sched_expedited() are also different. so we move synchronize_sched_expedited() to kernel/rcutree_plugin.h or include/linux/rcutiny.h instead of kerenl/rcupdate.c
---include/linux/rcupdate.h | 1
include/linux/rcutiny.h | 5 +++ include/linux/rcutree.h | 1 kernel/rcutree_plugin.h | 71 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched.c | 69 --------------------------------------------- 5 files changed, 77 insertions(+), 70 deletions(-)
include/linux/rcupdate.h +0 -1
@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,extern void synchronize_sched (void );extern void rcu_barrier_bh (void );extern void rcu_barrier_sched (void ); -extern void synchronize_sched_expedited (void );extern int sched_expedited_torture_stats (char *page);static inline void __rcu_read_lock_bh (void )
include/linux/rcutiny.h +5 -0
@@ -58,6 +58,11 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched (); } +static inline void synchronize_sched_expedited (void ) +{ +synchronize_sched (); +} +#ifdef CONFIG_TINY_RCU static inline void rcu_preempt_note_context_switch (void )
include/linux/rcutree.h +1 -0
@@ -47,6 +47,7 @@ static inline void exit_rcu(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ extern void synchronize_rcu_bh (void ); +extern void synchronize_sched_expedited (void );extern void synchronize_rcu_expedited (void );static inline void synchronize_rcu_bh_expedited (void )
kernel/rcutree_plugin.h +71 -0
83 lines not on this page · raw message
kernel/sched.c +0 -69
72 lines not on this page · raw message
Paul E. McKenney
AppliedQuoted 9 lines · Lai Jiangshan
The first version of
synchronize_sched_expedited()use the migration code of the scheduler code, so it have to be implemented insched.cbut now, the
synchronize_sched_expedited()does not use such code, it is time to move it out ofsched.c.Different rcu implementation'
synchronize_sched_expedited()are also different. so we movesynchronize_sched_expedited()tokernel/rcutree_plugin.horinclude/linux/rcutiny.hinstead ofkerenl/rcupdate.c
Queued, thank you!!!
Thanx, Paul
Quoted 214 lines
Signed-off-by Lai Jiangshan <[email protected]>---
include/linux/rcupdate.h| 1include/linux/rcutiny.h | 5 +++ include/linux/rcutree.h | 1 kernel/rcutree_plugin.h | 71 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched.c | 69 --------------------------------------------- 5 files changed, 77 insertions(+), 70 deletions(-)include/linux/rcupdate.h +0
-1@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,extern void synchronize_sched (void );extern void rcu_barrier_bh (void );extern void rcu_barrier_sched (void ); -extern void synchronize_sched_expedited (void );extern int sched_expedited_torture_stats (char *page);static inline void __rcu_read_lock_bh (void )include/linux/rcutiny.h +5
-0@@ -58,6 +58,11 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched (); } +static inline void synchronize_sched_expedited (void ) +{ +synchronize_sched (); +} +#ifdef CONFIG_TINY_RCU static inline void rcu_preempt_note_context_switch (void )include/linux/rcutree.h +1
-0@@ -47,6 +47,7 @@ static inline void exit_rcu(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ extern void synchronize_rcu_bh (void ); +extern void synchronize_sched_expedited (void );extern void synchronize_rcu_expedited (void );static inline void synchronize_rcu_bh_expedited (void )kernel/rcutree_plugin.h +71
-0@@ -25,6 +25,7 @@*/ #include <linux/delay.h> +#include <linux/stop_machine.h> /* * Check the RCU kernel configuration parameters and print informative @@ -581,6 +582,76 @@ void synchronize_rcu(void) }EXPORT_SYMBOL_GPL (synchronize_rcu); +#ifndef CONFIG_SMP + +void synchronize_sched_expedited (void ) +{ +cond_resched (); +} +EXPORT_SYMBOL_GPL (synchronize_sched_expedited); + +#else /* #ifndef CONFIG_SMP */ + +static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); + +static int synchronize_sched_expedited_cpu_stop (void *data) +{ +/* +* There must be a full memory barrier on each affected CPU +* between the time that try_stop_cpus() is called and the +* time that it returns. +* +* In the current initial implementation of cpu_stop, the +* above condition is already met when the control reaches +* this point and the following smp_mb() is not strictly +* necessary. Do smp_mb() anyway for documentation and +* robustness against future implementation changes. +*/ +smp_mb ();/* See above comment block. */ +return 0 ; +} + +/* +* Wait for an rcu-sched grace period to elapse, but use "big hammer" +* approach to force grace period to end quickly. This consumes +* significant time on all CPUs, and is thus not recommended for +* any sort of common-case code. +* +* Note that it is illegal to call this function while holding any +* lock that is acquired by a CPU-hotplug notifier. Failing to +* observe this restriction will result in deadlock. +*/ +void synchronize_sched_expedited (void ) +{ +int snap, trycount =0 ; + +smp_mb ();/* ensure prior mod happens before capturing snap. */ + snap =atomic_read (&synchronize_sched_expedited_count) +1 ; +get_online_cpus (); +while (try_stop_cpus (cpu_online_mask, + synchronize_sched_expedited_cpu_stop, +NULL ) == -EAGAIN ) { +put_online_cpus (); +if (trycount++ <10 ) +udelay (trycount *num_online_cpus ()); +else { +synchronize_sched (); +return ; + } +if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { +smp_mb ();/* ensure test happens before caller kfree */ +return ; + } +get_online_cpus (); + } +atomic_inc (&synchronize_sched_expedited_count); +smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ +put_online_cpus (); +} +EXPORT_SYMBOL_GPL (synchronize_sched_expedited); + +#endif /* #else #ifndef CONFIG_SMP */ +static DECLARE_WAIT_QUEUE_HEAD (sync_rcu_preempt_exp_wq);static long sync_rcu_preempt_exp_count;static DEFINE_MUTEX (sync_rcu_preempt_exp_mutex);kernel/sched.c +0
-69@@ -9332,72 +9332,3 @@ struct cgroup_subsys cpuacct_subsys = { };#endif /* CONFIG_CGROUP_CPUACCT */ -#ifndef CONFIG_SMP - -void synchronize_sched_expedited (void ) -{ -barrier (); -} -EXPORT_SYMBOL_GPL (synchronize_sched_expedited); - -#else /* #ifndef CONFIG_SMP */ - -static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); - -static int synchronize_sched_expedited_cpu_stop (void *data) -{ -/* -* There must be a full memory barrier on each affected CPU -* between the time that try_stop_cpus() is called and the -* time that it returns. -* -* In the current initial implementation of cpu_stop, the -* above condition is already met when the control reaches -* this point and the following smp_mb() is not strictly -* necessary. Do smp_mb() anyway for documentation and -* robustness against future implementation changes. -*/ -smp_mb ();/* See above comment block. */ -return 0 ; -} - -/* -* Wait for an rcu-sched grace period to elapse, but use "big hammer" -* approach to force grace period to end quickly. This consumes -* significant time on all CPUs, and is thus not recommended for -* any sort of common-case code. -* -* Note that it is illegal to call this function while holding any -* lock that is acquired by a CPU-hotplug notifier. Failing to -* observe this restriction will result in deadlock. -*/ -void synchronize_sched_expedited (void ) -{ -int snap, trycount =0 ; - -smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ; -get_online_cpus (); -while (try_stop_cpus (cpu_online_mask, - synchronize_sched_expedited_cpu_stop, -NULL ) == -EAGAIN ) { -put_online_cpus (); -if (trycount++ <10 ) -udelay (trycount *num_online_cpus ()); -else { -synchronize_sched (); -return ; - } -if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { -smp_mb ();/* ensure test happens before caller kfree */ -return ; - } -get_online_cpus (); - } -atomic_inc (&synchronize_sched_expedited_count); -smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ -put_online_cpus (); -} -EXPORT_SYMBOL_GPL (synchronize_sched_expedited); - -#endif /* #else #ifndef CONFIG_SMP */
Quoted 4 lines · Paul E. McKenney
Quoted 3 lines · Lai Jiangshan
The first version of
synchronize_sched_expedited()use the migration code of the scheduler code, so it have to be implemented insched.c
Quoted 8 lines
Quoted 6 lines
but now, the
synchronize_sched_expedited()does not use such code, it is time to move it out ofsched.c.Different rcu implementation'
synchronize_sched_expedited()are also different. so we movesynchronize_sched_expedited()tokernel/rcutree_plugin.horinclude/linux/rcutiny.hinstead ofkerenl/rcupdate.cQueued, thank you!!!
Hello again, Lai,
I hit the following build error during testing:
kernel/built-in.o: In function `.synchronize_rcu_expedited':
(.text+0x787d8): undefined reference to .synchronize_sched_expedited' kernel/built-in.o:(.toc1+0x1fe0): undefined reference to synchronize_sched_expedited'
This build uses defconfig with the following applied:
CONFIG_RCU_TRACE=yCONFIG_RCU_FAST_NO_HZ=yCONFIG_NO_HZ=yCONFIG_RCU_CPU_STALL_DETECTOR=yCONFIG_SMP=yCONFIG_RCU_FANOUT=8CONFIG_NR_CPUS=8CONFIG_RCU_FANOUT_EXACT=nCONFIG_HOTPLUG_CPU=yCONFIG_PREEMPT_NONE=yCONFIG_PREEMPT_VOLUNTARY=nCONFIG_PREEMPT=nCONFIG_TREE_RCU=yCONFIG_TREE_PREEMPT_RCU=nCONFIG_RCU_TORTURE_TEST=mCONFIG_MODULE_UNLOAD=yCONFIG_SYSFS_DEPRECATED_V2=yCONFIG_IKCONFIG=yCONFIG_IKCONFIG_PROC=y
Thoughts?
Thanx, Paul
Quoted 44 lines · Paul E. McKenney
Quoted 4 lines · Paul E. McKenney
Quoted 3 lines · Lai Jiangshan
The first version of
synchronize_sched_expedited()use the migration code of the scheduler code, so it have to be implemented insched.cQuoted 8 lines
Quoted 6 lines
but now, the
synchronize_sched_expedited()does not use such code, it is time to move it out ofsched.c.Different rcu implementation'
synchronize_sched_expedited()are also different. so we movesynchronize_sched_expedited()tokernel/rcutree_plugin.horinclude/linux/rcutiny.hinstead ofkerenl/rcupdate.cQueued, thank you!!!
Hello again, Lai,
I hit the following build error during testing:
kernel/built-in.o: In function `.synchronize_rcu_expedited':
(.text+0x787d8): undefined reference to.synchronize_sched_expedited' kernel/built-in.o:(.toc1+0x1fe0): undefined reference tosynchronize_sched_expedited'This build uses defconfig with the following applied:
CONFIG_RCU_TRACE=yCONFIG_RCU_FAST_NO_HZ=yCONFIG_NO_HZ=yCONFIG_RCU_CPU_STALL_DETECTOR=yCONFIG_SMP=yCONFIG_RCU_FANOUT=8CONFIG_NR_CPUS=8CONFIG_RCU_FANOUT_EXACT=nCONFIG_HOTPLUG_CPU=yCONFIG_PREEMPT_NONE=yCONFIG_PREEMPT_VOLUNTARY=nCONFIG_PREEMPT=nCONFIG_TREE_RCU=yCONFIG_TREE_PREEMPT_RCU=nCONFIG_RCU_TORTURE_TEST=mCONFIG_MODULE_UNLOAD=yCONFIG_SYSFS_DEPRECATED_V2=yCONFIG_IKCONFIG=yCONFIG_IKCONFIG_PROC=y
Quoted 5 lines
Thoughts?
Thanx, Paul
I moved the code into CONFIG_TREE_PREEMPT_RCU=y codes. fixed version:
---
include/linux/rcupdate.h +0 -1
@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,extern void synchronize_sched (void );extern void rcu_barrier_bh (void );extern void rcu_barrier_sched (void ); -extern void synchronize_sched_expedited (void );extern int sched_expedited_torture_stats (char *page);static inline void __rcu_read_lock_bh (void )
include/linux/rcutiny.h +5 -0
@@ -58,6 +58,11 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched (); } +static inline void synchronize_sched_expedited (void ) +{ +synchronize_sched (); +} +#ifdef CONFIG_TINY_RCU static inline void rcu_preempt_note_context_switch (void )
include/linux/rcutree.h +1 -0
@@ -47,6 +47,7 @@ static inline void exit_rcu(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ extern void synchronize_rcu_bh (void ); +extern void synchronize_sched_expedited (void );extern void synchronize_rcu_expedited (void );static inline void synchronize_rcu_bh_expedited (void )
kernel/rcutree_plugin.h +71 -0
83 lines not on this page · raw message
kernel/sched.c +0 -69
72 lines not on this page · raw message
Quoted 45 lines · Lai Jiangshan
Quoted 44 lines · Paul E. McKenney
Quoted 4 lines · Paul E. McKenney
Quoted 3 lines · Lai Jiangshan
The first version of
synchronize_sched_expedited()use the migration code of the scheduler code, so it have to be implemented insched.cQuoted 8 lines
Quoted 6 lines
but now, the
synchronize_sched_expedited()does not use such code, it is time to move it out ofsched.c.Different rcu implementation'
synchronize_sched_expedited()are also different. so we movesynchronize_sched_expedited()tokernel/rcutree_plugin.horinclude/linux/rcutiny.hinstead ofkerenl/rcupdate.cQueued, thank you!!!
Hello again, Lai,
I hit the following build error during testing:
kernel/built-in.o: In function `.synchronize_rcu_expedited':
(.text+0x787d8): undefined reference to.synchronize_sched_expedited' kernel/built-in.o:(.toc1+0x1fe0): undefined reference tosynchronize_sched_expedited'This build uses defconfig with the following applied:
CONFIG_RCU_TRACE=yCONFIG_RCU_FAST_NO_HZ=yCONFIG_NO_HZ=yCONFIG_RCU_CPU_STALL_DETECTOR=yCONFIG_SMP=yCONFIG_RCU_FANOUT=8CONFIG_NR_CPUS=8CONFIG_RCU_FANOUT_EXACT=nCONFIG_HOTPLUG_CPU=yCONFIG_PREEMPT_NONE=yCONFIG_PREEMPT_VOLUNTARY=nCONFIG_PREEMPT=nCONFIG_TREE_RCU=yCONFIG_TREE_PREEMPT_RCU=nCONFIG_RCU_TORTURE_TEST=mCONFIG_MODULE_UNLOAD=yCONFIG_SYSFS_DEPRECATED_V2=yCONFIG_IKCONFIG=yCONFIG_IKCONFIG_PROC=y
Quoted 7 lines
Quoted 5 lines
Thoughts?
Thanx, Paul
I moved the code into
CONFIG_TREE_PREEMPT_RCU=ycodes. fixed version:
Thank you, queued and am retesting.
Thanx, Paul
Quoted 213 lines
Signed-off-by Lai Jiangshan <[email protected]>---
include/linux/rcupdate.h +0
-1@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,extern void synchronize_sched (void );extern void rcu_barrier_bh (void );extern void rcu_barrier_sched (void ); -extern void synchronize_sched_expedited (void );extern int sched_expedited_torture_stats (char *page);static inline void __rcu_read_lock_bh (void )include/linux/rcutiny.h +5
-0@@ -58,6 +58,11 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched (); } +static inline void synchronize_sched_expedited (void ) +{ +synchronize_sched (); +} +#ifdef CONFIG_TINY_RCU static inline void rcu_preempt_note_context_switch (void )include/linux/rcutree.h +1
-0@@ -47,6 +47,7 @@ static inline void exit_rcu(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ extern void synchronize_rcu_bh (void ); +extern void synchronize_sched_expedited (void );extern void synchronize_rcu_expedited (void );static inline void synchronize_rcu_bh_expedited (void )kernel/rcutree_plugin.h +71
-0@@ -25,6 +25,7 @@*/ #include <linux/delay.h> +#include <linux/stop_machine.h> /* * Check the RCU kernel configuration parameters and print informative @@ -1014,6 +1015,76 @@ static void __init __rcu_init_preempt(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ +#ifndef CONFIG_SMP + +void synchronize_sched_expedited (void ) +{ +cond_resched (); +} +EXPORT_SYMBOL_GPL (synchronize_sched_expedited); + +#else /* #ifndef CONFIG_SMP */ + +static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); + +static int synchronize_sched_expedited_cpu_stop (void *data) +{ +/* +* There must be a full memory barrier on each affected CPU +* between the time that try_stop_cpus() is called and the +* time that it returns. +* +* In the current initial implementation of cpu_stop, the +* above condition is already met when the control reaches +* this point and the following smp_mb() is not strictly +* necessary. Do smp_mb() anyway for documentation and +* robustness against future implementation changes. +*/ +smp_mb ();/* See above comment block. */ +return 0 ; +} + +/* +* Wait for an rcu-sched grace period to elapse, but use "big hammer" +* approach to force grace period to end quickly. This consumes +* significant time on all CPUs, and is thus not recommended for +* any sort of common-case code. +* +* Note that it is illegal to call this function while holding any +* lock that is acquired by a CPU-hotplug notifier. Failing to +* observe this restriction will result in deadlock. +*/ +void synchronize_sched_expedited (void ) +{ +int snap, trycount =0 ; + +smp_mb ();/* ensure prior mod happens before capturing snap. */ + snap =atomic_read (&synchronize_sched_expedited_count) +1 ; +get_online_cpus (); +while (try_stop_cpus (cpu_online_mask, + synchronize_sched_expedited_cpu_stop, +NULL ) == -EAGAIN ) { +put_online_cpus (); +if (trycount++ <10 ) +udelay (trycount *num_online_cpus ()); +else { +synchronize_sched (); +return ; + } +if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { +smp_mb ();/* ensure test happens before caller kfree */ +return ; + } +get_online_cpus (); + } +atomic_inc (&synchronize_sched_expedited_count); +smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ +put_online_cpus (); +} +EXPORT_SYMBOL_GPL (synchronize_sched_expedited); + +#endif /* #else #ifndef CONFIG_SMP */ +#if !defined(CONFIG_RCU_FAST_NO_HZ )/* kernel/sched.c +0
-70@@ -9332,72 +9332,3 @@ struct cgroup_subsys cpuacct_subsys = { };#endif /* CONFIG_CGROUP_CPUACCT */ -#ifndef CONFIG_SMP - -void synchronize_sched_expedited (void ) -{ -barrier (); -} -EXPORT_SYMBOL_GPL (synchronize_sched_expedited); - -#else /* #ifndef CONFIG_SMP */ - -static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); - -static int synchronize_sched_expedited_cpu_stop (void *data) -{ -/* -* There must be a full memory barrier on each affected CPU -* between the time that try_stop_cpus() is called and the -* time that it returns. -* -* In the current initial implementation of cpu_stop, the -* above condition is already met when the control reaches -* this point and the following smp_mb() is not strictly -* necessary. Do smp_mb() anyway for documentation and -* robustness against future implementation changes. -*/ -smp_mb ();/* See above comment block. */ -return 0 ; -} - -/* -* Wait for an rcu-sched grace period to elapse, but use "big hammer" -* approach to force grace period to end quickly. This consumes -* significant time on all CPUs, and is thus not recommended for -* any sort of common-case code. -* -* Note that it is illegal to call this function while holding any -* lock that is acquired by a CPU-hotplug notifier. Failing to -* observe this restriction will result in deadlock. -*/ -void synchronize_sched_expedited (void ) -{ -int snap, trycount =0 ; - -smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ; -get_online_cpus (); -while (try_stop_cpus (cpu_online_mask, - synchronize_sched_expedited_cpu_stop, -NULL ) == -EAGAIN ) { -put_online_cpus (); -if (trycount++ <10 ) -udelay (trycount *num_online_cpus ()); -else { -synchronize_sched (); -return ; - } -if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { -smp_mb ();/* ensure test happens before caller kfree */ -return ; - } -get_online_cpus (); - } -atomic_inc (&synchronize_sched_expedited_count); -smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ -put_online_cpus (); -} -EXPORT_SYMBOL_GPL (synchronize_sched_expedited); - -#endif /* #else #ifndef CONFIG_SMP */ --To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Quoted 48 lines · Paul E. McKenney
Quoted 47 lines · Lai Jiangshan
Quoted 46 lines · Paul E. McKenney
Quoted 4 lines · Paul E. McKenney
Quoted 3 lines · Lai Jiangshan
The first version of
synchronize_sched_expedited()use the migration code of the scheduler code, so it have to be implemented insched.cQuoted 8 lines
Quoted 6 lines
but now, the
synchronize_sched_expedited()does not use such code, it is time to move it out ofsched.c.Different rcu implementation'
synchronize_sched_expedited()are also different. so we movesynchronize_sched_expedited()tokernel/rcutree_plugin.horinclude/linux/rcutiny.hinstead ofkerenl/rcupdate.cQueued, thank you!!!
Hello again, Lai,
I hit the following build error during testing:
kernel/built-in.o: In function `.synchronize_rcu_expedited':
(.text+0x787d8): undefined reference to.synchronize_sched_expedited' kernel/built-in.o:(.toc1+0x1fe0): undefined reference tosynchronize_sched_expedited'This build uses defconfig with the following applied:
CONFIG_RCU_TRACE=yCONFIG_RCU_FAST_NO_HZ=yCONFIG_NO_HZ=yCONFIG_RCU_CPU_STALL_DETECTOR=yCONFIG_SMP=yCONFIG_RCU_FANOUT=8CONFIG_NR_CPUS=8CONFIG_RCU_FANOUT_EXACT=nCONFIG_HOTPLUG_CPU=yCONFIG_PREEMPT_NONE=yCONFIG_PREEMPT_VOLUNTARY=nCONFIG_PREEMPT=nCONFIG_TREE_RCU=yCONFIG_TREE_PREEMPT_RCU=nCONFIG_RCU_TORTURE_TEST=mCONFIG_MODULE_UNLOAD=yCONFIG_SYSFS_DEPRECATED_V2=yCONFIG_IKCONFIG=yCONFIG_IKCONFIG_PROC=yThoughts?
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Thanx, Paul
I moved the code into
CONFIG_TREE_PREEMPT_RCU=ycodes. fixed version:Thank you, queued and am retesting.
And the new version works much better, thank you! Any news on the cpumask_any() issue?
Thanx, Paul
Quoted 213 lines
Quoted 213 lines
Signed-off-by Lai Jiangshan <[email protected]>---
include/linux/rcupdate.h +0
-1@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,extern void synchronize_sched (void );extern void rcu_barrier_bh (void );extern void rcu_barrier_sched (void ); -extern void synchronize_sched_expedited (void );extern int sched_expedited_torture_stats (char *page);static inline void __rcu_read_lock_bh (void )include/linux/rcutiny.h +5
-0@@ -58,6 +58,11 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched (); } +static inline void synchronize_sched_expedited (void ) +{ +synchronize_sched (); +} +#ifdef CONFIG_TINY_RCU static inline void rcu_preempt_note_context_switch (void )include/linux/rcutree.h +1
-0@@ -47,6 +47,7 @@ static inline void exit_rcu(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ extern void synchronize_rcu_bh (void ); +extern void synchronize_sched_expedited (void );extern void synchronize_rcu_expedited (void );static inline void synchronize_rcu_bh_expedited (void )kernel/rcutree_plugin.h +71
-0@@ -25,6 +25,7 @@*/ #include <linux/delay.h> +#include <linux/stop_machine.h> /* * Check the RCU kernel configuration parameters and print informative @@ -1014,6 +1015,76 @@ static void __init __rcu_init_preempt(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ +#ifndef CONFIG_SMP + +void synchronize_sched_expedited (void ) +{ +cond_resched (); +} +EXPORT_SYMBOL_GPL (synchronize_sched_expedited); + +#else /* #ifndef CONFIG_SMP */ + +static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); + +static int synchronize_sched_expedited_cpu_stop (void *data) +{ +/* +* There must be a full memory barrier on each affected CPU +* between the time that try_stop_cpus() is called and the +* time that it returns. +* +* In the current initial implementation of cpu_stop, the +* above condition is already met when the control reaches +* this point and the following smp_mb() is not strictly +* necessary. Do smp_mb() anyway for documentation and +* robustness against future implementation changes. +*/ +smp_mb ();/* See above comment block. */ +return 0 ; +} + +/* +* Wait for an rcu-sched grace period to elapse, but use "big hammer" +* approach to force grace period to end quickly. This consumes +* significant time on all CPUs, and is thus not recommended for +* any sort of common-case code. +* +* Note that it is illegal to call this function while holding any +* lock that is acquired by a CPU-hotplug notifier. Failing to +* observe this restriction will result in deadlock. +*/ +void synchronize_sched_expedited (void ) +{ +int snap, trycount =0 ; + +smp_mb ();/* ensure prior mod happens before capturing snap. */ + snap =atomic_read (&synchronize_sched_expedited_count) +1 ; +get_online_cpus (); +while (try_stop_cpus (cpu_online_mask, + synchronize_sched_expedited_cpu_stop, +NULL ) == -EAGAIN ) { +put_online_cpus (); +if (trycount++ <10 ) +udelay (trycount *num_online_cpus ()); +else { +synchronize_sched (); +return ; + } +if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { +smp_mb ();/* ensure test happens before caller kfree */ +return ; + } +get_online_cpus (); + } +atomic_inc (&synchronize_sched_expedited_count); +smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ +put_online_cpus (); +} +EXPORT_SYMBOL_GPL (synchronize_sched_expedited); + +#endif /* #else #ifndef CONFIG_SMP */ +#if !defined(CONFIG_RCU_FAST_NO_HZ )/* kernel/sched.c +0
-70@@ -9332,72 +9332,3 @@ struct cgroup_subsys cpuacct_subsys = { };#endif /* CONFIG_CGROUP_CPUACCT */ -#ifndef CONFIG_SMP - -void synchronize_sched_expedited (void ) -{ -barrier (); -} -EXPORT_SYMBOL_GPL (synchronize_sched_expedited); - -#else /* #ifndef CONFIG_SMP */ - -static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); - -static int synchronize_sched_expedited_cpu_stop (void *data) -{ -/* -* There must be a full memory barrier on each affected CPU -* between the time that try_stop_cpus() is called and the -* time that it returns. -* -* In the current initial implementation of cpu_stop, the -* above condition is already met when the control reaches -* this point and the following smp_mb() is not strictly -* necessary. Do smp_mb() anyway for documentation and -* robustness against future implementation changes. -*/ -smp_mb ();/* See above comment block. */ -return 0 ; -} - -/* -* Wait for an rcu-sched grace period to elapse, but use "big hammer" -* approach to force grace period to end quickly. This consumes -* significant time on all CPUs, and is thus not recommended for -* any sort of common-case code. -* -* Note that it is illegal to call this function while holding any -* lock that is acquired by a CPU-hotplug notifier. Failing to -* observe this restriction will result in deadlock. -*/ -void synchronize_sched_expedited (void ) -{ -int snap, trycount =0 ; - -smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ; -get_online_cpus (); -while (try_stop_cpus (cpu_online_mask, - synchronize_sched_expedited_cpu_stop, -NULL ) == -EAGAIN ) { -put_online_cpus (); -if (trycount++ <10 ) -udelay (trycount *num_online_cpus ()); -else { -synchronize_sched (); -return ; - } -if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { -smp_mb ();/* ensure test happens before caller kfree */ -return ; - } -get_online_cpus (); - } -atomic_inc (&synchronize_sched_expedited_count); -smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ -put_online_cpus (); -} -EXPORT_SYMBOL_GPL (synchronize_sched_expedited); - -#endif /* #else #ifndef CONFIG_SMP */ --To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/