preview of RCU patches for 2.6.38
Subsystems (3)
cover preview of RCU patches for 2.6.38
Hello!
This patchset contains RCU priority boosting for the tiny RCU implementations and a number of fixes and cleanups. The patches are as follows:
1. Add rcutorture tests to check for boosting.
2-6. RCU priority boosting for the tiny RCU implementations.
7. Fix a few naming holdouts from old Classic RCU.
8. Move synchronize_sched_expedited() out of sched.c
(from Lai Jiangshan).
9. Simplify the RCU callback orphan/adopt code for CPU hotplug
(from Lai Jiangshan).
10. Update RCU tracing documentation to reflect patch #9.
11. Fix a race condition in synchronize_sched_expedited(),
but Tejun Heo has suggested another approach that might be better longer term.
12. Make synchronize_srcu_expedited() spin for a bit to avoid
blocking -- again, there may be a better long-term fix.
For a testing-only version of this patchset from git, please see:
git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-2.6-rcu.git rcu/testing
Thanx, Paul
Documentation/RCU/trace.txt | 12 - b/Documentation/RCU/trace.txt | 132 ++++++++++++- b/include/linux/init_task.h | 9 b/include/linux/rcupdate.h | 1 b/include/linux/rcutiny.h | 8 b/include/linux/rcutree.h | 1 b/include/linux/sched.h | 11 - b/init/Kconfig | 39 +++ b/kernel/rcutiny.c | 71 +++++-- b/kernel/rcutiny_plugin.h | 15 - b/kernel/rcutorture.c | 270 ++++++++++++++++++++++++++- b/kernel/rcutree.c | 81 ++------ b/kernel/rcutree.h | 16 - b/kernel/rcutree_plugin.h | 71 +++++++ b/kernel/rcutree_trace.c | 8 b/kernel/sched.c | 69 ------ b/kernel/srcu.c | 8 include/linux/rcupdate.h | 1 include/linux/rcutiny.h | 5 include/linux/rcutree.h | 1 init/Kconfig | 1 kernel/rcutiny.c | 70 +++---- kernel/rcutiny_plugin.h | 418 ++++++++++++++++++++++++++++++++++++++++-- kernel/rcutree.c | 10 - kernel/rcutree_plugin.h | 28 ++ kernel/rcutree_trace.c | 4 26 files changed, 1090 insertions(+), 270 deletions(-)
RFC PATCH 1/12 rcu: add priority-inversion testing to rcutorture
Add an optional test to force long-term preemption of RCU read-side critical sections, controlled by new test_boost, test_boost_interval, and test_boost_duration module parameters. This is to be used to test RCU priority boosting.
--- kernel/rcutorture.c | 270 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 files changed, 259 insertions(+), 11 deletions(-)
kernel/rcutorture.c +259 -11
@@ -47,6 +47,7 @@#include <linux/srcu.h> #include <linux/slab.h> #include <asm/byteorder.h> +#include <linux/sched.h> MODULE_LICENSE ("GPL" ); MODULE_AUTHOR("Paul E. McKenney <[email protected]> and "
@@ -64,6 +65,9 @@ static int irqreader = 1; /* RCU readers from irq (timers). */static int fqs_duration =0 ;/* Duration of bursts (us), 0 to disable. */ static int fqs_holdoff =0 ;/* Hold time within burst (us). */ static int fqs_stutter =3 ;/* Wait time between bursts (s). */ +static int test_boost =1 ;/* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */ +static int test_boost_interval =7 ;/* Interval between boost tests, seconds. */ +static int test_boost_duration =4 ;/* Duration of each boost test, seconds. */ static char *torture_type ="rcu" ;/* What RCU implementation to torture. */ module_param (nreaders, int,0444 );
@@ -88,6 +92,12 @@ module_param(fqs_holdoff, int, 0444);MODULE_PARM_DESC (fqs_holdoff,"Holdoff time within fqs bursts (us)" );module_param (fqs_stutter, int,0444 );MODULE_PARM_DESC (fqs_stutter,"Wait time between fqs bursts (s)" ); +module_param (test_boost, int,0444 ); +MODULE_PARM_DESC (test_boost,"Test RCU prio boost: 0=no, 1=maybe, 2=yes." ); +module_param (test_boost_interval, int,0444 ); +MODULE_PARM_DESC (test_boost_interval,"Interval between boost tests, seconds." ); +module_param (test_boost_duration, int,0444 ); +MODULE_PARM_DESC (test_boost_duration,"Duration of each boost test, seconds." );module_param (torture_type, charp,0444 );MODULE_PARM_DESC (torture_type,"Type of RCU to torture (rcu, rcu_bh, srcu)" );
@@ -109,6 +119,7 @@ static struct task_struct *stats_task;static struct task_struct *shuffler_task;static struct task_struct *stutter_task;static struct task_struct *fqs_task; +static struct task_struct *boost_tasks[NR_CPUS ];#define RCU_TORTURE_PIPE_LEN 10
@@ -134,6 +145,12 @@ static atomic_t n_rcu_torture_alloc_fail;static atomic_t n_rcu_torture_free;static atomic_t n_rcu_torture_mberror;static atomic_t n_rcu_torture_error; +static long n_rcu_torture_boost_ktrerror; +static long n_rcu_torture_boost_rterror; +static long n_rcu_torture_boost_allocerror; +static long n_rcu_torture_boost_afferror; +static long n_rcu_torture_boost_failure; +static long n_rcu_torture_boosts;static long n_rcu_torture_timers;static struct list_head rcu_torture_removed;static cpumask_var_t shuffle_tmp_mask;
@@ -147,6 +164,16 @@ static int stutter_pause_test;#endif int rcutorture_runnable =RCUTORTURE_RUNNABLE_INIT ; +#ifdef CONFIG_RCU_BOOST +#define rcu_can_boost () 1 +#else /* #ifdef CONFIG_RCU_BOOST */ +#define rcu_can_boost () 0 +#endif /* #else #ifdef CONFIG_RCU_BOOST */ + +static unsigned long boost_starttime;/* jiffies of next boost test start. */ +DEFINE_MUTEX (boost_mutex);/* protect setting boost_starttime */ +/* and boost task create/destroy. */ +/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ #define FULLSTOP_DONTSTOP 0/* Normal operation. */
@@ -277,6 +304,7 @@ struct rcu_torture_ops {void (*fqs)(void );int (*stats)(char *page);int irq_capable; +int can_boost;char *name; };
@@ -366,6 +394,7 @@ static struct rcu_torture_ops rcu_ops = { .fqs = rcu_force_quiescent_state, .stats = NULL, .irq_capable = 1, + .can_boost = rcu_can_boost(), .name = "rcu" };
@@ -408,6 +437,7 @@ static struct rcu_torture_ops rcu_sync_ops = { .fqs = rcu_force_quiescent_state, .stats = NULL, .irq_capable = 1, + .can_boost = rcu_can_boost(), .name = "rcu_sync" };
@@ -424,6 +454,7 @@ static struct rcu_torture_ops rcu_expedited_ops = { .fqs = rcu_force_quiescent_state, .stats = NULL, .irq_capable = 1, + .can_boost = rcu_can_boost(), .name = "rcu_expedited" };
@@ -684,6 +715,110 @@ static struct rcu_torture_ops sched_expedited_ops = { };/* +* RCU torture priority-boost testing. Runs one real-time thread per +* CPU for moderate bursts, repeatedly registering RCU callbacks and +* spinning waiting for them to be invoked. If a given callback takes +* too long to be invoked, we assume that priority inversion has occurred. +*/ + +struct rcu_boost_inflight { +struct rcu_head rcu; +int inflight; +}; + +static void rcu_torture_boost_cb (struct rcu_head *head) +{ +struct rcu_boost_inflight *rbip = + container_of(head, struct rcu_boost_inflight, rcu); + +smp_mb ();/* Ensure RCU-core accesses precede clearing ->inflight */ + rbip->inflight =0 ; +} + +static int rcu_torture_boost (void *arg) +{ +unsigned long call_rcu_time; +unsigned long endtime; +unsigned long oldstarttime; +struct rcu_boost_inflight rbi = { .inflight =0 }; +struct sched_param sp; + +VERBOSE_PRINTK_STRING ("rcu_torture_boost started" ); + +/* Set real-time priority. */ + sp.sched_priority =1 ; +if (sched_setscheduler (current,SCHED_FIFO , &sp) <0 ) { +VERBOSE_PRINTK_STRING ("rcu_torture_boost RT prio failed!" ); + n_rcu_torture_boost_rterror++; + } + +/* Each pass through the following loop does one boost-test cycle. */ +do { +/* Wait for the next test interval. */ + oldstarttime = boost_starttime; +while (jiffies - oldstarttime >ULONG_MAX /2 ) { +schedule_timeout_uninterruptible (1 ); +rcu_stutter_wait ("rcu_torture_boost" ); +if (kthread_should_stop () || + fullstop !=FULLSTOP_DONTSTOP ) + goto checkwait; + } + +/* Do one boost-test interval. */ + endtime = oldstarttime + test_boost_duration *HZ ; + call_rcu_time = jiffies; +while (jiffies - endtime >ULONG_MAX /2 ) { +/* If we don't have a callback in flight, post one. */ +if (!rbi.inflight) { +smp_mb ();/* RCU core before ->inflight = 1. */ + rbi.inflight =1 ; +call_rcu (&rbi.rcu, rcu_torture_boost_cb); +if (jiffies - call_rcu_time > + test_boost_duration *HZ -HZ /2 ) { +VERBOSE_PRINTK_STRING ("rcu_torture_boost boosting failed" ); + n_rcu_torture_boost_failure++; + } + call_rcu_time = jiffies; + } +cond_resched (); +rcu_stutter_wait ("rcu_torture_boost" ); +if (kthread_should_stop () || + fullstop !=FULLSTOP_DONTSTOP ) + goto checkwait; + } + +/* +* Set the start time of the next test interval. +* Yes, this is vulnerable to long delays, but such +* delays simply cause a false negative for the next +* interval. Besides, we are running at RT priority, +* so delays should be relatively rare. +*/ +while (oldstarttime == boost_starttime) { +if (mutex_trylock (&boost_mutex)) { + boost_starttime = jiffies + + test_boost_interval *HZ ; + n_rcu_torture_boosts++; +mutex_unlock (&boost_mutex); +break ; + } +schedule_timeout_uninterruptible (1 ); + } + +/* Go do the stutter. */ +checkwait:rcu_stutter_wait ("rcu_torture_boost" ); + }while (!kthread_should_stop () && fullstop ==FULLSTOP_DONTSTOP ); + +/* Clean up and exit. */ +VERBOSE_PRINTK_STRING ("rcu_torture_boost task stopping" ); +rcutorture_shutdown_absorb ("rcu_torture_boost" ); +while (!kthread_should_stop () || rbi.inflight) +schedule_timeout_uninterruptible (1 ); +smp_mb ();/* order accesses to ->inflight before stack-frame death. */ +return 0 ; +} + +/* * RCU torture force-quiescent-state kthread. Repeatedly induces * bursts of calls to force_quiescent_state(), increasing the probability * of occurrence of some important types of race conditions.
@@ -933,7 +1068,8 @@ rcu_torture_printk(char *page) cnt +=sprintf (&page[cnt],"%s%s " , torture_type,TORTURE_FLAG ); cnt += sprintf(&page[cnt], "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d " - "rtmbe: %d nt: %ld", + "rtmbe: %d rtbke: %ld rtbre: %ld rtbae: %ld rtbafe: %ld " + "rtbf: %ld rtb: %ld nt: %ld", rcu_torture_current, rcu_torture_current_version, list_empty(&rcu_torture_freelist),
@@ -941,8 +1077,19 @@ rcu_torture_printk(char *page)atomic_read (&n_rcu_torture_alloc_fail),atomic_read (&n_rcu_torture_free),atomic_read (&n_rcu_torture_mberror), + n_rcu_torture_boost_ktrerror, + n_rcu_torture_boost_rterror, + n_rcu_torture_boost_allocerror, + n_rcu_torture_boost_afferror, + n_rcu_torture_boost_failure, + n_rcu_torture_boosts, n_rcu_torture_timers); -if (atomic_read (&n_rcu_torture_mberror) !=0 ) +if (atomic_read (&n_rcu_torture_mberror) !=0 || + n_rcu_torture_boost_ktrerror !=0 || + n_rcu_torture_boost_rterror !=0 || + n_rcu_torture_boost_allocerror !=0 || + n_rcu_torture_boost_afferror !=0 || + n_rcu_torture_boost_failure !=0 ) cnt +=sprintf (&page[cnt]," !!!" ); cnt +=sprintf (&page[cnt],"\n%s%s " , torture_type,TORTURE_FLAG );if (i >1 ) {
@@ -1094,22 +1241,91 @@ rcu_torture_stutter(void *arg) }static inline void -rcu_torture_print_module_parms (char *tag) +rcu_torture_print_module_parms (struct rcu_torture_ops *cur_ops,char *tag) {printk (KERN_ALERT "%s" TORTURE_FLAG "--- %s: nreaders=%d nfakewriters=%d " "stat_interval=%d verbose=%d test_no_idle_hz=%d " "shuffle_interval=%d stutter=%d irqreader=%d " -"fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d\n" , +"fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " +"test_boost=%d/%d test_boost_interval=%d " +"test_boost_duration=%d\n" , torture_type, tag, nrealreaders, nfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, - stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter); + stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, + test_boost, cur_ops->can_boost, + test_boost_interval, test_boost_duration); } -static struct notifier_block rcutorture_nb = { +static struct notifier_block rcutorture_shutdown_nb = { .notifier_call = rcutorture_shutdown_notify, }; +static void rcutorture_booster_cleanup (int cpu) +{ +struct task_struct *t; + +if (boost_tasks[cpu] ==NULL ) +return ; +mutex_lock (&boost_mutex); +VERBOSE_PRINTK_STRING ("Stopping rcu_torture_boost task" ); + t = boost_tasks[cpu]; + boost_tasks[cpu] =NULL ; +mutex_unlock (&boost_mutex); + +/* This must be outside of the mutex, otherwise deadlock! */ +kthread_stop (t); +} + +static int rcutorture_booster_init (int cpu) +{ +int retval; + +if (boost_tasks[cpu] !=NULL ) +return 0 ;/* Already created, nothing more to do. */ + +/* Don't allow time recalculation while creating a new task. */ +mutex_lock (&boost_mutex); +VERBOSE_PRINTK_STRING ("Creating rcu_torture_boost task" ); + boost_tasks[cpu] =kthread_create (rcu_torture_boost,NULL , +"rcu_torture_boost" ); +if (IS_ERR (boost_tasks[cpu])) { + retval =PTR_ERR (boost_tasks[cpu]); +VERBOSE_PRINTK_STRING ("rcu_torture_boost task create failed" ); + n_rcu_torture_boost_ktrerror++; + boost_tasks[cpu] =NULL ; +mutex_unlock (&boost_mutex); +return retval; + } +kthread_bind (boost_tasks[cpu], cpu); +wake_up_process (boost_tasks[cpu]); +mutex_unlock (&boost_mutex); +return 0 ; +} + +static int rcutorture_cpu_notify (struct notifier_block *self, +unsigned long action,void *hcpu) +{ +long cpu = (long )hcpu; + +switch (action) { +case CPU_ONLINE : +case CPU_DOWN_FAILED : + (void )rcutorture_booster_init (cpu); +break ; +case CPU_DOWN_PREPARE : +rcutorture_booster_cleanup (cpu); +break ; +default : +break ; + } +return NOTIFY_OK ; +} + +static struct notifier_block rcutorture_cpu_nb = { + .notifier_call = rcutorture_cpu_notify, +}; +static void rcu_torture_cleanup (void ) {
@@ -1127,7 +1343,7 @@ rcu_torture_cleanup(void) } fullstop =FULLSTOP_RMMOD ;mutex_unlock (&fullstop_mutex); -unregister_reboot_notifier (&rcutorture_nb); +unregister_reboot_notifier (&rcutorture_shutdown_nb);if (stutter_task) {VERBOSE_PRINTK_STRING ("Stopping rcu_torture_stutter task" );kthread_stop (stutter_task);
@@ -1184,6 +1400,12 @@ rcu_torture_cleanup(void)kthread_stop (fqs_task); } fqs_task =NULL ; +if ((test_boost ==1 && cur_ops->can_boost) || + test_boost ==2 ) { +unregister_cpu_notifier (&rcutorture_cpu_nb); +for_each_possible_cpu (i) +rcutorture_booster_cleanup (i); + }/* Wait for all RCU callbacks to fire. */
@@ -1195,9 +1417,9 @@ rcu_torture_cleanup(void)if (cur_ops->cleanup) cur_ops->cleanup ();if (atomic_read (&n_rcu_torture_error)) -rcu_torture_print_module_parms ("End of test: FAILURE" ); +rcu_torture_print_module_parms (cur_ops,"End of test: FAILURE" );else -rcu_torture_print_module_parms ("End of test: SUCCESS" ); +rcu_torture_print_module_parms (cur_ops,"End of test: SUCCESS" ); } static int __init
@@ -1242,7 +1464,7 @@ rcu_torture_init(void) nrealreaders = nreaders; else nrealreaders =2 *num_online_cpus (); -rcu_torture_print_module_parms ("Start of test" ); +rcu_torture_print_module_parms (cur_ops,"Start of test" ); fullstop =FULLSTOP_DONTSTOP ;/* Set up the freelist. */
@@ -1263,6 +1485,12 @@ rcu_torture_init(void)atomic_set (&n_rcu_torture_free,0 );atomic_set (&n_rcu_torture_mberror,0 );atomic_set (&n_rcu_torture_error,0 ); + n_rcu_torture_boost_ktrerror =0 ; + n_rcu_torture_boost_rterror =0 ; + n_rcu_torture_boost_allocerror =0 ; + n_rcu_torture_boost_afferror =0 ; + n_rcu_torture_boost_failure =0 ; + n_rcu_torture_boosts =0 ;for (i =0 ; i <RCU_TORTURE_PIPE_LEN +1 ; i++)atomic_set (&rcu_torture_wcount[i],0 );for_each_possible_cpu (cpu) {
@@ -1376,7 +1604,27 @@ rcu_torture_init(void) goto unwind; } } -register_reboot_notifier (&rcutorture_nb); +if (test_boost_interval <1 ) + test_boost_interval =1 ; +if (test_boost_duration <2 ) + test_boost_duration =2 ; +if ((test_boost ==1 && cur_ops->can_boost) || + test_boost ==2 ) { +int retval; + + boost_starttime = jiffies + test_boost_interval *HZ ; +register_cpu_notifier (&rcutorture_cpu_nb); +for_each_possible_cpu (i) { +if (cpu_is_offline (i)) +continue ;/* Heuristic: CPU can go offline. */ + retval =rcutorture_booster_init (i); +if (retval <0 ) { + firsterr = retval; + goto unwind; + } + } + } +register_reboot_notifier (&rcutorture_shutdown_nb);mutex_unlock (&fullstop_mutex);return 0 ;
Signature · 1 line
1.7.0.6
RFC PATCH 2/12 rcu: move TINY_RCU from softirq to kthread
In reply to Paul E. McKenneyIf RCU priority boosting is to be meaningful, callback invocation must be boosted in addition to preempted RCU readers. Otherwise, in presence of CPU real-time threads, the grace period ends, but the callbacks don't get invoked. If the callbacks don't get invoked, the associated memory doesn't get freed, so the system is still subject to OOM.
But it is not reasonable to priority-boost RCU_SOFTIRQ, so this commit moves the callback invocations to a kthread, which can be boosted easily.
--- include/linux/rcupdate.h | 1 - include/linux/rcutiny.h | 8 ++--- include/linux/rcutree.h | 1 + kernel/rcutiny.c | 71 +++++++++++++++++++++++++++++++++++++-------- kernel/rcutiny_plugin.h | 15 +++++---- 5 files changed, 70 insertions(+), 26 deletions(-)
include/linux/rcupdate.h +0 -1
@@ -118,7 +118,6 @@ static inline int rcu_preempt_depth(void)#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* Internal to kernel */ -extern void rcu_init (void );extern void rcu_sched_qs (int cpu);extern void rcu_bh_qs (int cpu);extern void rcu_check_callbacks (int cpu,int user);
include/linux/rcutiny.h +3 -5
@@ -27,7 +27,9 @@#include <linux/cache.h> -#define rcu_init_sched () do { } while (0) +static inline void rcu_init (void ) +{ +}#ifdef CONFIG_TINY_RCU
@@ -125,16 +127,12 @@ static inline void rcu_cpu_stall_reset(void) }#ifdef CONFIG_DEBUG_LOCK_ALLOC -extern int rcu_scheduler_active__read_mostly ;extern void rcu_scheduler_starting (void ); -#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -static inline void rcu_scheduler_starting (void ) { } -#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ #endif /* __LINUX_RCUTINY_H */
include/linux/rcutree.h +1 -0
@@ -30,6 +30,7 @@#ifndef __LINUX_RCUTREE_H#define __LINUX_RCUTREE_H +extern void rcu_init (void );extern void rcu_note_context_switch (int cpu);extern int rcu_needs_cpu (int cpu);extern void rcu_cpu_stall_reset (void );
kernel/rcutiny.c +58 -13
116 lines not on this page · raw message
kernel/rcutiny_plugin.h +8 -7
@@ -22,6 +22,8 @@* Author: Paul E. McKenney <[email protected]> */ +#include <linux/kthread.h> +#ifdef CONFIG_TINY_PREEMPT_RCU #include <linux/delay.h>
@@ -164,9 +166,9 @@ static void rcu_preempt_cpu_qs(void)if (!rcu_preempt_blocked_readers_any ()) rcu_preempt_ctrlblk.rcb.donetail = rcu_preempt_ctrlblk.nexttail; -/* If there are done callbacks, make RCU_SOFTIRQ process them. */ +/* If there are done callbacks, cause them to be invoked. */ if (*rcu_preempt_ctrlblk.rcb.donetail !=NULL ) -raise_softirq (RCU_SOFTIRQ ); +invoke_rcu_cbs (); }/*
@@ -374,7 +376,7 @@ static void rcu_preempt_check_callbacks(void)rcu_preempt_cpu_qs ();if (&rcu_preempt_ctrlblk.rcb.rcucblist != rcu_preempt_ctrlblk.rcb.donetail) -raise_softirq (RCU_SOFTIRQ ); +invoke_rcu_cbs ();if (rcu_preempt_gp_in_progress () &&rcu_cpu_blocking_cur_gp () &&rcu_preempt_running_reader ())
@@ -383,7 +385,7 @@ static void rcu_preempt_check_callbacks(void)/* * TINY_PREEMPT_RCU has an extra callback-list tail pointer to -* update, so this is invoked from __rcu_process_callbacks() to +* update, so this is invoked from rcu_process_callbacks() to * handle that case. Of course, it is invoked for all flavors of * RCU, but RCU callbacks can appear only on one of the lists, and * neither ->nexttail nor ->donetail can possibly be NULL, so there
@@ -400,7 +402,7 @@ static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk *rcp)*/ static void rcu_preempt_process_callbacks (void ) { -__rcu_process_callbacks (&rcu_preempt_ctrlblk.rcb); +rcu_process_callbacks (&rcu_preempt_ctrlblk.rcb); }/*
@@ -599,14 +601,13 @@ static void rcu_preempt_process_callbacks(void)#endif /* #else #ifdef CONFIG_TINY_PREEMPT_RCU */ #ifdef CONFIG_DEBUG_LOCK_ALLOC -#include <linux/kernel_stat.h> /* * During boot, we forgive RCU lockdep issues. After this function is * invoked, we start taking RCU lockdep issues seriously. */ -void rcu_scheduler_starting (void ) +void __init rcu_scheduler_starting (void ) {WARN_ON (nr_context_switches () >0 ); rcu_scheduler_active =1 ;
Signature · 1 line
1.7.0.6
RFC PATCH 3/12 rcu: priority boosting for TINY_PREEMPT_RCU
In reply to Paul E. McKenneyclean compile. FIXME: needs a decent changelog entry.
--- include/linux/init_task.h | 9 ++- include/linux/sched.h | 11 ++- init/Kconfig | 39 +++++++++ kernel/rcutiny.c | 66 ++++++--------- kernel/rcutiny_plugin.h | 206 +++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 279 insertions(+), 52 deletions(-)
include/linux/init_task.h +8 -1
@@ -81,6 +81,12 @@ extern struct group_info init_groups;*/ # define CAP_INIT_BSET CAP_FULL_SET +#ifdef CONFIG_RCU_BOOST +#define INIT_TASK_RCU_BOOST () \ + .rcu_boost_mutex = NULL, +#else +#define INIT_TASK_RCU_BOOST () +#endif #ifdef CONFIG_TREE_PREEMPT_RCU #define INIT_TASK_RCU_TREE_PREEMPT () \ .rcu_blocked_node = NULL,
@@ -92,7 +98,8 @@ extern struct group_info init_groups; .rcu_read_lock_nesting = 0, \ .rcu_read_unlock_special = 0, \ .rcu_node_entry = LIST_HEAD_INIT(tsk.rcu_node_entry), \ - INIT_TASK_RCU_TREE_PREEMPT() + INIT_TASK_RCU_TREE_PREEMPT() \ + INIT_TASK_RCU_BOOST()#else #define INIT_TASK_RCU_PREEMPT (tsk)#endif
include/linux/sched.h +9 -2
@@ -1210,6 +1210,9 @@ struct task_struct {#ifdef CONFIG_TREE_PREEMPT_RCU struct rcu_node *rcu_blocked_node;#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ +#ifdef CONFIG_RCU_BOOST +struct rt_mutex *rcu_boost_mutex; +#endif /* #ifdef CONFIG_RCU_BOOST */ #if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_TASK_DELAY_ACCT )struct sched_info sched_info;
@@ -1745,7 +1748,8 @@ extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *#ifdef CONFIG_PREEMPT_RCU #define RCU_READ_UNLOCK_BLOCKED (1 << 0)/* blocked while in RCU read-side. */ -#define RCU_READ_UNLOCK_NEED_QS (1 << 1)/* RCU core needs CPU response. */ +#define RCU_READ_UNLOCK_BOOSTED (1 << 1)/* boosted while in RCU read-side. */ +#define RCU_READ_UNLOCK_NEED_QS (1 << 2)/* RCU core needs CPU response. */ static inline void rcu_copy_process (struct task_struct *p) {
@@ -1753,7 +1757,10 @@ static inline void rcu_copy_process(struct task_struct *p) p->rcu_read_unlock_special =0 ;#ifdef CONFIG_TREE_PREEMPT_RCU p->rcu_blocked_node =NULL ; -#endif +#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ +#ifdef CONFIG_RCU_BOOST + p->rcu_boost_mutex =NULL ; +#endif /* #ifdef CONFIG_RCU_BOOST */ INIT_LIST_HEAD (&p->rcu_node_entry); }
init/Kconfig +39 -0
@@ -450,6 +450,45 @@ config TREE_RCU_TRACE TREE_PREEMPT_RCU implementations, permitting Makefile to trivially select kernel/rcutree_trace.c. +config RCU_BOOST + bool "Enable RCU priority boosting" + depends on TINY_PREEMPT_RCU + default n + help + This option boosts the priority of preempted RCU readers that + block the current preemptible RCU grace period for too long. + This option also prevents heavy loads from blocking RCU + callback invocation for all flavors of RCU. + + Say Y here if you are working with real-time apps or heavy loads + Say N here if you are unsure. + +config RCU_BOOST_PRIO + int "Real-time priority to boost RCU readers to" + range 1 99 + depends on RCU_BOOST + default 1 + help + This option specifies the real-time priority to which preempted + RCU readers are to be boosted. If you are working with CPU-bound + real-time applications, you should specify a priority higher then + the highest-priority CPU-bound application. + + Specify the real-time priority, or take the default if unsure. + +config RCU_BOOST_DELAY + int "Milliseconds to delay boosting after RCU grace-period start" + range 0 3000 + depends on RCU_BOOST + default 500 + help + This option specifies the time to wait after the beginning of + a given grace period before priority-boosting preempted RCU + readers blocking that grace period. Note that any RCU reader + blocking an expedited RCU grace period is boosted immediately. + + Accept the default if unsure. + endmenu # "RCU Subsystem" config IKCONFIG
kernel/rcutiny.c +26 -40
122 lines not on this page · raw message
kernel/rcutiny_plugin.h +197 -9
286 lines not on this page · raw message
Signature · 1 line
1.7.0.6
RFC PATCH 4/12 rcu: add tracing for TINY_RCU and TINY_PREEMPT_RCU
In reply to Paul E. McKenneyAdd tracing for the tiny RCU implementations, including statistics on boosting in the case of TINY_PREEMPT_RCU and RCU_BOOST.
--- init/Kconfig | 1 - kernel/rcutiny.c | 4 + kernel/rcutiny_plugin.h | 208 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 208 insertions(+), 5 deletions(-)
init/Kconfig +0 -1
@@ -384,7 +384,6 @@ config PREEMPT_RCU config RCU_TRACE bool "Enable tracing for RCU" - depends on TREE_RCU || TREE_PREEMPT_RCU help This option provides tracing in RCU which presents stats in debugfs for debugging RCU implementation.
kernel/rcutiny.c +4 -0
@@ -144,6 +144,7 @@ static void rcu_process_callbacks(struct rcu_ctrlblk *rcp) {struct rcu_head *next, *list;unsigned long flags; + RCU_TRACE(int cb_count = 0);/* If no RCU callbacks ready to invoke, just return. */ if (&rcp->rcucblist == rcp->donetail)
@@ -169,7 +170,9 @@ static void rcu_process_callbacks(struct rcu_ctrlblk *rcp) list->func (list);local_bh_enable (); list = next; +RCU_TRACE (cb_count++); } +RCU_TRACE (rcu_trace_sub_qlen (rcp, cb_count)); }/*
@@ -252,6 +255,7 @@ static void __call_rcu(struct rcu_head *head,local_irq_save (flags); *rcp->curtail = head; rcp->curtail = &head->next; +RCU_TRACE (rcp->qlen++);local_irq_restore (flags); }
kernel/rcutiny_plugin.h +204 -4
284 lines not on this page · raw message
Signature · 1 line
1.7.0.6
RFC PATCH 5/12 rcu: document TINY_RCU and TINY_PREEMPT_RCU tracing.
In reply to Paul E. McKenney--- Documentation/RCU/trace.txt | 132 ++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 124 insertions(+), 8 deletions(-)
Documentation/RCU/trace.txt +124 -8
@@ -1,18 +1,22 @@ CONFIG_RCU_TRACE debugfs Files and Formats -The rcutree implementation of RCU provides debugfs trace output that -summarizes counters and state. This information is useful for debugging -RCU itself, and can sometimes also help to debug abuses of RCU. -The following sections describe the debugfs files and formats. +The rcutree and rcutiny implementations of RCU provide debugfs trace +output that summarizes counters and state. This information is useful for +debugging RCU itself, and can sometimes also help to debug abuses of RCU. +The following sections describe the debugfs files and formats, first +for rcutree and next for rcutiny. -Hierarchical RCU debugfs Files and Formats +CONFIG_TREE_RCU and CONFIG_TREE_PREEMPT_RCU debugfs Files and Formats -This implementation of RCU provides three debugfs files under the +These implementations of RCU provides five debugfs files under the top-level directory RCU: rcu/rcudata (which displays fields in struct -rcu_data), rcu/rcugp (which displays grace-period counters), and -rcu/rcuhier (which displays the struct rcu_node hierarchy). +rcu_data), rcu/rcudata.csv (which is a .csv spreadsheet version of +rcu/rcudata), rcu/rcugp (which displays grace-period counters), +rcu/rcuhier (which displays the struct rcu_node hierarchy), and +rcu/rcu_pending (which displays counts of the reasons that the +rcu_pending() function decided that there was core RCU work to do). The output of "cat rcu/rcudata" looks as follows:
@@ -326,3 +330,115 @@ o "nn" is the number of times that this CPU needed nothing. Alert readers will note that the rcu "nn" number for a given CPU very closely matches the rcu_bh "np" number for that same CPU. This is due to short-circuit evaluation in rcu_pending(). + + +CONFIG_TINY_RCU and CONFIG_TINY_PREEMPT_RCU debugfs Files and Formats + +These implementations of RCU provides a single debugfs file under the +top-level directory RCU, namely rcu/rcudata, which displays fields in +rcu_bh_ctrlblk, rcu_sched_ctrlblk and, for CONFIG_TINY_PREEMPT_RCU, +rcu_preempt_ctrlblk. + +The output of "cat rcu/rcudata" is as follows: + +rcu_preempt: qlen=24 gp=1097669 g197/p197/c197 tasks=... + ttb=. btg=no ntb=184 neb=0 nnb=183 j=01f7 bt=0274 + normal balk: nt=1097669 gt=0 bt=371 b=0 ny=25073378 nos=0 + exp balk: bt=0 nos=0 +rcu_sched: qlen: 0 +rcu_bh: qlen: 0 + +This is split into rcu_preempt, rcu_sched, and rcu_bh sections, with the +rcu_preempt section appearing only in CONFIG_TINY_PREEMPT_RCU builds. +The last three lines of the rcu_preempt section appear only in +CONFIG_RCU_BOOST kernel builds. The fields are as follows: + +o "qlen" is the number of RCU callbacks currently waiting either + for an RCU grace period or waiting to be invoked. This is the + only field present for rcu_sched and rcu_bh, due to the + short-circuiting of grace period in those two cases. + +o "gp" is the number of grace periods that have completed. + +o "g197/p197/c197" displays the grace-period state, with the + "g" number being the number of grace periods that have started + (mod 256), the "p" number being the number of grace periods + that the CPU has responded to (also mod 256), and the "c" + number being the number of grace periods that have completed + (once again mode 256). + + Why have both "gp" and "g"? Because the data flowing into + "gp" is only present in a CONFIG_RCU_TRACE kernel. + +o "tasks" is a set of bits. The first bit is "T" if there are + currently tasks that have recently blocked within an RCU + read-side critical section, the second bit is "N" if any of the + aforementioned tasks are blocking the current RCU grace period, + and the third bit is "E" if any of the aforementioned tasks are + blocking the current expedited grace period. Each bit is "." + if the corresponding condition does not hold. + +o "ttb" is a single bit. It is "B" if any of the blocked tasks + need to be priority boosted and "." otherwise. + +o "btg" indicates whether boosting has been carried out during + the current grace period, with "exp" indicating that boosting + is in progress for an expedited grace period, "no" indicating + that boosting has not yet started for a normal grace period, + "begun" indicating that boosting has bebug for a normal grace + period, and "done" indicating that boosting has completed for + a normal grace period. + +o "ntb" is the total number of tasks subjected to RCU priority boosting + periods since boot. + +o "neb" is the number of expedited grace periods that have had + to resort to RCU priority boosting since boot. + +o "nnb" is the number of normal grace periods that have had + to resort to RCU priority boosting since boot. + +o "j" is the low-order 12 bits of the jiffies counter in hexadecimal. + +o "bt" is the low-order 12 bits of the value that the jiffies counter + will have at the next time that boosting is scheduled to begin. + +o In the line beginning with "normal balk", the fields are as follows: + + o "nt" is the number of times that the system balked from + boosting because there were no blocked tasks to boost. + Note that the system will balk from boosting even if the + grace period is overdue when the currently running task + is looping within an RCU read-side critical section. + There is no point in boosting in this case, because + boosting a running task won't make it run any faster. + + o "gt" is the number of times that the system balked + from boosting because, although there were blocked tasks, + none of them were preventing the current grace period + from completing. + + o "bt" is the number of times that the system balked + from boosting because boosting was already in progress. + + o "b" is the number of times that the system balked from + boosting because boosting had already completed for + the grace period in question. + + o "ny" is the number of times that the system balked from + boosting because it was not yet time to start boosting + the grace period in question. + + o "nos" is the number of times that the system balked from + boosting for inexplicable ("not otherwise specified") + reasons. This can actually happen due to races involving + increments of the jiffies counter. + +o In the line beginning with "exp balk", the fields are as follows: + + o "bt" is the number of times that the system balked from + boosting because there were no blocked tasks to boost. + + o "nos" is the number of times that the system balked from + boosting for inexplicable ("not otherwise specified") + reasons.
Signature · 1 line
1.7.0.6
RFC PATCH 6/12 rcu: Distinguish between boosting and boosted
In reply to Paul E. McKenneyRCU priority boosting's tracing did not distinguish between ongoing boosting and completion of boosting. This commit therefore adds this capability.
--- kernel/rcutiny_plugin.h | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
kernel/rcutiny_plugin.h +4 -0
@@ -229,6 +229,9 @@ static void show_tiny_preempt_stats(struct seq_file *m)seq_puts (m,"no" );break ;case 1 : +seq_puts (m,"begun" ); +break ; +case 2 :seq_puts (m,"done" );break ;default :
@@ -284,6 +287,7 @@ static int rcu_boost(void)raw_local_irq_restore (flags);rt_mutex_lock (&mtx);RCU_TRACE (rcu_preempt_ctrlblk.n_tasks_boosted++); + rcu_preempt_ctrlblk.boosted_this_gp++;rt_mutex_unlock (&mtx);return rcu_preempt_ctrlblk.boost_tasks !=NULL ; }
Signature · 1 line
1.7.0.6
RFC PATCH 7/12 rcu: get rid of obsolete "classic" names in TREE_RCU tracing
In reply to Paul E. McKenneyThe TREE_RCU tracing had obsolete rcuclassic_trace_init() and rcuclassic_trace_cleanup() function names. This commit brings them up to date: rcutree_trace_init() and rcutree_trace_cleanup(), respectively.
--- kernel/rcutree_trace.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-)
kernel/rcutree_trace.c +4 -4
@@ -300,7 +300,7 @@ static const struct file_operations rcu_pending_fops = {static struct dentry *rcudir; -static int __init rcuclassic_trace_init (void ) +static int __init rcutree_trace_init (void ) {struct dentry *retval;
@@ -337,14 +337,14 @@ free_out:return 1 ; } -static void __exit rcuclassic_trace_cleanup(void) +static void __exit rcutree_trace_cleanup(void) {debugfs_remove_recursive (rcudir); } -module_init (rcuclassic_trace_init); -module_exit (rcuclassic_trace_cleanup); +module_init (rcutree_trace_init); +module_exit (rcutree_trace_cleanup);MODULE_AUTHOR ("Paul E. McKenney" );MODULE_DESCRIPTION ("Read-Copy Update tracing for hierarchical implementation" );
Signature · 1 line
1.7.0.6
RFC PATCH 8/12 rcu,cleanup: move synchronize_sched_expedited() out of sched.c
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
The first version of synchronize_sched_expedited() used the migration code in the scheduler, and was therefore implemented in kernel/sched.c. However, the more recent version of this code no longer uses the migration code, so this commit moves it to the main RCU source files.
--- include/linux/rcupdate.h | 1 - include/linux/rcutiny.h | 5 +++ include/linux/rcutree.h | 1 + kernel/rcutree_plugin.h | 71 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched.c | 69 -------------------------------------------- 5 files changed, 77 insertions(+), 70 deletions(-)
include/linux/rcupdate.h +0 -1
@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,extern void synchronize_sched (void );extern void rcu_barrier_bh (void );extern void rcu_barrier_sched (void ); -extern void synchronize_sched_expedited (void );extern int sched_expedited_torture_stats (char *page);static inline void __rcu_read_lock_bh (void )
include/linux/rcutiny.h +5 -0
@@ -60,6 +60,11 @@ static inline void synchronize_rcu_bh_expedited(void)synchronize_sched (); } +static inline void synchronize_sched_expedited (void ) +{ +synchronize_sched (); +} +#ifdef CONFIG_TINY_RCU static inline void rcu_preempt_note_context_switch (void )
include/linux/rcutree.h +1 -0
@@ -48,6 +48,7 @@ static inline void exit_rcu(void)#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ extern void synchronize_rcu_bh (void ); +extern void synchronize_sched_expedited (void );extern void synchronize_rcu_expedited (void );static inline void synchronize_rcu_bh_expedited (void )
kernel/rcutree_plugin.h +71 -0
83 lines not on this page · raw message
kernel/sched.c +0 -69
72 lines not on this page · raw message
Signature · 1 line
1.7.0.6
RFC PATCH 9/12 rcu,cleanup: simplify the code when cpu is dying
In reply to Paul E. McKenneyFrom: Lai Jiangshan <[email protected]>
When we handle the CPU_DYING notifier, the whole system is stopped except for the current CPU. We therefore need no synchronization with the other CPUs. This allows us to move any orphaned RCU callbacks directly to the list of any online CPU without needing to run them through the global orphan lists. These global orphan lists can therefore be dispensed with. This commit makes thes changes, though currently victimizes CPU 0 @@@.
--- kernel/rcutree.c | 81 ++++++++++++---------------------------------- kernel/rcutree.h | 16 ++------- kernel/rcutree_plugin.h | 8 ++-- kernel/rcutree_trace.c | 4 +- 4 files changed, 31 insertions(+), 78 deletions(-)
kernel/rcutree.c +21 -60
@@ -67,9 +67,6 @@ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; .gpnum = -300, \ .completed = -300, \ .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \ - .orphan_cbs_list = NULL, \ - .orphan_cbs_tail = &structname.orphan_cbs_list, \ - .orphan_qlen = 0, \ .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \ .n_force_qs = 0, \ .n_force_qs_ngp = 0, \
@@ -984,53 +981,31 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)#ifdef CONFIG_HOTPLUG_CPU /* -* Move a dying CPU's RCU callbacks to the ->orphan_cbs_list for the -* specified flavor of RCU. The callbacks will be adopted by the next -* _rcu_barrier() invocation or by the CPU_DEAD notifier, whichever -* comes first. Because this is invoked from the CPU_DYING notifier, -* irqs are already disabled. +* Move a dying CPU's RCU callbacks to online CPU's callback list. +* Synchronization is not required because this function executes +* in stop_machine() context. */ -static void rcu_send_cbs_to_orphanage (struct rcu_state *rsp) +static void rcu_send_cbs_to_online (struct rcu_state *rsp) {int i; +/* current DYING CPU is cleared in the cpu_online_mask */ +int receive_cpu =cpumask_any (cpu_online_mask);struct rcu_data *rdp =this_cpu_ptr (rsp->rda); +struct rcu_data *receive_rdp =per_cpu_ptr (rsp->rda, receive_cpu);if (rdp->nxtlist ==NULL )return ;/* irqs disabled, so comparison is stable. */ -raw_spin_lock (&rsp->onofflock);/* irqs already disabled. */ - *rsp->orphan_cbs_tail = rdp->nxtlist; - rsp->orphan_cbs_tail = rdp->nxttail[RCU_NEXT_TAIL ]; + + *receive_rdp->nxttail[RCU_NEXT_TAIL ] = rdp->nxtlist; + receive_rdp->nxttail[RCU_NEXT_TAIL ] = rdp->nxttail[RCU_NEXT_TAIL ]; + receive_rdp->qlen += rdp->qlen; + receive_rdp->n_cbs_adopted += rdp->qlen; + rdp->n_cbs_orphaned += rdp->qlen; + rdp->nxtlist =NULL ;for (i =0 ; i <RCU_NEXT_SIZE ; i++) rdp->nxttail[i] = &rdp->nxtlist; - rsp->orphan_qlen += rdp->qlen; - rdp->n_cbs_orphaned += rdp->qlen; rdp->qlen =0 ; -raw_spin_unlock (&rsp->onofflock);/* irqs remain disabled. */ -} - -/* -* Adopt previously orphaned RCU callbacks. -*/ -static void rcu_adopt_orphan_cbs (struct rcu_state *rsp) -{ -unsigned long flags; -struct rcu_data *rdp; - -raw_spin_lock_irqsave (&rsp->onofflock, flags); - rdp =this_cpu_ptr (rsp->rda); -if (rsp->orphan_cbs_list ==NULL ) { -raw_spin_unlock_irqrestore (&rsp->onofflock, flags); -return ; - } - *rdp->nxttail[RCU_NEXT_TAIL ] = rsp->orphan_cbs_list; - rdp->nxttail[RCU_NEXT_TAIL ] = rsp->orphan_cbs_tail; - rdp->qlen += rsp->orphan_qlen; - rdp->n_cbs_adopted += rsp->orphan_qlen; - rsp->orphan_cbs_list =NULL ; - rsp->orphan_cbs_tail = &rsp->orphan_cbs_list; - rsp->orphan_qlen =0 ; -raw_spin_unlock_irqrestore (&rsp->onofflock, flags); }/*
@@ -1081,8 +1056,6 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)raw_spin_unlock_irqrestore (&rnp->lock, flags);if (need_report &RCU_OFL_TASKS_EXP_GP )rcu_report_exp_rnp (rsp, rnp); - -rcu_adopt_orphan_cbs (rsp); }/*
@@ -1100,11 +1073,7 @@ static void rcu_offline_cpu(int cpu)#else /* #ifdef CONFIG_HOTPLUG_CPU */ -static void rcu_send_cbs_to_orphanage (struct rcu_state *rsp) -{ -} - -static void rcu_adopt_orphan_cbs (struct rcu_state *rsp) +static void rcu_send_cbs_to_online (struct rcu_state *rsp) { }
@@ -1702,10 +1671,7 @@ static void _rcu_barrier(struct rcu_state *rsp,* early. */ atomic_set (&rcu_barrier_cpu_count,1 ); -preempt_disable ();/* stop CPU_DYING from filling orphan_cbs_list */ -rcu_adopt_orphan_cbs (rsp);on_each_cpu (rcu_barrier_func, (void *)call_rcu_func,1 ); -preempt_enable ();/* CPU_DYING can again fill orphan_cbs_list */ if (atomic_dec_and_test (&rcu_barrier_cpu_count))complete (&rcu_barrier_completion);wait_for_completion (&rcu_barrier_completion);
@@ -1831,18 +1797,13 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,case CPU_DYING :case CPU_DYING_FROZEN :/* -* preempt_disable() in _rcu_barrier() prevents stop_machine(), -* so when "on_each_cpu(rcu_barrier_func, (void *)type, 1);" -* returns, all online cpus have queued rcu_barrier_func(). -* The dying CPU clears its cpu_online_mask bit and -* moves all of its RCU callbacks to ->orphan_cbs_list -* in the context of stop_machine(), so subsequent calls -* to _rcu_barrier() will adopt these callbacks and only -* then queue rcu_barrier_func() on all remaining CPUs. +* The whole machine is "stopped" except this cpu, so we can +* touch any data without introducing corruption. And we send +* the callbacks to an attribute chosen online cpu. */ -rcu_send_cbs_to_orphanage (&rcu_bh_state); -rcu_send_cbs_to_orphanage (&rcu_sched_state); -rcu_preempt_send_cbs_to_orphanage (); +rcu_send_cbs_to_online (&rcu_bh_state); +rcu_send_cbs_to_online (&rcu_sched_state); +rcu_preempt_send_cbs_to_online ();break ;case CPU_DEAD :case CPU_DEAD_FROZEN :
kernel/rcutree.h +4 -12
@@ -203,8 +203,8 @@ struct rcu_data {long qlen_last_fqs_check;/* qlen at last check for QS forcing */ unsigned long n_cbs_invoked;/* count of RCU cbs invoked. */ -unsigned long n_cbs_orphaned;/* RCU cbs sent to orphanage. */ -unsigned long n_cbs_adopted;/* RCU cbs adopted from orphanage. */ +unsigned long n_cbs_orphaned;/* RCU cbs orphaned by dying CPU */ +unsigned long n_cbs_adopted;/* RCU cbs adopted from dying CPU */ unsigned long n_force_qs_snap;/* did other CPU force QS recently? */ long blimit;/* Upper limit on a processed batch */
@@ -309,15 +309,7 @@ struct rcu_state {/* End of fields guarded by root rcu_node's lock. */ raw_spinlock_t onofflock;/* exclude on/offline and */ -/* starting new GP. Also */ -/* protects the following */ -/* orphan_cbs fields. */ -struct rcu_head *orphan_cbs_list;/* list of rcu_head structs */ -/* orphaned by all CPUs in */ -/* a given leaf rcu_node */ -/* going offline. */ -struct rcu_head **orphan_cbs_tail;/* And tail pointer. */ -long orphan_qlen;/* Number of orphaned cbs. */ +/* starting new GP. */ raw_spinlock_t fqslock;/* Only one task forcing */ /* quiescent states. */ unsigned long jiffies_force_qs;/* Time at which to invoke */
@@ -390,7 +382,7 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp);static int rcu_preempt_pending (int cpu);static int rcu_preempt_needs_cpu (int cpu);static void __cpuinit rcu_preempt_init_percpu_data (int cpu); -static void rcu_preempt_send_cbs_to_orphanage (void ); +static void rcu_preempt_send_cbs_to_online (void );static void __init __rcu_init_preempt (void );static void rcu_needs_cpu_flush (void );
kernel/rcutree_plugin.h +4 -4
@@ -774,11 +774,11 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu) }/* -* Move preemptable RCU's callbacks to ->orphan_cbs_list. +* Move preemptable DYING RCU's callbacks to other online CPU. */ -static void rcu_preempt_send_cbs_to_orphanage (void ) +static void rcu_preempt_send_cbs_to_online (void ) { -rcu_send_cbs_to_orphanage (&rcu_preempt_state); +rcu_send_cbs_to_online (&rcu_preempt_state); }/*
@@ -1002,7 +1002,7 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu)/* * Because there is no preemptable RCU, there are no callbacks to move. */ -static void rcu_preempt_send_cbs_to_orphanage (void ) +static void rcu_preempt_send_cbs_to_online (void ) { }
kernel/rcutree_trace.c +2 -2
@@ -166,13 +166,13 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp) gpnum = rsp->gpnum;seq_printf (m,"c=%lu g=%lu s=%d jfq=%ld j=%x " -"nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld\n" , +"nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu\n" , rsp->completed, gpnum, rsp->signaled, (long )(rsp->jiffies_force_qs - jiffies), (int )(jiffies &0xffff ), rsp->n_force_qs, rsp->n_force_qs_ngp, rsp->n_force_qs - rsp->n_force_qs_ngp, - rsp->n_force_qs_lh, rsp->orphan_qlen); + rsp->n_force_qs_lh);for (rnp = &rsp->node[0 ]; rnp - &rsp->node[0 ] <NUM_RCU_NODES ; rnp++) {if (rnp->level != level) {seq_puts (m,"\n" );
Signature · 1 line
1.7.0.6
RFC PATCH 10/12 rcu: update documentation/comments for Lai's adoption patch
In reply to Paul E. McKenneyLai's RCU-callback immediate-adoption patch changes the RCU tracing output, so update tracing.txt. Also update a few comments to clarify the synchronization design.
--- Documentation/RCU/trace.txt | 12 ++++-------- kernel/rcutree.c | 10 ++++++---- kernel/rcutree_plugin.h | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-)
Documentation/RCU/trace.txt +4 -8
@@ -134,7 +134,8 @@ o "ci" is the number of RCU callbacks that have been invoked for been registered in absence of CPU-hotplug activity. o "co" is the number of RCU callbacks that have been orphaned due to - this CPU going offline. + this CPU going offline. These orphaned callbacks have been moved + to an arbitrarily chosen online CPU. o "ca" is the number of RCU callbacks that have been adopted due to other CPUs going offline. Note that ci+co-ca+ql is the number of
@@ -172,12 +173,12 @@ o "gpnum" is the number of grace periods that have started. It is The output of "cat rcu/rcuhier" looks as follows, with very long lines: -c=6902 g=6903 s=2 jfq=3 j=72c7 nfqs=13142/nfqsng=0(13142) fqlh=6 oqlen=0 +c=6902 g=6903 s=2 jfq=3 j=72c7 nfqs=13142/nfqsng=0(13142) fqlh=6 1/1 .>. 0:127 ^0 3/3 .>. 0:35 ^0 0/0 .>. 36:71 ^1 0/0 .>. 72:107 ^2 0/0 .>. 108:127 ^3 3/3f .>. 0:5 ^0 2/3 .>. 6:11 ^1 0/0 .>. 12:17 ^2 0/0 .>. 18:23 ^3 0/0 .>. 24:29 ^4 0/0 .>. 30:35 ^5 0/0 .>. 36:41 ^0 0/0 .>. 42:47 ^1 0/0 .>. 48:53 ^2 0/0 .>. 54:59 ^3 0/0 .>. 60:65 ^4 0/0 .>. 66:71 ^5 0/0 .>. 72:77 ^0 0/0 .>. 78:83 ^1 0/0 .>. 84:89 ^2 0/0 .>. 90:95 ^3 0/0 .>. 96:101 ^4 0/0 .>. 102:107 ^5 0/0 .>. 108:113 ^0 0/0 .>. 114:119 ^1 0/0 .>. 120:125 ^2 0/0 .>. 126:127 ^3 rcu_bh: -c=-226 g=-226 s=1 jfq=-5701 j=72c7 nfqs=88/nfqsng=0(88) fqlh=0 oqlen=0 +c=-226 g=-226 s=1 jfq=-5701 j=72c7 nfqs=88/nfqsng=0(88) fqlh=0 0/1 .>. 0:127 ^0 0/3 .>. 0:35 ^0 0/0 .>. 36:71 ^1 0/0 .>. 72:107 ^2 0/0 .>. 108:127 ^3 0/3f .>. 0:5 ^0 0/3 .>. 6:11 ^1 0/0 .>. 12:17 ^2 0/0 .>. 18:23 ^3 0/0 .>. 24:29 ^4 0/0 .>. 30:35 ^5 0/0 .>. 36:41 ^0 0/0 .>. 42:47 ^1 0/0 .>. 48:53 ^2 0/0 .>. 54:59 ^3 0/0 .>. 60:65 ^4 0/0 .>. 66:71 ^5 0/0 .>. 72:77 ^0 0/0 .>. 78:83 ^1 0/0 .>. 84:89 ^2 0/0 .>. 90:95 ^3 0/0 .>. 96:101 ^4 0/0 .>. 102:107 ^5 0/0 .>. 108:113 ^0 0/0 .>. 114:119 ^1 0/0 .>. 120:125 ^2 0/0 .>. 126:127 ^3
@@ -216,11 +217,6 @@ o "fqlh" is the number of calls to force_quiescent_state() that exited immediately (without even being counted in nfqs above) due to contention on ->fqslock. -o "oqlen" is the number of callbacks on the "orphan" callback - list. RCU callbacks are placed on this list by CPUs going - offline, and are "adopted" either by the CPU helping the outgoing - CPU or by the next rcu_barrier*() call, whichever comes first. - o Each element of the form "1/1 0:127 ^0" represents one struct rcu_node. Each line represents one level of the hierarchy, from root to leaves. It is best to think of the rcu_data structures
kernel/rcutree.c +6 -4
@@ -1668,7 +1668,9 @@ static void _rcu_barrier(struct rcu_state *rsp,* decrement rcu_barrier_cpu_count -- otherwise the first CPU * might complete its grace period before all of the other CPUs * did their increment, causing this function to return too -* early. +* early. Note that on_each_cpu() disables irqs, which prevents +* any CPUs from coming online or going offline until each online +* CPU has queued its RCU-barrier callback. */ atomic_set (&rcu_barrier_cpu_count,1 );on_each_cpu (rcu_barrier_func, (void *)call_rcu_func,1 );
@@ -1797,9 +1799,9 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,case CPU_DYING :case CPU_DYING_FROZEN :/* -* The whole machine is "stopped" except this cpu, so we can -* touch any data without introducing corruption. And we send -* the callbacks to an attribute chosen online cpu. +* The whole machine is "stopped" except this CPU, so we can +* touch any data without introducing corruption. We send the +* dying CPU's callbacks to an arbitrarily chosen online CPU. */ rcu_send_cbs_to_online (&rcu_bh_state);rcu_send_cbs_to_online (&rcu_sched_state);
kernel/rcutree_plugin.h +1 -1
@@ -774,7 +774,7 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu) }/* -* Move preemptable DYING RCU's callbacks to other online CPU. +* Move preemptable RCU's callbacks from dying CPU to other online CPU. */ static void rcu_preempt_send_cbs_to_online (void ) {
Signature · 1 line
1.7.0.6
RFC PATCH 11/12 rcu: fix race condition in synchronize_sched_expedited()
In reply to Paul E. McKenneyThe new (early 2010) implementation of synchronize_sched_expedited() uses try_stop_cpu() to force a context switch on every CPU. It also permits concurrent calls to synchronize_sched_expedited() to share a single call to try_stop_cpu() through use of an atomically incremented synchronize_sched_expedited_count variable. Unfortunately, this is subject to failure as follows:
o Task A invokes synchronize_sched_expedited(), try_stop_cpus()
succeeds, but Task A is preempted before getting to the atomic increment of synchronize_sched_expedited_count.
o Task B also invokes synchronize_sched_expedited(), with exactly
the same outcome as Task A.
o Task C also invokes synchronize_sched_expedited(), again with
exactly the same outcome as Tasks A and B.
o Task D also invokes synchronize_sched_expedited(), but only
gets as far as acquiring the mutex within try_stop_cpus() before being preempted, interrupted, or otherwise delayed.
o Task E also invokes synchronize_sched_expedited(), but only
gets to the snapshotting of synchronize_sched_expedited_count.
o Tasks A, B, and C all increment synchronize_sched_expedited_count.
o Task E fails to get the mutex, so checks the new value
of synchronize_sched_expedited_count. It finds that the value has increased, so (wrongly) assumes that its work has been done, returning despite there having been no expedited grace period since it began.
The solution is to have the lowest-numbered CPU atomically increment the synchronize_sched_expedited_count variable within the synchronize_sched_expedited_cpu_stop() function, which is under the protection of the mutex acquired by try_stop_cpus(). However, this also requires that piggybacking tasks wait for three rather than two instances of try_stop_cpu(), because we cannot control the order in which the per-CPU callback function occur.
--- kernel/rcutree_plugin.h | 18 ++++++++++++++++-- 1 files changed, 16 insertions(+), 2 deletions(-)
kernel/rcutree_plugin.h +16 -2
@@ -1041,6 +1041,8 @@ static int synchronize_sched_expedited_cpu_stop(void *data)* robustness against future implementation changes. */ smp_mb ();/* See above comment block. */ +if (cpumask_first (cpu_online_mask) ==smp_processor_id ()) +atomic_inc (&synchronize_sched_expedited_count);return 0 ; }
@@ -1053,13 +1055,26 @@ static int synchronize_sched_expedited_cpu_stop(void *data)* Note that it is illegal to call this function while holding any * lock that is acquired by a CPU-hotplug notifier. Failing to * observe this restriction will result in deadlock. +* +* The synchronize_sched_expedited_cpu_stop() function is called +* in stop-CPU context, but in order to keep overhead down to a dull +* roar, we don't force this function to wait for its counterparts +* on other CPUs. One instance of this function will increment the +* synchronize_sched_expedited_count variable per call to +* try_stop_cpus(), but there is no guarantee what order this instance +* will occur in. The worst case is that it is last on one call +* to try_stop_cpus(), and the first on the next call. This means +* that piggybacking requires that synchronize_sched_expedited_count +* be incremented by 3: this guarantees that the piggybacking +* task has waited through an entire cycle of context switches, +* even in the worst case. */ void synchronize_sched_expedited (void ) {int snap, trycount =0 ;smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ; + snap =atomic_read (&synchronize_sched_expedited_count) +2 ;get_online_cpus (); while (try_stop_cpus(cpu_online_mask, synchronize_sched_expedited_cpu_stop,
@@ -1077,7 +1092,6 @@ void synchronize_sched_expedited(void) }get_online_cpus (); } -atomic_inc (&synchronize_sched_expedited_count);smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ put_online_cpus (); }
Signature · 1 line
1.7.0.6
Re: [PATCH RFC tip/core/rcu 11/12] rcu: fix race condition in synchronize_sched_expedited()
Hello, Paul.
Quoted 32 lines · Paul E. McKenney
The new (early 2010) implementation of
synchronize_sched_expedited()usestry_stop_cpu()to force a context switch on every CPU. It also permits concurrent calls tosynchronize_sched_expedited()to share a single call totry_stop_cpu()through use of an atomically incrementedsynchronize_sched_expedited_countvariable. Unfortunately, this is subject to failure as follows:o Task A invokes
synchronize_sched_expedited(),try_stop_cpus()
succeeds, but Task A is preempted before getting to the atomic increment ofsynchronize_sched_expedited_count.o Task B also invokes
synchronize_sched_expedited(), with exactly
the same outcome as Task A.o Task C also invokes
synchronize_sched_expedited(), again with
exactly the same outcome as Tasks A and B.o Task D also invokes
synchronize_sched_expedited(), but only
gets as far as acquiring the mutex withintry_stop_cpus()before being preempted, interrupted, or otherwise delayed.o Task E also invokes
synchronize_sched_expedited(), but only
gets to the snapshotting ofsynchronize_sched_expedited_count.o Tasks A, B, and C all increment
synchronize_sched_expedited_count.o Task E fails to get the mutex, so checks the new value
ofsynchronize_sched_expedited_count. It finds that the value has increased, so (wrongly) assumes that its work has been done, returning despite there having been no expedited grace period since it began.
Quoted 7 lines
The solution is to have the lowest-numbered CPU atomically increment the
synchronize_sched_expedited_countvariable within thesynchronize_sched_expedited_cpu_stop()function, which is under the protection of the mutex acquired bytry_stop_cpus(). However, this also requires that piggybacking tasks wait for three rather than two instances oftry_stop_cpu(), because we cannot control the order in which the per-CPU callback function occur.
How about something like the following? It's slightly bigger but I think it's a bit easier to understand. Thanks.
kernel/sched.c +35 -6
@@ -9342,7 +9342,8 @@ EXPORT_SYMBOL_GPL(synchronize_sched_expedited);#else /* #ifndef CONFIG_SMP */ -static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); +static atomic_t sync_sched_expedited_token =ATOMIC_INIT (0 ); +static atomic_t sync_sched_expedited_done =ATOMIC_INIT (0 );static int synchronize_sched_expedited_cpu_stop (void *data) {
@@ -9373,11 +9374,18 @@ static int synchronize_sched_expedited_cpu_stop(void *data)*/ void synchronize_sched_expedited (void ) { -int snap, trycount =0 ; +int my_tok, tok, t, trycount =0 ; + +smp_mb ();/* ensure prior mod happens before getting token. */ + +/* +* Get a token. This is used to coordinate with other +* concurrent syncers and consolidate multiple syncs. +*/ + my_tok = tok =atomic_inc_return (&sync_sched_expedited_token); -smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ;get_online_cpus (); +while (try_stop_cpus (cpu_online_mask, synchronize_sched_expedited_cpu_stop,NULL ) == -EAGAIN ) {
@@ -9388,13 +9396,34 @@ void synchronize_sched_expedited(void)synchronize_sched ();return ; } -if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { + +/* +* If the done count reached @my_tok, we know at least +* one synchronization happened since we entered this +* function. +*/ +if (atomic_read (&sync_sched_expedited_done) - my_tok >=0 ) {smp_mb ();/* ensure test happens before caller kfree */ return ; } +get_online_cpus (); + +/* about to retry, get the latest token value */ + tok =atomic_read (&sync_sched_expedited_token); } -atomic_inc (&synchronize_sched_expedited_count); + +/* +* We now know that everything upto @tok is synchronized. +* Update done counter which should always monotonically +* increase (with wrapping considered). +*/ +do { + t =atomic_read (&sync_sched_expedited_done); +if (t - tok >=0 ) +break ; + }while (atomic_cmpxchg (&sync_sched_expedited_done, t, tok) != t); +smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ put_online_cpus (); }
Lai Jiangshan
Acked-byRe: [PATCH RFC tip/core/rcu 11/12] rcu: fix race condition in synchronize_sched_expedited()
Quoted 5 lines · Tejun Heo
Hello, Paul.
How about something like the following? It's slightly bigger but I think it's a bit easier to understand. Thanks.
Hello, Paul, Tejun,
I think this approach is good and much better when several tasks call synchronize_sched_expedited() at the same time.
Quoted 75 lines
kernel/sched.c +35
-6@@ -9342,7 +9342,8 @@ EXPORT_SYMBOL_GPL(synchronize_sched_expedited);#else /* #ifndef CONFIG_SMP */ -static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); +static atomic_t sync_sched_expedited_token =ATOMIC_INIT (0 ); +static atomic_t sync_sched_expedited_done =ATOMIC_INIT (0 );static int synchronize_sched_expedited_cpu_stop (void *data) {@@ -9373,11 +9374,18 @@ static int synchronize_sched_expedited_cpu_stop(void *data)*/ void synchronize_sched_expedited (void ) { -int snap, trycount =0 ; +int my_tok, tok, t, trycount =0 ; + +smp_mb ();/* ensure prior mod happens before getting token. */ + +/* +* Get a token. This is used to coordinate with other +* concurrent syncers and consolidate multiple syncs. +*/ + my_tok = tok =atomic_inc_return (&sync_sched_expedited_token); -smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ;get_online_cpus (); +while (try_stop_cpus (cpu_online_mask, synchronize_sched_expedited_cpu_stop,NULL ) == -EAGAIN ) {@@ -9388,13 +9396,34 @@ void synchronize_sched_expedited(void)synchronize_sched ();return ; } -if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { + +/* +* If the done count reached @my_tok, we know at least +* one synchronization happened since we entered this +* function. +*/ +if (atomic_read (&sync_sched_expedited_done) - my_tok >=0 ) {smp_mb ();/* ensure test happens before caller kfree */ return ; } +get_online_cpus (); + +/* about to retry, get the latest token value */ + tok =atomic_read (&sync_sched_expedited_token); } -atomic_inc (&synchronize_sched_expedited_count); + +/* +* We now know that everything upto @tok is synchronized. +* Update done counter which should always monotonically +* increase (with wrapping considered). +*/ +do { + t =atomic_read (&sync_sched_expedited_done); +if (t - tok >=0 ) +break ; + }while (atomic_cmpxchg (&sync_sched_expedited_done, t, tok) != t); +smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ put_online_cpus (); }
Re: [PATCH RFC tip/core/rcu 11/12] rcu: fix race condition in synchronize_sched_expedited()
Quoted 11 lines · Lai Jiangshan
Quoted 5 lines · Tejun Heo
Hello, Paul.
How about something like the following? It's slightly bigger but I think it's a bit easier to understand. Thanks.
Hello, Paul, Tejun,
I think this approach is good and much better when several tasks call
synchronize_sched_expedited()at the same time.
I am becoming more comfortable with it as well. Tejun, what kind of testing did you do? Lai, could you please run it on your systems?
Thanx, Paul
Quoted 78 lines
Acked-by Lai Jiangshan <[email protected]>Quoted 75 lines
kernel/sched.c +35
-6@@ -9342,7 +9342,8 @@ EXPORT_SYMBOL_GPL(synchronize_sched_expedited);#else /* #ifndef CONFIG_SMP */ -static atomic_t synchronize_sched_expedited_count =ATOMIC_INIT (0 ); +static atomic_t sync_sched_expedited_token =ATOMIC_INIT (0 ); +static atomic_t sync_sched_expedited_done =ATOMIC_INIT (0 );static int synchronize_sched_expedited_cpu_stop (void *data) {@@ -9373,11 +9374,18 @@ static int synchronize_sched_expedited_cpu_stop(void *data)*/ void synchronize_sched_expedited (void ) { -int snap, trycount =0 ; +int my_tok, tok, t, trycount =0 ; + +smp_mb ();/* ensure prior mod happens before getting token. */ + +/* +* Get a token. This is used to coordinate with other +* concurrent syncers and consolidate multiple syncs. +*/ + my_tok = tok =atomic_inc_return (&sync_sched_expedited_token); -smp_mb ();/* ensure prior mod happens before capturing snap. */ - snap =atomic_read (&synchronize_sched_expedited_count) +1 ;get_online_cpus (); +while (try_stop_cpus (cpu_online_mask, synchronize_sched_expedited_cpu_stop,NULL ) == -EAGAIN ) {@@ -9388,13 +9396,34 @@ void synchronize_sched_expedited(void)synchronize_sched ();return ; } -if (atomic_read (&synchronize_sched_expedited_count) - snap >0 ) { + +/* +* If the done count reached @my_tok, we know at least +* one synchronization happened since we entered this +* function. +*/ +if (atomic_read (&sync_sched_expedited_done) - my_tok >=0 ) {smp_mb ();/* ensure test happens before caller kfree */ return ; } +get_online_cpus (); + +/* about to retry, get the latest token value */ + tok =atomic_read (&sync_sched_expedited_token); } -atomic_inc (&synchronize_sched_expedited_count); + +/* +* We now know that everything upto @tok is synchronized. +* Update done counter which should always monotonically +* increase (with wrapping considered). +*/ +do { + t =atomic_read (&sync_sched_expedited_done); +if (t - tok >=0 ) +break ; + }while (atomic_cmpxchg (&sync_sched_expedited_done, t, tok) != t); +smp_mb__after_atomic_inc ();/* ensure post-GP actions seen after GP. */ put_online_cpus (); }
Re: [PATCH RFC tip/core/rcu 11/12] rcu: fix race condition in synchronize_sched_expedited()
Hello, Paul, Lai.
Quoted 8 lines · Paul E. McKenney
Quoted 7 lines · Lai Jiangshan
Quoted 5 lines · Tejun Heo
Hello, Paul.
How about something like the following? It's slightly bigger but I think it's a bit easier to understand. Thanks.
Quoted 7 lines
Quoted 4 lines
Hello, Paul, Tejun,
I think this approach is good and much better when several tasks call
synchronize_sched_expedited()at the same time.I am becoming more comfortable with it as well. Tejun, what kind of testing did you do? Lai, could you please run it on your systems?
I just compile tested it (so no SOB). Please feel free to take it and shape it into a proper patch. Oh, I think we can drop both mb()'s at the top and bottom as both atomic_inc_return() and atomic_cmpxchg() imply full memory barrier.
Thanks.
Signature · 1 line
tejun
Re: [PATCH RFC tip/core/rcu 11/12] rcu: fix race condition in synchronize_sched_expedited()
Quoted 16 lines · Tejun Heo
Hello, Paul, Lai.
Quoted 13 lines · Paul E. McKenney
Quoted 11 lines · Lai Jiangshan
Quoted 5 lines · Tejun Heo
Hello, Paul.
How about something like the following? It's slightly bigger but I think it's a bit easier to understand. Thanks.
Hello, Paul, Tejun,
I think this approach is good and much better when several tasks call
synchronize_sched_expedited()at the same time.
Quoted 7 lines
Quoted 2 lines
I am becoming more comfortable with it as well. Tejun, what kind of testing did you do? Lai, could you please run it on your systems?
I just compile tested it (so no SOB). Please feel free to take it and shape it into a proper patch. Oh, I think we can drop both mb()'s at the top and bottom as both
atomic_inc_return()andatomic_cmpxchg()imply full memory barrier.
Actually, the memory barriers are still one source of discomfort to me. I am concerned about the path out of the function that skips the atomic_cmpxchg(), which seem to happen if some concurrent invocation advances the "done" counter past us before we get around to checking it. I agree on the atomic_inc_return() upon entry to the function, though.
And this is going to need some serious testing either way. ;-)
Thanx, Paul
Re: [PATCH RFC tip/core/rcu 11/12] rcu: fix race condition in synchronize_sched_expedited()
Hello,
Quoted 5 lines · Paul E. McKenney
Actually, the memory barriers are still one source of discomfort to me. I am concerned about the path out of the function that skips the
atomic_cmpxchg(), which seem to happen if some concurrent invocation advances the "done" counter past us before we get around to checking it. I agree on theatomic_inc_return()upon entry to the function, though.
Oooh, you're right. It needs to be smp_mb(). We can put it right before break but I don't think it would worth the ugliness.
Quoted 1 line
And this is going to need some serious testing either way. ;-)
Definitely. This is a scary piece of code. :-)
Signature · 1 line
tejun
RFC PATCH 12/12 rcu: Make synchronize_srcu_expedited() fast if running readers
In reply to Paul E. McKenneyThe synchronize_srcu_expedited() function is currently quick if there are no active readers, but will delay a full jiffy if there are any. If these readers leave their SRCU read-side critical sections quickly, this is way too long to wait. So this commit first waits ten microseconds, and only then falls back to jiffy-at-a-time waiting.
--- kernel/srcu.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-)
kernel/srcu.c +7 -1
@@ -31,6 +31,7 @@#include <linux/rcupdate.h> #include <linux/sched.h> #include <linux/smp.h> +#include <linux/delay.h> #include <linux/srcu.h> static int init_srcu_struct_fields (struct srcu_struct *sp)
@@ -203,9 +204,14 @@ static void __synchronize_srcu(struct srcu_struct *sp, void (*sync_func)(void))* all srcu_read_lock() calls using the old counters have completed. * Their corresponding critical sections might well be still * executing, but the srcu_read_lock() primitives themselves -* will have finished executing. +* will have finished executing. We initially give readers +* an arbitrarily chosen 10 microseconds to get out of their +* SRCU read-side critical sections, then loop waiting 1/HZ +* seconds per iteration. */ +if (srcu_readers_active_idx (sp, idx)) +udelay (10 );while (srcu_readers_active_idx (sp, idx))schedule_timeout_interruptible (1 );
Signature · 1 line
1.7.0.6