RCU fixes
Subsystems (3)
Linus,
Please pull the latest core-urgent-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git core-urgent-for-linus
This is mainly the (functional) revert to softirqs, with no kthreads left. I'm sending this now after just one day of testing thinking that we'd probably want this in -rc4.
Thanks,
Ingo
------------------>
Paul E. McKenney (3):
rcu: Simplify curing of load woes
rcu: use softirq instead of kthreads except when RCU_BOOST=y
rcu: Move RCU_BOOST #ifdefs to header file
Shaohua Li (1):
rcu: Use softirq to address performance regression
Documentation/filesystems/proc.txt | 1 + include/linux/interrupt.h | 1 + include/trace/events/irq.h | 3 +- kernel/rcutree.c | 398 ++------------------------------- kernel/rcutree.h | 12 +- kernel/rcutree_plugin.h | 419 ++++++++++++++++++++++++++++++++++- kernel/rcutree_trace.c | 32 ++- kernel/softirq.c | 2 +- tools/perf/util/trace-event-parse.c | 1 + 9 files changed, 466 insertions(+), 403 deletions(-)
Documentation/filesystems/proc.txt +1 -0
@@ -843,6 +843,7 @@ Provides counts of softirq handlers serviced since boot time, for each cpu. TASKLET: 0 0 0 290 SCHED: 27035 26983 26971 26746 HRTIMER: 0 0 0 0 + RCU: 1678 1769 2178 2250 1.3 IDE devices in /proc/ide
include/linux/interrupt.h +1 -0
@@ -414,6 +414,7 @@ enumTASKLET_SOFTIRQ ,SCHED_SOFTIRQ ,HRTIMER_SOFTIRQ , +RCU_SOFTIRQ ,/* Preferable RCU should always be the last softirq */ NR_SOFTIRQS };
include/trace/events/irq.h +2 -1
@@ -20,7 +20,8 @@ struct softirq_action;softirq_name (BLOCK_IOPOLL ), \softirq_name (TASKLET ), \softirq_name (SCHED ), \ - softirq_name(HRTIMER)) +softirq_name (HRTIMER ), \ + softirq_name(RCU))/** * irq_handler_entry - called immediately before the irq action handler
kernel/rcutree.c +19 -379
473 lines not on this page · raw message
kernel/rcutree.h +11 -1
@@ -369,6 +369,7 @@ struct rcu_state {/* period because */ /* force_quiescent_state() */ /* was running. */ +u8 boost;/* Subject to priority boost. */ unsigned long gpnum;/* Current gp number. */ unsigned long completed;/* # of last completed gp. */
@@ -426,6 +427,7 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);#ifdef CONFIG_HOTPLUG_CPU static void rcu_report_unblock_qs_rnp (struct rcu_node *rnp,unsigned long flags); +static void rcu_stop_cpu_kthread (int cpu);#endif /* #ifdef CONFIG_HOTPLUG_CPU */ static void rcu_print_detail_task_stall (struct rcu_state *rsp);static void rcu_print_task_stall (struct rcu_node *rnp);
@@ -450,11 +452,19 @@ static void rcu_preempt_send_cbs_to_online(void);static void __init __rcu_init_preempt (void );static void rcu_needs_cpu_flush (void );static void rcu_initiate_boost (struct rcu_node *rnp,unsigned long flags); +static void rcu_preempt_boost_start_gp (struct rcu_node *rnp); +static void invoke_rcu_callbacks_kthread (void ); +#ifdef CONFIG_RCU_BOOST +static void rcu_preempt_do_callbacks (void );static void rcu_boost_kthread_setaffinity (struct rcu_node *rnp,cpumask_var_t cm); -static void rcu_preempt_boost_start_gp (struct rcu_node *rnp);static int __cpuinit rcu_spawn_one_boost_kthread (struct rcu_state *rsp,struct rcu_node *rnp,int rnp_index); +static void invoke_rcu_node_kthread (struct rcu_node *rnp); +static void rcu_yield (void (*f)(unsigned long ),unsigned long arg); +#endif /* #ifdef CONFIG_RCU_BOOST */ +static void rcu_cpu_kthread_setrt (int cpu,int to_rt); +static void __cpuinit rcu_prepare_kthreads (int cpu);#endif /* #ifndef RCU_TREE_NONCORE */
kernel/rcutree_plugin.h +407 -12
475 lines not on this page · raw message
kernel/rcutree_trace.c +23 -9
@@ -46,6 +46,8 @@#define RCU_TREE_NONCORE #include "rcutree.h" +#ifdef CONFIG_RCU_BOOST + DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_cpu); DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
@@ -58,6 +60,8 @@ static char convert_kthread_status(unsigned int kthread_status) return "SRWOY"[kthread_status]; } +#endif /* #ifdef CONFIG_RCU_BOOST */ +static void print_one_rcu_data (struct seq_file *m,struct rcu_data *rdp) {if (!rdp->beenonline)
@@ -76,7 +80,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) rdp->dynticks_fqs);#endif /* #ifdef CONFIG_NO_HZ */ seq_printf (m," of=%lu ri=%lu" , rdp->offline_fqs, rdp->resched_ipi); - seq_printf(m, " ql=%ld qs=%c%c%c%c kt=%d/%c/%d ktl=%x b=%ld", + seq_printf(m, " ql=%ld qs=%c%c%c%c", rdp->qlen, ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] != rdp->nxttail[RCU_NEXT_TAIL]],
@@ -84,13 +88,16 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) rdp->nxttail[RCU_NEXT_READY_TAIL]],".W" [rdp->nxttail[RCU_DONE_TAIL ] != rdp->nxttail[RCU_WAIT_TAIL ]], -".D" [&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL ]], + ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]]); +#ifdef CONFIG_RCU_BOOST +seq_printf (m," kt=%d/%c/%d ktl=%x" ,per_cpu (rcu_cpu_has_work, rdp->cpu),convert_kthread_status (per_cpu (rcu_cpu_kthread_status, rdp->cpu)),per_cpu (rcu_cpu_kthread_cpu, rdp->cpu), -per_cpu (rcu_cpu_kthread_loops, rdp->cpu) &0xffff , - rdp->blimit); +per_cpu (rcu_cpu_kthread_loops, rdp->cpu) &0xffff ); +#endif /* #ifdef CONFIG_RCU_BOOST */ +seq_printf (m," b=%ld" , rdp->blimit);seq_printf (m," ci=%lu co=%lu ca=%lu\n" , rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted); }
@@ -147,18 +154,21 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp) rdp->dynticks_fqs);#endif /* #ifdef CONFIG_NO_HZ */ seq_printf (m,",%lu,%lu" , rdp->offline_fqs, rdp->resched_ipi); -seq_printf (m,",%ld,\"%c%c%c%c\",%d,\"%c\",%ld" , rdp->qlen, +seq_printf (m,",%ld,\"%c%c%c%c\"" , rdp->qlen,".N" [rdp->nxttail[RCU_NEXT_READY_TAIL ] != rdp->nxttail[RCU_NEXT_TAIL ]],".R" [rdp->nxttail[RCU_WAIT_TAIL ] != rdp->nxttail[RCU_NEXT_READY_TAIL ]],".W" [rdp->nxttail[RCU_DONE_TAIL ] != rdp->nxttail[RCU_WAIT_TAIL ]], -".D" [&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL ]], +".D" [&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL ]]); +#ifdef CONFIG_RCU_BOOST +seq_printf (m,",%d,\"%c\"" ,per_cpu (rcu_cpu_has_work, rdp->cpu),convert_kthread_status (per_cpu (rcu_cpu_kthread_status, - rdp->cpu)), - rdp->blimit); + rdp->cpu))); +#endif /* #ifdef CONFIG_RCU_BOOST */ +seq_printf (m,",%ld" , rdp->blimit);seq_printf (m,",%lu,%lu,%lu\n" , rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted); }
@@ -169,7 +179,11 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)#ifdef CONFIG_NO_HZ seq_puts (m,"\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\"," );#endif /* #ifdef CONFIG_NO_HZ */ -seq_puts (m,"\"of\",\"ri\",\"ql\",\"b\",\"ci\",\"co\",\"ca\"\n" ); +seq_puts (m,"\"of\",\"ri\",\"ql\",\"qs\"" ); +#ifdef CONFIG_RCU_BOOST +seq_puts (m,"\"kt\",\"ktl\"" ); +#endif /* #ifdef CONFIG_RCU_BOOST */ +seq_puts (m,",\"b\",\"ci\",\"co\",\"ca\"\n" );#ifdef CONFIG_TREE_PREEMPT_RCU seq_puts (m,"\"rcu_preempt:\"\n" );PRINT_RCU_DATA (rcu_preempt_data, print_one_rcu_data_csv, m);
kernel/softirq.c +1 -1
@@ -58,7 +58,7 @@ DEFINE_PER_CPU(struct task_struct *, ksoftirqd);char *softirq_to_name[NR_SOFTIRQS ] = {"HI" ,"TIMER" ,"NET_TX" ,"NET_RX" ,"BLOCK" ,"BLOCK_IOPOLL" , -"TASKLET" ,"SCHED" ,"HRTIMER" +"TASKLET" ,"SCHED" ,"HRTIMER" ,"RCU" };/*
tools/perf/util/trace-event-parse.c +1 -0
@@ -2187,6 +2187,7 @@ static const struct flag flags[] = { { "TASKLET_SOFTIRQ", 6 }, { "SCHED_SOFTIRQ", 7 }, { "HRTIMER_SOFTIRQ", 8 }, + { "RCU_SOFTIRQ", 9 }, { "HRTIMER_NORESTART", 0 }, { "HRTIMER_RESTART", 1 },