NMI-safe lightweight parallel-update -rt RCU
PATCH 1/1 NMI-safe lightweight parallel-update -rt RCU
Hello!
Just work in progress, not recommended for inclusion. Seems stable under rigorous rcutorture testing, so should be OK for experimentation.
This snapshot has an implementation of rcu_read_lock() and rcu_read_unlock() that may be invoked from NMI/SMI handlers, and that do not contain any heavyweight atomics or memory barriers (though they still do momentarily disable IRQs). The grace-period computation is now fully parallel, which will become important with upcoming multicore CPUs.
Next steps: (1) port to -mm. (2) bring forward to latest -rt. (3) remove ugly debug code. (4) integrate with CPU hotplug (which should finally be easy to do). (5) integrate with dynticks, hopefully so as to not require waking up dynticked CPUs unnecessarily. (6) get RCU priority boosting re-merged (which I have done for testing purposes a couple of times along the way). (7) apply some fixes to keep malevolent compilers from optimizing RCU out of existence (e.g., volatile cast in rcu_dereference() -- been working with the concurrency people in the C/C++ standards committees, and they have an amazingly permissive attitude towards compiler optimizations...). Not a complete list, and not necessarily in this order.
Thoughts?
Thanx, Paul
---
include/linux/rcuclassic.h | 3 include/linux/rcupreempt.h | 2 include/linux/rcupreempt_trace.h | 36 + include/linux/sched.h | 3 kernel/rcuclassic.c | 2 kernel/rcupreempt.c | 735 +++++++++++++++++++++++++++------------ kernel/rcupreempt_trace.c | 127 ++++++ kernel/rcutorture.c | 13 8 files changed, 683 insertions(+), 238 deletions(-)
include/linux/rcuclassic.h +0 -3
@@ -144,8 +144,5 @@ extern void rcu_check_callbacks(int cpu,extern void rcu_restart_cpu (int cpu);extern long rcu_batches_completed (void ); -struct softirq_action ; -extern void rcu_process_callbacks (struct softirq_action *unused); -#endif /* __KERNEL__ */ #endif /* __LINUX_RCUCLASSIC_H */
include/linux/rcupreempt.h +0 -2
@@ -65,7 +65,5 @@ extern long rcu_batches_completed(void);struct softirq_action ; -extern void rcu_process_callbacks (struct softirq_action *unused); -#endif /* __KERNEL__ */ #endif /* __LINUX_RCUPREEMPT_H */
include/linux/rcupreempt_trace.h +26 -10
@@ -53,12 +53,20 @@ struct rcupreempt_trace {long done_remove;atomic_t done_invoked;long rcu_check_callbacks; -atomic_t rcu_try_flip1; -long rcu_try_flip2; -long rcu_try_flip3; +atomic_t rcu_try_flip_1;atomic_t rcu_try_flip_e1; -long rcu_try_flip_e2; -long rcu_try_flip_e3; +long rcu_try_flip_i1; +long rcu_try_flip_ie1; +long rcu_try_flip_g1; +long rcu_try_flip_a1; +long rcu_try_flip_ae1; +long rcu_try_flip_a2; +long rcu_try_flip_z1; +long rcu_try_flip_ze1; +long rcu_try_flip_z2; +long rcu_try_flip_m1; +long rcu_try_flip_me1; +long rcu_try_flip_m2; };#ifdef CONFIG_RCU_TRACE
@@ -69,12 +77,20 @@ struct rcupreempt_trace {extern void rcupreempt_trace_move2done (struct rcupreempt_trace *trace);extern void rcupreempt_trace_move2wait (struct rcupreempt_trace *trace); -extern void rcupreempt_trace_try_flip1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_1 (struct rcupreempt_trace *trace);extern void rcupreempt_trace_try_flip_e1 (struct rcupreempt_trace *trace); -extern void rcupreempt_trace_try_flip_e2 (struct rcupreempt_trace *trace); -extern void rcupreempt_trace_try_flip_e3 (struct rcupreempt_trace *trace); -extern void rcupreempt_trace_try_flip2 (struct rcupreempt_trace *trace); -extern void rcupreempt_trace_try_flip3 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_i1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_ie1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_g1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_a1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_ae1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_a2 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_z1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_ze1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_z2 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_m1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_me1 (struct rcupreempt_trace *trace); +extern void rcupreempt_trace_try_flip_m2 (struct rcupreempt_trace *trace);extern void rcupreempt_trace_check_callbacks (struct rcupreempt_trace *trace);extern void rcupreempt_trace_done_remove (struct rcupreempt_trace *trace);extern void rcupreempt_trace_invoke (struct rcupreempt_trace *trace);
include/linux/sched.h +1 -2
@@ -1034,8 +1034,7 @@ struct task_struct {#ifdef CONFIG_PREEMPT_RCU int rcu_read_lock_nesting; -atomic_t *rcu_flipctr1; -atomic_t *rcu_flipctr2; +int rcu_flipctr_idx;#endif #if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_TASK_DELAY_ACCT )
kernel/rcuclassic.c +1 -1
@@ -424,7 +424,7 @@ static void __rcu_process_callbacks(strurcu_do_batch (rdp); } -void rcu_process_callbacks (struct softirq_action *unused) +static void rcu_process_callbacks (struct softirq_action *unused) {__rcu_process_callbacks (&rcu_ctrlblk, &__get_cpu_var (rcu_data));__rcu_process_callbacks (&rcu_bh_ctrlblk, &__get_cpu_var (rcu_bh_data));
kernel/rcupreempt.c +525 -210
899 lines not on this page · raw message
kernel/rcupreempt_trace.c +117 -10
149 lines not on this page · raw message
kernel/rcutorture.c +13 -0
@@ -258,6 +258,15 @@ static void rcu_torture_deferred_free(stcall_rcu (&p->rtort_rcu, rcu_torture_cb); } +#if 1 +static int rcu_stats (char *page) +{ +int rcu_preempt_trace_dump_stats (char *page); + +return rcu_preempt_trace_dump_stats (page); +} +#endif /* #if 1 */ +static struct rcu_torture_ops rcu_ops = { .init =NULL , .cleanup =NULL ,
@@ -267,7 +276,11 @@ static struct rcu_torture_ops rcu_ops = .completed = rcu_torture_completed, .deferredfree = rcu_torture_deferred_free, .sync = synchronize_rcu, +#if 1 + .stats = rcu_stats, +#else /* #if 1 */ .stats = NULL, +#endif /* #else #if 1 */ .name = "rcu" };
Quoted 5 lines · Paul E. McKenney
Hello!
Just work in progress, not recommended for inclusion. Seems stable under rigorous rcutorture testing, so should be OK for experimentation.
nice stuff!
i'm wondering, could you try to do this as a splitup against -mm? I think we should get people exposed to the idea of RCU_PREEMPT in the upstream kernel a bit more. The netfilter folks promised some preparatory enhancements to make it easier, a few months ago :)
Ingo
Quoted 9 lines · Ingo Molnar
Quoted 5 lines · Paul E. McKenney
Hello!
Just work in progress, not recommended for inclusion. Seems stable under rigorous rcutorture testing, so should be OK for experimentation.
Quoted 6 lines
nice stuff!
i'm wondering, could you try to do this as a splitup against -mm? I think we should get people exposed to the idea of
RCU_PREEMPTin the upstream kernel a bit more. The netfilter folks promised some preparatory enhancements to make it easier, a few months ago :)
Good point!
Besides, I might as well test out my assertion that I have made integration with CPU hotplug easier, being as that was one of the reasons that it got kicked out earlier this year, IIRC. ;-)
Thanx, Paul