Alternative RCU implementation for short grace periods and realtime response
PATCH 1/1 Alternative RCU implementation for short grace periods and realtime response
Hello!
The issue of realtime response and RCU short grace periods arose again recently:
http://marc.theaimsgroup.com/?l=linux-kernel&m=112938253403552&w=2
and was resolved (in the short term) in favor for short grace periods. This patch provides an alternative RCU implementation with limited scalability, but which provides both short grace periods and realtime response. The default RCU implementation remains "classic RCU" (as it should unless/until the performance and scalability of the alternative counter-based RCU implentation improves to the point that it is indistinguishable from that of classic RCU -- and, yes, there is work going on in this direction).
This alternative implementation can be selected with a config parameter (CONFIG_RCU_COUNTER vs. the default CONFIG_RCU_CLASSIC), and provides the hooks needed for the RCU torture tests. This implementation passes the torture tests on a number of platforms, and has seen extensive use in Ingo Molnar's -rt tree over the past few months.
Signed-off-by <[email protected]>
include/linux/rcupdate.h | 25 ++ include/linux/sched.h | 5 kernel/Kconfig.preempt | 33 +++ kernel/Makefile | 4 kernel/rcucommon.h | 73 +++++++ kernel/rcucounter.c | 481 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/rcupdate.c | 63 ------ 7 files changed, 627 insertions(+), 57 deletions(-)
include/linux/rcupdate.h +25 -0
@@ -59,6 +59,7 @@ struct rcu_head { } while (0) +#ifdef CONFIG_RCU_CLASSIC /* Global control variables for rcupdate callback mechanism. */ struct rcu_ctrlblk {
@@ -193,6 +194,18 @@ static inline int rcu_pending(int cpu)*/ #define rcu_read_unlock () preempt_enable() +#else /* #ifdef CONFIG_RCU_CLASSIC */ + +#define rcu_qsctr_inc (cpu) +#define rcu_bh_qsctr_inc (cpu) +#define call_rcu_bh (head, rcu) call_rcu(head, rcu) + +extern void rcu_read_lock (void ); +extern void rcu_read_unlock (void ); +extern int rcu_pending (int cpu); + +#endif /* #else #ifdef CONFIG_RCU_CLASSIC */ +/* * So where is rcu_write_lock()? It does not exist, as there is no * way for writers to lock out RCU readers. This is a feature, not
@@ -214,14 +227,22 @@ static inline int rcu_pending(int cpu)* can use just rcu_read_lock(). * */ +#ifdef CONFIG_RCU_CLASSIC #define rcu_read_lock_bh () local_bh_disable() +#else /* #ifdef CONFIG_RCU_CLASSIC */ +#define rcu_read_lock_bh () { rcu_read_lock(); local_bh_disable(); } +#endif /* #else #ifdef CONFIG_RCU_CLASSIC */ /* * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section * * See rcu_read_lock_bh() for more information. */ +#ifdef CONFIG_RCU_CLASSIC #define rcu_read_unlock_bh () local_bh_enable() +#else /* #ifdef CONFIG_RCU_CLASSIC */ +#define rcu_read_unlock_bh () { local_bh_enable(); rcu_read_unlock(); } +#endif /* #else #ifdef CONFIG_RCU_CLASSIC */ /** * rcu_dereference - fetch an RCU-protected pointer in an
@@ -270,7 +291,11 @@ static inline int rcu_pending(int cpu)* synchronize_kernel() API. In contrast, synchronize_rcu() only * guarantees that rcu_read_lock() sections will have completed. */ +#ifdef CONFIG_RCU_CLASSIC #define synchronize_sched () synchronize_rcu() +#else /* #ifdef CONFIG_RCU_CLASSIC */ +extern void synchronize_sched (void ); +#endif /* #else #ifdef CONFIG_RCU_CLASSIC */ extern void rcu_init (void );extern void rcu_check_callbacks (int cpu,int user);
include/linux/sched.h +5 -0
@@ -666,6 +666,11 @@ struct task_struct {cpumask_t cpus_allowed;unsigned int time_slice, first_time_slice; +#ifdef CONFIG_RCU_COUNTER +int rcu_read_lock_nesting; +atomic_t *rcu_flipctr1; +atomic_t *rcu_flipctr2; +#endif #ifdef CONFIG_SCHEDSTATS struct sched_info sched_info;#endif
kernel/Kconfig.preempt +33 -0
@@ -63,3 +63,36 @@ config PREEMPT_BKL Say Y here if you are building a kernel for a desktop system. Say N if you are unsure. +choice + prompt "RCU infrastructure implementation" + default RCU_CLASSIC + +config RCU_CLASSIC + bool "Classic RCU implementation" + help + This is the classic Linux implementation of the RCU + infrastructure, intended for excellent performance and + scalability on machines that are not severely memory-constrained + and that do not require aggressive realtime latencies. + + Select this option if you are building a kernel for a server or + scientific/computation system, or if you want to maximize the + raw processing power of the kernel, irrespective of scheduling + latencies. + +config RCU_COUNTER + bool "Counter-based RCU implementation" + help + This implementation of the RCU infrastructure manipulates + per-CPU counters to mark the beginnings and ends of RCU + read-side critical sections. It is intended for aggressive + realtime environments and for machines that are either severely + memory-constrained or subject to exteme denial-of-service + attacks. Note that this implementation permits RCU + read-side critical sections to be preempted. + + Select this if you are experimenting with the new RCU + implementation or if RCU_CLASSIC is not meeting your realtime + needs or your memory constraints. + +endchoice
kernel/Makefile +3 -1
@@ -6,7 +6,7 @@ obj-y = sched.o fork.o exec_domain.o exit.o itimer.o time.o softirq.o resource.o \ sysctl.o capability.o ptrace.o timer.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o \ - rcupdate.o intermodule.o extable.o params.o posix-timers.o \ + intermodule.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o obj-$(CONFIG_FUTEX) += futex.o
@@ -32,6 +32,8 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softl obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_SECCOMP) += seccomp.o +obj-$(CONFIG_RCU_CLASSIC) += rcupdate.o +obj-$(CONFIG_RCU_COUNTER) += rcucounter.o ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) # According to Alan Modra <[email protected]>, the -fno-omit-frame-pointer is
kernel/rcucommon.h +73 -0
73 lines not on this page · raw message
kernel/rcucounter.c +481 -0
481 lines not on this page · raw message
kernel/rcupdate.c +7 -56
96 lines not on this page · raw message