RCU: Preemptible RCU
Subsystems (5)
cover RCU: Preemptible RCU
Work in progress, still not for inclusion.
This is a respin of the following prior postings:
http://lkml.org/lkml/2007/8/7/276 (the four initial preemptible RCU patches)
http://lkml.org/lkml/2007/8/17/262 (hotplug CPU for preemptible RCU)
http://lkml.org/lkml/2007/8/22/348 (RCU priority boosting)
http://lkml.org/lkml/2007/8/22/373 (rcutorture for RCU priority boosting)
This release adds an additional patch that makes the RCU priority booster more friendly to power-savings measures by causing the booster task to sleep if there is no cause to believe that priority boosting will be required in the near future. An additional fixup and documentation patch is forthcoming. And of course, the whole thing will be rebased to a more recent version of Linux.
These patches take prior feedback into account, including the bit about running checkpatch.pl. ;-)
Thanx, Paul
RFC PATCH 1/8 RCU: Split API to permit multiple RCU implementations
Work in progress, not for inclusion.
This patch re-organizes the RCU code to enable multiple implementations of RCU. Users of RCU continues to include rcupdate.h and the RCU interfaces remain the same. This is in preparation for subsequently merging the preemptible RCU implementation.
Signed-off-by Dipankar Sarma <[email protected]>
Signed-off-by Paul E. McKenney <[email protected]>
---
include/linux/rcuclassic.h | 149 +++++++++++ include/linux/rcupdate.h | 151 +++--------- kernel/Makefile | 2 kernel/rcuclassic.c | 558 ++++++++++++++++++++++++++++++++++++++++++++ kernel/rcupdate.c | 561 ++------------------------------------------- 5 files changed, 779 insertions(+), 642 deletions(-)
include/linux/rcuclassic.h +149 -0
@@ -0,0 +1,149 @@ +/* +* Read-Copy Update mechanism for mutual exclusion (classic version) +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* +* Copyright IBM Corporation, 2001 +* +* Author: Dipankar Sarma <[email protected]> +* +* Based on the original work by Paul McKenney <[email protected]> +* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. +* Papers: +* http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf +* http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) +* +* For detailed explanation of Read-Copy Update mechanism see - +* Documentation/RCU +* +*/ + +#ifndef __LINUX_RCUCLASSIC_H +#define __LINUX_RCUCLASSIC_H + +#ifdef __KERNEL__ + +#include <linux/cache.h> +#include <linux/spinlock.h> +#include <linux/threads.h> +#include <linux/percpu.h> +#include <linux/cpumask.h> +#include <linux/seqlock.h> + + +/* Global control variables for rcupdate callback mechanism. */ +struct rcu_ctrlblk { +long cur;/* Current batch number. */ +long completed;/* Number of the last completed batch */ +int next_pending;/* Is the next batch already waiting? */ + +int signaled; + +spinlock_t lock____cacheline_internodealigned_in_smp ; +cpumask_t cpumask;/* CPUs that need to switch in order */ +/* for current batch to proceed. */ +}____cacheline_internodealigned_in_smp ; + +/* Is batch a before batch b ? */ +static inline int rcu_batch_before (long a,long b) +{ +return (a - b) <0 ; +} + +/* Is batch a after batch b ? */ +static inline int rcu_batch_after (long a,long b) +{ +return (a - b) >0 ; +} + +/* +* Per-CPU data for Read-Copy UPdate. +* nxtlist - new callbacks are added here +* curlist - current batch for which quiescent cycle started if any +*/ +struct rcu_data { +/* 1) quiescent state handling : */ +long quiescbatch;/* Batch # for grace period */ +int passed_quiesc;/* User-mode/idle loop etc. */ +int qs_pending;/* core waits for quiesc state */ + +/* 2) batch handling */ +long batch;/* Batch # for current RCU batch */ +struct rcu_head *nxtlist; +struct rcu_head **nxttail; +long qlen;/* # of queued callbacks */ +struct rcu_head *curlist; +struct rcu_head **curtail; +struct rcu_head *donelist; +struct rcu_head **donetail; +long blimit;/* Upper limit on a processed batch */ +int cpu; +struct rcu_head barrier; +}; + +DECLARE_PER_CPU(struct rcu_data, rcu_data); +DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); + +/* +* Increment the quiescent state counter. +* The counter is a bit degenerated: We do not need to know +* how many quiescent states passed, just if there was at least +* one since the start of the grace period. Thus just a flag. +*/ +static inline void rcu_qsctr_inc (int cpu) +{ +struct rcu_data *rdp = &per_cpu (rcu_data, cpu); + rdp->passed_quiesc =1 ; +} +static inline void rcu_bh_qsctr_inc (int cpu) +{ +struct rcu_data *rdp = &per_cpu (rcu_bh_data, cpu); + rdp->passed_quiesc =1 ; +} + +extern int rcu_pending (int cpu); +extern int rcu_needs_cpu (int cpu); + +#define __rcu_read_lock () \ + do { \ + preempt_disable(); \ + __acquire(RCU); \ + } while (0) +#define __rcu_read_unlock () \ + do { \ + __release(RCU); \ + preempt_enable(); \ + } while (0) +#define __rcu_read_lock_bh () \ + do { \ + local_bh_disable(); \ + __acquire(RCU_BH); \ + } while (0) +#define __rcu_read_unlock_bh () \ + do { \ + __release(RCU_BH); \ + local_bh_enable(); \ + } while (0) + +#define __synchronize_sched () synchronize_rcu() + +extern void __rcu_init (void ); +extern void rcu_check_callbacks (int cpu,int user); +extern void rcu_restart_cpu (int cpu); +extern long rcu_batches_completed (void ); +extern long rcu_batches_completed_bh (void ); + +#endif /* __KERNEL__ */ +#endif /* __LINUX_RCUCLASSIC_H */
include/linux/rcupdate.h +45 -106
204 lines not on this page · raw message
kernel/Makefile +1 -1
@@ -6,7 +6,7 @@ obj-y = sched.o fork.o exec_domain.o exit.o itimer.o time.o softirq.o resource.o \ sysctl.o capability.o ptrace.o timer.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o \ - rcupdate.o extable.o params.o posix-timers.o \ + rcupdate.o rcuclassic.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o latency.o nsproxy.o srcu.o die_notifier.o
kernel/rcuclassic.c +558 -0
558 lines not on this page · raw message
kernel/rcupdate.c +26 -535
623 lines not on this page · raw message
RFC PATCH 2/8 RCU: Fix barriers
Work in progress, not for inclusion.
Fix rcu_barrier() to work properly in preemptive kernel environment. Also, the ordering of callback must be preserved while moving callbacks to another CPU during CPU hotplug.
Signed-off-by Dipankar Sarma <[email protected]>
Signed-off-by Paul E. McKenney <[email protected]>
---
rcuclassic.c | 2 +- rcupdate.c | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-)
kernel/rcuclassic.c +1 -1
@@ -349,9 +349,9 @@ static void __rcu_offline_cpu(struct rcuif (rcp->cur != rcp->completed)cpu_quiet (rdp->cpu, rcp);spin_unlock_bh (&rcp->lock); +rcu_move_batch (this_rdp, rdp->donelist, rdp->donetail);rcu_move_batch (this_rdp, rdp->curlist, rdp->curtail);rcu_move_batch (this_rdp, rdp->nxtlist, rdp->nxttail); -rcu_move_batch (this_rdp, rdp->donelist, rdp->donetail); }static void rcu_offline_cpu (int cpu)
kernel/rcupdate.c +10 -0
@@ -115,7 +115,17 @@ void rcu_barrier(void)mutex_lock (&rcu_barrier_mutex);init_completion (&rcu_barrier_completion);atomic_set (&rcu_barrier_cpu_count,0 ); +/* +* The queueing of callbacks in all CPUs must be atomic with +* respect to RCU, otherwise one CPU may queue a callback, +* wait for a grace period, decrement barrier count and call +* complete(), while other CPUs have not yet queued anything. +* So, we need to make sure that grace periods cannot complete +* until all the callbacks are queued. +*/ +rcu_read_lock ();on_each_cpu (rcu_barrier_func,NULL ,0 ,1 ); +rcu_read_unlock ();wait_for_completion (&rcu_barrier_completion);mutex_unlock (&rcu_barrier_mutex); }
RFC PATCH 3/8 RCU: Preemptible RCU
Work in progress, not for inclusion.
This patch implements a new version of RCU which allows its read-side critical sections to be preempted. It uses a set of counter pairs to keep track of the read-side critical sections and flips them when all tasks exit read-side critical section. The details of this implementation can be found in this paper -
http://www.rdrop.com/users/paulmck/RCU/OLSrtRCU.2006.08.11a.pdf
This patch was developed as a part of the -rt kernel development and meant to provide better latencies when read-side critical sections of RCU don't disable preemption. As a consequence of keeping track of RCU readers, the readers have a slight overhead (optimizations in the paper). This implementation co-exists with the "classic" RCU implementations and can be switched to at compiler.
Also includes RCU tracing summarized in debugfs and RCU_SOFTIRQ for the preemptible variant of RCU.
Signed-off-by Dipankar Sarma <[email protected]>
Signed-off-by Steven Rostedt <[email protected]> (for RCU_SOFTIRQ)
Signed-off-by Paul McKenney <[email protected]>
---
include/linux/interrupt.h | 1 include/linux/rcuclassic.h | 2 include/linux/rcupdate.h | 7 include/linux/rcupreempt.h | 78 +++ include/linux/rcupreempt_trace.h | 100 +++++ include/linux/sched.h | 5 kernel/Kconfig.preempt | 38 + kernel/Makefile | 7 kernel/fork.c | 4 kernel/rcupreempt.c | 767 +++++++++++++++++++++++++++++++++++++++ kernel/rcupreempt_trace.c | 330 ++++++++++++++++ 11 files changed, 1336 insertions(+), 3 deletions(-)
include/linux/interrupt.h +1 -0
@@ -269,6 +269,7 @@ enum#ifdef CONFIG_HIGH_RES_TIMERS HRTIMER_SOFTIRQ , #endif + RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */ };/* softirq mask and active fields moved to irq_cpustat_t in
include/linux/rcuclassic.h +0 -2
@@ -142,8 +142,6 @@ extern int rcu_needs_cpu(int cpu);extern void __rcu_init (void );extern void rcu_check_callbacks (int cpu,int user);extern void rcu_restart_cpu (int cpu); -extern long rcu_batches_completed (void ); -extern long rcu_batches_completed_bh (void );#endif /* __KERNEL__ */ #endif /* __LINUX_RCUCLASSIC_H */
include/linux/rcupdate.h +7 -0
@@ -52,7 +52,11 @@ struct rcu_head {void (*func)(struct rcu_head *head); }; +#ifdef CONFIG_CLASSIC_RCU #include <linux/rcuclassic.h> +#else /* #ifdef CONFIG_CLASSIC_RCU */ +#include <linux/rcupreempt.h> +#endif /* #else #ifdef CONFIG_CLASSIC_RCU */ #define RCU_HEAD_INIT { .next = NULL, .func = NULL }#define RCU_HEAD (head) struct rcu_head head = RCU_HEAD_INIT
@@ -218,10 +222,13 @@ extern void FASTCALL(call_rcu_bh(struct/* Exported common interfaces */ extern void synchronize_rcu (void );extern void rcu_barrier (void ); +extern long rcu_batches_completed (void ); +extern long rcu_batches_completed_bh (void );/* Internal to kernel */ extern void rcu_init (void );extern void rcu_check_callbacks (int cpu,int user); +extern int rcu_needs_cpu (int cpu);#endif /* __KERNEL__ */ #endif /* __LINUX_RCUPDATE_H */
include/linux/rcupreempt.h +78 -0
78 lines not on this page · raw message
include/linux/rcupreempt_trace.h +100 -0
100 lines not on this page · raw message
include/linux/sched.h +5 -0
@@ -850,6 +850,11 @@ struct task_struct {cpumask_t cpus_allowed;unsigned int time_slice, first_time_slice; +#ifdef CONFIG_PREEMPT_RCU +int rcu_read_lock_nesting; +int rcu_flipctr_idx; +#endif /* #ifdef CONFIG_PREEMPT_RCU */ +#if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_TASK_DELAY_ACCT )struct sched_info sched_info;#endif
kernel/fork.c +4 -0
@@ -1032,6 +1032,10 @@ static struct task_struct *copy_process(INIT_LIST_HEAD (&p->children);INIT_LIST_HEAD (&p->sibling); +#ifdef CONFIG_PREEMPT_RCU + p->rcu_read_lock_nesting =0 ; + p->rcu_flipctr_idx =0 ; +#endif /* #ifdef CONFIG_PREEMPT_RCU */ p->vfork_done =NULL ;spin_lock_init (&p->alloc_lock);
kernel/Kconfig.preempt +38 -0
@@ -63,3 +63,41 @@ config PREEMPT_BKL Say Y here if you are building a kernel for a desktop system. Say N if you are unsure. +choice + prompt "RCU implementation type:" + default CLASSIC_RCU + +config CLASSIC_RCU + bool "Classic RCU" + help + This option selects the classic RCU implementation that is + designed for best read-side performance on non-realtime + systems. + + Say Y if you are unsure. + +config PREEMPT_RCU + bool "Preemptible RCU" + depends on PREEMPT + help + This option reduces the latency of the kernel by making certain + RCU sections preemptible. Normally RCU code is non-preemptible, if + this option is selected then read-only RCU sections become + preemptible. This helps latency, but may expose bugs due to + now-naive assumptions about each RCU read-side critical section + remaining on a given CPU through its execution. + + Say N if you are unsure. + +endchoice + +config RCU_TRACE + bool "Enable tracing for RCU - currently stats in debugfs" + select DEBUG_FS + default y + help + This option provides tracing in RCU which presents stats + in debugfs for debugging RCU implementation. + + Say Y here if you want to enable RCU tracing + Say N if you are unsure.
kernel/Makefile +6 -1
@@ -6,7 +6,7 @@ obj-y = sched.o fork.o exec_domain.o exit.o itimer.o time.o softirq.o resource.o \ sysctl.o capability.o ptrace.o timer.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o \ - rcupdate.o rcuclassic.o extable.o params.o posix-timers.o \ + rcupdate.o extable.o params.o posix-timers.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o latency.o nsproxy.o srcu.o die_notifier.o
@@ -46,6 +46,11 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softl obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o +obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o +obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o +ifeq ($(CONFIG_PREEMPT_RCU),y) +obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o +endif obj-$(CONFIG_RELAY) += relay.o obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_UTS_NS) += utsname.o
kernel/rcupreempt.c +767 -0
767 lines not on this page · raw message
kernel/rcupreempt_trace.c +330 -0
330 lines not on this page · raw message
RFC PATCH 4/8 RCU: synchronize_sched() workaround for CPU hotplug
Work in progress, not for inclusion.
The combination of CPU hotplug and PREEMPT_RCU has resulted in deadlocks due to the migration-based implementation of synchronize_sched() in -rt. This experimental patch maps synchronize_sched() back onto Classic RCU, eliminating the migration, thus hopefully also eliminating the deadlocks. It is not clear that this is a good long-term approach, but it will at least permit people doing CPU hotplug in -rt kernels additional wiggle room in their design and implementation.
The basic approach is to cause the -rt kernel to incorporate rcuclassic.c as well as rcupreempt.c, but to #ifdef out the conflicting portions of rcuclassic.c so that only the code needed to implement synchronize_sched() remains in a PREEMPT_RT build. Invocations of grace-period detection from the scheduling-clock interrupt go to rcuclassic.c, which then invokes the corresponding functions in rcupreempt.c (with _rt suffix added to keep the linker happy). Also applies the RCU_SOFTIRQ to classic RCU. The bulk of this patch just moves code around, but likely increases scheduling-clock latency.
If this patch does turn out to be the right approach, the #ifdefs in kernel/rcuclassic.c might be dealt with. ;-) At current writing, Gautham Shenoy's most recent CPU-hotplug fixes seem likely to obsolete this patch (which would be a very good thing indeed!).
Signed-off-by Steven Rostedt <[email protected]> (for RCU_SOFTIRQ)
Signed-off-by Paul E. McKenney <[email protected]>
---
include/linux/rcuclassic.h | 79 +++++-------------------------------- include/linux/rcupdate.h | 30 ++++++++++++-- include/linux/rcupreempt.h | 27 ++++++------ kernel/Makefile | 2 kernel/rcuclassic.c | 95 ++++++++++++++++++++++++++++++++++++--------- kernel/rcupdate.c | 22 ++++++++-- kernel/rcupreempt.c | 50 +++++------------------ 7 files changed, 158 insertions(+), 147 deletions(-)
include/linux/rcuclassic.h +12 -67
@@ -42,80 +42,19 @@#include <linux/cpumask.h> #include <linux/seqlock.h> - -/* Global control variables for rcupdate callback mechanism. */ -struct rcu_ctrlblk { -long cur;/* Current batch number. */ -long completed;/* Number of the last completed batch */ -int next_pending;/* Is the next batch already waiting? */ - -int signaled; - -spinlock_t lock____cacheline_internodealigned_in_smp ; -cpumask_t cpumask;/* CPUs that need to switch in order */ -/* for current batch to proceed. */ -}____cacheline_internodealigned_in_smp ; - -/* Is batch a before batch b ? */ -static inline int rcu_batch_before (long a,long b) -{ -return (a - b) <0 ; -} - -/* Is batch a after batch b ? */ -static inline int rcu_batch_after (long a,long b) -{ -return (a - b) >0 ; -} +DECLARE_PER_CPU (int, rcu_data_bh_passed_quiesc);/* -* Per-CPU data for Read-Copy UPdate. -* nxtlist - new callbacks are added here -* curlist - current batch for which quiescent cycle started if any -*/ -struct rcu_data { -/* 1) quiescent state handling : */ -long quiescbatch;/* Batch # for grace period */ -int passed_quiesc;/* User-mode/idle loop etc. */ -int qs_pending;/* core waits for quiesc state */ - -/* 2) batch handling */ -long batch;/* Batch # for current RCU batch */ -struct rcu_head *nxtlist; -struct rcu_head **nxttail; -long qlen;/* # of queued callbacks */ -struct rcu_head *curlist; -struct rcu_head **curtail; -struct rcu_head *donelist; -struct rcu_head **donetail; -long blimit;/* Upper limit on a processed batch */ -int cpu; -struct rcu_head barrier; -}; - -DECLARE_PER_CPU(struct rcu_data, rcu_data); -DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); - -/* -* Increment the quiescent state counter. +* Increment the bottom-half quiescent state counter. * The counter is a bit degenerated: We do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period. Thus just a flag. */ -static inline void rcu_qsctr_inc (int cpu) -{ -struct rcu_data *rdp = &per_cpu (rcu_data, cpu); - rdp->passed_quiesc =1 ; -}static inline void rcu_bh_qsctr_inc (int cpu) { -struct rcu_data *rdp = &per_cpu (rcu_bh_data, cpu); - rdp->passed_quiesc =1 ; +per_cpu (rcu_data_bh_passed_quiesc, cpu) =1 ; } -extern int rcu_pending (int cpu); -extern int rcu_needs_cpu (int cpu); -#define __rcu_read_lock () \ do { \ preempt_disable(); \
@@ -139,9 +78,15 @@ extern int rcu_needs_cpu(int cpu);#define __synchronize_sched () synchronize_rcu() -extern void __rcu_init (void ); -extern void rcu_check_callbacks (int cpu,int user); -extern void rcu_restart_cpu (int cpu); +#define rcu_advance_callbacks_rt (cpu, user) do { } while (0) +#define rcu_check_callbacks_rt (cpu, user) do { } while (0) +#define rcu_init_rt () do { } while (0) +#define rcu_needs_cpu_rt (cpu) 0 +#define rcu_pending_rt (cpu) 0 +#define rcu_process_callbacks_rt (unused) do { } while (0) + +extern void FASTCALL (call_rcu_classic (struct rcu_head *head, +void (*func)(struct rcu_head *head)));#endif /* __KERNEL__ */ #endif /* __LINUX_RCUCLASSIC_H */
include/linux/rcupdate.h +26 -4
@@ -197,8 +197,11 @@ struct rcu_head {* delimited by rcu_read_lock() and rcu_read_unlock(), * and may be nested. */ -extern void FASTCALL (call_rcu (struct rcu_head *head, -void (*func)(struct rcu_head *head))); +#ifdef CONFIG_CLASSIC_RCU +#define call_rcu (head, func) call_rcu_classic(head, func) +#else /* #ifdef CONFIG_CLASSIC_RCU */ +#define call_rcu (head, func) call_rcu_preempt(head, func) +#endif /* #else #ifdef CONFIG_CLASSIC_RCU */ /** * call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
@@ -226,9 +229,28 @@ extern long rcu_batches_completed(void);extern long rcu_batches_completed_bh (void );/* Internal to kernel */ -extern void rcu_init (void );extern void rcu_check_callbacks (int cpu,int user); -extern int rcu_needs_cpu (int cpu); +extern long rcu_batches_completed (void ); +extern long rcu_batches_completed_bh (void ); +extern void rcu_check_callbacks (int cpu,int user); +extern void rcu_init (void ); +extern int rcu_needs_cpu (int cpu); +extern int rcu_pending (int cpu); +struct softirq_action ; +extern void rcu_restart_cpu (int cpu); + +DECLARE_PER_CPU (int, rcu_data_passed_quiesc); + +/* +* Increment the quiescent state counter. +* The counter is a bit degenerated: We do not need to know +* how many quiescent states passed, just if there was at least +* one since the start of the grace period. Thus just a flag. +*/ +static inline void rcu_qsctr_inc (int cpu) +{ +per_cpu (rcu_data_passed_quiesc, cpu) =1 ; +}#endif /* __KERNEL__ */ #endif /* __LINUX_RCUPDATE_H */
include/linux/rcupreempt.h +14 -13
@@ -42,25 +42,26 @@#include <linux/cpumask.h> #include <linux/seqlock.h> -#define rcu_qsctr_inc (cpu) -#define rcu_bh_qsctr_inc (cpu)#define call_rcu_bh (head, rcu) call_rcu(head, rcu) - -extern void __rcu_read_lock (void ); -extern void __rcu_read_unlock (void ); -extern int rcu_pending (int cpu); -extern int rcu_needs_cpu (int cpu); - +#define rcu_bh_qsctr_inc (cpu) do { } while (0)#define __rcu_read_lock_bh () { rcu_read_lock(); local_bh_disable(); }#define __rcu_read_unlock_bh () { local_bh_enable(); rcu_read_unlock(); } -#define __rcu_read_lock_nesting () (current->rcu_read_lock_nesting) +extern void FASTCALL (call_rcu_classic (struct rcu_head *head, +void (*func)(struct rcu_head *head))); +extern void FASTCALL (call_rcu_preempt (struct rcu_head *head, +void (*func)(struct rcu_head *head))); +extern void __rcu_read_lock (void ); +extern void __rcu_read_unlock (void );extern void __synchronize_sched (void ); - -extern void __rcu_init (void ); -extern void rcu_check_callbacks (int cpu,int user); -extern void rcu_restart_cpu (int cpu); +extern void rcu_advance_callbacks_rt (int cpu,int user); +extern void rcu_check_callbacks_rt (int cpu,int user); +extern void rcu_init_rt (void ); +extern int rcu_needs_cpu_rt (int cpu); +extern int rcu_pending_rt (int cpu); +struct softirq_action ; +extern void rcu_process_callbacks_rt (struct softirq_action *unused);#ifdef CONFIG_RCU_TRACE struct rcupreempt_trace ;
kernel/Makefile +1 -1
@@ -47,7 +47,7 @@ obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o -obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o +obj-$(CONFIG_PREEMPT_RCU) += rcuclassic.o rcupreempt.o ifeq ($(CONFIG_PREEMPT_RCU),y) obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o endif
kernel/rcuclassic.c +77 -18
220 lines not on this page · raw message
kernel/rcupdate.c +17 -5
@@ -52,6 +52,7 @@ struct rcu_synchronize {struct completion completion; }; +DEFINE_PER_CPU (int, rcu_data_passed_quiesc); static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};static atomic_t rcu_barrier_cpu_count;static DEFINE_MUTEX (rcu_barrier_mutex);
@@ -88,6 +89,22 @@ void synchronize_rcu(void) }EXPORT_SYMBOL_GPL (synchronize_rcu); +#ifdef CONFIG_PREEMPT_RCU + +/* +* Map synchronize_sched() to the classic RCU implementation. +*/ +void __synchronize_sched (void ) +{ +struct rcu_synchronize rcu; + +init_completion (&rcu.completion); +call_rcu_classic (&rcu.head, wakeme_after_rcu); +wait_for_completion (&rcu.completion); +} +EXPORT_SYMBOL_GPL (__synchronize_sched); +#endif /* #ifdef CONFIG_PREEMPT_RCU */ +static void rcu_barrier_callback (struct rcu_head *notused) {if (atomic_dec_and_test (&rcu_barrier_cpu_count))
@@ -131,8 +148,3 @@ void rcu_barrier(void)wait_for_completion (&rcu_barrier_completion);mutex_unlock (&rcu_barrier_mutex); } - -void __init rcu_init (void ) -{ -__rcu_init (); -}
kernel/rcupreempt.c +11 -39
115 lines not on this page · raw message
RFC PATCH 5/8 RCU: CPU hotplug support for preemptible RCU
Work in progress, not for inclusion.
This patch allows preemptible RCU to tolerate CPU-hotplug operations. It accomplishes this by maintaining a local copy of a map of online CPUs, which it accesses under its own lock.
Signed-off-by Paul E. McKenney <[email protected]>
---
include/linux/rcuclassic.h | 2 include/linux/rcupreempt.h | 2 kernel/rcuclassic.c | 8 +++ kernel/rcupreempt.c | 93 +++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 100 insertions(+), 5 deletions(-)
include/linux/rcuclassic.h +2 -0
@@ -143,6 +143,8 @@ extern int rcu_needs_cpu(int cpu);#define rcu_check_callbacks_rt (cpu, user)#define rcu_init_rt ()#define rcu_needs_cpu_rt (cpu) 0 +#define rcu_offline_cpu_rt (cpu) +#define rcu_online_cpu_rt (cpu)#define rcu_pending_rt (cpu) 0#define rcu_process_callbacks_rt (unused)
include/linux/rcupreempt.h +2 -0
@@ -59,6 +59,8 @@ extern void rcu_advance_callbacks_rt(intextern void rcu_check_callbacks_rt (int cpu,int user);extern void rcu_init_rt (void );extern int rcu_needs_cpu_rt (int cpu); +extern void rcu_offline_cpu_rt (int cpu); +extern void rcu_online_cpu_rt (int cpu);extern int rcu_pending_rt (int cpu);struct softirq_action ;extern void rcu_process_callbacks_rt (struct softirq_action *unused);
kernel/rcuclassic.c +7 -1
@@ -414,14 +414,19 @@ static void __rcu_offline_cpu(struct rcustatic void rcu_offline_cpu (int cpu) {struct rcu_data *this_rdp = &get_cpu_var (rcu_data); +#ifdef CONFIG_CLASSIC_RCU struct rcu_data *this_bh_rdp = &get_cpu_var (rcu_bh_data); +#endif /* #ifdef CONFIG_CLASSIC_RCU */ __rcu_offline_cpu (this_rdp, &rcu_ctrlblk, &per_cpu (rcu_data, cpu)); +#ifdef CONFIG_CLASSIC_RCU __rcu_offline_cpu (this_bh_rdp, &rcu_bh_ctrlblk, &per_cpu (rcu_bh_data, cpu)); -put_cpu_var (rcu_data);put_cpu_var (rcu_bh_data); +#endif /* #ifdef CONFIG_CLASSIC_RCU */ +put_cpu_var (rcu_data); +rcu_offline_cpu_rt (cpu); }#else
@@ -571,6 +576,7 @@ static void __devinit rcu_online_cpu(int rdp->passed_quiesc = &per_cpu (rcu_data_passed_quiesc, cpu);rcu_init_percpu_data (cpu, &rcu_bh_ctrlblk, bh_rdp); bh_rdp->passed_quiesc = &per_cpu (rcu_data_bh_passed_quiesc, cpu); +rcu_online_cpu_rt (cpu); } static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
kernel/rcupreempt.c +89 -4
129 lines not on this page · raw message
RFC PATCH 6/8 RCU priority boosting for preemptible RCU
Work in progress, not for inclusion.
RCU priority boosting is needed when running a workload that might include CPU-bound user tasks running at realtime priorities with preemptible RCU. In this situation, RCU priority boosting is needed to avoid OOM.
Please note that because Classic RCU does not permit RCU read-side critical sections to be preempted, there is no need to boost the priority of Classic RCU readers. Boosting the priority of a running process does not make it run any faster, at least not on any hardware that I am aware of. ;-)
Signed-off-by Paul E. McKenney <[email protected]>
---
include/linux/init_task.h | 13 include/linux/rcupdate.h | 17 + include/linux/rcupreempt.h | 20 + include/linux/sched.h | 24 + init/main.c | 1 kernel/Kconfig.preempt | 14 - kernel/fork.c | 6 kernel/rcupreempt.c | 608 ++++++++++++++++++++++++++++++++++++++++++--- kernel/rtmutex.c | 7 kernel/sched.c | 5 lib/Kconfig.debug | 34 ++ 11 files changed, 703 insertions(+), 46 deletions(-)
include/linux/init_task.h +13 -0
@@ -87,6 +87,17 @@ extern struct nsproxy init_nsproxy; .signalfd_list = LIST_HEAD_INIT(sighand.signalfd_list), \ } +#ifdef CONFIG_PREEMPT_RCU_BOOST +#define INIT_RCU_BOOST_PRIO .rcu_prio = MAX_PRIO, +#define INIT_PREEMPT_RCU_BOOST (tsk) \ + .rcub_rbdp = NULL, \ + .rcub_state = RCU_BOOST_IDLE, \ + .rcub_entry = LIST_HEAD_INIT(tsk.rcub_entry), +#else /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ +#define INIT_RCU_BOOST_PRIO +#define INIT_PREEMPT_RCU_BOOST (tsk) +#endif /* #else #ifdef CONFIG_PREEMPT_RCU_BOOST */ +extern struct group_info init_groups;#define INIT_STRUCT_PID { \
@@ -125,6 +136,7 @@ extern struct group_info init_groups; .prio = MAX_PRIO-20, \ .static_prio = MAX_PRIO-20, \ .normal_prio =MAX_PRIO -20 , \ +INIT_RCU_BOOST_PRIO \ .policy =SCHED_NORMAL , \ .cpus_allowed = CPU_MASK_ALL, \ .mm = NULL, \
@@ -169,6 +181,7 @@ extern struct group_info init_groups; }, \INIT_TRACE_IRQFLAGS \ INIT_LOCKDEP \ + INIT_PREEMPT_RCU_BOOST(tsk) \ }
include/linux/rcupdate.h +17 -0
@@ -252,5 +252,22 @@ static inline void rcu_qsctr_inc(int cpuper_cpu (rcu_data_passed_quiesc, cpu) =1 ; } +#ifdef CONFIG_PREEMPT_RCU_BOOST +extern void init_rcu_boost_late (void ); +extern void __rcu_preempt_boost (void ); +#define rcu_preempt_boost ()/* cpp to avoid #include hell. */ \ + do { \ + if (unlikely(current->rcu_read_lock_nesting > 0)) \ + __rcu_preempt_boost(); \ + } while (0) +#else /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ +static inline void init_rcu_boost_late (void ) +{ +} +static inline void rcu_preempt_boost (void ) +{ +} +#endif /* #else #ifdef CONFIG_PREEMPT_RCU_BOOST */ +#endif /* __KERNEL__ */ #endif /* __LINUX_RCUPDATE_H */
include/linux/rcupreempt.h +20 -0
@@ -42,6 +42,26 @@#include <linux/cpumask.h> #include <linux/seqlock.h> +#ifdef CONFIG_PREEMPT_RCU_BOOST +/* +* Task state with respect to being RCU-boosted. This state is changed +* by the task itself in response to the following three events: +* 1. Preemption (or block on lock) while in RCU read-side critical section. +* 2. Outermost rcu_read_unlock() for blocked RCU read-side critical section. +* +* The RCU-boost task also updates the state when boosting priority. +*/ +enum rcu_boost_state { +RCU_BOOST_IDLE =0 ,/* Not yet blocked if in RCU read-side. */ +RCU_BOOST_BLOCKED =1 ,/* Blocked from RCU read-side. */ +RCU_BOOSTED =2 ,/* Boosting complete. */ +RCU_BOOST_INVALID =3 ,/* For bogus state sightings. */ +}; + +#define N_RCU_BOOST_STATE (RCU_BOOST_INVALID + 1) + +#endif /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ +#define call_rcu_bh (head, rcu) call_rcu(head, rcu)#define rcu_bh_qsctr_inc (cpu) do { } while (0)#define __rcu_read_lock_bh () { rcu_read_lock(); local_bh_disable(); }
include/linux/sched.h +24 -0
@@ -546,6 +546,22 @@ struct signal_struct {#define is_rt_policy (p) ((p) != SCHED_NORMAL && (p) != SCHED_BATCH)#define has_rt_policy (p) unlikely(is_rt_policy((p)->policy)) +#ifdef CONFIG_PREEMPT_RCU_BOOST +#define set_rcu_prio (p, prio)/* cpp to avoid #include hell */ \ + do { \ + (p)->rcu_prio = (prio); \ + } while (0) +#define get_rcu_prio (p) (p)->rcu_prio/* cpp to avoid #include hell */ +#else /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ +static inline void set_rcu_prio (struct task_struct *p,int prio) +{ +} +static inline int get_rcu_prio (struct task_struct *p) +{ +return MAX_PRIO ; +} +#endif /* #else #ifdef CONFIG_PREEMPT_RCU_BOOST */ +/* * Some day this will be a full-fledged user tracking system.. */
@@ -834,6 +850,9 @@ struct task_struct {#endif int load_weight;/* for niceness load balancing purposes */ int prio, static_prio, normal_prio; +#ifdef CONFIG_PREEMPT_RCU_BOOST +int rcu_prio; +#endif /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ struct list_head run_list;struct prio_array *array;
@@ -858,6 +877,11 @@ struct task_struct {#if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_TASK_DELAY_ACCT )struct sched_info sched_info;#endif +#ifdef CONFIG_PREEMPT_RCU_BOOST +struct rcu_boost_dat *rcub_rbdp; +enum rcu_boost_state rcub_state; +struct list_head rcub_entry; +#endif /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ struct list_head tasks;/*
init/main.c +1 -0
@@ -722,6 +722,7 @@ static void __init do_basic_setup(void)driver_init ();init_irq_proc ();do_initcalls (); +init_rcu_boost_late (); }static void __init do_pre_smp_initcalls (void )
kernel/fork.c +6 -0
@@ -1036,6 +1036,12 @@ static struct task_struct *copy_process( p->rcu_read_lock_nesting =0 ; p->rcu_flipctr_idx =0 ;#endif /* #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_PREEMPT_RCU_BOOST + p->rcu_prio =MAX_PRIO ; + p->rcub_rbdp =NULL ; + p->rcub_state =RCU_BOOST_IDLE ; +INIT_LIST_HEAD (&p->rcub_entry); +#endif /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ p->vfork_done =NULL ;spin_lock_init (&p->alloc_lock);
kernel/Kconfig.preempt +7 -7
@@ -91,13 +91,13 @@ config PREEMPT_RCU endchoice -config RCU_TRACE - bool "Enable tracing for RCU - currently stats in debugfs" - select DEBUG_FS - default y +config PREEMPT_RCU_BOOST + bool "Enable priority boosting of RCU read-side critical sections" + depends on PREEMPT_RCU + default n help - This option provides tracing in RCU which presents stats - in debugfs for debugging RCU implementation. + This option permits priority boosting of RCU read-side critical + sections that have been preempted in order to prevent indefinite + delay of grace periods in face of runaway non-realtime processes. - Say Y here if you want to enable RCU tracing Say N if you are unsure.
kernel/rcupreempt.c +572 -36
773 lines not on this page · raw message
kernel/rtmutex.c +4 -3
@@ -111,11 +111,12 @@ static inline void mark_rt_mutex_waiters*/ int rt_mutex_getprio (struct task_struct *task) { +int prio =min (task->normal_prio,get_rcu_prio (task)); +if (likely (!task_has_pi_waiters (task))) -return task->normal_prio; +return prio; -return min (task_top_pi_waiter (task)->pi_list_entry.prio, - task->normal_prio); +return min (task_top_pi_waiter (task)->pi_list_entry.prio, prio); }/*
kernel/sched.c +5 -0
@@ -1702,6 +1702,7 @@ void fastcall sched_fork(struct task_str* Make sure we do not leak PI boosting priority to the child: */ p->prio = current->normal_prio; +set_rcu_prio (p,MAX_PRIO );INIT_LIST_HEAD (&p->run_list); p->array =NULL ;
@@ -1784,6 +1785,7 @@ void fastcall wake_up_new_task(struct taelse { p->prio = current->prio; p->normal_prio = current->normal_prio; +set_rcu_prio (p,MAX_PRIO );list_add_tail (&p->run_list, ¤t->run_list); p->array = current->array; p->array->nr_active++;
@@ -3590,6 +3592,8 @@ asmlinkage void __sched schedule(void) }profile_hit (SCHED_PROFILING ,__builtin_return_address (0 )); +rcu_preempt_boost (); + need_resched:preempt_disable (); prev = current;
@@ -5060,6 +5064,7 @@ void __cpuinit init_idle(struct task_str idle->sleep_avg =0 ; idle->array =NULL ; idle->prio = idle->normal_prio =MAX_PRIO ; +set_rcu_prio (idle,MAX_PRIO ); idle->state =TASK_RUNNING ; idle->cpus_allowed =cpumask_of_cpu (cpu);set_task_cpu (idle, cpu);
lib/Kconfig.debug +34 -0
@@ -391,6 +391,40 @@ config RCU_TORTURE_TEST Say M if you want the RCU torture tests to build as a module. Say N if you are unsure. +config RCU_TRACE + bool "Enable tracing for RCU - currently stats in debugfs" + select DEBUG_FS + depends on DEBUG_KERNEL + default y + help + This option provides tracing in RCU which presents stats + in debugfs for debugging RCU implementation. + + Say Y here if you want to enable RCU tracing + Say N if you are unsure. + +config PREEMPT_RCU_BOOST_STATS + bool "Enable RCU priority-boosting statistic printing" + depends on PREEMPT_RCU_BOOST + depends on DEBUG_KERNEL + default n + help + This option enables debug printk()s of RCU boost statistics, + which are normally only used to debug RCU priority boost + implementations. + + Say N if you are unsure. + +config PREEMPT_RCU_BOOST_STATS_INTERVAL + int "RCU priority-boosting statistic printing interval (seconds)" + depends on PREEMPT_RCU_BOOST_STATS + default 100 + range 10 86400 + help + This option controls the timing of debug printk()s of RCU boost + statistics, which are normally only used to debug RCU priority + boost implementations. + config LKDTM tristate "Linux Kernel Dump Test Tool Module" depends on DEBUG_KERNEL
RFC PATCH 7/8 RCU: rcutorture testing for RCU priority boosting
Work in progress, not for inclusion. Still uses xtime because this patch is still against 2.6.22.
This patch modifies rcutorture to also torture RCU priority boosting. The torturing involves forcing RCU read-side critical sections (already performed as part of the torturing of RCU) to run for extremely long time periods, increasing the probability of their being preempted and thus needing priority boosting. The fact that rcutorture's "nreaders" module parameter defaults to twice the number of CPUs helps ensure lots of the needed preemption.
Signed-off-by Paul E. McKenney <[email protected]>
---
rcutorture.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 14 deletions(-)
kernel/rcutorture.c +77 -14
@@ -58,6 +58,7 @@ static int stat_interval; /* Interval bestatic int verbose;/* Print more debug info. */ static int test_no_idle_hz;/* Test RCU's support for tickless idle CPUs. */ static int shuffle_interval =5 ;/* Interval between shuffles (in sec)*/ +static int preempt_torture;/* Realtime task preempts torture readers. */ static char *torture_type ="rcu" ;/* What RCU implementation to torture. */ module_param (nreaders, int,0444 );
@@ -72,6 +73,8 @@ module_param(test_no_idle_hz, bool, 0444MODULE_PARM_DESC (test_no_idle_hz,"Test support for tickless idle CPUs" );module_param (shuffle_interval, int,0444 );MODULE_PARM_DESC (shuffle_interval,"Number of seconds between shuffles" ); +module_param (preempt_torture, bool,0444 ); +MODULE_PARM_DESC (preempt_torture,"Enable realtime preemption torture" );module_param (torture_type, charp,0444 );MODULE_PARM_DESC (torture_type,"Type of RCU to torture (rcu, rcu_bh, srcu)" );
@@ -194,6 +197,8 @@ struct rcu_torture_ops {int (*completed)(void );void (*deferredfree)(struct rcu_torture *p);void (*sync)(void ); +long (*preemptstart)(void ); +void (*preemptend)(void );int (*stats)(char *page);char *name; };
@@ -258,16 +263,75 @@ static void rcu_torture_deferred_free(stcall_rcu (&p->rtort_rcu, rcu_torture_cb); } +static struct task_struct *rcu_preeempt_task; +static unsigned long rcu_torture_preempt_errors; + +static int rcu_torture_preempt (void *arg) +{ +int completedstart; +int err; +time_t gcstart; +struct sched_param sp; + + sp.sched_priority =MAX_RT_PRIO -1 ; + err =sched_setscheduler (current,SCHED_RR , &sp); +if (err !=0 ) +printk (KERN_ALERT "rcu_torture_preempt() priority err: %d\n" , + err); + current->flags |=PF_NOFREEZE ; + +do { + completedstart =rcu_torture_completed (); + gcstart = xtime.tv_sec; +while ((xtime.tv_sec - gcstart <10 ) && + (rcu_torture_completed () == completedstart)) +cond_resched (); +if (rcu_torture_completed () == completedstart) + rcu_torture_preempt_errors++; +schedule_timeout_interruptible (HZ ); + }while (!kthread_should_stop ()); +return 0 ; +} + +static long rcu_preempt_start (void ) +{ +long retval =0 ; + + rcu_preeempt_task =kthread_run (rcu_torture_preempt,NULL , +"rcu_torture_preempt" ); +if (IS_ERR (rcu_preeempt_task)) { +VERBOSE_PRINTK_ERRSTRING ("Failed to create preempter" ); + retval =PTR_ERR (rcu_preeempt_task); + rcu_preeempt_task =NULL ; + } +return retval; +} + +static void rcu_preempt_end (void ) +{ +if (rcu_preeempt_task !=NULL ) { +VERBOSE_PRINTK_STRING ("Stopping rcu_preempt task" ); +kthread_stop (rcu_preeempt_task); + } + rcu_preeempt_task =NULL ; +} + +static int rcu_preempt_stats (char *page) +{ +return sprintf (page, +"Preemption stalls: %lu\n" , rcu_torture_preempt_errors); +} +static struct rcu_torture_ops rcu_ops = { - .init =NULL , - .cleanup =NULL , .readlock = rcu_torture_read_lock, .readdelay = rcu_read_delay, .readunlock = rcu_torture_read_unlock, .completed = rcu_torture_completed, .deferredfree = rcu_torture_deferred_free, .sync = synchronize_rcu, - .stats =NULL , + .preemptstart = rcu_preempt_start, + .preemptend = rcu_preempt_end, + .stats = rcu_preempt_stats, .name ="rcu" };
@@ -299,14 +363,12 @@ static void rcu_sync_torture_init(void)static struct rcu_torture_ops rcu_sync_ops = { .init = rcu_sync_torture_init, - .cleanup =NULL , .readlock = rcu_torture_read_lock, .readdelay = rcu_read_delay, .readunlock = rcu_torture_read_unlock, .completed = rcu_torture_completed, .deferredfree = rcu_sync_torture_deferred_free, .sync = synchronize_rcu, - .stats =NULL , .name ="rcu_sync" };
@@ -358,28 +420,23 @@ static void rcu_bh_torture_synchronize(v }static struct rcu_torture_ops rcu_bh_ops = { - .init =NULL , - .cleanup =NULL , .readlock = rcu_bh_torture_read_lock, .readdelay = rcu_read_delay,/* just reuse rcu's version. */ .readunlock = rcu_bh_torture_read_unlock, .completed = rcu_bh_torture_completed, .deferredfree = rcu_bh_torture_deferred_free, .sync = rcu_bh_torture_synchronize, - .stats =NULL , .name ="rcu_bh" };static struct rcu_torture_ops rcu_bh_sync_ops = { .init = rcu_sync_torture_init, - .cleanup =NULL , .readlock = rcu_bh_torture_read_lock, .readdelay = rcu_read_delay,/* just reuse rcu's version. */ .readunlock = rcu_bh_torture_read_unlock, .completed = rcu_bh_torture_completed, .deferredfree = rcu_sync_torture_deferred_free, .sync = rcu_bh_torture_synchronize, - .stats =NULL , .name ="rcu_bh_sync" };
@@ -491,14 +548,12 @@ static void sched_torture_synchronize(vostatic struct rcu_torture_ops sched_ops = { .init = rcu_sync_torture_init, - .cleanup =NULL , .readlock = sched_torture_read_lock, .readdelay = rcu_read_delay,/* just reuse rcu's version. */ .readunlock = sched_torture_read_unlock, .completed = sched_torture_completed, .deferredfree = rcu_sync_torture_deferred_free, .sync = sched_torture_synchronize, - .stats =NULL , .name ="sched" };
@@ -793,9 +848,10 @@ rcu_torture_print_module_parms(char *tagprintk (KERN_ALERT "%s" TORTURE_FLAG "--- %s: nreaders=%d nfakewriters=%d " "stat_interval=%d verbose=%d test_no_idle_hz=%d " -"shuffle_interval = %d\n" , +"shuffle_interval=%d preempt_torture=%d\n" , torture_type, tag, nrealreaders, nfakewriters, - stat_interval, verbose, test_no_idle_hz, shuffle_interval); + stat_interval, verbose, test_no_idle_hz, shuffle_interval, + preempt_torture); } static void
@@ -848,6 +904,8 @@ rcu_torture_cleanup(void)kthread_stop (stats_task); } stats_task =NULL ; +if (preempt_torture && (cur_ops->preemptend !=NULL )) + cur_ops->preemptend ();/* Wait for all RCU callbacks to fire. */ rcu_barrier ();
@@ -990,6 +1048,11 @@ rcu_torture_init(void) goto unwind; } } +if (preempt_torture && (cur_ops->preemptstart !=NULL )) { + firsterr = cur_ops->preemptstart (); +if (firsterr !=0 ) + goto unwind; + }return 0 ; unwind:
RFC PATCH 8/8 RCU: Make RCU priority boosting consume less power
Work in progress, not for inclusion.
This patch modified the RCU priority booster to explicitly sleep when there are no RCU readers in need of priority boosting. This should be a power-consumption improvement over the one-second polling cycle in the underlying RCU priority-boosting patch.
Signed-off-by Paul E. McKenney <[email protected]>
---
include/linux/rcupreempt.h | 15 ++++++ kernel/rcupreempt.c | 102 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 2 deletions(-)
include/linux/rcupreempt.h +15 -0
@@ -60,6 +60,21 @@ enum rcu_boost_state {#define N_RCU_BOOST_STATE (RCU_BOOST_INVALID + 1) +/* +* RCU-booster state with respect to sleeping. The RCU booster +* sleeps when no task has recently been seen sleeping in an RCU +* read-side critical section, and is awakened when a new sleeper +* appears. +*/ +enum rcu_booster_state { +RCU_BOOSTER_ACTIVE =0 ,/* RCU booster actively scanning. */ +RCU_BOOSTER_DROWSY =1 ,/* RCU booster is considering sleeping. */ +RCU_BOOSTER_SLEEPING =2 ,/* RCU booster is asleep. */ +RCU_BOOSTER_INVALID =3 ,/* For bogus state sightings. */ +}; + +#define N_RCU_BOOSTER_STATE (RCU_BOOSTER_INVALID + 1) +#endif /* #ifdef CONFIG_PREEMPT_RCU_BOOST */ #define call_rcu_bh (head, rcu) call_rcu(head, rcu)
kernel/rcupreempt.c +100 -2
192 lines not on this page · raw message