Skip to contentSkip to conversation

random RCU fixes and updates

PATCH In review Paul E. McKenney 2 participants 11 messages Sep 23, 2010

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover random RCU fixes and updates

Hello!

This patchset contains fixes for problems identified in -next testing and some statistics and documentation updates. The patches are as follows:

1. Update stallwarn.txt to include stall warnings caused by
real-time processes (yes, even in mainline kernels).
2. Fix some performance bugs in TINY_PREEMPT_RCU.
3. Fix an overactive rcu_bootup_announce_oddness() warning.
4. Eliminate a redundant #ifdef (from Christian Dietrich).
5. Fix the new sparse-aware primitives to correctly handle
macro arguments with side effects.
6. Fix some issues sparse found in rcutorture.c.
7. Add some tracing data to help compute estimates of how long
RCU callbacks are queued.
8. Fix up the Kconfig message for PROVE_RCU_REPEATEDLY.

For a testing-only version of this patchset from git, please see:

git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-2.6-rcu.git rcu/testing

Thanx, Paul

 Documentation/RCU/stallwarn.txt |   18 ++++++++++++++++++
 Documentation/RCU/trace.txt     |   13 ++++++++++++-
 include/linux/rcupdate.h        |   14 +++++++++++---
 kernel/rcutiny_plugin.h         |   12 ++++++++----
 kernel/rcutorture.c             |    8 +++++---
 kernel/rcutree.c                |    3 +++
 kernel/rcutree.h                |    3 +++
 kernel/rcutree_plugin.h         |    2 +-
 kernel/rcutree_trace.c          |   10 +++++++---
 kernel/srcu.c                   |    2 --
 lib/Kconfig.debug               |    4 ++++
 11 files changed, 72 insertions(+), 17 deletions(-)
#1 lore raw

PATCH 1/8 rcu: upgrade stallwarn.txt documentation for CPU-bound RT processes

CPU-bound real-time processes can cause RCU CPU stall warnings, and much other trouble as well. Document the fact that they can cause RCU CPU stall warnings.

---
 Documentation/RCU/stallwarn.txt |   18 ++++++++++++++++++
 1 files changed, 18 insertions(+), 0 deletions(-)
Documentation/RCU/stallwarn.txt +18 -0
index 44c6dcc..862c08e 100644
@@ -80,6 +80,24 @@ o	A CPU looping with bottom halves disabled.  This condition can
 o	For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the kernel
 	without invoking schedule().
 
+o	A CPU-bound real-time task in a CONFIG_PREEMPT kernel, which might
+	happen to preempt a low-priority task in the middle of an RCU
+	read-side critical section.   This is especially damaging if
+	that low-priority task is not permitted to run on any other CPU,
+	in which case the next RCU grace period can never complete, which
+	will eventually cause the system to run out of memory and hang.
+	While the system is in the process of running itself out of
+	memory, you might see stall-warning messages.
+
+o	A CPU-bound real-time task in a CONFIG_PREEMPT_RT kernel that
+	is running at a higher priority than the RCU softirq threads.
+	This will prevent RCU callbacks from ever being invoked,
+	and in a CONFIG_TREE_PREEMPT_RCU kernel will further prevent
+	RCU grace periods from ever completing.  Either way, the
+	system will eventually run out of memory and hang.  In the
+	CONFIG_TREE_PREEMPT_RCU case, you might see stall-warning
+	messages.
+
 o	A bug in the RCU implementation.
 
 o	A hardware failure.  This is quite unlikely, but has occurred
Signature · 1 line
1.7.0.6
#2 lore raw

PATCH 2/8 rcu: performance fixes to TINY_PREEMPT_RCU callback checking

In reply to Paul E. McKenney

This commit tightens up checks in rcu_preempt_check_callbacks() to avoid unnecessary special handling at rcu_read_unlock() time.

---
 kernel/rcutiny_plugin.h |   12 ++++++++----
 1 files changed, 8 insertions(+), 4 deletions(-)
kernel/rcutiny_plugin.h +8 -4
index c5bea11..6ceca4f 100644
@@ -72,7 +72,7 @@ static void rcu_report_exp_done(void);
 /*
  * Return true if the CPU has not yet responded to the current grace period.
  */
-static int rcu_cpu_cur_gp(void)
+static int rcu_cpu_blocking_cur_gp(void)
 {
 	return rcu_preempt_ctrlblk.gpcpu != rcu_preempt_ctrlblk.gpnum;
 }
@@ -229,7 +229,7 @@ void rcu_preempt_note_context_switch(void)
 		 * cannot end.
 		 */
 		list_add(&t->rcu_node_entry, &rcu_preempt_ctrlblk.blkd_tasks);
-		if (rcu_cpu_cur_gp())
+		if (rcu_cpu_blocking_cur_gp())
 			rcu_preempt_ctrlblk.gp_tasks = &t->rcu_node_entry;
 	}
 
@@ -368,12 +368,16 @@ static void rcu_preempt_check_callbacks(void)
 {
 	struct task_struct *t = current;
 
-	if (!rcu_preempt_running_reader() && rcu_preempt_gp_in_progress())
+	if (rcu_preempt_gp_in_progress() &&
+	    (!rcu_preempt_running_reader() ||
+	     !rcu_cpu_blocking_cur_gp()))
 		rcu_preempt_cpu_qs();
 	if (&rcu_preempt_ctrlblk.rcb.rcucblist !=
 	    rcu_preempt_ctrlblk.rcb.donetail)
 		raise_softirq(RCU_SOFTIRQ);
-	if (rcu_preempt_gp_in_progress() && rcu_preempt_running_reader())
+	if (rcu_preempt_gp_in_progress() &&
+	    rcu_cpu_blocking_cur_gp() &&
+	    rcu_preempt_running_reader())
 		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
 }
 
Signature · 1 line
1.7.0.6
#3 lore raw

PATCH 3/8 rcu: fix _oddness handling of verbose stall warnings

In reply to Paul E. McKenney

CONFIG_RCU_CPU_STALL_VERBOSE depends on CONFIG_TREE_PREEMPT_RCU, but rcu_bootup_announce_oddness() complains if CONFIG_RCU_CPU_STALL_VERBOSE is not set even in the case of CONFIG_TREE_RCU. This commit therefore fixes rcu_bootup_announce_oddness() to avoid insisting on impossibilities.

---
 kernel/rcutree_plugin.h |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
kernel/rcutree_plugin.h +1 -1
index e9e0bc7..71a4147 100644
@@ -57,7 +57,7 @@ static void __init rcu_bootup_announce_oddness(void)
 	printk(KERN_INFO
 	       "\tRCU-based detection of stalled CPUs is disabled.\n");
 #endif
-#ifndef CONFIG_RCU_CPU_STALL_VERBOSE
+#if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
 	printk(KERN_INFO "\tVerbose stalled-CPUs detection is disabled.\n");
 #endif
 #if NUM_RCU_LVL_4 != 0
Signature · 1 line
1.7.0.6
#4 lore raw

PATCH 4/8 kernel: Removing undead ifdef CONFIG_DEBUG_LOCK_ALLOC

In reply to Paul E. McKenney

From: Christian Dietrich <[email protected]>

The CONFIG_DEBUG_LOCK_ALLOC ifdef isn't necessary at this point, because it is checked in an outer ifdef level already and has no effect here.

---
 kernel/srcu.c |    2 --
 1 files changed, 0 insertions(+), 2 deletions(-)
kernel/srcu.c +0 -2
index 2980da3..c71e075 100644
@@ -46,11 +46,9 @@ static int init_srcu_struct_fields(struct srcu_struct *sp)
 int __init_srcu_struct(struct srcu_struct *sp, const char *name,
 		       struct lock_class_key *key)
 {
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
 	/* Don't re-initialize a lock while it is held. */
 	debug_check_no_locks_freed((void *)sp, sizeof(*sp));
 	lockdep_init_map(&sp->dep_map, name, key, 0);
-#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 	return init_srcu_struct_fields(sp);
 }
 EXPORT_SYMBOL_GPL(__init_srcu_struct);
Signature · 1 line
1.7.0.6
#5 lore raw

PATCH 5/8 rcu: only one evaluation of arg in rcu_dereference_check() unless sparse

In reply to Paul E. McKenney

The current version of the __rcu_access_pointer(), __rcu_dereference_check(), and __rcu_dereference_protected() macros evaluate their "p" argument three times, not counting typeof()s. This is bad news if that argument contains a side effect. This commit therefore evaluates this argument only once in normal kernel builds. However, the straightforward approach defeats sparse's RCU-pointer checking, so when __CHECKER__ is defined, the additional pair of evaluations of the "p" argument are performed in order to permit sparse to detect misuse of RCU-protected pointers.

---
 include/linux/rcupdate.h |   14 +++++++++++---
 1 files changed, 11 insertions(+), 3 deletions(-)
include/linux/rcupdate.h +11 -3
index 89414d6..03cda7b 100644
@@ -310,24 +310,32 @@ extern int rcu_my_thread_group_empty(void);
  * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
  * the future.
  */
+
+#ifdef __CHECKER__
+#define rcu_dereference_sparse(p, space) \
+	((void)(((typeof(*p) space *)p) == p))
+#else /* #ifdef __CHECKER__ */
+#define rcu_dereference_sparse(p, space)
+#endif /* #else #ifdef __CHECKER__ */
+
 #define __rcu_access_pointer(p, space) \
 	({ \
 		typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
-		(void) (((typeof (*p) space *)p) == p); \
+		rcu_dereference_sparse(p, space); \
 		((typeof(*p) __force __kernel *)(_________p1)); \
 	})
 #define __rcu_dereference_check(p, c, space) \
 	({ \
 		typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
 		rcu_lockdep_assert(c); \
-		(void) (((typeof (*p) space *)p) == p); \
+		rcu_dereference_sparse(p, space); \
 		smp_read_barrier_depends(); \
 		((typeof(*p) __force __kernel *)(_________p1)); \
 	})
 #define __rcu_dereference_protected(p, c, space) \
 	({ \
 		rcu_lockdep_assert(c); \
-		(void) (((typeof (*p) space *)p) == p); \
+		rcu_dereference_sparse(p, space); \
 		((typeof(*p) __force __kernel *)(p)); \
 	})
 
Signature · 1 line
1.7.0.6
Acked-by on v1 5/8
#6 lore raw

Re: [PATCH tip/core/rcu 5/8] rcu: only one evaluation of arg in rcu_dereference_check() unless sparse

Quoted 11 lines · Paul E. McKenney

The current version of the __rcu_access_pointer(), __rcu_dereference_check(), and __rcu_dereference_protected() macros evaluate their "p" argument three times, not counting typeof()s. This is bad news if that argument contains a side effect. This commit therefore evaluates this argument only once in normal kernel builds. However, the straightforward approach defeats sparse's RCU-pointer checking, so when __CHECKER__ is defined, the additional pair of evaluations of the "p" argument are performed in order to permit sparse to detect misuse of RCU-protected pointers.

Sorry for not following up with the alternative approach yet. We can always do that later, for your patch should go into 2.6.36 to fix any problems arising from unintended macro side-effects.

#7 lore raw

Re: [PATCH tip/core/rcu 5/8] rcu: only one evaluation of arg in rcu_dereference_check() unless sparse

Quoted 10 lines · Arnd Bergmann
Quoted 9 lines · Paul E. McKenney

The current version of the __rcu_access_pointer(), __rcu_dereference_check(), and __rcu_dereference_protected() macros evaluate their "p" argument three times, not counting typeof()s. This is bad news if that argument contains a side effect. This commit therefore evaluates this argument only once in normal kernel builds. However, the straightforward approach defeats sparse's RCU-pointer checking, so when __CHECKER__ is defined, the additional pair of evaluations of the "p" argument are performed in order to permit sparse to detect misuse of RCU-protected pointers.

Quoted 8 lines
Quoted 2 lines

Sorry for not following up with the alternative approach yet. We can always do that later, for your patch should go into 2.6.36 to fix any problems arising from unintended macro side-effects.

No problem, when you get me a better approach, I will happily queue it!

Thanx, Paul

#8 lore raw

PATCH 6/8 rcu: fix sparse errors in rcutorture.c

In reply to Paul E. McKenney

Add the sparse __rcu address-space identifier and make a couple of variables static.

---
 kernel/rcutorture.c |    8 +++++---
 1 files changed, 5 insertions(+), 3 deletions(-)
kernel/rcutorture.c +5 -3
index 7297102..9afa047 100644
@@ -120,7 +120,7 @@ struct rcu_torture {
 };
 
 static LIST_HEAD(rcu_torture_freelist);
-static struct rcu_torture *rcu_torture_current;
+static struct rcu_torture __rcu *rcu_torture_current;
 static long rcu_torture_current_version;
 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 static DEFINE_SPINLOCK(rcu_torture_lock);
@@ -153,7 +153,8 @@ int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
 #define FULLSTOP_SHUTDOWN 1	/* System shutdown with rcutorture running. */
 #define FULLSTOP_RMMOD    2	/* Normal rmmod of rcutorture. */
 static int fullstop = FULLSTOP_RMMOD;
-DEFINE_MUTEX(fullstop_mutex);	/* Protect fullstop transitions and spawning */
+static DEFINE_MUTEX(fullstop_mutex);
+				/* Protect fullstop transitions and spawning */
 				/*  of kthreads. */
 
 /*
@@ -737,7 +738,8 @@ rcu_torture_writer(void *arg)
 			continue;
 		rp->rtort_pipe_count = 0;
 		udelay(rcu_random(&rand) & 0x3ff);
-		old_rp = rcu_torture_current;
+		old_rp = rcu_dereference_check(rcu_torture_current,
+					       current == writer_task);
 		rp->rtort_mbtest = 1;
 		rcu_assign_pointer(rcu_torture_current, rp);
 		smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
Signature · 1 line
1.7.0.6
#9 lore raw

PATCH 7/8 rcu: Add tracing data to support queueing models

In reply to Paul E. McKenney

The current tracing data is not sufficient to deduce the average time that a callback spends waiting for a grace period to end. Add three per-CPU counters recording the number of callbacks invoked (ci), the number of callbacks orphaned (co), and the number of callbacks adopted (ca). Given the existing callback queue length (ql), the average wait time in absence of CPU hotplug operations is ql/ci. The units of wait time will be in terms of the duration over which ci was measured.

In the presence of CPU hotplug operations, there is room for argument, but ql/(ci-co+ca) won't steer you too far wrong.

Also fixes a typo called out by Lucas De Marchi <[email protected]>.

---
 Documentation/RCU/trace.txt |   13 ++++++++++++-
 kernel/rcutree.c            |    3 +++
 kernel/rcutree.h            |    3 +++
 kernel/rcutree_trace.c      |   10 +++++++---
 4 files changed, 25 insertions(+), 4 deletions(-)
Documentation/RCU/trace.txt +12 -1
index efd8cc9..a851118 100644
@@ -125,6 +125,17 @@ o	"b" is the batch limit for this CPU.  If more than this number
 	of RCU callbacks is ready to invoke, then the remainder will
 	be deferred.
 
+o	"ci" is the number of RCU callbacks that have been invoked for
+	this CPU.  Note that ci+ql is the number of callbacks that have
+	been registered in absence of CPU-hotplug activity.
+
+o	"co" is the number of RCU callbacks that have been orphaned due to
+	this CPU going offline.
+
+o	"ca" is the number of RCU callbacks that have been adopted due to
+	other CPUs going offline.  Note that ci+co-ca+ql is the number of
+	RCU callbacks registered on this CPU.
+
 There is also an rcu/rcudata.csv file with the same information in
 comma-separated-variable spreadsheet format.
 
@@ -180,7 +191,7 @@ o	"s" is the "signaled" state that drives force_quiescent_state()'s
 
 o	"jfq" is the number of jiffies remaining for this grace period
 	before force_quiescent_state() is invoked to help push things
-	along.  Note that CPUs in dyntick-idle mode thoughout the grace
+	along.  Note that CPUs in dyntick-idle mode throughout the grace
 	period will not report on their own, but rather must be check by
 	some other CPU via force_quiescent_state().
 
kernel/rcutree.c +3 -0
index 42140a8..e750735 100644
@@ -1004,6 +1004,7 @@ static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
 	for (i = 0; i < RCU_NEXT_SIZE; i++)
 		rdp->nxttail[i] = &rdp->nxtlist;
 	rsp->orphan_qlen += rdp->qlen;
+	rdp->n_cbs_orphaned += rdp->qlen;
 	rdp->qlen = 0;
 	raw_spin_unlock(&rsp->onofflock);  /* irqs remain disabled. */
 }
@@ -1025,6 +1026,7 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
 	*rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_list;
 	rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_tail;
 	rdp->qlen += rsp->orphan_qlen;
+	rdp->n_cbs_adopted += rsp->orphan_qlen;
 	rsp->orphan_cbs_list = NULL;
 	rsp->orphan_cbs_tail = &rsp->orphan_cbs_list;
 	rsp->orphan_qlen = 0;
@@ -1156,6 +1158,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
 
 	/* Update count, and requeue any remaining callbacks. */
 	rdp->qlen -= count;
+	rdp->n_cbs_invoked += count;
 	if (list != NULL) {
 		*tail = rdp->nxtlist;
 		rdp->nxtlist = list;
kernel/rcutree.h +3 -0
index 7918ba6..91d4170 100644
@@ -202,6 +202,9 @@ struct rcu_data {
 	long		qlen;		/* # of queued callbacks */
 	long		qlen_last_fqs_check;
 					/* qlen at last check for QS forcing */
+	unsigned long	n_cbs_invoked;	/* count of RCU cbs invoked. */
+	unsigned long	n_cbs_orphaned;	/* RCU cbs sent to orphanage. */
+	unsigned long	n_cbs_adopted;	/* RCU cbs adopted from orphanage. */
 	unsigned long	n_force_qs_snap;
 					/* did other CPU force QS recently? */
 	long		blimit;		/* Upper limit on a processed batch */
kernel/rcutree_trace.c +7 -3
index 458e032..d15430b 100644
@@ -64,7 +64,9 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 		   rdp->dynticks_fqs);
 #endif /* #ifdef CONFIG_NO_HZ */
 	seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
-	seq_printf(m, " ql=%ld b=%ld\n", rdp->qlen, rdp->blimit);
+	seq_printf(m, " ql=%ld b=%ld", rdp->qlen, rdp->blimit);
+	seq_printf(m, " ci=%lu co=%lu ca=%lu\n",
+		   rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
 }
 
 #define PRINT_RCU_DATA(name, func, m) \
@@ -119,7 +121,9 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
 		   rdp->dynticks_fqs);
 #endif /* #ifdef CONFIG_NO_HZ */
 	seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
-	seq_printf(m, ",%ld,%ld\n", rdp->qlen, rdp->blimit);
+	seq_printf(m, ",%ld,%ld", rdp->qlen, rdp->blimit);
+	seq_printf(m, ",%lu,%lu,%lu\n",
+		   rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
 }
 
 static int show_rcudata_csv(struct seq_file *m, void *unused)
@@ -128,7 +132,7 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
 #ifdef CONFIG_NO_HZ
 	seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
 #endif /* #ifdef CONFIG_NO_HZ */
-	seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\"\n");
+	seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\",\"ci\",\"co\",\"ca\"\n");
 #ifdef CONFIG_TREE_PREEMPT_RCU
 	seq_puts(m, "\"rcu_preempt:\"\n");
 	PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m);
Signature · 1 line
1.7.0.6
#10 lore raw

PATCH 8/8 rcu: Add advice to PROVE_RCU_REPEATEDLY kernel config parameter

In reply to Paul E. McKenney

The PROVE_RCU_REPEATEDLY has no "Say Y"/"Say N" advice, so this commit adds it.

---
 lib/Kconfig.debug |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
lib/Kconfig.debug +4 -0
index 52c2172..9886cf5 100644
@@ -539,6 +539,10 @@ config PROVE_RCU_REPEATEDLY
 	 disabling, allowing multiple RCU-lockdep warnings to be printed
 	 on a single reboot.
 
+	 Say Y to allow multiple RCU-lockdep warnings per boot.
+
+	 Say N if you are unsure.
+
 config SPARSE_RCU_POINTER
 	bool "RCU debugging: sparse-based checks for pointer usage"
 	default n
Signature · 1 line
1.7.0.6